From 68f9bebccbba17672976e70e5ff27c5a3caf3b2e Mon Sep 17 00:00:00 2001 From: Ahmad Bilal Khalid Date: Thu, 11 Jul 2019 13:34:21 +0500 Subject: [PATCH 01/10] Shutdown VM, ListUserFiles added. Able to parse units included in specs --- Pipfile | 1 + Pipfile.lock | 75 +++++++++++++++++++++++++++---------------------- main.py | 49 +++++++++++++++++++++++++++++++- specs_parser.py | 25 +++++++++++++++++ 4 files changed, 115 insertions(+), 35 deletions(-) create mode 100644 specs_parser.py diff --git a/Pipfile b/Pipfile index daeed8a..273e7a6 100644 --- a/Pipfile +++ b/Pipfile @@ -13,6 +13,7 @@ flask = "*" flask-restful = "*" etcd3 = "*" gunicorn = "*" +bitmath = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 86ac1a7..6c93789 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "87f5447c7fa8d96dacc7c638075cc31842ef676f6d1c35214e1960572d44e929" + "sha256": "fa257b2f0b607e37897d62a624b656fdf219f74dc0a943f481ba6ef0d1a92c90" }, "pipfile-spec": 6, "requires": { @@ -23,6 +23,13 @@ ], "version": "==7.0.0" }, + "bitmath": { + "hashes": [ + "sha256:293325f01e65defe966853111df11d39215eb705a967cb115851da8c4cfa3eb8" + ], + "index": "pypi", + "version": "==1.3.3.1" + }, "certifi": { "hashes": [ "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", @@ -69,40 +76,40 @@ }, "grpcio": { "hashes": [ - "sha256:0232add03144dd3cf9b660e2718244cb8e175370dca4d3855cb4e489a7811b53", - "sha256:0f20e6dcb1b8662cdca033bb97c0a8116a5343e3ebc7f71c5fe7f89039978350", - "sha256:10b07a623d33d4966f45c85d410bc6a79c5ac6341f06c3beda6c22be12cbfe07", - "sha256:10c0476d5a52d21f402fc073745dc43b87cc8e080a1f49bbff4e1059019310fb", - "sha256:289dae0b35c59d191c524e976dd0a6f8c995d2062e72621eb866ad0f4472a635", - "sha256:2be726f16142d358a0df1e81d583d6820ee561a7856a79cca2fbe49989308be7", - "sha256:4338d2a81f5b4ca022e085040b3cfce19419a5ce44aa7e6810ac1df05365bed7", - "sha256:4c535b46f20e66bee3097583231977e721acdfcb1671d1490c99b7be8902ce18", - "sha256:557154aef70a0e979700cc9528bc8b606b668084a29a0d57dbc4b06b078a2f1c", - "sha256:5bfdd7e6647498f979dc46583723c852d97b25afe995d55aa1c76a5f9816bc1f", - "sha256:87d8943ae7aa6ca5bbad732867d7f17d2550e4966a0c15b52088e8b579422e47", - "sha256:89d8719d8de4d137678f7caa979e1b0a6fd4026f8096ceef8c2d164bbabefaf2", - "sha256:9c3f4af989ce860710ac1864dc2e867dd87e6cee51a2368df1b253596868e52f", - "sha256:9da52c3c728883aee429bb7c315049f50b2139f680cd86bb1165418e4f93a982", - "sha256:9e9736659987beab42d18525ed10d21f80a1ba8389eac03425fbfd5684e6bbf0", - "sha256:9ebcbb1a054cab362d29d3be571d43d6b9b23302d9fc4b43e5327000da1680a9", - "sha256:a93e08636623e24c939851e2e0c0140b14f524b2980c9cdc4ea52b70a871c7e0", - "sha256:ac322d86d1a079e0a118d544443ee16f320af0062c191b4754c0c6ec2fc79310", - "sha256:b1fb101459868f52df6b61e7bb13375e50badf17a160e39fe1d51ae19e53f461", - "sha256:b39aac96cceac624a23d540473835086a3ffa77c91030189988c073488434493", - "sha256:b65507bc273c6dbf539175a786a344cc0ac78d50e5584f72c6599733f8a3301f", - "sha256:be5bb6e47417e537c884a2e2ff2e1a8b2c064a998fcfdfcc67528d4e63e7ebaf", - "sha256:c92de6a28a909c4f460dc1bbbcb50d676cf0b1f40224b222761f73fdd851b522", - "sha256:c9f5962eb7fa7607b20eb0e4f59ed35829bd600fc0eacb626a6db83229a3e445", - "sha256:d00bdf9c546ed6e649f785c55b05288e8b2dbb6bf2eb74b6c579fa0d591d35bd", - "sha256:da804b1dd8293bd9d61b1e6ea989c887ba042a808a4fbdd80001cfa059aafed2", - "sha256:ead6c5aa3e807345913649c3be395aaca2bbb2d225f18b8f31f37eab225508f6", - "sha256:eb4d81550ce6f826af4ec6e8d98be347fe96291d718bf115c3f254621ae8d98d", - "sha256:ef6a18ec8fd32ec81748fe720544ea2fb2d2dc50fd6d06739d5e2eb8f0626a1c", - "sha256:fad42835656e0b6d3b7ffc900598e776722e30f43b7234a48f2576ca30f31a47", - "sha256:fb98dbfee0d963b49ae5754554028cf62e6bd695f22de16d242ba9d2f0b7339b", - "sha256:fb9cd9bb8d26dc17c2dd715a46bca3a879ec8283879b164e85863110dc6e3b2a" + "sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2", + "sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce", + "sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb", + "sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e", + "sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941", + "sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e", + "sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce", + "sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf", + "sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f", + "sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303", + "sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2", + "sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf", + "sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e", + "sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369", + "sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87", + "sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e", + "sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97", + "sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2", + "sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369", + "sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67", + "sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174", + "sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f", + "sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b", + "sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e", + "sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0", + "sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad", + "sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa", + "sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03", + "sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1", + "sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348", + "sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530", + "sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc" ], - "version": "==1.21.1" + "version": "==1.22.0" }, "gunicorn": { "hashes": [ diff --git a/main.py b/main.py index 8aa1470..8cba8e1 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,5 @@ # TODO # convert etcd3 usage to etcd3_wrapper - import etcd3 import json @@ -10,6 +9,7 @@ from flask_restful import Resource, Api, reqparse from decouple import config from uuid import uuid4 from etcd3_wrapper import Etcd3Wrapper +from specs_parser import SpecsParser app = Flask(__name__) api = Api(app) @@ -45,6 +45,9 @@ add_vmid_args(add_otp_args(startvm_argparser)) uservm_argparser = reqparse.RequestParser() add_otp_args(uservm_argparser) +# Specs parser +specs_parser = SpecsParser(exceptional_devices=["cpu"]) + def is_image_valid(image_uuid): images = client.get_prefix("/v1/image/") @@ -61,6 +64,10 @@ class CreateVM(Resource): if check_otp(name, realm, token) == 200: # User is good if is_image_valid(image_uuid): + if not specs_parser.transform_specs(specs): + return {"message": f"""Invalid unit - Please use following units {specs_parser.get_allowed_units()}"""}, 400 + + print(specs) vm_key = f"/v1/vm/{uuid4().hex}" vm_entry = {"owner": name, "specs": specs, @@ -205,6 +212,23 @@ class ResumeVM(Resource): return {"message": "Invalid Credentials"}, 400 +class ShutdownVM(Resource): + def post(self): + args = startvm_argparser.parse_args() + name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid + + if check_otp(name, realm, token) == 200: + vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) + if vm: + vm.value["status"] = "REQUESTED_SHUTDOWN" + client.put(vm.key, json.dumps(vm.value)) + return {"message": f"VM Shutdown Queued"} + else: + return {"message": "No such VM found"} + else: + return {"message": "Invalid Credentials"}, 400 + + class ListUserVM(Resource): def get(self): args = uservm_argparser.parse_args() @@ -228,17 +252,40 @@ class ListUserVM(Resource): return {"message": "Invalid Credentials"}, 400 +class ListUserFiles(Resource): + def get(self): + args = uservm_argparser.parse_args() + name, realm, token = args.name, args.realm, args.token + + if check_otp(name, realm, token) == 200: + files = client.get_prefix(f"/v1/files/", value_in_json=True) + if files: + return_files = [] + user_files = list(filter(lambda f: f.value["owner"] == name, files)) + for file in user_files: + return_files.append({ + "filename": vm.value["filename"] + }) + return {"message": return_files}, 200 + else: + return {"message": "No File found"}, 404 + else: + return {"message": "Invalid Credentials"}, 400 + api.add_resource(CreateVM, "/vm/create") api.add_resource(DeleteVM, "/vm/delete") api.add_resource(VmStatus, "/vm/status") + api.add_resource(StartVM, "/vm/start") api.add_resource(SuspendVM, "/vm/suspend") api.add_resource(ResumeVM, "/vm/resume") +api.add_resource(ShutdownVM, "/vm/shutdown") api.add_resource(CreateImage, "/image/create") api.add_resource(ListPublicImages, "/image/list-public") api.add_resource(ListUserVM, "/user/vms") +api.add_resource(ListUserFiles, "/user/files") if __name__ == "__main__": app.run(host="::", debug=True) diff --git a/specs_parser.py b/specs_parser.py new file mode 100644 index 0000000..0cd9894 --- /dev/null +++ b/specs_parser.py @@ -0,0 +1,25 @@ +import bitmath + +class SpecsParser(object): + def __init__(self, exceptional_devices, allowed_unit = 10): + self.exceptional_devices = exceptional_devices + self.allowed_unit = allowed_unit + + def transform_specs(self, specs): + try: + for device in filter(lambda x: x not in self.exceptional_devices, specs): + parsed = bitmath.parse_string_unsafe(specs[device]) + if parsed.base != self.allowed_unit: + return False + specs[device] = int(parsed.to_Byte()) + return True + except ValueError as _: + return False + + def get_allowed_units(self): + if self.allowed_unit == 10: + unit_prefix = bitmath.SI_PREFIXES + else: + unit_prefix = bitmath.NIST_PREFIXES + + return list(map(lambda u: u.upper() + "B", unit_prefix)) From c474282b3372b75d50efd5c559f2a6ee62b44a29 Mon Sep 17 00:00:00 2001 From: Ahmad Bilal Khalid Date: Thu, 18 Jul 2019 17:10:17 +0500 Subject: [PATCH 02/10] implemented add host, authenticate user before performing action of vm --- Pipfile | 1 + helper.py | 14 ++--- main.py | 144 +++++++++++++++++++++++++++++++++++++----------- specs_parser.py | 3 +- 4 files changed, 123 insertions(+), 39 deletions(-) diff --git a/Pipfile b/Pipfile index 273e7a6..bd5f667 100644 --- a/Pipfile +++ b/Pipfile @@ -14,6 +14,7 @@ flask-restful = "*" etcd3 = "*" gunicorn = "*" bitmath = "*" +pylint = "*" [requires] python_version = "3.7" diff --git a/helper.py b/helper.py index 4b580f1..9d3a254 100644 --- a/helper.py +++ b/helper.py @@ -8,22 +8,22 @@ from pyotp import TOTP def check_otp(name, realm, token): try: data = { - "auth_name": config('AUTH_NAME', ''), - "auth_token": TOTP(config('AUTH_SEED', '')).now(), - "auth_realm": config('AUTH_REALM', ''), + "auth_name": config("AUTH_NAME", ""), + "auth_token": TOTP(config("AUTH_SEED", "")).now(), + "auth_realm": config("AUTH_REALM", ""), "name": name, "realm": realm, - "token": token + "token": token, } except binascii.Error: return 400 response = requests.post( "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( - OTP_SERVER=config('OTP_SERVER', ''), - OTP_VERIFY_ENDPOINT=config('OTP_VERIFY_ENDPOINT', 'verify/') + OTP_SERVER=config("OTP_SERVER", ""), + OTP_VERIFY_ENDPOINT=config("OTP_VERIFY_ENDPOINT", "verify/"), ), - data=data + data=data, ) return response.status_code diff --git a/main.py b/main.py index 8cba8e1..5617eb3 100644 --- a/main.py +++ b/main.py @@ -1,6 +1,7 @@ # TODO -# convert etcd3 usage to etcd3_wrapper -import etcd3 +# 1. Allow user of realm ungleich-admin to perform any action on +# any user vm. + import json from helper import check_otp, add_otp_args, add_vmid_args @@ -10,11 +11,11 @@ from decouple import config from uuid import uuid4 from etcd3_wrapper import Etcd3Wrapper from specs_parser import SpecsParser +from functools import wraps app = Flask(__name__) api = Api(app) -etcd_client = etcd3.client(host=config("ETCD_HOST"), port=int(config("ETCD_PORT"))) client = Etcd3Wrapper() # CreateVM argparser @@ -48,6 +49,12 @@ add_otp_args(uservm_argparser) # Specs parser specs_parser = SpecsParser(exceptional_devices=["cpu"]) +# AddHost argparser +addhost_argparser = reqparse.RequestParser() +addhost_argparser.add_argument("hostname", type=str, required=True) +addhost_argparser.add_argument("specs", type=dict, required=True) +add_otp_args(addhost_argparser) + def is_image_valid(image_uuid): images = client.get_prefix("/v1/image/") @@ -57,25 +64,36 @@ def is_image_valid(image_uuid): class CreateVM(Resource): def post(self): createvm_args = createvm_argparser.parse_args() - name, realm, token, specs, image_uuid = createvm_args.name, createvm_args.realm,\ - createvm_args.token, createvm_args.specs,\ - createvm_args.image_uuid + name, realm, token, specs, image_uuid = ( + createvm_args.name, + createvm_args.realm, + createvm_args.token, + createvm_args.specs, + createvm_args.image_uuid, + ) if check_otp(name, realm, token) == 200: # User is good if is_image_valid(image_uuid): if not specs_parser.transform_specs(specs): - return {"message": f"""Invalid unit - Please use following units {specs_parser.get_allowed_units()}"""}, 400 + return ( + { + "message": f"""Invalid unit - Please use following units {specs_parser.get_allowed_units()}""" + }, + 400, + ) print(specs) vm_key = f"/v1/vm/{uuid4().hex}" - vm_entry = {"owner": name, - "specs": specs, - "hostname": "", - "status": "REQUESTED_NEW", - "image_uuid": image_uuid} + vm_entry = { + "owner": name, + "specs": specs, + "hostname": "", + "status": "REQUESTED_NEW", + "image_uuid": image_uuid, + } - etcd_client.put(vm_key, json.dumps(vm_entry)) + client.put(vm_key, vm_entry, value_in_json=True) return {"message": "VM Creation Queued"}, 200 else: @@ -87,18 +105,23 @@ class CreateVM(Resource): class DeleteVM(Resource): def post(self): deletevm_args = deletevm_argparser.parse_args() - name, realm, token, vmid = deletevm_args.name, deletevm_args.realm,\ - deletevm_args.token, deletevm_args.vmid + name, realm, token, vmid = ( + deletevm_args.name, + deletevm_args.realm, + deletevm_args.token, + deletevm_args.vmid, + ) if check_otp(name, realm, token) == 200: # User is good - vmentry_etcd = etcd_client.get(f"/v1/vm/{vmid}")[0] + vmentry_etcd = client.get(f"/v1/vm/{vmid}").value if vmentry_etcd: vmentry_etcd = json.loads(vmentry_etcd) vmentry_etcd["status"] = "REQUESTED_DELETE" - etcd_client.put(f"/v1/vm/{vmid}", json.dumps(vmentry_etcd)) + client.put(f"/v1/vm/{vmid}", vmentry_etcd, + value_in_json=True) return {"message": "VM Deletion Queued"}, 200 else: @@ -110,7 +133,7 @@ class DeleteVM(Resource): class VmStatus(Resource): def get(self): args = vmstatus_argparser.parse_args() - r = etcd_client.get(f"/v1/vm/{args.vmid}")[0] + r = client.get(f"/v1/vm/{args.vmid}").value print(r) if r: r = dict(json.loads(r.decode("utf-8"))) @@ -127,13 +150,18 @@ class CreateImage(Resource): file_entry = client.get(f"/v1/files/{image_file_uuid}") if file_entry is None: - return { - "Message": - f"Image File with uuid '{image_file_uuid}' Not Found"}, 400 + return ( + {"Message": f"Image File with uuid '{image_file_uuid}' Not Found"}, + 400, + ) file_entry_value = json.loads(file_entry.value) - image_store = list(filter(lambda s: json.loads(s.value)["name"] == image_store_name, image_stores)) + image_store = list( + filter( + lambda s: json.loads(s.value)["name"] == image_store_name, image_stores + ) + ) if not image_store: return {"Message": f"Store '{image_store_name}' does not exists"}, 400 @@ -144,7 +172,7 @@ class CreateImage(Resource): "filename": file_entry_value["filename"], "name": args.name, "store_name": image_store_name, - "visibility": "public" + "visibility": "public", } client.put(f"/v1/image/{image_file_uuid}", json.dumps(image_entry_json)) @@ -168,6 +196,8 @@ class StartVM(Resource): if check_otp(name, realm, token) == 200: vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) + if vm.value["owner"] != name: + return {"message": "Invalid User"} if vm: vm.value["status"] = "REQUESTED_START" client.put(vm.key, json.dumps(vm.value)) @@ -185,6 +215,8 @@ class SuspendVM(Resource): if check_otp(name, realm, token) == 200: vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) + if vm.value["owner"] != name: + return {"message": "Invalid User"} if vm: vm.value["status"] = "REQUESTED_SUSPEND" client.put(vm.key, json.dumps(vm.value)) @@ -202,6 +234,8 @@ class ResumeVM(Resource): if check_otp(name, realm, token) == 200: vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) + if vm.value["owner"] != name: + return {"message": "Invalid User"} if vm: vm.value["status"] = "REQUESTED_RESUME" client.put(vm.key, json.dumps(vm.value)) @@ -219,6 +253,9 @@ class ShutdownVM(Resource): if check_otp(name, realm, token) == 200: vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) + if vm.value["owner"] != name: + return {"message": "Invalid User"} + if vm: vm.value["status"] = "REQUESTED_SHUTDOWN" client.put(vm.key, json.dumps(vm.value)) @@ -240,11 +277,13 @@ class ListUserVM(Resource): return_vms = [] user_vms = list(filter(lambda v: v.value["owner"] == name, vms)) for vm in user_vms: - return_vms.append({ - "vm_uuid": vm.key.split("/")[-1], - "specs": vm.value["specs"], - "status": vm.value["status"] - }) + return_vms.append( + { + "vm_uuid": vm.key.split("/")[-1], + "specs": vm.value["specs"], + "status": vm.value["status"], + } + ) return {"message": return_vms}, 200 else: return {"message": "No VM found"}, 404 @@ -263,15 +302,55 @@ class ListUserFiles(Resource): return_files = [] user_files = list(filter(lambda f: f.value["owner"] == name, files)) for file in user_files: - return_files.append({ - "filename": vm.value["filename"] - }) + return_files.append( + { + "filename": file.value["filename"], + "uuid": file.key.split("/")[-1], + } + ) return {"message": return_files}, 200 else: return {"message": "No File found"}, 404 else: return {"message": "Invalid Credentials"}, 400 + +class CreateHost(Resource): + def post(self): + args = addhost_argparser.parse_args() + name, realm, token, specs, hostname = ( + args.name, + args.realm, + args.token, + args.specs, + args.hostname, + ) + + if realm == "ungleich-admin" and check_otp(name, realm, token) == 200: + # User is good + if not specs_parser.transform_specs(specs): + return ( + { + "message": f"""Invalid unit - Please use following units {specs_parser.get_allowed_units()}""" + }, + 400, + ) + + print(specs) + host_key = f"/v1/host/{uuid4().hex}" + host_entry = { + "specs": specs, + "hostname": hostname, + "status": "DEAD", + "last_heart_beat": "", + } + client.put(host_key, host_entry, value_in_json=True) + + return {"message": "Host Created"}, 200 + else: + return {"message": "Invalid Credentials/Insufficient Permission"}, 400 + + api.add_resource(CreateVM, "/vm/create") api.add_resource(DeleteVM, "/vm/delete") api.add_resource(VmStatus, "/vm/status") @@ -287,5 +366,8 @@ api.add_resource(ListPublicImages, "/image/list-public") api.add_resource(ListUserVM, "/user/vms") api.add_resource(ListUserFiles, "/user/files") +api.add_resource(CreateHost, "/host/create") + + if __name__ == "__main__": app.run(host="::", debug=True) diff --git a/specs_parser.py b/specs_parser.py index 0cd9894..c2996af 100644 --- a/specs_parser.py +++ b/specs_parser.py @@ -1,7 +1,8 @@ import bitmath + class SpecsParser(object): - def __init__(self, exceptional_devices, allowed_unit = 10): + def __init__(self, exceptional_devices, allowed_unit=10): self.exceptional_devices = exceptional_devices self.allowed_unit = allowed_unit From cdf3c741ee99d310082c4fc1407e05bef4c88388 Mon Sep 17 00:00:00 2001 From: Ahmad Bilal Khalid Date: Thu, 18 Jul 2019 19:09:26 +0500 Subject: [PATCH 03/10] last_heart_beat -> last_hearbeat --- main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/main.py b/main.py index 5617eb3..f9dbdde 100644 --- a/main.py +++ b/main.py @@ -342,7 +342,7 @@ class CreateHost(Resource): "specs": specs, "hostname": hostname, "status": "DEAD", - "last_heart_beat": "", + "last_heartbeat": "", } client.put(host_key, host_entry, value_in_json=True) From 715128a138a5db9129e6a9693a80f24bae7ce768 Mon Sep 17 00:00:00 2001 From: Ahmad Bilal Khalid Date: Sat, 20 Jul 2019 15:19:39 +0500 Subject: [PATCH 04/10] check owner of vm after ensuring that vm exists --- main.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/main.py b/main.py index f9dbdde..b4a9043 100644 --- a/main.py +++ b/main.py @@ -196,9 +196,9 @@ class StartVM(Resource): if check_otp(name, realm, token) == 200: vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm.value["owner"] != name: - return {"message": "Invalid User"} if vm: + if vm.value["owner"] != name: + return {"message": "Invalid User"} vm.value["status"] = "REQUESTED_START" client.put(vm.key, json.dumps(vm.value)) return {"message": f"VM Start Queued"} @@ -215,9 +215,9 @@ class SuspendVM(Resource): if check_otp(name, realm, token) == 200: vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm.value["owner"] != name: - return {"message": "Invalid User"} if vm: + if vm.value["owner"] != name: + return {"message": "Invalid User"} vm.value["status"] = "REQUESTED_SUSPEND" client.put(vm.key, json.dumps(vm.value)) return {"message": f"VM Suspension Queued"} @@ -234,9 +234,9 @@ class ResumeVM(Resource): if check_otp(name, realm, token) == 200: vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm.value["owner"] != name: - return {"message": "Invalid User"} if vm: + if vm.value["owner"] != name: + return {"message": "Invalid User"} vm.value["status"] = "REQUESTED_RESUME" client.put(vm.key, json.dumps(vm.value)) return {"message": f"VM Resume Queued"} @@ -253,10 +253,10 @@ class ShutdownVM(Resource): if check_otp(name, realm, token) == 200: vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm.value["owner"] != name: - return {"message": "Invalid User"} - if vm: + if vm.value["owner"] != name: + return {"message": "Invalid User"} + vm.value["status"] = "REQUESTED_SHUTDOWN" client.put(vm.key, json.dumps(vm.value)) return {"message": f"VM Shutdown Queued"} From 31a5c3c4a74f1e5c72de1686a147444d76c5ee32 Mon Sep 17 00:00:00 2001 From: Ahmed Bilal Khalid Date: Tue, 30 Jul 2019 20:45:05 +0500 Subject: [PATCH 05/10] Files -> File --- Pipfile | 1 + Pipfile.lock | 145 ++++++++++++++++++++++++++++++++++++++++----------- main.py | 7 +-- 3 files changed, 119 insertions(+), 34 deletions(-) diff --git a/Pipfile b/Pipfile index bd5f667..73dba6e 100644 --- a/Pipfile +++ b/Pipfile @@ -15,6 +15,7 @@ etcd3 = "*" gunicorn = "*" bitmath = "*" pylint = "*" +transitions = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 6c93789..2451cc2 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "fa257b2f0b607e37897d62a624b656fdf219f74dc0a943f481ba6ef0d1a92c90" + "sha256": "bdace76cfe8c967d3e22d6d44b46fc749d8742af5d2d250bc8e0627718c4bfd0" }, "pipfile-spec": 6, "requires": { @@ -23,6 +23,13 @@ ], "version": "==7.0.0" }, + "astroid": { + "hashes": [ + "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", + "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" + ], + "version": "==2.2.5" + }, "bitmath": { "hashes": [ "sha256:293325f01e65defe966853111df11d39215eb705a967cb115851da8c4cfa3eb8" @@ -60,11 +67,11 @@ }, "flask": { "hashes": [ - "sha256:ad7c6d841e64296b962296c2c2dabc6543752985727af86a975072dea984b6f3", - "sha256:e7d32475d1de5facaa55e3958bc4ec66d3762076b074296aa50ef8fdc5b9df61" + "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", + "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" ], "index": "pypi", - "version": "==1.0.3" + "version": "==1.1.1" }, "flask-restful": { "hashes": [ @@ -126,6 +133,13 @@ ], "version": "==2.8" }, + "isort": { + "hashes": [ + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" + ], + "version": "==4.3.21" + }, "itsdangerous": { "hashes": [ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", @@ -140,6 +154,29 @@ ], "version": "==2.10.1" }, + "lazy-object-proxy": { + "hashes": [ + "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", + "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", + "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", + "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", + "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", + "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", + "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", + "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", + "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", + "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", + "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", + "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", + "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", + "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", + "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", + "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", + "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", + "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" + ], + "version": "==1.4.1" + }, "markupsafe": { "hashes": [ "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", @@ -173,36 +210,51 @@ ], "version": "==1.1.1" }, + "mccabe": { + "hashes": [ + "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", + "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" + ], + "version": "==0.6.1" + }, "protobuf": { "hashes": [ - "sha256:03f43eac9d5b651f976e91cf46a25b75e5779d98f0f4114b0abfed83376d75f8", - "sha256:0c94b21e6de01362f91a86b372555d22a60b59708599ca9d5032ae9fdf8e3538", - "sha256:2d2a9f30f61f4063fadd7fb68a2510a6939b43c0d6ceeec5c4704f22225da28e", - "sha256:34a0b05fca061e4abb77dd180209f68d8637115ff319f51e28a6a9382d69853a", - "sha256:358710fd0db25372edcf1150fa691f48376a134a6c69ce29f38f185eea7699e6", - "sha256:41e47198b94c27ba05a08b4a95160656105745c462af574e4bcb0807164065c0", - "sha256:8c61cc8a76e9d381c665aecc5105fa0f1878cf7db8b5cd17202603bcb386d0fc", - "sha256:a6eebc4db759e58fdac02efcd3028b811effac881d8a5bad1996e4e8ee6acb47", - "sha256:a9c12f7c98093da0a46ba76ec40ace725daa1ac4038c41e4b1466afb5c45bb01", - "sha256:cb95068492ba0859b8c9e61fa8ba206a83c64e5d0916fb4543700b2e2b214115", - "sha256:cd98476ce7bb4dcd6a7b101f5eecdc073dafea19f311e36eb8fba1a349346277", - "sha256:ce64cfbea18c535176bdaa10ba740c0fc4c6d998a3f511c17bedb0ae4b3b167c", - "sha256:dcbb59eac73fd454e8f2c5fba9e3d3320fd4707ed6a9d3ea3717924a6f0903ea", - "sha256:dd67f34458ae716029e2a71ede998e9092493b62a519236ca52e3c5202096c87", - "sha256:e3c96056eb5b7284a20e256cb0bf783c8f36ad82a4ae5434a7b7cd02384144a7", - "sha256:f612d584d7a27e2f39e7b17878430a959c1bc09a74ba09db096b468558e5e126", - "sha256:f6de8a7d6122297b81566e5bd4df37fd5d62bec14f8f90ebff8ede1c9726cd0a", - "sha256:fa529d9261682b24c2aaa683667253175c9acebe0a31105394b221090da75832" + "sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f", + "sha256:139b7eadcca0a861d60b523cb37d9475505e0dfb07972436b15407c2b968d87e", + "sha256:15f683006cb77fb849b1f561e509b03dd2b7dcc749086b8dd1831090d0ba4740", + "sha256:2ad566b7b7cdd8717c7af1825e19f09e8fef2787b77fcb979588944657679604", + "sha256:35cfcf97642ef62108e10a9431c77733ec7eaab8e32fe4653de20403429907cb", + "sha256:387822859ecdd012fdc25ec879f7f487da6e1d5b1ae6115e227e6be208836f71", + "sha256:4df14cbe1e7134afcfdbb9f058949e31c466de27d9b2f7fb4da9e0b67231b538", + "sha256:586c4ca37a7146d4822c700059f150ac3445ce0aef6f3ea258640838bb892dc2", + "sha256:58b11e530e954d29ab3180c48dc558a409f705bf16739fd4e0d3e07924ad7add", + "sha256:63c8c98ccb8c95f41c18fb829aeeab21c6249adee4ed75354125bdc44488f30e", + "sha256:72edcbacd0c73eef507d2ff1af99a6c27df18e66a3ff4351e401182e4de62b03", + "sha256:83dc8a561b3b954fd7002c690bb83278b8d1742a1e28abba9aaef28b0c8b437d", + "sha256:913171ecc84c2726b86574e40549a0ea619d569657c5a5ff782a3be7d81401a5", + "sha256:aabb7c741d3416671c3e6fe7c52970a226e6a8274417a97d7d795f953fadef36", + "sha256:b3452bbda12b1cbe2187d416779de07b2ab4c497d83a050e43c344778763721d", + "sha256:c5d5b8d4a9212338297fa1fa44589f69b470c0ba1d38168b432d577176b386a8", + "sha256:d86ee389c2c4fc3cebabb8ce83a8e97b6b3b5dc727b7419c1ccdc7b6e545a233", + "sha256:f2db8c754de788ab8be5e108e1e967c774c0942342b4f8aaaf14063889a6cfdc" ], - "version": "==3.8.0" + "version": "==3.9.0" + }, + "pylint": { + "hashes": [ + "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", + "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" + ], + "index": "pypi", + "version": "==2.3.1" }, "pyotp": { "hashes": [ - "sha256:1e3dc3d16919c4efac528d1dbecc17de1a97c4ecfdacb89d7726ed2c6645adff", - "sha256:be0ffeabddaa5ee53e7204e7740da842d070cf69168247a3d0c08541b84de602" + "sha256:c88f37fd47541a580b744b42136f387cdad481b560ef410c0d85c957eb2a2bc0", + "sha256:fc537e8acd985c5cbf51e11b7d53c42276fee017a73aec7c07380695671ca1a1" ], "index": "pypi", - "version": "==2.2.7" + "version": "==2.3.0" }, "python-decouple": { "hashes": [ @@ -240,6 +292,35 @@ ], "version": "==5.0.4" }, + "transitions": { + "hashes": [ + "sha256:00bfa91b0cfe3f649731b2f28478c4314bb9e830c600b422973f1b3a704ac710", + "sha256:afe0f498cf1f3f3b0fc13562011b8895a172df8f891dbb5118923d46e78a96d7" + ], + "index": "pypi", + "version": "==0.6.9" + }, + "typed-ast": { + "hashes": [ + "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", + "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", + "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", + "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", + "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", + "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", + "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", + "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", + "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", + "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", + "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", + "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", + "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", + "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", + "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" + ], + "markers": "implementation_name == 'cpython'", + "version": "==1.4.0" + }, "urllib3": { "hashes": [ "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", @@ -249,10 +330,16 @@ }, "werkzeug": { "hashes": [ - "sha256:865856ebb55c4dcd0630cdd8f3331a1847a819dda7e8c750d3db6f2aa6c0209c", - "sha256:a0b915f0815982fb2a09161cb8f31708052d0951c3ba433ccc5e1aa276507ca6" + "sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4", + "sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6" ], - "version": "==0.15.4" + "version": "==0.15.5" + }, + "wrapt": { + "hashes": [ + "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" + ], + "version": "==1.11.2" } }, "develop": {} diff --git a/main.py b/main.py index b4a9043..3755cdb 100644 --- a/main.py +++ b/main.py @@ -7,12 +7,9 @@ import json from helper import check_otp, add_otp_args, add_vmid_args from flask import Flask from flask_restful import Resource, Api, reqparse -from decouple import config from uuid import uuid4 from etcd3_wrapper import Etcd3Wrapper from specs_parser import SpecsParser -from functools import wraps - app = Flask(__name__) api = Api(app) @@ -148,7 +145,7 @@ class CreateImage(Resource): image_file_uuid = args.uuid image_store_name = args.image_store - file_entry = client.get(f"/v1/files/{image_file_uuid}") + file_entry = client.get(f"/v1/file/{image_file_uuid}") if file_entry is None: return ( {"Message": f"Image File with uuid '{image_file_uuid}' Not Found"}, @@ -297,7 +294,7 @@ class ListUserFiles(Resource): name, realm, token = args.name, args.realm, args.token if check_otp(name, realm, token) == 200: - files = client.get_prefix(f"/v1/files/", value_in_json=True) + files = client.get_prefix(f"/v1/file/", value_in_json=True) if files: return_files = [] user_files = list(filter(lambda f: f.value["owner"] == name, files)) From 47b0ba77195ab6e95fa1510e332b3c39b4c999d9 Mon Sep 17 00:00:00 2001 From: meow Date: Thu, 1 Aug 2019 15:04:40 +0500 Subject: [PATCH 06/10] Cleaning --- .gitignore | 2 + common_fields.py | 64 ++++++++ create_image_store.py | 23 +++ helper.py | 12 -- main.py | 343 +++++++++++------------------------------- schemas.py | 153 +++++++++++++++++++ 6 files changed, 326 insertions(+), 271 deletions(-) create mode 100644 common_fields.py create mode 100755 create_image_store.py create mode 100644 schemas.py diff --git a/.gitignore b/.gitignore index 5929a22..209604a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ __pycache__/ venv/ settings.json +ucloud_common +etcd3_wrapper diff --git a/common_fields.py b/common_fields.py new file mode 100644 index 0000000..4450bb6 --- /dev/null +++ b/common_fields.py @@ -0,0 +1,64 @@ +from etcd3_wrapper import Etcd3Wrapper +from specs_parser import SpecsParser + +specs_parser = SpecsParser(exceptional_devices=["cpu"]) + + +class Field(object): + def __init__(self, _name, _type, _value=None): + self.name = _name + self.value = _value + self.type = _type + self.__errors = [] + + def validation(self): + return True + + def is_valid(self): + if self.value == KeyError: + self.add_error(f"'{self.name}' field is a required field") + else: + if not isinstance(self.value, self.type): + self.add_error(f"Incorrect Type for '{self.name}' field") + else: + self.validation() + + if self.__errors: + return False + return True + + def get_errors(self): + return self.__errors + + def add_error(self, error): + self.__errors.append(error) + + +class VmUUIDField(Field): + def __init__(self, data): + self.uuid = data.get("uuid", KeyError) + + super().__init__("uuid", str, self.uuid) + + self.validation = self.vm_uuid_validation + + def vm_uuid_validation(self): + client = Etcd3Wrapper() + + r = client.get(f"/v1/vm/{self.uuid}") + if not r: + self.add_error(f"VM with uuid {self.uuid} does not exists") + + +class SpecsField(Field): + def __init__(self, data): + self.specs = data.get("specs", KeyError) + + super().__init__("specs", dict, self.specs) + + self.validation = self.specs_validation + + def specs_validation(self): + if not specs_parser.transform_specs(self.specs): + self.add_error("Invalid unit - " + f"Please use following units {specs_parser.get_allowed_units()}") diff --git a/create_image_store.py b/create_image_store.py new file mode 100755 index 0000000..7d5fc56 --- /dev/null +++ b/create_image_store.py @@ -0,0 +1,23 @@ +import json +from uuid import uuid4 +from etcd3_wrapper import Etcd3Wrapper + +client = Etcd3Wrapper() + +data = { + "is_public": True, + "type": "ceph", + "name": "images", + "description": "first ever public image-store", + "attributes": { + "list": [], + "key": [], + "pool": "images", + } +} + +client.put( + f"/v1/image_store/{uuid4().hex}", + json.dumps(data), +) + diff --git a/helper.py b/helper.py index 9d3a254..819ffe6 100644 --- a/helper.py +++ b/helper.py @@ -26,15 +26,3 @@ def check_otp(name, realm, token): data=data, ) return response.status_code - - -def add_otp_args(parser): - parser.add_argument("name", required=True) - parser.add_argument("realm", required=True) - parser.add_argument("token", required=True) - return parser - - -def add_vmid_args(parser): - parser.add_argument("vmid", required=True) - return parser diff --git a/main.py b/main.py index 3755cdb..8bad54e 100644 --- a/main.py +++ b/main.py @@ -4,176 +4,74 @@ import json -from helper import check_otp, add_otp_args, add_vmid_args -from flask import Flask -from flask_restful import Resource, Api, reqparse +from flask import Flask, request +from flask_restful import Resource, Api from uuid import uuid4 from etcd3_wrapper import Etcd3Wrapper -from specs_parser import SpecsParser + +from ucloud_common.vm import VmPool, VMStatus +from schemas import (CreateVMSchema, VMStatusSchema, + CreateImageSchema, VmActionSchema, + OTPSchema, CreateHostSchema) + app = Flask(__name__) api = Api(app) client = Etcd3Wrapper() - -# CreateVM argparser -createvm_argparser = reqparse.RequestParser() -createvm_argparser.add_argument("specs", type=dict, required=True) -createvm_argparser.add_argument("image_uuid", type=str, required=True) -add_otp_args(createvm_argparser) - -# CreateImage argparser -createimage_argparser = reqparse.RequestParser() -createimage_argparser.add_argument("uuid", type=str, required=True) -createimage_argparser.add_argument("name", type=str, required=True) -createimage_argparser.add_argument("image_store", type=str, required=True) - -# DeleteVM argparser -deletevm_argparser = reqparse.RequestParser() -add_vmid_args(add_otp_args(deletevm_argparser)) - -# VMStatus argparser -vmstatus_argparser = reqparse.RequestParser() -add_vmid_args(vmstatus_argparser) - -# StartVM argparser -startvm_argparser = reqparse.RequestParser() -add_vmid_args(add_otp_args(startvm_argparser)) - -# UserVM argparser -uservm_argparser = reqparse.RequestParser() -add_otp_args(uservm_argparser) - -# Specs parser -specs_parser = SpecsParser(exceptional_devices=["cpu"]) - -# AddHost argparser -addhost_argparser = reqparse.RequestParser() -addhost_argparser.add_argument("hostname", type=str, required=True) -addhost_argparser.add_argument("specs", type=dict, required=True) -add_otp_args(addhost_argparser) - - -def is_image_valid(image_uuid): - images = client.get_prefix("/v1/image/") - return image_uuid in [i.key.split("/")[-1] for i in images] +vm_pool = VmPool(client, "/v1/vm") class CreateVM(Resource): def post(self): - createvm_args = createvm_argparser.parse_args() - name, realm, token, specs, image_uuid = ( - createvm_args.name, - createvm_args.realm, - createvm_args.token, - createvm_args.specs, - createvm_args.image_uuid, - ) - - if check_otp(name, realm, token) == 200: - # User is good - if is_image_valid(image_uuid): - if not specs_parser.transform_specs(specs): - return ( - { - "message": f"""Invalid unit - Please use following units {specs_parser.get_allowed_units()}""" - }, - 400, - ) - - print(specs) - vm_key = f"/v1/vm/{uuid4().hex}" - vm_entry = { - "owner": name, - "specs": specs, - "hostname": "", - "status": "REQUESTED_NEW", - "image_uuid": image_uuid, - } - - client.put(vm_key, vm_entry, value_in_json=True) - - return {"message": "VM Creation Queued"}, 200 - else: - return {"message": "Image uuid not valid"} + data = request.json + validator = CreateVMSchema(data) + if validator.is_valid(): + vm_key = f"/v1/vm/{uuid4().hex}" + vm_entry = { + "owner": data["name"], + "specs": data["specs"], + "hostname": "", + "status": "REQUESTED_NEW", + "image_uuid": data["image_uuid"], + } + client.put(vm_key, vm_entry, value_in_json=True) + return {"message": "VM Creation Queued"}, 200 else: - return {"message": "Invalid Credentials"}, 400 - - -class DeleteVM(Resource): - def post(self): - deletevm_args = deletevm_argparser.parse_args() - name, realm, token, vmid = ( - deletevm_args.name, - deletevm_args.realm, - deletevm_args.token, - deletevm_args.vmid, - ) - - if check_otp(name, realm, token) == 200: - # User is good - - vmentry_etcd = client.get(f"/v1/vm/{vmid}").value - if vmentry_etcd: - vmentry_etcd = json.loads(vmentry_etcd) - vmentry_etcd["status"] = "REQUESTED_DELETE" - - client.put(f"/v1/vm/{vmid}", vmentry_etcd, - value_in_json=True) - - return {"message": "VM Deletion Queued"}, 200 - else: - return {"message": "Invalid VM ID"} - else: - return {"message": "Invalid Credentials"}, 400 + return validator.get_errors(), 400 class VmStatus(Resource): def get(self): - args = vmstatus_argparser.parse_args() - r = client.get(f"/v1/vm/{args.vmid}").value - print(r) - if r: - r = dict(json.loads(r.decode("utf-8"))) - return r - return {"Message": "Not Found"} + data = request.json + validator = VMStatusSchema(data) + if validator.is_valid(): + vm = vm_pool.get(f"/v1/vm/{data['uuid']}") + return str(vm) + else: + return validator.get_errors(), 400 class CreateImage(Resource): def post(self): - image_stores = list(client.get_prefix("/v1/image_store/")) - args = createimage_argparser.parse_args() - image_file_uuid = args.uuid - image_store_name = args.image_store + data = request.json + validator = CreateImageSchema(data) + if validator.is_valid(): + file_entry = client.get(f"/v1/file/{data['uuid']}") + file_entry_value = json.loads(file_entry.value) - file_entry = client.get(f"/v1/file/{image_file_uuid}") - if file_entry is None: - return ( - {"Message": f"Image File with uuid '{image_file_uuid}' Not Found"}, - 400, - ) + image_entry_json = { + "status": "TO_BE_CREATED", + "owner": file_entry_value["owner"], + "filename": file_entry_value["filename"], + "name": data["name"], + "store_name": data["image_store"], + "visibility": "public", + } + client.put(f"/v1/image/{data['uuid']}", json.dumps(image_entry_json)) - file_entry_value = json.loads(file_entry.value) - - image_store = list( - filter( - lambda s: json.loads(s.value)["name"] == image_store_name, image_stores - ) - ) - if not image_store: - return {"Message": f"Store '{image_store_name}' does not exists"}, 400 - - image_store = image_store[0] - image_entry_json = { - "status": "TO_BE_CREATED", - "owner": file_entry_value["owner"], - "filename": file_entry_value["filename"], - "name": args.name, - "store_name": image_store_name, - "visibility": "public", - } - client.put(f"/v1/image/{image_file_uuid}", json.dumps(image_entry_json)) - - return {"Message": "Image successfully created"} + return {"Message": "Image successfully created"} + else: + return validator.get_errors(), 400 class ListPublicImages(Resource): @@ -186,93 +84,41 @@ class ListPublicImages(Resource): return r, 200 -class StartVM(Resource): +class VMAction(Resource): def post(self): - args = startvm_argparser.parse_args() - name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid + data = request.json + validator = VmActionSchema(data) + action = data["action"] - if check_otp(name, realm, token) == 200: - vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm: - if vm.value["owner"] != name: - return {"message": "Invalid User"} - vm.value["status"] = "REQUESTED_START" - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Start Queued"} - else: - return {"message": "No such VM found"} + if validator.is_valid(): + vm = vm_pool.get(f"/v1/vm/{data['uuid']}") + if action == "start": + vm.status = VMStatus.requested_start + elif action == "suspend": + vm.status = VMStatus.requested_suspend + elif action == "resume": + vm.status = VMStatus.requested_resume + elif action == "shutdown": + vm.status = VMStatus.requested_shutdown + elif action == "delete": + vm.status = VMStatus.requested_delete + + client.put(vm.key, json.dumps(vm.value)) + return {"message": f"VM Start Queued"} else: - return {"message": "Invalid Credentials"}, 400 - - -class SuspendVM(Resource): - def post(self): - args = startvm_argparser.parse_args() - name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid - - if check_otp(name, realm, token) == 200: - vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm: - if vm.value["owner"] != name: - return {"message": "Invalid User"} - vm.value["status"] = "REQUESTED_SUSPEND" - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Suspension Queued"} - else: - return {"message": "No such VM found"} - else: - return {"message": "Invalid Credentials"}, 400 - - -class ResumeVM(Resource): - def post(self): - args = startvm_argparser.parse_args() - name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid - - if check_otp(name, realm, token) == 200: - vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm: - if vm.value["owner"] != name: - return {"message": "Invalid User"} - vm.value["status"] = "REQUESTED_RESUME" - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Resume Queued"} - else: - return {"message": "No such VM found"} - else: - return {"message": "Invalid Credentials"}, 400 - - -class ShutdownVM(Resource): - def post(self): - args = startvm_argparser.parse_args() - name, realm, token, vm_uuid = args.name, args.realm, args.token, args.vmid - - if check_otp(name, realm, token) == 200: - vm = client.get(f"/v1/vm/{vm_uuid}", value_in_json=True) - if vm: - if vm.value["owner"] != name: - return {"message": "Invalid User"} - - vm.value["status"] = "REQUESTED_SHUTDOWN" - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Shutdown Queued"} - else: - return {"message": "No such VM found"} - else: - return {"message": "Invalid Credentials"}, 400 + return validator.get_errors(), 400 class ListUserVM(Resource): def get(self): - args = uservm_argparser.parse_args() - name, realm, token = args.name, args.realm, args.token + data = request.json + validator = OTPSchema(data) - if check_otp(name, realm, token) == 200: + if validator.is_valid(): vms = client.get_prefix(f"/v1/vm/", value_in_json=True) if vms: return_vms = [] - user_vms = list(filter(lambda v: v.value["owner"] == name, vms)) + user_vms = list(filter(lambda v: v.value["owner"] == data["name"], vms)) for vm in user_vms: return_vms.append( { @@ -285,19 +131,19 @@ class ListUserVM(Resource): else: return {"message": "No VM found"}, 404 else: - return {"message": "Invalid Credentials"}, 400 + return validator.get_errors(), 400 class ListUserFiles(Resource): def get(self): - args = uservm_argparser.parse_args() - name, realm, token = args.name, args.realm, args.token + data = request.json + validator = OTPSchema(data) - if check_otp(name, realm, token) == 200: + if validator.is_valid(): files = client.get_prefix(f"/v1/file/", value_in_json=True) if files: return_files = [] - user_files = list(filter(lambda f: f.value["owner"] == name, files)) + user_files = list(filter(lambda f: f.value["owner"] == data["name"], files)) for file in user_files: return_files.append( { @@ -309,35 +155,18 @@ class ListUserFiles(Resource): else: return {"message": "No File found"}, 404 else: - return {"message": "Invalid Credentials"}, 400 + return validator.get_errors(), 400 class CreateHost(Resource): def post(self): - args = addhost_argparser.parse_args() - name, realm, token, specs, hostname = ( - args.name, - args.realm, - args.token, - args.specs, - args.hostname, - ) - - if realm == "ungleich-admin" and check_otp(name, realm, token) == 200: - # User is good - if not specs_parser.transform_specs(specs): - return ( - { - "message": f"""Invalid unit - Please use following units {specs_parser.get_allowed_units()}""" - }, - 400, - ) - - print(specs) + data = request.json + validator = CreateHostSchema(data) + if validator.is_valid(): host_key = f"/v1/host/{uuid4().hex}" host_entry = { - "specs": specs, - "hostname": hostname, + "specs": data["specs"], + "hostname": data["hostname"], "status": "DEAD", "last_heartbeat": "", } @@ -345,17 +174,13 @@ class CreateHost(Resource): return {"message": "Host Created"}, 200 else: - return {"message": "Invalid Credentials/Insufficient Permission"}, 400 + return validator.get_errors(), 400 api.add_resource(CreateVM, "/vm/create") -api.add_resource(DeleteVM, "/vm/delete") api.add_resource(VmStatus, "/vm/status") -api.add_resource(StartVM, "/vm/start") -api.add_resource(SuspendVM, "/vm/suspend") -api.add_resource(ResumeVM, "/vm/resume") -api.add_resource(ShutdownVM, "/vm/shutdown") +api.add_resource(VMAction, "/vm/action") api.add_resource(CreateImage, "/image/create") api.add_resource(ListPublicImages, "/image/list-public") diff --git a/schemas.py b/schemas.py new file mode 100644 index 0000000..a11d900 --- /dev/null +++ b/schemas.py @@ -0,0 +1,153 @@ +import json + +from common_fields import Field, VmUUIDField, SpecsField +from helper import check_otp +from etcd3_wrapper import Etcd3Wrapper + +client = Etcd3Wrapper() + + +class BaseSchema(object): + def __init__(self, data, fields=None): + _ = data + + self.__errors = [] + if fields is None: + self.fields = [] + else: + self.fields = fields + + def validation(self): + # custom validation is optional + return True + + def is_valid(self): + for field in self.fields: + field.is_valid() + self.add_field_errors(field) + + for parent in self.__class__.__bases__: + try: + parent.validation(self) + except AttributeError: + pass + if not self.__errors: + self.validation() + + if self.__errors: + return False + return True + + def get_errors(self): + return {"Message": self.__errors} + + def add_field_errors(self, field: Field): + self.__errors += field.get_errors() + + def add_error(self, error): + self.__errors.append(error) + + +class OTPSchema(BaseSchema): + def __init__(self, data: dict, fields=None): + self.name = Field("name", str, data.get("name", KeyError)) + self.realm = Field("realm", str, data.get("realm", KeyError)) + self.token = Field("token", str, data.get("token", KeyError)) + + _fields = [self.name, self.realm, self.token] + if fields: + _fields += fields + super().__init__(data=data, fields=_fields) + + def validation(self): + rc = check_otp(self.name.value, self.realm.value, self.token.value) + if rc != 200: + self.add_error("Wrong Credentials") + + +class CreateVMSchema(OTPSchema): + def __init__(self, data): + self.specs = SpecsField(data) + + self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError)) + self.image_uuid.validation = self.image_uuid_validation + + fields = [self.specs, self.image_uuid] + super().__init__(data=data, fields=fields) + + def image_uuid_validation(self): + images = client.get_prefix("/v1/image/") + + if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]: + self.add_error("Image UUID not valid") + + +class VMStatusSchema(BaseSchema): + def __init__(self, data): + self.uuid = VmUUIDField(data) + + fields = [self.uuid] + + super().__init__(data, fields) + + +class CreateImageSchema(BaseSchema): + def __init__(self, data): + # Fields + self.uuid: Field = Field("uuid", str, data.get("uuid", KeyError)) + self.name = Field("name", str, data.get("name", KeyError)) + self.image_store = Field("image_store", str, data.get("image_store", KeyError)) + # Validations + self.uuid.validation = self.file_uuid_validation + + # All Fields + fields = [self.uuid, self.name, self.image_store] + super().__init__(data, fields) + + def file_uuid_validation(self): + file_entry = client.get(f"/v1/file/{self.uuid.value}") + if file_entry is None: + self.add_error(f"Image File with uuid '{self.uuid.value}' Not Found") + + def image_store_name_validation(self): + image_stores = list(client.get_prefix("/v1/image_store/")) + + image_store = next(filter(lambda s: json.loads(s.value)["name"] == self.image_store.value, + image_stores)) + if not image_store: + self.add_error(f"Store '{self.image_store.value}' does not exists") + + +class VmActionSchema(OTPSchema): + def __init__(self, data): + self.uuid = VmUUIDField(data) + self.action = Field("action", str, data.get("action", KeyError)) + + self.action.validation = self.action_validation + + fields = [self.uuid, self.action] + super().__init__(data=data, fields=fields) + + def action_validation(self): + allowed_actions = ["start", "shutdown", "suspend", "resume", "delete"] + if self.action.value not in allowed_actions: + self.add_error(f"Invalid Action. Allowed Actions are {allowed_actions}") + + def validation(self): + vm = client.get(f"/v1/vm/{self.uuid.value}", value_in_json=True) + if vm.value["owner"] != self.name: + self.add_error("Invalid User") + + +class CreateHostSchema(OTPSchema): + def __init__(self, data): + self.specs = SpecsField(data) + self.hostname = Field("hostname", str, data.get("hostname", KeyError)) + + fields = [self.specs, self.hostname] + + super().__init__(data=data, fields=fields) + + def validation(self): + if self.realm.value != "ungleich-admin": + self.add_error("Invalid Credentials/Insufficient Permission") From 080933b1404ab78deecd9990d9f8f393d783f6dc Mon Sep 17 00:00:00 2001 From: meow Date: Sun, 11 Aug 2019 22:01:27 +0500 Subject: [PATCH 07/10] Rolled out new request mechanism, vm migration view added --- .gitmodules | 3 -- etcd3_wrapper | 1 - main.py | 117 ++++++++++++++++++++++++++++++++++++++------------ schemas.py | 53 ++++++++++++++++++++--- 4 files changed, 137 insertions(+), 37 deletions(-) delete mode 160000 etcd3_wrapper diff --git a/.gitmodules b/.gitmodules index 316b9ea..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "etcd3_wrapper"] - path = etcd3_wrapper - url = https://code.ungleich.ch/ahmedbilal/etcd3_wrapper diff --git a/etcd3_wrapper b/etcd3_wrapper deleted file mode 160000 index cb2a416..0000000 --- a/etcd3_wrapper +++ /dev/null @@ -1 +0,0 @@ -Subproject commit cb2a416a17d6789e613ba3b9957917770f4211e1 diff --git a/main.py b/main.py index 8bad54e..5132372 100644 --- a/main.py +++ b/main.py @@ -3,45 +3,63 @@ # any user vm. import json +import subprocess from flask import Flask, request from flask_restful import Resource, Api from uuid import uuid4 from etcd3_wrapper import Etcd3Wrapper +from os.path import join from ucloud_common.vm import VmPool, VMStatus +from ucloud_common.host import HostPool +from ucloud_common.request import RequestEntry, RequestPool, RequestType from schemas import (CreateVMSchema, VMStatusSchema, CreateImageSchema, VmActionSchema, - OTPSchema, CreateHostSchema) + OTPSchema, CreateHostSchema, + VmMigrationSchema) app = Flask(__name__) api = Api(app) client = Etcd3Wrapper() vm_pool = VmPool(client, "/v1/vm") +host_pool = HostPool(client, "/v1/host") +request_pool = RequestPool(client, "/v1/request") class CreateVM(Resource): - def post(self): + @staticmethod + def post(): data = request.json validator = CreateVMSchema(data) if validator.is_valid(): - vm_key = f"/v1/vm/{uuid4().hex}" + # Create VM Entry under /v1/vm/ + vm_uuid = uuid4().hex + vm_key = f"/v1/vm/{vm_uuid}" vm_entry = { "owner": data["name"], "specs": data["specs"], "hostname": "", - "status": "REQUESTED_NEW", + "status": "", "image_uuid": data["image_uuid"], + "log": [] } client.put(vm_key, vm_entry, value_in_json=True) + + # Create ScheduleVM Request + r = RequestEntry.from_scratch(type=RequestType.ScheduleVM, + uuid=vm_uuid) + request_pool.put(r) + return {"message": "VM Creation Queued"}, 200 else: return validator.get_errors(), 400 class VmStatus(Resource): - def get(self): + @staticmethod + def get(): data = request.json validator = VMStatusSchema(data) if validator.is_valid(): @@ -52,7 +70,8 @@ class VmStatus(Resource): class CreateImage(Resource): - def post(self): + @staticmethod + def post(): data = request.json validator = CreateImageSchema(data) if validator.is_valid(): @@ -69,13 +88,14 @@ class CreateImage(Resource): } client.put(f"/v1/image/{data['uuid']}", json.dumps(image_entry_json)) - return {"Message": "Image successfully created"} + return {"message": "Image successfully created"} else: return validator.get_errors(), 400 class ListPublicImages(Resource): - def get(self): + @staticmethod + def get(): images = client.get_prefix("/v1/image/") r = {} for image in images: @@ -85,32 +105,64 @@ class ListPublicImages(Resource): class VMAction(Resource): - def post(self): + @staticmethod + def post(): data = request.json validator = VmActionSchema(data) - action = data["action"] if validator.is_valid(): - vm = vm_pool.get(f"/v1/vm/{data['uuid']}") - if action == "start": - vm.status = VMStatus.requested_start - elif action == "suspend": - vm.status = VMStatus.requested_suspend - elif action == "resume": - vm.status = VMStatus.requested_resume - elif action == "shutdown": - vm.status = VMStatus.requested_shutdown - elif action == "delete": - vm.status = VMStatus.requested_delete + vm_entry = vm_pool.get(f"/v1/vm/{data['uuid']}") + action = data["action"] - client.put(vm.key, json.dumps(vm.value)) - return {"message": f"VM Start Queued"} + if action == "start": + vm_entry.status = VMStatus.requested_start + vm_pool.put(vm_entry) + action = "schedule" + + if action == "delete" and vm_entry.hostname == "": + try: + path_without_protocol = vm_entry.path[vm_entry.path.find(":")+1:] + rc = subprocess.call(f"rbd rm {path_without_protocol}".split(" ")) + except FileNotFoundError: + return {"message": "VM image does not exists"} + else: + if rc == 0: + client.client.delete(vm_entry.key) + return {"message": "VM successfully deleted"} + else: + return {"message": "Some error occurred while deleting VM"} + + r = RequestEntry.from_scratch(type=f"{action.title()}VM", + uuid=data['uuid'], + hostname=vm_entry.hostname) + request_pool.put(r) + return {"message": f"VM {action.title()} Queued"}, 200 + else: + return validator.get_errors(), 400 + + +class VMMigration(Resource): + @staticmethod + def post(): + data = request.json + validator = VmMigrationSchema(data) + + if validator.is_valid(): + vm = vm_pool.get(data['uuid']) + + r = RequestEntry.from_scratch(type=RequestType.ScheduleVM, + uuid=vm.uuid, + destination=join("/v1/host", data["destination"]), + migration=True) + request_pool.put(r) + return {"message": f"VM Migration Initialization Queued"}, 200 else: return validator.get_errors(), 400 class ListUserVM(Resource): - def get(self): + @staticmethod + def get(): data = request.json validator = OTPSchema(data) @@ -135,7 +187,8 @@ class ListUserVM(Resource): class ListUserFiles(Resource): - def get(self): + @staticmethod + def get(): data = request.json validator = OTPSchema(data) @@ -159,7 +212,8 @@ class ListUserFiles(Resource): class CreateHost(Resource): - def post(self): + @staticmethod + def post(): data = request.json validator = CreateHostSchema(data) if validator.is_valid(): @@ -177,10 +231,19 @@ class CreateHost(Resource): return validator.get_errors(), 400 +class ListHost(Resource): + @staticmethod + def get(): + hosts = host_pool.hosts + r = {host.key: {"status": host.status, "specs": host.specs, "hostname": host.hostname} for host in hosts} + return r, 200 + + api.add_resource(CreateVM, "/vm/create") api.add_resource(VmStatus, "/vm/status") api.add_resource(VMAction, "/vm/action") +api.add_resource(VMMigration, "/vm/migrate") api.add_resource(CreateImage, "/image/create") api.add_resource(ListPublicImages, "/image/list-public") @@ -189,7 +252,7 @@ api.add_resource(ListUserVM, "/user/vms") api.add_resource(ListUserFiles, "/user/files") api.add_resource(CreateHost, "/host/create") - +api.add_resource(ListHost, "/host/list") if __name__ == "__main__": app.run(host="::", debug=True) diff --git a/schemas.py b/schemas.py index a11d900..37f2f8a 100644 --- a/schemas.py +++ b/schemas.py @@ -1,10 +1,15 @@ import json from common_fields import Field, VmUUIDField, SpecsField +from ucloud_common.host import HostPool, HostStatus +from ucloud_common.vm import VmPool, VMStatus from helper import check_otp from etcd3_wrapper import Etcd3Wrapper +from os.path import join client = Etcd3Wrapper() +host_pool = HostPool(client, "/v1/host") +vm_pool = VmPool(client, "/v1/vm") class BaseSchema(object): @@ -39,7 +44,7 @@ class BaseSchema(object): return True def get_errors(self): - return {"Message": self.__errors} + return {"message": self.__errors} def add_field_errors(self, field: Field): self.__errors += field.get_errors() @@ -125,19 +130,55 @@ class VmActionSchema(OTPSchema): self.action.validation = self.action_validation - fields = [self.uuid, self.action] - super().__init__(data=data, fields=fields) + _fields = [self.uuid, self.action] + super().__init__(data=data, fields=_fields) def action_validation(self): - allowed_actions = ["start", "shutdown", "suspend", "resume", "delete"] + allowed_actions = ["start", "stop", "delete"] if self.action.value not in allowed_actions: self.add_error(f"Invalid Action. Allowed Actions are {allowed_actions}") def validation(self): - vm = client.get(f"/v1/vm/{self.uuid.value}", value_in_json=True) - if vm.value["owner"] != self.name: + vm = vm_pool.get(self.uuid.value) + if vm.value["owner"] != self.name.value: self.add_error("Invalid User") + if self.action.value == "start" and vm.status == VMStatus.running and vm.hostname != "": + self.add_error("VM Already Running") + + if self.action.value == "stop" and vm.status == VMStatus.stopped: + self.add_error("VM Already Stopped") + + +class VmMigrationSchema(OTPSchema): + def __init__(self, data): + self.uuid = VmUUIDField(data) + self.destination = Field("destination", str, data.get("destination", KeyError)) + + self.destination.validation = self.destination_validation + + fields = [self.destination] + super().__init__(data=data, fields=fields) + + def destination_validation(self): + host_key = self.destination.value + host = host_pool.get(host_key) + if not host: + self.add_error(f"No Such Host ({self.destination.value}) exists") + elif host.status != HostStatus.alive: + self.add_error("Destination Host is dead") + + def validation(self): + vm = vm_pool.get(self.uuid.value) + if vm.owner != self.name.value: + self.add_error("Invalid User") + + if vm.status != VMStatus.running: + self.add_error("Can't migrate non-running VM") + + if vm.hostname == join("/v1/host", self.destination.value): + self.add_error("Destination host couldn't be same as Source Host") + class CreateHostSchema(OTPSchema): def __init__(self, data): From 9aeb05987bd1b84e7ac851ce8b84f31647e436bd Mon Sep 17 00:00:00 2001 From: Ahmed Bilal Khalid Date: Mon, 12 Aug 2019 18:36:38 +0500 Subject: [PATCH 08/10] ListUserVM result now includes hostname of VM --- main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/main.py b/main.py index 5132372..c7175e5 100644 --- a/main.py +++ b/main.py @@ -177,6 +177,7 @@ class ListUserVM(Resource): "vm_uuid": vm.key.split("/")[-1], "specs": vm.value["specs"], "status": vm.value["status"], + "hostname": vm.value["hostname"] } ) return {"message": return_vms}, 200 From 4a3c81852a64eb2704d979f9a4e75b595ad13ab3 Mon Sep 17 00:00:00 2001 From: Ahmed Bilal <49-ahmedbilal@users.noreply.code.ungleich.ch> Date: Tue, 3 Sep 2019 18:01:40 +0200 Subject: [PATCH 09/10] Merge into master --- .gitignore | 0 .gitmodules | 0 Pipfile | 2 + Pipfile.lock | 346 ------------------------------------------ README.md | 0 common_fields.py | 4 +- config.py | 4 + create_image_store.py | 4 +- helper.py | 0 main.py | 11 +- schemas.py | 9 +- specs_parser.py | 2 +- 12 files changed, 18 insertions(+), 364 deletions(-) mode change 100644 => 100755 .gitignore mode change 100644 => 100755 .gitmodules mode change 100644 => 100755 Pipfile delete mode 100644 Pipfile.lock mode change 100644 => 100755 README.md mode change 100644 => 100755 common_fields.py create mode 100644 config.py mode change 100644 => 100755 helper.py mode change 100644 => 100755 main.py mode change 100644 => 100755 schemas.py mode change 100644 => 100755 specs_parser.py diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 diff --git a/.gitmodules b/.gitmodules old mode 100644 new mode 100755 diff --git a/Pipfile b/Pipfile old mode 100644 new mode 100755 index 73dba6e..88496f8 --- a/Pipfile +++ b/Pipfile @@ -4,6 +4,7 @@ url = "https://pypi.org/simple" verify_ssl = true [dev-packages] +prospector = "*" [packages] pyotp = "*" @@ -11,6 +12,7 @@ python-decouple = "*" requests = "*" flask = "*" flask-restful = "*" +grpcio = "*" etcd3 = "*" gunicorn = "*" bitmath = "*" diff --git a/Pipfile.lock b/Pipfile.lock deleted file mode 100644 index 2451cc2..0000000 --- a/Pipfile.lock +++ /dev/null @@ -1,346 +0,0 @@ -{ - "_meta": { - "hash": { - "sha256": "bdace76cfe8c967d3e22d6d44b46fc749d8742af5d2d250bc8e0627718c4bfd0" - }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.7" - }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "aniso8601": { - "hashes": [ - "sha256:513d2b6637b7853806ae79ffaca6f3e8754bdd547048f5ccc1420aec4b714f1e", - "sha256:d10a4bf949f619f719b227ef5386e31f49a2b6d453004b21f02661ccc8670c7b" - ], - "version": "==7.0.0" - }, - "astroid": { - "hashes": [ - "sha256:6560e1e1749f68c64a4b5dee4e091fce798d2f0d84ebe638cf0e0585a343acf4", - "sha256:b65db1bbaac9f9f4d190199bb8680af6f6f84fd3769a5ea883df8a91fe68b4c4" - ], - "version": "==2.2.5" - }, - "bitmath": { - "hashes": [ - "sha256:293325f01e65defe966853111df11d39215eb705a967cb115851da8c4cfa3eb8" - ], - "index": "pypi", - "version": "==1.3.3.1" - }, - "certifi": { - "hashes": [ - "sha256:046832c04d4e752f37383b628bc601a7ea7211496b4638f6514d0e5b9acc4939", - "sha256:945e3ba63a0b9f577b1395204e13c3a231f9bc0223888be653286534e5873695" - ], - "version": "==2019.6.16" - }, - "chardet": { - "hashes": [ - "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae", - "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691" - ], - "version": "==3.0.4" - }, - "click": { - "hashes": [ - "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" - ], - "version": "==7.0" - }, - "etcd3": { - "hashes": [ - "sha256:25a524b9f032c6631ff0097532907dea81243eaa63c3744510fd1598cc4e0e87" - ], - "index": "pypi", - "version": "==0.10.0" - }, - "flask": { - "hashes": [ - "sha256:13f9f196f330c7c2c5d7a5cf91af894110ca0215ac051b5844701f2bfd934d52", - "sha256:45eb5a6fd193d6cf7e0cf5d8a5b31f83d5faae0293695626f539a823e93b13f6" - ], - "index": "pypi", - "version": "==1.1.1" - }, - "flask-restful": { - "hashes": [ - "sha256:ecd620c5cc29f663627f99e04f17d1f16d095c83dc1d618426e2ad68b03092f8", - "sha256:f8240ec12349afe8df1db168ea7c336c4e5b0271a36982bff7394f93275f2ca9" - ], - "index": "pypi", - "version": "==0.3.7" - }, - "grpcio": { - "hashes": [ - "sha256:03b78b4e7dcdfe3e257bb528cc93923f9cbbab6d5babf15a60d21e9a4a70b1a2", - "sha256:1ce0ccfbdfe84387dbcbf44adb4ae16ec7ae70e166ffab478993eb1ea1cba3ce", - "sha256:22e167a9406d73dd19ffe8ed6a485f17e6eac82505be8c108897f15e68badcbb", - "sha256:31d0aeca8d8ee2301c62c5c340e0889d653b1280d68f9fa203982cb6337b050e", - "sha256:44c7f99ca17ebbcc96fc54ed00b454d8313f1eac28c563098d8b901025aff941", - "sha256:5471444f53f9db6a1f1f11f5dbc173228881df8446380b6b98f90afb8fd8348e", - "sha256:561bca3b1bde6d6564306eb05848fd155136e9c3a25d2961129b1e2edba22fce", - "sha256:5bf58e1d2c2f55365c06e8cb5abe067b88ca2e5550fb62009c41df4b54505acf", - "sha256:6b7163d1e85d76b0815df63fcc310daec02b44532bb433f743142d4febcb181f", - "sha256:766d79cddad95f5f6020037fe60ea8b98578afdf0c59d5a60c106c1bdd886303", - "sha256:770b7372d5ca68308ff66d7baee53369fa5ce985f84bcb6aa1948c1f2f7b02f2", - "sha256:7ab178da777fc0f55b6aef5a755f99726e8e4b75e3903954df07b27059b54fcf", - "sha256:8078305e77c2f6649d36b24d8778096413e474d9d7892c6f92cfb589c9d71b2e", - "sha256:85600b63a386d860eeaa955e9335e18dd0d7e5477e9214825abf2c2884488369", - "sha256:857d9b939ae128be1c0c792eb885c7ff6a386b9dea899ac4b06f4d90a31f9d87", - "sha256:87a41630c90c179fa5c593400f30a467c498972c702f348d41e19dafeb1d319e", - "sha256:8805d486c6128cc0fcc8ecf16c4095d99a8693a541ef851429ab334e028a4a97", - "sha256:8d71b7a89c306a41ccc7741fc9409b14f5b86727455c2a1c0c7cfcb0f784e1f2", - "sha256:9e1b80bd65f8f160880cb4dad7f55697f6d37b2d7f251fc0c2128e811928f369", - "sha256:9e290c84a145ae2411ee0ec9913c41cd7500e2e7485fe93632434d84ef4fda67", - "sha256:9ec9f88b5bc94bd99372f27cdd53af1c92ba06717380b127733b953cfb181174", - "sha256:a0a02a8b4ba6deadf706d5f849539b3685b72b186a3c9ef5d43e8972ed60fb6f", - "sha256:a4059c59519f5940e01a071f74ae2a60ea8f6185b03d22a09d40c7959a36b16b", - "sha256:a6e028c2a6da2ebfa2365a5b32531d311fbfec0e3600fc27e901b64f0ff7e54e", - "sha256:adcdebf9f8463df4120c427cf6c9aed39258bccd03ed37b6939e7a145d64d6e0", - "sha256:bdec982610259d07156a58f80b8c3e69be7751a9208bc577b059c5193d087fad", - "sha256:cefc4d4251ffb73feb303d4b7e9d6c367cb60f2db16d259ea28b114045f965aa", - "sha256:d4145c8aa6afbac10ad27e408f7ce15992fe89ba5d0b4abca31c0c2729864c03", - "sha256:da76dc5ad719ee99de5ea28a5629ff92172cbb4a70d8a6ae3a5b7a53c7382ce1", - "sha256:dde2452c08ef8b6426ccab6b5b6de9f06d836d9937d6870e68153cbf8cb49348", - "sha256:e3d88091d2539a4868750914a6fe7b9ec50e42b913851fc1b77423b5bd918530", - "sha256:f9c67cfe6278499d7f83559dc6322a8bbb108e307817a3d7acbfea807b3603cc" - ], - "version": "==1.22.0" - }, - "gunicorn": { - "hashes": [ - "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", - "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" - ], - "index": "pypi", - "version": "==19.9.0" - }, - "idna": { - "hashes": [ - "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", - "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" - ], - "version": "==2.8" - }, - "isort": { - "hashes": [ - "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", - "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" - ], - "version": "==4.3.21" - }, - "itsdangerous": { - "hashes": [ - "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", - "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" - ], - "version": "==1.1.0" - }, - "jinja2": { - "hashes": [ - "sha256:065c4f02ebe7f7cf559e49ee5a95fb800a9e4528727aec6f24402a5374c65013", - "sha256:14dd6caf1527abb21f08f86c784eac40853ba93edb79552aa1e4b8aef1b61c7b" - ], - "version": "==2.10.1" - }, - "lazy-object-proxy": { - "hashes": [ - "sha256:159a745e61422217881c4de71f9eafd9d703b93af95618635849fe469a283661", - "sha256:23f63c0821cc96a23332e45dfaa83266feff8adc72b9bcaef86c202af765244f", - "sha256:3b11be575475db2e8a6e11215f5aa95b9ec14de658628776e10d96fa0b4dac13", - "sha256:3f447aff8bc61ca8b42b73304f6a44fa0d915487de144652816f950a3f1ab821", - "sha256:4ba73f6089cd9b9478bc0a4fa807b47dbdb8fad1d8f31a0f0a5dbf26a4527a71", - "sha256:4f53eadd9932055eac465bd3ca1bd610e4d7141e1278012bd1f28646aebc1d0e", - "sha256:64483bd7154580158ea90de5b8e5e6fc29a16a9b4db24f10193f0c1ae3f9d1ea", - "sha256:6f72d42b0d04bfee2397aa1862262654b56922c20a9bb66bb76b6f0e5e4f9229", - "sha256:7c7f1ec07b227bdc561299fa2328e85000f90179a2f44ea30579d38e037cb3d4", - "sha256:7c8b1ba1e15c10b13cad4171cfa77f5bb5ec2580abc5a353907780805ebe158e", - "sha256:8559b94b823f85342e10d3d9ca4ba5478168e1ac5658a8a2f18c991ba9c52c20", - "sha256:a262c7dfb046f00e12a2bdd1bafaed2408114a89ac414b0af8755c696eb3fc16", - "sha256:acce4e3267610c4fdb6632b3886fe3f2f7dd641158a843cf6b6a68e4ce81477b", - "sha256:be089bb6b83fac7f29d357b2dc4cf2b8eb8d98fe9d9ff89f9ea6012970a853c7", - "sha256:bfab710d859c779f273cc48fb86af38d6e9210f38287df0069a63e40b45a2f5c", - "sha256:c10d29019927301d524a22ced72706380de7cfc50f767217485a912b4c8bd82a", - "sha256:dd6e2b598849b3d7aee2295ac765a578879830fb8966f70be8cd472e6069932e", - "sha256:e408f1eacc0a68fed0c08da45f31d0ebb38079f043328dce69ff133b95c29dc1" - ], - "version": "==1.4.1" - }, - "markupsafe": { - "hashes": [ - "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", - "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", - "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", - "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", - "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", - "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", - "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", - "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", - "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", - "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", - "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", - "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", - "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", - "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", - "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", - "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", - "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", - "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", - "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", - "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", - "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", - "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", - "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", - "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", - "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", - "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", - "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", - "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7" - ], - "version": "==1.1.1" - }, - "mccabe": { - "hashes": [ - "sha256:ab8a6258860da4b6677da4bd2fe5dc2c659cff31b3ee4f7f5d64e79735b80d42", - "sha256:dd8d182285a0fe56bace7f45b5e7d1a6ebcbf524e8f3bd87eb0f125271b8831f" - ], - "version": "==0.6.1" - }, - "protobuf": { - "hashes": [ - "sha256:05c36022fef3c7d3562ac22402965c0c2b9fe8421f459bb377323598996e407f", - "sha256:139b7eadcca0a861d60b523cb37d9475505e0dfb07972436b15407c2b968d87e", - "sha256:15f683006cb77fb849b1f561e509b03dd2b7dcc749086b8dd1831090d0ba4740", - "sha256:2ad566b7b7cdd8717c7af1825e19f09e8fef2787b77fcb979588944657679604", - "sha256:35cfcf97642ef62108e10a9431c77733ec7eaab8e32fe4653de20403429907cb", - "sha256:387822859ecdd012fdc25ec879f7f487da6e1d5b1ae6115e227e6be208836f71", - "sha256:4df14cbe1e7134afcfdbb9f058949e31c466de27d9b2f7fb4da9e0b67231b538", - "sha256:586c4ca37a7146d4822c700059f150ac3445ce0aef6f3ea258640838bb892dc2", - "sha256:58b11e530e954d29ab3180c48dc558a409f705bf16739fd4e0d3e07924ad7add", - "sha256:63c8c98ccb8c95f41c18fb829aeeab21c6249adee4ed75354125bdc44488f30e", - "sha256:72edcbacd0c73eef507d2ff1af99a6c27df18e66a3ff4351e401182e4de62b03", - "sha256:83dc8a561b3b954fd7002c690bb83278b8d1742a1e28abba9aaef28b0c8b437d", - "sha256:913171ecc84c2726b86574e40549a0ea619d569657c5a5ff782a3be7d81401a5", - "sha256:aabb7c741d3416671c3e6fe7c52970a226e6a8274417a97d7d795f953fadef36", - "sha256:b3452bbda12b1cbe2187d416779de07b2ab4c497d83a050e43c344778763721d", - "sha256:c5d5b8d4a9212338297fa1fa44589f69b470c0ba1d38168b432d577176b386a8", - "sha256:d86ee389c2c4fc3cebabb8ce83a8e97b6b3b5dc727b7419c1ccdc7b6e545a233", - "sha256:f2db8c754de788ab8be5e108e1e967c774c0942342b4f8aaaf14063889a6cfdc" - ], - "version": "==3.9.0" - }, - "pylint": { - "hashes": [ - "sha256:5d77031694a5fb97ea95e828c8d10fc770a1df6eb3906067aaed42201a8a6a09", - "sha256:723e3db49555abaf9bf79dc474c6b9e2935ad82230b10c1138a71ea41ac0fff1" - ], - "index": "pypi", - "version": "==2.3.1" - }, - "pyotp": { - "hashes": [ - "sha256:c88f37fd47541a580b744b42136f387cdad481b560ef410c0d85c957eb2a2bc0", - "sha256:fc537e8acd985c5cbf51e11b7d53c42276fee017a73aec7c07380695671ca1a1" - ], - "index": "pypi", - "version": "==2.3.0" - }, - "python-decouple": { - "hashes": [ - "sha256:1317df14b43efee4337a4aa02914bf004f010cd56d6c4bd894e6474ec8c4fe2d" - ], - "index": "pypi", - "version": "==3.1" - }, - "pytz": { - "hashes": [ - "sha256:303879e36b721603cc54604edcac9d20401bdbe31e1e4fdee5b9f98d5d31dfda", - "sha256:d747dd3d23d77ef44c6a3526e274af6efeb0a6f1afd5a69ba4d5be4098c8e141" - ], - "version": "==2019.1" - }, - "requests": { - "hashes": [ - "sha256:11e007a8a2aa0323f5a921e9e6a2d7e4e67d9877e85773fba9ba6419025cbeb4", - "sha256:9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590f48c010551dc6c4b31" - ], - "index": "pypi", - "version": "==2.22.0" - }, - "six": { - "hashes": [ - "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", - "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" - ], - "version": "==1.12.0" - }, - "tenacity": { - "hashes": [ - "sha256:a0c3c5f7ae0c33f5556c775ca059c12d6fd8ab7121613a713e8b7d649908571b", - "sha256:b87c1934daa0b2ccc7db153c37b8bf91d12f165936ade8628e7b962b92dc7705" - ], - "version": "==5.0.4" - }, - "transitions": { - "hashes": [ - "sha256:00bfa91b0cfe3f649731b2f28478c4314bb9e830c600b422973f1b3a704ac710", - "sha256:afe0f498cf1f3f3b0fc13562011b8895a172df8f891dbb5118923d46e78a96d7" - ], - "index": "pypi", - "version": "==0.6.9" - }, - "typed-ast": { - "hashes": [ - "sha256:18511a0b3e7922276346bcb47e2ef9f38fb90fd31cb9223eed42c85d1312344e", - "sha256:262c247a82d005e43b5b7f69aff746370538e176131c32dda9cb0f324d27141e", - "sha256:2b907eb046d049bcd9892e3076c7a6456c93a25bebfe554e931620c90e6a25b0", - "sha256:354c16e5babd09f5cb0ee000d54cfa38401d8b8891eefa878ac772f827181a3c", - "sha256:4e0b70c6fc4d010f8107726af5fd37921b666f5b31d9331f0bd24ad9a088e631", - "sha256:630968c5cdee51a11c05a30453f8cd65e0cc1d2ad0d9192819df9978984529f4", - "sha256:66480f95b8167c9c5c5c87f32cf437d585937970f3fc24386f313a4c97b44e34", - "sha256:71211d26ffd12d63a83e079ff258ac9d56a1376a25bc80b1cdcdf601b855b90b", - "sha256:95bd11af7eafc16e829af2d3df510cecfd4387f6453355188342c3e79a2ec87a", - "sha256:bc6c7d3fa1325a0c6613512a093bc2a2a15aeec350451cbdf9e1d4bffe3e3233", - "sha256:cc34a6f5b426748a507dd5d1de4c1978f2eb5626d51326e43280941206c209e1", - "sha256:d755f03c1e4a51e9b24d899561fec4ccaf51f210d52abdf8c07ee2849b212a36", - "sha256:d7c45933b1bdfaf9f36c579671fec15d25b06c8398f113dab64c18ed1adda01d", - "sha256:d896919306dd0aa22d0132f62a1b78d11aaf4c9fc5b3410d3c666b818191630a", - "sha256:ffde2fbfad571af120fcbfbbc61c72469e72f550d676c3342492a9dfdefb8f12" - ], - "markers": "implementation_name == 'cpython'", - "version": "==1.4.0" - }, - "urllib3": { - "hashes": [ - "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1", - "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232" - ], - "version": "==1.25.3" - }, - "werkzeug": { - "hashes": [ - "sha256:87ae4e5b5366da2347eb3116c0e6c681a0e939a33b2805e2c0cbd282664932c4", - "sha256:a13b74dd3c45f758d4ebdb224be8f1ab8ef58b3c0ffc1783a8c7d9f4f50227e6" - ], - "version": "==0.15.5" - }, - "wrapt": { - "hashes": [ - "sha256:565a021fd19419476b9362b05eeaa094178de64f8361e44468f9e9d7843901e1" - ], - "version": "==1.11.2" - } - }, - "develop": {} -} diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/common_fields.py b/common_fields.py old mode 100644 new mode 100755 index 4450bb6..9b0950d --- a/common_fields.py +++ b/common_fields.py @@ -1,5 +1,5 @@ -from etcd3_wrapper import Etcd3Wrapper from specs_parser import SpecsParser +from config import etcd_client as client specs_parser = SpecsParser(exceptional_devices=["cpu"]) @@ -43,8 +43,6 @@ class VmUUIDField(Field): self.validation = self.vm_uuid_validation def vm_uuid_validation(self): - client = Etcd3Wrapper() - r = client.get(f"/v1/vm/{self.uuid}") if not r: self.add_error(f"VM with uuid {self.uuid} does not exists") diff --git a/config.py b/config.py new file mode 100644 index 0000000..833dc0d --- /dev/null +++ b/config.py @@ -0,0 +1,4 @@ +from etcd3_wrapper import Etcd3Wrapper +from decouple import config + +etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) diff --git a/create_image_store.py b/create_image_store.py index 7d5fc56..04bc01f 100755 --- a/create_image_store.py +++ b/create_image_store.py @@ -1,8 +1,7 @@ import json from uuid import uuid4 -from etcd3_wrapper import Etcd3Wrapper +from config import etcd_client as client -client = Etcd3Wrapper() data = { "is_public": True, @@ -20,4 +19,3 @@ client.put( f"/v1/image_store/{uuid4().hex}", json.dumps(data), ) - diff --git a/helper.py b/helper.py old mode 100644 new mode 100755 diff --git a/main.py b/main.py old mode 100644 new mode 100755 index c7175e5..4cc2e9f --- a/main.py +++ b/main.py @@ -8,8 +8,8 @@ import subprocess from flask import Flask, request from flask_restful import Resource, Api from uuid import uuid4 -from etcd3_wrapper import Etcd3Wrapper from os.path import join +from config import etcd_client as client from ucloud_common.vm import VmPool, VMStatus from ucloud_common.host import HostPool @@ -22,7 +22,6 @@ from schemas import (CreateVMSchema, VMStatusSchema, app = Flask(__name__) api = Api(app) -client = Etcd3Wrapper() vm_pool = VmPool(client, "/v1/vm") host_pool = HostPool(client, "/v1/host") request_pool = RequestPool(client, "/v1/request") @@ -35,6 +34,7 @@ class CreateVM(Resource): validator = CreateVMSchema(data) if validator.is_valid(): # Create VM Entry under /v1/vm/ + # TODO: !!!Generate Mac Address on creation of VM vm_uuid = uuid4().hex vm_key = f"/v1/vm/{vm_uuid}" vm_entry = { @@ -43,7 +43,8 @@ class CreateVM(Resource): "hostname": "", "status": "", "image_uuid": data["image_uuid"], - "log": [] + "log": [], + "storage_attachment": [] } client.put(vm_key, vm_entry, value_in_json=True) @@ -228,8 +229,8 @@ class CreateHost(Resource): client.put(host_key, host_entry, value_in_json=True) return {"message": "Host Created"}, 200 - else: - return validator.get_errors(), 400 + + return validator.get_errors(), 400 class ListHost(Resource): diff --git a/schemas.py b/schemas.py old mode 100644 new mode 100755 index 37f2f8a..2e46b11 --- a/schemas.py +++ b/schemas.py @@ -4,18 +4,15 @@ from common_fields import Field, VmUUIDField, SpecsField from ucloud_common.host import HostPool, HostStatus from ucloud_common.vm import VmPool, VMStatus from helper import check_otp -from etcd3_wrapper import Etcd3Wrapper +from config import etcd_client as client from os.path import join -client = Etcd3Wrapper() host_pool = HostPool(client, "/v1/host") vm_pool = VmPool(client, "/v1/vm") -class BaseSchema(object): +class BaseSchema: def __init__(self, data, fields=None): - _ = data - self.__errors = [] if fields is None: self.fields = [] @@ -118,7 +115,7 @@ class CreateImageSchema(BaseSchema): image_stores = list(client.get_prefix("/v1/image_store/")) image_store = next(filter(lambda s: json.loads(s.value)["name"] == self.image_store.value, - image_stores)) + image_stores), None) if not image_store: self.add_error(f"Store '{self.image_store.value}' does not exists") diff --git a/specs_parser.py b/specs_parser.py old mode 100644 new mode 100755 index c2996af..3f05486 --- a/specs_parser.py +++ b/specs_parser.py @@ -1,7 +1,7 @@ import bitmath -class SpecsParser(object): +class SpecsParser: def __init__(self, exceptional_devices, allowed_unit=10): self.exceptional_devices = exceptional_devices self.allowed_unit = allowed_unit From 08cbecebdbfdb5c2a545f67b31793e6a1be9fc35 Mon Sep 17 00:00:00 2001 From: Ahmed Bilal <49-ahmedbilal@users.noreply.code.ungleich.ch> Date: Sat, 7 Sep 2019 12:38:58 +0200 Subject: [PATCH 10/10] allow ucloud-api to also be able work without ceph i.e use filesystem --- .gitignore | 1 + config.py | 13 +++++++++++++ main.py | 20 +++++++++++++++----- schemas.py | 2 ++ 4 files changed, 31 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 209604a..a94f03d 100755 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ venv/ settings.json ucloud_common etcd3_wrapper +log.txt diff --git a/config.py b/config.py index 833dc0d..603501c 100644 --- a/config.py +++ b/config.py @@ -1,4 +1,17 @@ +import logging + from etcd3_wrapper import Etcd3Wrapper from decouple import config +logging.basicConfig( + level=logging.DEBUG, + filename="log.txt", + filemode="a", + format="%(asctime)s: %(levelname)s - %(message)s", + datefmt="%d-%b-%y %H:%M:%S", +) + + +WITHOUT_CEPH = config("WITHOUT_CEPH", False, cast=bool) + etcd_client = Etcd3Wrapper(host=config("ETCD_URL")) diff --git a/main.py b/main.py index 4cc2e9f..7c2a61d 100755 --- a/main.py +++ b/main.py @@ -4,12 +4,14 @@ import json import subprocess +import os from flask import Flask, request from flask_restful import Resource, Api from uuid import uuid4 from os.path import join from config import etcd_client as client +from config import WITHOUT_CEPH, logging from ucloud_common.vm import VmPool, VMStatus from ucloud_common.host import HostPool @@ -123,15 +125,23 @@ class VMAction(Resource): if action == "delete" and vm_entry.hostname == "": try: path_without_protocol = vm_entry.path[vm_entry.path.find(":")+1:] - rc = subprocess.call(f"rbd rm {path_without_protocol}".split(" ")) - except FileNotFoundError: - return {"message": "VM image does not exists"} - else: - if rc == 0: + + if WITHOUT_CEPH: + command_to_delete = ["rm", os.path.join("/var/vm", vm_entry.uuid)] + else: + command_to_delete = ["rbd", "rm", path_without_protocol] + + subprocess.check_output(command_to_delete, stderr=subprocess.PIPE) + except subprocess.CalledProcessError as e: + if "No such file" in e.stderr.decode("utf-8"): client.client.delete(vm_entry.key) return {"message": "VM successfully deleted"} else: + logging.exception(e) return {"message": "Some error occurred while deleting VM"} + else: + client.client.delete(vm_entry.key) + return {"message": "VM successfully deleted"} r = RequestEntry.from_scratch(type=f"{action.title()}VM", uuid=data['uuid'], diff --git a/schemas.py b/schemas.py index 2e46b11..90acd50 100755 --- a/schemas.py +++ b/schemas.py @@ -99,8 +99,10 @@ class CreateImageSchema(BaseSchema): self.uuid: Field = Field("uuid", str, data.get("uuid", KeyError)) self.name = Field("name", str, data.get("name", KeyError)) self.image_store = Field("image_store", str, data.get("image_store", KeyError)) + # Validations self.uuid.validation = self.file_uuid_validation + self.image_store.validation = self.image_store_name_validation # All Fields fields = [self.uuid, self.name, self.image_store]