Implemented support for VM operations through VM name, Allow user of realm ungleich-admin to do operation on someone's else VM (for providing support etc), Fix few hidden issues, Refactoring/Formating + Little bit Documenting

Pipfile.lock

@@ -54,7 +54,7 @@
         "etcd3-wrapper": {
             "editable": true,
             "git": "https://code.ungleich.ch/ungleich-public/etcd3_wrapper.git",
-            "ref": "b1893fc286e9ed59876a526d81094edd796d6815"
+            "ref": "76fb0bdf797199e9ea161dad1d004eea9b4520f8"
         },
         "flask": {
             "hashes": [
@@ -240,17 +240,17 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
+                "sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb",
-                "sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
+                "sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47"
             ],
-            "version": "==1.25.3"
+            "version": "==1.25.5"
         },
         "werkzeug": {
             "hashes": [
-                "sha256:00d32beac38fcd48d329566f80d39f10ec2ed994efbecfb8dd4b320062d05902",
+                "sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7",
-                "sha256:0a24d43be6a7dce81bae05292356176d6c46d63e42a0dd3f9504b210a9cfaa43"
+                "sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"
             ],
-            "version": "==0.15.6"
+            "version": "==0.16.0"
         }
     },
     "develop": {

common_fields.py

@@ -62,5 +62,7 @@ class SpecsField(Field):
 
     def specs_validation(self):
         if not specs_parser.transform_specs(self.specs):
-            self.add_error("Invalid unit - "
+            self.add_error(
-                           "Please use following units {}".format(specs_parser.get_allowed_units()))
+                "Invalid unit - "
+                "Please use following units {}".format(specs_parser.get_allowed_units())
+            )

config.py

@@ -3,6 +3,11 @@ import logging
 from etcd3_wrapper import Etcd3Wrapper
 from decouple import config
 
+
+from ucloud_common.vm import VmPool
+from ucloud_common.host import HostPool
+from ucloud_common.request import RequestPool
+
 logging.basicConfig(
     level=logging.DEBUG,
     filename="log.txt",
@@ -21,3 +26,7 @@ IMAGE_PREFIX = config("IMAGE_PREFIX")
 IMAGE_STORE_PREFIX = config("IMAGE_STORE_PREFIX")
 
 etcd_client = Etcd3Wrapper(host=config("ETCD_URL"))
+
+VM_POOL = VmPool(etcd_client, VM_PREFIX)
+HOST_POOL = HostPool(etcd_client, HOST_PREFIX)
+REQUEST_POOL = RequestPool(etcd_client, REQUEST_PREFIX)

create_image_store.py

@@ -11,14 +11,7 @@ data = {
     "type": "ceph",
     "name": "images",
     "description": "first ever public image-store",
-    "attributes": {
+    "attributes": {"list": [], "key": [], "pool": "images"},
-        "list": [],
-        "key": [],
-        "pool": "images",
-    }
 }
 
-client.put(
+client.put(os.path.join(IMAGE_STORE_PREFIX, uuid4().hex), json.dumps(data))
-    os.path.join(IMAGE_STORE_PREFIX, uuid4().hex),
-    json.dumps(data),
-)

helper.py

@@ -3,6 +3,7 @@ import requests
 
 from decouple import config
 from pyotp import TOTP
+from config import VM_POOL
 
 
 def check_otp(name, realm, token):
@@ -26,3 +27,19 @@ def check_otp(name, realm, token):
         data=data,
     )
     return response.status_code
+
+
+def resolve_vm_name(name, owner):
+    """
+    Input: name of vm, owner of vm
+    Output: uuid of vm if found otherwise None
+    """
+    result = next(
+        filter(
+            lambda vm: vm.value["owner"] == owner and vm.value["name"] == name,
+            VM_POOL.vms,
+        ),
+        None,
+    )
+    if result:
+        return result.key.split("/")[-1]

main.py

@@ -1,7 +1,7 @@
-# TODO
+# TODO:  realm should be part of user's name. Because, there can be multiple
-#  1. Allow user of realm ungleich-admin to perform any action on
+#       name but with different realms. For Example, nico exists in both
-#     any user vm.
+#       ungleich-admin and ungleich-twitter realm. So, to differentiate between
-
+#       them, we need to make realm part of user's name
 import json
 import subprocess
 import os
@@ -10,29 +10,34 @@ from uuid import uuid4
 
 from flask import Flask, request
 from flask_restful import Resource, Api
-from ucloud_common.vm import VmPool, VMStatus
+from ucloud_common.vm import VMStatus
-from ucloud_common.host import HostPool
+from ucloud_common.request import RequestEntry, RequestType
-from ucloud_common.request import (RequestEntry,
-                                   RequestPool,
-                                   RequestType)
 
 from config import etcd_client as client
-from config import (WITHOUT_CEPH, VM_PREFIX,
+from config import (
-                    HOST_PREFIX, REQUEST_PREFIX,
+    WITHOUT_CEPH,
-                    FILE_PREFIX, IMAGE_PREFIX,
+    VM_PREFIX,
-                    logging)
+    HOST_PREFIX,
-from schemas import (CreateVMSchema, VMStatusSchema,
+    FILE_PREFIX,
-                     CreateImageSchema, VmActionSchema,
+    IMAGE_PREFIX,
-                     OTPSchema, CreateHostSchema,
+    logging,
-                     VmMigrationSchema)
+    REQUEST_POOL,
+    VM_POOL,
+    HOST_POOL,
+)
+from schemas import (
+    CreateVMSchema,
+    VMStatusSchema,
+    CreateImageSchema,
+    VmActionSchema,
+    OTPSchema,
+    CreateHostSchema,
+    VmMigrationSchema,
+)
 
 app = Flask(__name__)
 api = Api(app)
 
-VM_POOL = VmPool(client, VM_PREFIX)
-HOST_POOL = HostPool(client, HOST_PREFIX)
-REQUEST_POOL = RequestPool(client, REQUEST_PREFIX)
-
 
 class CreateVM(Resource):
     @staticmethod
@@ -43,6 +48,7 @@ class CreateVM(Resource):
             vm_uuid = uuid4().hex
             vm_key = os.path.join(VM_PREFIX, vm_uuid)
             vm_entry = {
+                "name": data["vm_name"],
                 "owner": data["name"],
                 "specs": data["specs"],
                 "hostname": "",
@@ -50,13 +56,12 @@ class CreateVM(Resource):
                 "image_uuid": data["image_uuid"],
                 "log": [],
                 "storage_attachment": [],
-                "vnc_socket": ""
+                "vnc_socket": "",
             }
             client.put(vm_key, vm_entry, value_in_json=True)
 
             # Create ScheduleVM Request
-            r = RequestEntry.from_scratch(type=RequestType.ScheduleVM,
+            r = RequestEntry.from_scratch(type=RequestType.ScheduleVM, uuid=vm_uuid)
-                                          uuid=vm_uuid)
             REQUEST_POOL.put(r)
 
             return {"message": "VM Creation Queued"}, 200
@@ -69,7 +74,7 @@ class VmStatus(Resource):
         data = request.json
         validator = VMStatusSchema(data)
         if validator.is_valid():
-            vm = VM_POOL.get(os.path.join(VM_PREFIX, data['uuid']))
+            vm = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"]))
             return str(vm)
         else:
             return validator.get_errors(), 400
@@ -81,7 +86,7 @@ class CreateImage(Resource):
         data = request.json
         validator = CreateImageSchema(data)
         if validator.is_valid():
-            file_entry = client.get(os.path.join(FILE_PREFIX, data['uuid']))
+            file_entry = client.get(os.path.join(FILE_PREFIX, data["uuid"]))
             file_entry_value = json.loads(file_entry.value)
 
             image_entry_json = {
@@ -92,7 +97,9 @@ class CreateImage(Resource):
                 "store_name": data["image_store"],
                 "visibility": "public",
             }
-            client.put(os.path.join(IMAGE_PREFIX, data['uuid']), json.dumps(image_entry_json))
+            client.put(
+                os.path.join(IMAGE_PREFIX, data["uuid"]), json.dumps(image_entry_json)
+            )
 
             return {"message": "Image successfully created"}
         return validator.get_errors(), 400
@@ -115,7 +122,7 @@ class VMAction(Resource):
         validator = VmActionSchema(data)
 
         if validator.is_valid():
-            vm_entry = VM_POOL.get(os.path.join(VM_PREFIX, data['uuid']))
+            vm_entry = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"]))
             action = data["action"]
 
             if action == "start":
@@ -125,10 +132,13 @@ class VMAction(Resource):
 
             if action == "delete" and vm_entry.hostname == "":
                 try:
-                    path_without_protocol = vm_entry.path[vm_entry.path.find(":")+1:]
+                    path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1 :]
 
                     if WITHOUT_CEPH:
-                        command_to_delete = ["rm", os.path.join("/var/vm", vm_entry.uuid)]
+                        command_to_delete = [
+                            "rm",
+                            os.path.join("/var/vm", vm_entry.uuid),
+                        ]
                     else:
                         command_to_delete = ["rbd", "rm", path_without_protocol]
 
@@ -144,9 +154,11 @@ class VMAction(Resource):
                     client.client.delete(vm_entry.key)
                     return {"message": "VM successfully deleted"}
 
-            r = RequestEntry.from_scratch(type="{}VM".format(action.title()),
+            r = RequestEntry.from_scratch(
-                                          uuid=data['uuid'],
+                type="{}VM".format(action.title()),
-                                          hostname=vm_entry.hostname)
+                uuid=data["uuid"],
+                hostname=vm_entry.hostname,
+            )
             REQUEST_POOL.put(r)
             return {"message": "VM {} Queued".format(action.title())}, 200
         else:
@@ -160,12 +172,14 @@ class VMMigration(Resource):
         validator = VmMigrationSchema(data)
 
         if validator.is_valid():
-            vm = VM_POOL.get(data['uuid'])
+            vm = VM_POOL.get(data["uuid"])
 
-            r = RequestEntry.from_scratch(type=RequestType.ScheduleVM,
+            r = RequestEntry.from_scratch(
+                type=RequestType.ScheduleVM,
                 uuid=vm.uuid,
                 destination=os.path.join(HOST_PREFIX, data["destination"]),
-                                          migration=True)
+                migration=True,
+            )
             REQUEST_POOL.put(r)
             return {"message": "VM Migration Initialization Queued"}, 200
         else:
@@ -180,22 +194,25 @@ class ListUserVM(Resource):
 
         if validator.is_valid():
             vms = client.get_prefix(VM_PREFIX, value_in_json=True)
-            if vms:
             return_vms = []
-                user_vms = list(filter(lambda v: v.value["owner"] == data["name"], vms))
+            user_vms = filter(lambda v: v.value["owner"] == data["name"], vms)
             for vm in user_vms:
                 return_vms.append(
                     {
+                        "name": vm.value["name"],
                         "vm_uuid": vm.key.split("/")[-1],
                         "specs": vm.value["specs"],
                         "status": vm.value["status"],
                         "hostname": vm.value["hostname"],
-                            "vnc_socket": None or vm.value["vnc_socket"]
+                        "vnc_socket": None
+                        if vm.value.get("vnc_socket", None) == None
+                        else vm.value["vnc_socket"],
                     }
                 )
+            if return_vms:
                 return {"message": return_vms}, 200
-            else:
             return {"message": "No VM found"}, 404
+
         else:
             return validator.get_errors(), 400
 
@@ -246,7 +263,14 @@ class ListHost(Resource):
     @staticmethod
     def get():
         hosts = HOST_POOL.hosts
-        r = {host.key: {"status": host.status, "specs": host.specs, "hostname": host.hostname} for host in hosts}
+        r = {
+            host.key: {
+                "status": host.status,
+                "specs": host.specs,
+                "hostname": host.hostname,
+            }
+            for host in hosts
+        }
         return r, 200
 
 

schemas.py

@@ -1,3 +1,19 @@
+"""
+This module contain classes thats validates and intercept/modify
+data coming from ucloud-cli (user)
+
+It was primarily developed as an alternative to argument parser
+of Flask_Restful which is going to be deprecated. I also tried
+marshmallow for that purpose but it was an overkill (because it
+do validation + serialization + deserialization) and little
+inflexible for our purpose.
+"""
+
+# TODO: Fix error message when user's mentioned VM (referred by name)
+#       does not exists.
+#       
+#       Currently, it says uuid is a required field.
+
 import json
 import os
 
@@ -5,10 +21,9 @@ from ucloud_common.host import HostPool, HostStatus
 from ucloud_common.vm import VmPool, VMStatus
 
 from common_fields import Field, VmUUIDField, SpecsField
-from helper import check_otp
+from helper import check_otp, resolve_vm_name
 from config import etcd_client as client
-from config import (HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX,
+from config import HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX, FILE_PREFIX, IMAGE_STORE_PREFIX
-                    FILE_PREFIX, IMAGE_STORE_PREFIX)
 
 HOST_POOL = HostPool(client, HOST_PREFIX)
 VM_POOL = VmPool(client, VM_PREFIX)
@@ -70,30 +85,7 @@ class OTPSchema(BaseSchema):
             self.add_error("Wrong Credentials")
 
 
-class CreateVMSchema(OTPSchema):
+########################## Image Operations ###############################################
-    def __init__(self, data):
-        self.specs = SpecsField(data)
-
-        self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError))
-        self.image_uuid.validation = self.image_uuid_validation
-
-        fields = [self.specs, self.image_uuid]
-        super().__init__(data=data, fields=fields)
-
-    def image_uuid_validation(self):
-        images = client.get_prefix(IMAGE_PREFIX)
-
-        if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]:
-            self.add_error("Image UUID not valid")
-
-
-class VMStatusSchema(BaseSchema):
-    def __init__(self, data):
-        self.uuid = VmUUIDField(data)
-
-        fields = [self.uuid]
-
-        super().__init__(data, fields)
 
 
 class CreateImageSchema(BaseSchema):
@@ -114,46 +106,149 @@ class CreateImageSchema(BaseSchema):
     def file_uuid_validation(self):
         file_entry = client.get(os.path.join(FILE_PREFIX, self.uuid.value))
         if file_entry is None:
-            self.add_error("Image File with uuid '{}' Not Found".format(self.uuid.value))
+            self.add_error(
+                "Image File with uuid '{}' Not Found".format(self.uuid.value)
+            )
 
     def image_store_name_validation(self):
         image_stores = list(client.get_prefix(IMAGE_STORE_PREFIX))
 
-        image_store = next(filter(lambda s: json.loads(s.value)["name"] == self.image_store.value,
+        image_store = next(
-                                  image_stores), None)
+            filter(
+                lambda s: json.loads(s.value)["name"] == self.image_store.value,
+                image_stores,
+            ),
+            None,
+        )
         if not image_store:
             self.add_error("Store '{}' does not exists".format(self.image_store.value))
 
 
+########################## Host Operations ###############################################
+
+
+class CreateHostSchema(OTPSchema):
+    def __init__(self, data):
+        self.specs = SpecsField(data)
+        self.hostname = Field("hostname", str, data.get("hostname", KeyError))
+
+        fields = [self.specs, self.hostname]
+
+        super().__init__(data=data, fields=fields)
+
+    def validation(self):
+        if self.realm.value != "ungleich-admin":
+            self.add_error("Invalid Credentials/Insufficient Permission")
+
+
+########################## VM Operations ###############################################
+
+
+class CreateVMSchema(OTPSchema):
+    def __init__(self, data):
+        self.specs = SpecsField(data)
+
+        self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError))
+        self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError))
+
+        self.image_uuid.validation = self.image_uuid_validation
+        self.vm_name.validation = self.vm_name_validation
+
+        fields = [self.vm_name, self.specs, self.image_uuid]
+        super().__init__(data=data, fields=fields)
+
+    def image_uuid_validation(self):
+        images = client.get_prefix(IMAGE_PREFIX)
+
+        if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]:
+            self.add_error("Image UUID not valid")
+
+    def vm_name_validation(self):
+        if resolve_vm_name(name=self.vm_name.value, owner=self.name.value):
+            self.add_error(
+                'VM with same name "{}" already exists'.format(self.vm_name.value)
+            )
+
+
+class VMStatusSchema(OTPSchema):
+    def __init__(self, data):
+        data["uuid"] = (
+            resolve_vm_name(
+                name=data.get("vm_name", None),
+                owner=(data.get("in_support_of", None) or data.get("name", None)),
+            )
+            or KeyError
+        )
+        self.uuid = VmUUIDField(data)
+
+        fields = [self.uuid]
+
+        super().__init__(data, fields)
+
+    def validation(self):
+        vm = VM_POOL.get(self.uuid.value)
+        if not (
+            vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
+        ):
+            self.add_error("Invalid User")
+
+
 class VmActionSchema(OTPSchema):
     def __init__(self, data):
+        data["uuid"] = (
+            resolve_vm_name(
+                name=data.get("vm_name", None),
+                owner=(data.get("in_support_of", None) or data.get("name", None)),
+            )
+            or KeyError
+        )
         self.uuid = VmUUIDField(data)
         self.action = Field("action", str, data.get("action", KeyError))
 
         self.action.validation = self.action_validation
 
         _fields = [self.uuid, self.action]
+
         super().__init__(data=data, fields=_fields)
 
     def action_validation(self):
         allowed_actions = ["start", "stop", "delete"]
         if self.action.value not in allowed_actions:
-            self.add_error("Invalid Action. Allowed Actions are {}".format(allowed_actions))
+            self.add_error(
+                "Invalid Action. Allowed Actions are {}".format(allowed_actions)
+            )
 
     def validation(self):
         vm = VM_POOL.get(self.uuid.value)
-        if vm.value["owner"] != self.name.value:
+        if not (
+            vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
+        ):
             self.add_error("Invalid User")
 
-        if self.action.value == "start" and vm.status == VMStatus.running and vm.hostname != "":
+        if (
+            self.action.value == "start"
+            and vm.status == VMStatus.running
+            and vm.hostname != ""
+        ):
             self.add_error("VM Already Running")
 
-        if self.action.value == "stop" and vm.status == VMStatus.stopped:
+        if self.action.value == "stop":
+            if vm.status == VMStatus.stopped:
                 self.add_error("VM Already Stopped")
+            elif vm.status != VMStatus.running:
+                self.add_error("Cannot stop non-running VM")
 
 
 class VmMigrationSchema(OTPSchema):
     def __init__(self, data):
+        data["uuid"] = (
+            resolve_vm_name(
+                name=data.get("vm_name", None),
+                owner=(data.get("in_support_of", None) or data.get("name", None)),
+            )
+            or KeyError
+        )
+
         self.uuid = VmUUIDField(data)
         self.destination = Field("destination", str, data.get("destination", KeyError))
 
@@ -172,7 +267,9 @@ class VmMigrationSchema(OTPSchema):
 
     def validation(self):
         vm = VM_POOL.get(self.uuid.value)
-        if vm.owner != self.name.value:
+        if not (
+            vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
+        ):
             self.add_error("Invalid User")
 
         if vm.status != VMStatus.running:
@@ -180,17 +277,3 @@ class VmMigrationSchema(OTPSchema):
 
         if vm.hostname == os.path.join(HOST_PREFIX, self.destination.value):
             self.add_error("Destination host couldn't be same as Source Host")
-
-
-class CreateHostSchema(OTPSchema):
-    def __init__(self, data):
-        self.specs = SpecsField(data)
-        self.hostname = Field("hostname", str, data.get("hostname", KeyError))
-
-        fields = [self.specs, self.hostname]
-
-        super().__init__(data=data, fields=fields)
-
-    def validation(self):
-        if self.realm.value != "ungleich-admin":
-            self.add_error("Invalid Credentials/Insufficient Permission")