uncloud/ucloud-api
2
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Implemented support for VM operations through VM name, Allow user of realm ungleich-admin to do operation on someone's else VM (for providing support etc), Fix few hidden issues, Refactoring/Formating + Little bit Documenting

Browse source
This commit is contained in:
ahmadbilalkhalid 2019-09-22 23:51:33 +05:00
commit f18dbbe62b
7 changed files with 250 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

14
Pipfile.lock generated
View file

@ -54,7 +54,7 @@
"etcd3-wrapper": {
"editable": true,
"git": "https://code.ungleich.ch/ungleich-public/etcd3_wrapper.git",
"ref": "b1893fc286e9ed59876a526d81094edd796d6815"
"ref": "76fb0bdf797199e9ea161dad1d004eea9b4520f8"
},
"flask": {
"hashes": [
@ -240,17 +240,17 @@
},
"urllib3": {
"hashes": [
"sha256:b246607a25ac80bedac05c6f282e3cdaf3afb65420fd024ac94435cabe6e18d1",
"sha256:dbe59173209418ae49d485b87d1681aefa36252ee85884c31346debd19463232"
"sha256:2f3eadfea5d92bc7899e75b5968410b749a054b492d5a6379c1344a1481bc2cb",
"sha256:9c6c593cb28f52075016307fc26b0a0f8e82bc7d1ff19aaaa959b91710a56c47"
],
"version": "==1.25.3"
"version": "==1.25.5"
},
"werkzeug": {
"hashes": [
"sha256:00d32beac38fcd48d329566f80d39f10ec2ed994efbecfb8dd4b320062d05902",
"sha256:0a24d43be6a7dce81bae05292356176d6c46d63e42a0dd3f9504b210a9cfaa43"
"sha256:7280924747b5733b246fe23972186c6b348f9ae29724135a6dfc1e53cea433e7",
"sha256:e5f4a1f98b52b18a93da705a7458e55afb26f32bff83ff5d19189f92462d65c4"
],
"version": "==0.15.6"
"version": "==0.16.0"
}
},
"develop": {

6
common_fields.py
View file

@ -62,5 +62,7 @@ class SpecsField(Field):
def specs_validation(self):
if not specs_parser.transform_specs(self.specs):
self.add_error("Invalid unit - "
"Please use following units {}".format(specs_parser.get_allowed_units()))
self.add_error(
"Invalid unit - "
"Please use following units {}".format(specs_parser.get_allowed_units())
)

9
config.py
View file

@ -3,6 +3,11 @@ import logging
from etcd3_wrapper import Etcd3Wrapper
from decouple import config
from ucloud_common.vm import VmPool
from ucloud_common.host import HostPool
from ucloud_common.request import RequestPool
logging.basicConfig(
level=logging.DEBUG,
filename="log.txt",
@ -21,3 +26,7 @@ IMAGE_PREFIX = config("IMAGE_PREFIX")
IMAGE_STORE_PREFIX = config("IMAGE_STORE_PREFIX")
etcd_client = Etcd3Wrapper(host=config("ETCD_URL"))
VM_POOL = VmPool(etcd_client, VM_PREFIX)
HOST_POOL = HostPool(etcd_client, HOST_PREFIX)
REQUEST_POOL = RequestPool(etcd_client, REQUEST_PREFIX)

11
create_image_store.py
View file

@ -11,14 +11,7 @@ data = {
"type": "ceph",
"name": "images",
"description": "first ever public image-store",
"attributes": {
"list": [],
"key": [],
"pool": "images",
}
"attributes": {"list": [], "key": [], "pool": "images"},
}
client.put(
os.path.join(IMAGE_STORE_PREFIX, uuid4().hex),
json.dumps(data),
)
client.put(os.path.join(IMAGE_STORE_PREFIX, uuid4().hex), json.dumps(data))

17
helper.py
View file

@ -3,6 +3,7 @@ import requests
from decouple import config
from pyotp import TOTP
from config import VM_POOL
def check_otp(name, realm, token):
@ -26,3 +27,19 @@ def check_otp(name, realm, token):
data=data,
)
return response.status_code
def resolve_vm_name(name, owner):
"""
Input: name of vm, owner of vm
Output: uuid of vm if found otherwise None
"""
result = next(
filter(
lambda vm: vm.value["owner"] == owner and vm.value["name"] == name,
VM_POOL.vms,
),
None,
)
if result:
return result.key.split("/")[-1]

104
main.py
View file

@ -1,7 +1,7 @@
# TODO
# 1. Allow user of realm ungleich-admin to perform any action on
# any user vm.
# TODO: realm should be part of user's name. Because, there can be multiple
# name but with different realms. For Example, nico exists in both
# ungleich-admin and ungleich-twitter realm. So, to differentiate between
# them, we need to make realm part of user's name
import json
import subprocess
import os
@ -10,29 +10,34 @@ from uuid import uuid4
from flask import Flask, request
from flask_restful import Resource, Api
from ucloud_common.vm import VmPool, VMStatus
from ucloud_common.host import HostPool
from ucloud_common.request import (RequestEntry,
RequestPool,
RequestType)
from ucloud_common.vm import VMStatus
from ucloud_common.request import RequestEntry, RequestType
from config import etcd_client as client
from config import (WITHOUT_CEPH, VM_PREFIX,
HOST_PREFIX, REQUEST_PREFIX,
FILE_PREFIX, IMAGE_PREFIX,
logging)
from schemas import (CreateVMSchema, VMStatusSchema,
CreateImageSchema, VmActionSchema,
OTPSchema, CreateHostSchema,
VmMigrationSchema)
from config import (
WITHOUT_CEPH,
VM_PREFIX,
HOST_PREFIX,
FILE_PREFIX,
IMAGE_PREFIX,
logging,
REQUEST_POOL,
VM_POOL,
HOST_POOL,
)
from schemas import (
CreateVMSchema,
VMStatusSchema,
CreateImageSchema,
VmActionSchema,
OTPSchema,
CreateHostSchema,
VmMigrationSchema,
)
app = Flask(__name__)
api = Api(app)
VM_POOL = VmPool(client, VM_PREFIX)
HOST_POOL = HostPool(client, HOST_PREFIX)
REQUEST_POOL = RequestPool(client, REQUEST_PREFIX)
class CreateVM(Resource):
@staticmethod
@ -43,6 +48,7 @@ class CreateVM(Resource):
vm_uuid = uuid4().hex
vm_key = os.path.join(VM_PREFIX, vm_uuid)
vm_entry = {
"name": data["vm_name"],
"owner": data["name"],
"specs": data["specs"],
"hostname": "",
@ -50,13 +56,12 @@ class CreateVM(Resource):
"image_uuid": data["image_uuid"],
"log": [],
"storage_attachment": [],
"vnc_socket": ""
"vnc_socket": "",
}
client.put(vm_key, vm_entry, value_in_json=True)
# Create ScheduleVM Request
r = RequestEntry.from_scratch(type=RequestType.ScheduleVM,
uuid=vm_uuid)
r = RequestEntry.from_scratch(type=RequestType.ScheduleVM, uuid=vm_uuid)
REQUEST_POOL.put(r)
return {"message": "VM Creation Queued"}, 200
@ -69,7 +74,7 @@ class VmStatus(Resource):
data = request.json
validator = VMStatusSchema(data)
if validator.is_valid():
vm = VM_POOL.get(os.path.join(VM_PREFIX, data['uuid']))
vm = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"]))
return str(vm)
else:
return validator.get_errors(), 400
@ -81,7 +86,7 @@ class CreateImage(Resource):
data = request.json
validator = CreateImageSchema(data)
if validator.is_valid():
file_entry = client.get(os.path.join(FILE_PREFIX, data['uuid']))
file_entry = client.get(os.path.join(FILE_PREFIX, data["uuid"]))
file_entry_value = json.loads(file_entry.value)
image_entry_json = {
@ -92,7 +97,9 @@ class CreateImage(Resource):
"store_name": data["image_store"],
"visibility": "public",
}
client.put(os.path.join(IMAGE_PREFIX, data['uuid']), json.dumps(image_entry_json))
client.put(
os.path.join(IMAGE_PREFIX, data["uuid"]), json.dumps(image_entry_json)
)
return {"message": "Image successfully created"}
return validator.get_errors(), 400
@ -115,7 +122,7 @@ class VMAction(Resource):
validator = VmActionSchema(data)
if validator.is_valid():
vm_entry = VM_POOL.get(os.path.join(VM_PREFIX, data['uuid']))
vm_entry = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"]))
action = data["action"]
if action == "start":
@ -128,7 +135,10 @@ class VMAction(Resource):
path_without_protocol = vm_entry.path[vm_entry.path.find(":") + 1 :]
if WITHOUT_CEPH:
command_to_delete = ["rm", os.path.join("/var/vm", vm_entry.uuid)]
command_to_delete = [
"rm",
os.path.join("/var/vm", vm_entry.uuid),
]
else:
command_to_delete = ["rbd", "rm", path_without_protocol]
@ -144,9 +154,11 @@ class VMAction(Resource):
client.client.delete(vm_entry.key)
return {"message": "VM successfully deleted"}
r = RequestEntry.from_scratch(type="{}VM".format(action.title()),
uuid=data['uuid'],
hostname=vm_entry.hostname)
r = RequestEntry.from_scratch(
type="{}VM".format(action.title()),
uuid=data["uuid"],
hostname=vm_entry.hostname,
)
REQUEST_POOL.put(r)
return {"message": "VM {} Queued".format(action.title())}, 200
else:
@ -160,12 +172,14 @@ class VMMigration(Resource):
validator = VmMigrationSchema(data)
if validator.is_valid():
vm = VM_POOL.get(data['uuid'])
vm = VM_POOL.get(data["uuid"])
r = RequestEntry.from_scratch(type=RequestType.ScheduleVM,
r = RequestEntry.from_scratch(
type=RequestType.ScheduleVM,
uuid=vm.uuid,
destination=os.path.join(HOST_PREFIX, data["destination"]),
migration=True)
migration=True,
)
REQUEST_POOL.put(r)
return {"message": "VM Migration Initialization Queued"}, 200
else:
@ -180,22 +194,25 @@ class ListUserVM(Resource):
if validator.is_valid():
vms = client.get_prefix(VM_PREFIX, value_in_json=True)
if vms:
return_vms = []
user_vms = list(filter(lambda v: v.value["owner"] == data["name"], vms))
user_vms = filter(lambda v: v.value["owner"] == data["name"], vms)
for vm in user_vms:
return_vms.append(
{
"name": vm.value["name"],
"vm_uuid": vm.key.split("/")[-1],
"specs": vm.value["specs"],
"status": vm.value["status"],
"hostname": vm.value["hostname"],
"vnc_socket": None or vm.value["vnc_socket"]
"vnc_socket": None
if vm.value.get("vnc_socket", None) == None
else vm.value["vnc_socket"],
}
)
if return_vms:
return {"message": return_vms}, 200
else:
return {"message": "No VM found"}, 404
else:
return validator.get_errors(), 400
@ -246,7 +263,14 @@ class ListHost(Resource):
@staticmethod
def get():
hosts = HOST_POOL.hosts
r = {host.key: {"status": host.status, "specs": host.specs, "hostname": host.hostname} for host in hosts}
r = {
host.key: {
"status": host.status,
"specs": host.specs,
"hostname": host.hostname,
}
for host in hosts
}
return r, 200

181
schemas.py
View file

@ -1,3 +1,19 @@
"""
This module contain classes thats validates and intercept/modify
data coming from ucloud-cli (user)
It was primarily developed as an alternative to argument parser
of Flask_Restful which is going to be deprecated. I also tried
marshmallow for that purpose but it was an overkill (because it
do validation + serialization + deserialization) and little
inflexible for our purpose.
"""
# TODO: Fix error message when user's mentioned VM (referred by name)
# does not exists.
#
# Currently, it says uuid is a required field.
import json
import os
@ -5,10 +21,9 @@ from ucloud_common.host import HostPool, HostStatus
from ucloud_common.vm import VmPool, VMStatus
from common_fields import Field, VmUUIDField, SpecsField
from helper import check_otp
from helper import check_otp, resolve_vm_name
from config import etcd_client as client
from config import (HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX,
FILE_PREFIX, IMAGE_STORE_PREFIX)
from config import HOST_PREFIX, VM_PREFIX, IMAGE_PREFIX, FILE_PREFIX, IMAGE_STORE_PREFIX
HOST_POOL = HostPool(client, HOST_PREFIX)
VM_POOL = VmPool(client, VM_PREFIX)
@ -70,30 +85,7 @@ class OTPSchema(BaseSchema):
self.add_error("Wrong Credentials")
class CreateVMSchema(OTPSchema):
def __init__(self, data):
self.specs = SpecsField(data)
self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError))
self.image_uuid.validation = self.image_uuid_validation
fields = [self.specs, self.image_uuid]
super().__init__(data=data, fields=fields)
def image_uuid_validation(self):
images = client.get_prefix(IMAGE_PREFIX)
if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]:
self.add_error("Image UUID not valid")
class VMStatusSchema(BaseSchema):
def __init__(self, data):
self.uuid = VmUUIDField(data)
fields = [self.uuid]
super().__init__(data, fields)
########################## Image Operations ###############################################
class CreateImageSchema(BaseSchema):
@ -114,46 +106,149 @@ class CreateImageSchema(BaseSchema):
def file_uuid_validation(self):
file_entry = client.get(os.path.join(FILE_PREFIX, self.uuid.value))
if file_entry is None:
self.add_error("Image File with uuid '{}' Not Found".format(self.uuid.value))
self.add_error(
"Image File with uuid '{}' Not Found".format(self.uuid.value)
)
def image_store_name_validation(self):
image_stores = list(client.get_prefix(IMAGE_STORE_PREFIX))
image_store = next(filter(lambda s: json.loads(s.value)["name"] == self.image_store.value,
image_stores), None)
image_store = next(
filter(
lambda s: json.loads(s.value)["name"] == self.image_store.value,
image_stores,
),
None,
)
if not image_store:
self.add_error("Store '{}' does not exists".format(self.image_store.value))
########################## Host Operations ###############################################
class CreateHostSchema(OTPSchema):
def __init__(self, data):
self.specs = SpecsField(data)
self.hostname = Field("hostname", str, data.get("hostname", KeyError))
fields = [self.specs, self.hostname]
super().__init__(data=data, fields=fields)
def validation(self):
if self.realm.value != "ungleich-admin":
self.add_error("Invalid Credentials/Insufficient Permission")
########################## VM Operations ###############################################
class CreateVMSchema(OTPSchema):
def __init__(self, data):
self.specs = SpecsField(data)
self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError))
self.image_uuid = Field("image_uuid", str, data.get("image_uuid", KeyError))
self.image_uuid.validation = self.image_uuid_validation
self.vm_name.validation = self.vm_name_validation
fields = [self.vm_name, self.specs, self.image_uuid]
super().__init__(data=data, fields=fields)
def image_uuid_validation(self):
images = client.get_prefix(IMAGE_PREFIX)
if self.image_uuid.value not in [i.key.split("/")[-1] for i in images]:
self.add_error("Image UUID not valid")
def vm_name_validation(self):
if resolve_vm_name(name=self.vm_name.value, owner=self.name.value):
self.add_error(
'VM with same name "{}" already exists'.format(self.vm_name.value)
)
class VMStatusSchema(OTPSchema):
def __init__(self, data):
data["uuid"] = (
resolve_vm_name(
name=data.get("vm_name", None),
owner=(data.get("in_support_of", None) or data.get("name", None)),
)
or KeyError
)
self.uuid = VmUUIDField(data)
fields = [self.uuid]
super().__init__(data, fields)
def validation(self):
vm = VM_POOL.get(self.uuid.value)
if not (
vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
):
self.add_error("Invalid User")
class VmActionSchema(OTPSchema):
def __init__(self, data):
data["uuid"] = (
resolve_vm_name(
name=data.get("vm_name", None),
owner=(data.get("in_support_of", None) or data.get("name", None)),
)
or KeyError
)
self.uuid = VmUUIDField(data)
self.action = Field("action", str, data.get("action", KeyError))
self.action.validation = self.action_validation
_fields = [self.uuid, self.action]
super().__init__(data=data, fields=_fields)
def action_validation(self):
allowed_actions = ["start", "stop", "delete"]
if self.action.value not in allowed_actions:
self.add_error("Invalid Action. Allowed Actions are {}".format(allowed_actions))
self.add_error(
"Invalid Action. Allowed Actions are {}".format(allowed_actions)
)
def validation(self):
vm = VM_POOL.get(self.uuid.value)
if vm.value["owner"] != self.name.value:
if not (
vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
):
self.add_error("Invalid User")
if self.action.value == "start" and vm.status == VMStatus.running and vm.hostname != "":
if (
self.action.value == "start"
and vm.status == VMStatus.running
and vm.hostname != ""
):
self.add_error("VM Already Running")
if self.action.value == "stop" and vm.status == VMStatus.stopped:
if self.action.value == "stop":
if vm.status == VMStatus.stopped:
self.add_error("VM Already Stopped")
elif vm.status != VMStatus.running:
self.add_error("Cannot stop non-running VM")
class VmMigrationSchema(OTPSchema):
def __init__(self, data):
data["uuid"] = (
resolve_vm_name(
name=data.get("vm_name", None),
owner=(data.get("in_support_of", None) or data.get("name", None)),
)
or KeyError
)
self.uuid = VmUUIDField(data)
self.destination = Field("destination", str, data.get("destination", KeyError))
@ -172,7 +267,9 @@ class VmMigrationSchema(OTPSchema):
def validation(self):
vm = VM_POOL.get(self.uuid.value)
if vm.owner != self.name.value:
if not (
vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
):
self.add_error("Invalid User")
if vm.status != VMStatus.running:
@ -180,17 +277,3 @@ class VmMigrationSchema(OTPSchema):
if vm.hostname == os.path.join(HOST_PREFIX, self.destination.value):
self.add_error("Destination host couldn't be same as Source Host")
class CreateHostSchema(OTPSchema):
def __init__(self, data):
self.specs = SpecsField(data)
self.hostname = Field("hostname", str, data.get("hostname", KeyError))
fields = [self.specs, self.hostname]
super().__init__(data=data, fields=fields)
def validation(self):
if self.realm.value != "ungleich-admin":
self.add_error("Invalid Credentials/Insufficient Permission")