ahmedbilal/uncloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'master' of code.ungleich.ch:ucloud/ucloud

Browse source
This commit is contained in:
llnu 2019-12-08 13:32:01 +01:00
commit 6c56a7a7c6
6 changed files with 75 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
conf/ucloud.conf
View file

@ -1,12 +1,33 @@
# This section contains default values for all other sections # This section contains default values for all other sections
[DEFAULT] [DEFAULT]
AUTH_NAME = "replace me"
AUTH_SEED = "replace me"
AUTH_REALM = "replace me"
NETWORK_PREFIX = moo NETWORK_PREFIX = moo
OTP_VERIFY_ENDPOINT = verify/
[api] [api]
NETWORK_PREFIX = foo NETWORK_PREFIX = foo
[woo] [network]
NETWORK_PREFIX = foo PREFIX_LENGTH = 64
PREFIX = 2001:db8::/48
[noval] [netbox]
NETBOX_URL = https://replace-me.example.com
NETBOX_TOKEN = replace me
[etcd]
FILE_PREFIX = file/
HOST_PREFIx = host/
IMAGE_PREFIX = image/
IMAGE_STORE_PREFIX = imagestore/
NETWORK_PREFIX = network/
REQUEST_PREFIX = request/
USER_PREFIX = user/
VM_PREFIX = vm/

5
ucloud/api/common_fields.py
View file

@ -1,7 +1,6 @@
import os import os
from ucloud.config import etcd_client, env_vars from ucloud.config import etcd_client, config
class Optional: class Optional:
pass pass
@ -48,6 +47,6 @@ class VmUUIDField(Field):
self.validation = self.vm_uuid_validation self.validation = self.vm_uuid_validation
def vm_uuid_validation(self): def vm_uuid_validation(self):
r = etcd_client.get(os.path.join(env_vars.get('VM_PREFIX'), self.uuid)) r = etcd_client.get(os.path.join(config['api']['VM_PREFIX'], self.uuid))
if not r: if not r:
self.add_error("VM with uuid {} does not exists".format(self.uuid)) self.add_error("VM with uuid {} does not exists".format(self.uuid))

4
ucloud/api/create_image_store.py
View file

@ -3,7 +3,7 @@ import os
from uuid import uuid4 from uuid import uuid4
from ucloud.config import etcd_client, env_vars from ucloud.config import etcd_client, config
data = { data = {
"is_public": True, "is_public": True,
@ -13,4 +13,4 @@ data = {
"attributes": {"list": [], "key": [], "pool": "images"}, "attributes": {"list": [], "key": [], "pool": "images"},
} }
etcd_client.put(os.path.join(env_vars.get('IMAGE_STORE_PREFIX'), uuid4().hex), json.dumps(data)) etcd_client.put(os.path.join(config['api']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data))

26
ucloud/api/helper.py
View file

@ -7,15 +7,15 @@ import requests
from pyotp import TOTP from pyotp import TOTP
from ucloud.config import vm_pool, env_vars from ucloud.config import vm_pool, config
def check_otp(name, realm, token): def check_otp(name, realm, token):
try: try:
data = { data = {
"auth_name": env_vars.get("AUTH_NAME"), "auth_name": config['api']["AUTH_NAME"],
"auth_token": TOTP(env_vars.get("AUTH_SEED")).now(), "auth_token": TOTP(config['api']["AUTH_SEED"]).now(),
"auth_realm": env_vars.get("AUTH_REALM"), "auth_realm": config['api']["AUTH_REALM"],
"name": name, "name": name,
"realm": realm, "realm": realm,
"token": token, "token": token,
@ -25,8 +25,8 @@ def check_otp(name, realm, token):
response = requests.post( response = requests.post(
"{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format(
OTP_SERVER=env_vars.get("OTP_SERVER", ""), OTP_SERVER=config['api']["OTP_SERVER"],
OTP_VERIFY_ENDPOINT=env_vars.get("OTP_VERIFY_ENDPOINT", "verify/"), OTP_VERIFY_ENDPOINT=config['api']["OTP_VERIFY_ENDPOINT"]
), ),
json=data, json=data,
) )
@ -35,7 +35,7 @@ def check_otp(name, realm, token):
def resolve_vm_name(name, owner): def resolve_vm_name(name, owner):
"""Return UUID of Virtual Machine of name == name and owner == owner """Return UUID of Virtual Machine of name == name and owner == owner
Input: name of vm, owner of vm. Input: name of vm, owner of vm.
Output: uuid of vm if found otherwise None Output: uuid of vm if found otherwise None
""" """
@ -54,7 +54,7 @@ def resolve_vm_name(name, owner):
def resolve_image_name(name, etcd_client): def resolve_image_name(name, etcd_client):
"""Return image uuid given its name and its store """Return image uuid given its name and its store
* If the provided name is not in correct format * If the provided name is not in correct format
i.e {store_name}:{image_name} return ValueError i.e {store_name}:{image_name} return ValueError
* If no such image found then return KeyError * If no such image found then return KeyError
@ -70,9 +70,9 @@ def resolve_image_name(name, etcd_client):
""" """
Examples, where it would work and where it would raise exception Examples, where it would work and where it would raise exception
"images:alpine" --> ["images", "alpine"] "images:alpine" --> ["images", "alpine"]
"images" --> ["images"] it would raise Exception as non enough value to unpack "images" --> ["images"] it would raise Exception as non enough value to unpack
"images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception "images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception
as too many values to unpack as too many values to unpack
""" """
@ -80,7 +80,7 @@ def resolve_image_name(name, etcd_client):
except Exception: except Exception:
raise ValueError("Image name not in correct format i.e {store_name}:{image_name}") raise ValueError("Image name not in correct format i.e {store_name}:{image_name}")
images = etcd_client.get_prefix(env_vars.get('IMAGE_PREFIX'), value_in_json=True) images = etcd_client.get_prefix(config['api']['IMAGE_PREFIX'], value_in_json=True)
# Try to find image with name == image_name and store_name == store_name # Try to find image with name == image_name and store_name == store_name
try: try:
@ -119,14 +119,14 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='
def get_ip_addr(mac_address, device): def get_ip_addr(mac_address, device):
"""Return IP address of a device provided its mac address / link local address """Return IP address of a device provided its mac address / link local address
and the device with which it is connected. and the device with which it is connected.
For Example, if we call get_ip_addr(mac_address="52:54:00:12:34:56", device="br0") For Example, if we call get_ip_addr(mac_address="52:54:00:12:34:56", device="br0")
the following two scenarios can happen the following two scenarios can happen
1. It would return None if we can't be able to find device whose mac_address is equal 1. It would return None if we can't be able to find device whose mac_address is equal
to the arg:mac_address or the mentioned arg:device does not exists or the ip address to the arg:mac_address or the mentioned arg:device does not exists or the ip address
we found is local. we found is local.
2. It would return ip_address of device whose mac_address is equal to arg:mac_address 2. It would return ip_address of device whose mac_address is equal to arg:mac_address
and is connected/neighbor of arg:device and is connected/neighbor of arg:device
""" """
try: try:
output = sp.check_output(['ip', '-6', 'neigh', 'show', 'dev', device], stderr=sp.PIPE) output = sp.check_output(['ip', '-6', 'neigh', 'show', 'dev', device], stderr=sp.PIPE)

56
ucloud/api/main.py
View file

@ -10,7 +10,7 @@ from flask_restful import Resource, Api
from ucloud.common import counters from ucloud.common import counters
from ucloud.common.vm import VMStatus from ucloud.common.vm import VMStatus
from ucloud.common.request import RequestEntry, RequestType from ucloud.common.request import RequestEntry, RequestType
from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, env_vars, image_storage_handler) from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, config, image_storage_handler)
from . import schemas from . import schemas
from .helper import generate_mac, mac2ipv6 from .helper import generate_mac, mac2ipv6
from . import logger from . import logger
@ -28,7 +28,7 @@ class CreateVM(Resource):
validator = schemas.CreateVMSchema(data) validator = schemas.CreateVMSchema(data)
if validator.is_valid(): if validator.is_valid():
vm_uuid = uuid4().hex vm_uuid = uuid4().hex
vm_key = join_path(env_vars.get("VM_PREFIX"), vm_uuid) vm_key = join_path(config['api']["VM_PREFIX"), vm_uuid)
specs = { specs = {
"cpu": validator.specs["cpu"], "cpu": validator.specs["cpu"],
"ram": validator.specs["ram"], "ram": validator.specs["ram"],
@ -56,7 +56,7 @@ class CreateVM(Resource):
# Create ScheduleVM Request # Create ScheduleVM Request
r = RequestEntry.from_scratch( r = RequestEntry.from_scratch(
type=RequestType.ScheduleVM, uuid=vm_uuid, type=RequestType.ScheduleVM, uuid=vm_uuid,
request_prefix=env_vars.get("REQUEST_PREFIX") request_prefix=config['api']["REQUEST_PREFIX")
) )
request_pool.put(r) request_pool.put(r)
@ -71,7 +71,7 @@ class VmStatus(Resource):
validator = schemas.VMStatusSchema(data) validator = schemas.VMStatusSchema(data)
if validator.is_valid(): if validator.is_valid():
vm = vm_pool.get( vm = vm_pool.get(
join_path(env_vars.get("VM_PREFIX"), data["uuid"]) join_path(config['api']["VM_PREFIX"), data["uuid"])
) )
vm_value = vm.value.copy() vm_value = vm.value.copy()
vm_value["ip"] = [] vm_value["ip"] = []
@ -79,7 +79,7 @@ class VmStatus(Resource):
network_name, mac, tap = network_mac_and_tap network_name, mac, tap = network_mac_and_tap
network = etcd_client.get( network = etcd_client.get(
join_path( join_path(
env_vars.get("NETWORK_PREFIX"), config['api']["NETWORK_PREFIX"),
data["name"], data["name"],
network_name, network_name,
), ),
@ -100,7 +100,7 @@ class CreateImage(Resource):
validator = schemas.CreateImageSchema(data) validator = schemas.CreateImageSchema(data)
if validator.is_valid(): if validator.is_valid():
file_entry = etcd_client.get( file_entry = etcd_client.get(
join_path(env_vars.get("FILE_PREFIX"), data["uuid"]) join_path(config['api']["FILE_PREFIX"), data["uuid"])
) )
file_entry_value = json.loads(file_entry.value) file_entry_value = json.loads(file_entry.value)
@ -113,7 +113,7 @@ class CreateImage(Resource):
"visibility": "public", "visibility": "public",
} }
etcd_client.put( etcd_client.put(
join_path(env_vars.get("IMAGE_PREFIX"), data["uuid"]), join_path(config['etcd']["IMAGE_PREFIX"), data["uuid"]),
json.dumps(image_entry_json), json.dumps(image_entry_json),
) )
@ -125,7 +125,7 @@ class ListPublicImages(Resource):
@staticmethod @staticmethod
def get(): def get():
images = etcd_client.get_prefix( images = etcd_client.get_prefix(
env_vars.get("IMAGE_PREFIX"), value_in_json=True config['etcd']["IMAGE_PREFIX"), value_in_json=True
) )
r = { r = {
"images": [] "images": []
@ -148,7 +148,7 @@ class VMAction(Resource):
if validator.is_valid(): if validator.is_valid():
vm_entry = vm_pool.get( vm_entry = vm_pool.get(
join_path(env_vars.get("VM_PREFIX"), data["uuid"]) join_path(config['etcd']["VM_PREFIX"), data["uuid"])
) )
action = data["action"] action = data["action"]
@ -172,7 +172,7 @@ class VMAction(Resource):
type="{}VM".format(action.title()), type="{}VM".format(action.title()),
uuid=data["uuid"], uuid=data["uuid"],
hostname=vm_entry.hostname, hostname=vm_entry.hostname,
request_prefix=env_vars.get("REQUEST_PREFIX") request_prefix=config['etcd']["REQUEST_PREFIX"]
) )
request_pool.put(r) request_pool.put(r)
return {"message": "VM {} Queued".format(action.title())}, 200 return {"message": "VM {} Queued".format(action.title())}, 200
@ -193,10 +193,10 @@ class VMMigration(Resource):
type=RequestType.ScheduleVM, type=RequestType.ScheduleVM,
uuid=vm.uuid, uuid=vm.uuid,
destination=join_path( destination=join_path(
env_vars.get("HOST_PREFIX"), validator.destination.value config['etcd']["HOST_PREFIX"], validator.destination.value
), ),
migration=True, migration=True,
request_prefix=env_vars.get("REQUEST_PREFIX") request_prefix=config['etcd']["REQUEST_PREFIX"]
) )
request_pool.put(r) request_pool.put(r)
return {"message": "VM Migration Initialization Queued"}, 200 return {"message": "VM Migration Initialization Queued"}, 200
@ -212,7 +212,7 @@ class ListUserVM(Resource):
if validator.is_valid(): if validator.is_valid():
vms = etcd_client.get_prefix( vms = etcd_client.get_prefix(
env_vars.get("VM_PREFIX"), value_in_json=True config['etcd']["VM_PREFIX"], value_in_json=True
) )
return_vms = [] return_vms = []
user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) user_vms = filter(lambda v: v.value["owner"] == data["name"], vms)
@ -246,7 +246,7 @@ class ListUserFiles(Resource):
if validator.is_valid(): if validator.is_valid():
files = etcd_client.get_prefix( files = etcd_client.get_prefix(
env_vars.get("FILE_PREFIX"), value_in_json=True config['etcd']["FILE_PREFIX"], value_in_json=True
) )
return_files = [] return_files = []
user_files = list( user_files = list(
@ -270,7 +270,7 @@ class CreateHost(Resource):
data = request.json data = request.json
validator = schemas.CreateHostSchema(data) validator = schemas.CreateHostSchema(data)
if validator.is_valid(): if validator.is_valid():
host_key = join_path(env_vars.get("HOST_PREFIX"), uuid4().hex) host_key = join_path(config['etcd']["HOST_PREFIX"], uuid4().hex)
host_entry = { host_entry = {
"specs": data["specs"], "specs": data["specs"],
"hostname": data["hostname"], "hostname": data["hostname"],
@ -309,7 +309,7 @@ class GetSSHKeys(Resource):
# {user_prefix}/{realm}/{name}/key/ # {user_prefix}/{realm}/{name}/key/
etcd_key = join_path( etcd_key = join_path(
env_vars.get('USER_PREFIX'), config['etcd']['USER_PREFIX'],
data["realm"], data["realm"],
data["name"], data["name"],
"key", "key",
@ -326,7 +326,7 @@ class GetSSHKeys(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name} # {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path( etcd_key = join_path(
env_vars.get('USER_PREFIX'), config['etcd']['USER_PREFIX'),
data["realm"], data["realm"],
data["name"], data["name"],
"key", "key",
@ -355,7 +355,7 @@ class AddSSHKey(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name} # {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path( etcd_key = join_path(
env_vars.get("USER_PREFIX"), config['etcd']["USER_PREFIX"],
data["realm"], data["realm"],
data["name"], data["name"],
"key", "key",
@ -385,7 +385,7 @@ class RemoveSSHKey(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name} # {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path( etcd_key = join_path(
env_vars.get("USER_PREFIX"), config['etcd']["USER_PREFIX"],
data["realm"], data["realm"],
data["name"], data["name"],
"key", "key",
@ -421,17 +421,17 @@ class CreateNetwork(Resource):
} }
if validator.user.value: if validator.user.value:
nb = pynetbox.api( nb = pynetbox.api(
url=env_vars.get("NETBOX_URL"), url=config['netbox']["NETBOX_URL"],
token=env_vars.get("NETBOX_TOKEN"), token=config['netbox']["NETBOX_TOKEN"],
) )
nb_prefix = nb.ipam.prefixes.get( nb_prefix = nb.ipam.prefixes.get(
prefix=env_vars.get("PREFIX") prefix=config['network']["PREFIX"]
) )
prefix = nb_prefix.available_prefixes.create( prefix = nb_prefix.available_prefixes.create(
data={ data={
"prefix_length": env_vars.get( "prefix_length": config['network'][
"PREFIX_LENGTH", cast=int "PREFIX_LENGTH"]
), ),
"description": '{}\'s network "{}"'.format( "description": '{}\'s network "{}"'.format(
data["name"], data["network_name"] data["name"], data["network_name"]
@ -444,7 +444,7 @@ class CreateNetwork(Resource):
network_entry["ipv6"] = "fd00::/64" network_entry["ipv6"] = "fd00::/64"
network_key = join_path( network_key = join_path(
env_vars.get("NETWORK_PREFIX"), config['network']["NETWORK_PREFIX"],
data["name"], data["name"],
data["network_name"], data["network_name"],
) )
@ -462,7 +462,7 @@ class ListUserNetwork(Resource):
if validator.is_valid(): if validator.is_valid():
prefix = join_path( prefix = join_path(
env_vars.get("NETWORK_PREFIX"), data["name"] config['network']["NETWORK_PREFIX"], data["name"]
) )
networks = etcd_client.get_prefix(prefix, value_in_json=True) networks = etcd_client.get_prefix(prefix, value_in_json=True)
user_networks = [] user_networks = []
@ -498,7 +498,7 @@ api.add_resource(CreateNetwork, "/network/create")
def main(): def main():
image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'), value_in_json=True)) image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX'], value_in_json=True))
if len(image_stores) == 0: if len(image_stores) == 0:
data = { data = {
"is_public": True, "is_public": True,
@ -508,7 +508,7 @@ def main():
"attributes": {"list": [], "key": [], "pool": "images"}, "attributes": {"list": [], "key": [], "pool": "images"},
} }
etcd_client.put(join_path(env_vars.get('IMAGE_STORE_PREFIX'), uuid4().hex), json.dumps(data)) etcd_client.put(join_path(config['etcd']['IMAGE_STORE_PREFIX'], uuid4().hex), json.dumps(data))
app.run(host="::", debug=True) app.run(host="::", debug=True)

12
ucloud/api/schemas.py
View file

@ -21,7 +21,7 @@ import bitmath
from ucloud.common.host import HostStatus from ucloud.common.host import HostStatus
from ucloud.common.vm import VMStatus from ucloud.common.vm import VMStatus
from ucloud.config import etcd_client, env_vars, vm_pool, host_pool from ucloud.config import etcd_client, config, vm_pool, host_pool
from . import helper from . import helper
from .common_fields import Field, VmUUIDField from .common_fields import Field, VmUUIDField
from .helper import check_otp, resolve_vm_name from .helper import check_otp, resolve_vm_name
@ -102,14 +102,14 @@ class CreateImageSchema(BaseSchema):
super().__init__(data, fields) super().__init__(data, fields)
def file_uuid_validation(self): def file_uuid_validation(self):
file_entry = etcd_client.get(os.path.join(env_vars.get('FILE_PREFIX'), self.uuid.value)) file_entry = etcd_client.get(os.path.join(config['etcd']['FILE_PREFIX'], self.uuid.value))
if file_entry is None: if file_entry is None:
self.add_error( self.add_error(
"Image File with uuid '{}' Not Found".format(self.uuid.value) "Image File with uuid '{}' Not Found".format(self.uuid.value)
) )
def image_store_name_validation(self): def image_store_name_validation(self):
image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'))) image_stores = list(etcd_client.get_prefix(config['etcd']['IMAGE_STORE_PREFIX']))
image_store = next( image_store = next(
filter( filter(
@ -235,7 +235,7 @@ class CreateVMSchema(OTPSchema):
if _network: if _network:
for net in _network: for net in _network:
network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'), network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'],
self.name.value, self.name.value,
net), value_in_json=True) net), value_in_json=True)
if not network: if not network:
@ -400,7 +400,7 @@ class VmMigrationSchema(OTPSchema):
if vm.status != VMStatus.running: if vm.status != VMStatus.running:
self.add_error("Can't migrate non-running VM") self.add_error("Can't migrate non-running VM")
if vm.hostname == os.path.join(env_vars.get('HOST_PREFIX'), self.destination.value): if vm.hostname == os.path.join(config['etcd']['HOST_PREFIX'], self.destination.value):
self.add_error("Destination host couldn't be same as Source Host") self.add_error("Destination host couldn't be same as Source Host")
@ -442,7 +442,7 @@ class CreateNetwork(OTPSchema):
super().__init__(data, fields=fields) super().__init__(data, fields=fields)
def network_name_validation(self): def network_name_validation(self):
network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'), network = etcd_client.get(os.path.join(config['etcd']['NETWORK_PREFIX'],
self.name.value, self.name.value,
self.network_name.value), self.network_name.value),
value_in_json=True) value_in_json=True)