fnux/uncloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Compare commits

base: fnux:master
Branches Tags
fnux:master
fnux:uncloud-api-refactoring
fnux:0.0.6
fnux:0.0.5
fnux:0.0.4
fnux:0.0.3
fnux:0.0.2
fnux:0.0.1
..
compare: fnux:uncloud-api-refactoring
Branches Tags
fnux:master
fnux:uncloud-api-refactoring
fnux:0.0.6
fnux:0.0.5
fnux:0.0.4
fnux:0.0.3
fnux:0.0.2
fnux:0.0.1

2 commits
master ... uncloud-ap

Author SHA1 Message Date
meow
f1bb1ee3ca 2nd commit 2020-01-10 00:33:35 +05:00
meow
e5dd5e45c6 initial work 2020-01-10 00:03:10 +05:00
70 changed files with 719 additions and 2150 deletions
Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -16,5 +16,3 @@ uncloud/version.py
build/ build/
venv/ venv/
dist/ dist/
*.iso

12
bin/gen-version
View file

@ -1,22 +1,22 @@
#!/bin/sh #!/bin/sh
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
# #
# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org) # 2019 Nico Schottelius (nico-ucloud at schottelius.org)
# #
# This file is part of uncloud. # This file is part of ucloud.
# #
# uncloud is free software: you can redistribute it and/or modify # ucloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by # it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or # the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version. # (at your option) any later version.
# #
# uncloud is distributed in the hope that it will be useful, # ucloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of # but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details. # GNU General Public License for more details.
# #
# You should have received a copy of the GNU General Public License # You should have received a copy of the GNU General Public License
# along with uncloud. If not, see <http://www.gnu.org/licenses/>. # along with ucloud. If not, see <http://www.gnu.org/licenses/>.
# #
# #
@ -26,4 +26,4 @@ dir=${0%/*}
# Ensure version is present - the bundled/shipped version contains a static version, # Ensure version is present - the bundled/shipped version contains a static version,
# the git version contains a dynamic version # the git version contains a dynamic version
printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py

4
bin/uncloud-run-reinstall
View file

@ -24,6 +24,6 @@
dir=${0%/*} dir=${0%/*}
${dir}/gen-version; ${dir}/gen-version;
pip uninstall -y uncloud >/dev/null pip uninstall -y uncloud
python setup.py install >/dev/null python setup.py install
${dir}/uncloud "$@" ${dir}/uncloud "$@"

36
docs/source/hacking.rst
View file

@ -1,36 +0,0 @@
Hacking
=======
Using uncloud in hacking (aka development) mode.
Get the code
------------
.. code-block:: sh
:linenos:
git clone https://code.ungleich.ch/uncloud/uncloud.git
Install python requirements
---------------------------
You need to have python3 installed.
.. code-block:: sh
:linenos:
cd uncloud!
python -m venv venv
. ./venv/bin/activate
./bin/uncloud-run-reinstall
Install os requirements
-----------------------
Install the following software packages: **dnsmasq**.
If you already have a working IPv6 SLAAC and DNS setup,
this step can be skipped.
Note that you need at least one /64 IPv6 network to run uncloud.

66
docs/source/vm-images.rst
View file

@ -1,66 +0,0 @@
VM images
==================================
Overview
---------
ucloud tries to be least invasise towards VMs and only require
strictly necessary changes for running in a virtualised
environment. This includes configurations for:
* Configuring the network
* Managing access via ssh keys
* Resizing the attached disk(s)
Upstream images
---------------
The 'official' uncloud images are defined in the `uncloud/images
<https://code.ungleich.ch/uncloud/images>`_ repository.
How to make you own Uncloud images
----------------------------------
.. note::
It is fairly easy to create your own images for uncloud, as the common
operations (which are detailed below) can be automatically handled by the
`uncloud/uncloud-init <https://code.ungleich.ch/uncloud/uncloud-init>`_ tool.
Network configuration
~~~~~~~~~~~~~~~~~~~~~
All VMs in ucloud are required to support IPv6. The primary network
configuration is always done using SLAAC. A VM thus needs only to be
configured to
* accept router advertisements on all network interfaces
* use the router advertisements to configure the network interfaces
* accept the DNS entries from the router advertisements
Configuring SSH keys
~~~~~~~~~~~~~~~~~~~~
To be able to access the VM, ucloud support provisioning SSH keys.
To accept ssh keys in your VM, request the URL
*http://metadata/ssh_keys*. Add the content to the appropriate user's
**authorized_keys** file. Below you find sample code to accomplish
this task:
.. code-block:: sh
tmp=$(mktemp)
curl -s http://metadata/ssk_keys > "$tmp"
touch ~/.ssh/authorized_keys # ensure it exists
cat ~/.ssh/authorized_keys >> "$tmp"
sort "$tmp" | uniq > ~/.ssh/authorized_keys
Disk resize
~~~~~~~~~~~
In virtualised environments, the disk sizes might grow. The operating
system should detect disks that are bigger than the existing partition
table and resize accordingly. This task is os specific.
ucloud does not support shrinking disks due to the complexity and
intra OS dependencies.

90
scripts/uncloud
View file

@ -3,86 +3,56 @@ import logging
import sys import sys
import importlib import importlib
import argparse import argparse
import os import multiprocessing as mp
from etcd3.exceptions import ConnectionFailedError
from uncloud.common import settings
from uncloud import UncloudException from uncloud import UncloudException
from uncloud.common.cli import resolve_otp_credentials from contextlib import suppress
# Components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner',
'imagescanner', 'metadata', 'configure', 'hack']
ALL_COMPONENTS = ETCD_COMPONENTS.copy() def exception_hook(exc_type, exc_value, exc_traceback):
ALL_COMPONENTS.append('oneshot') logging.getLogger(__name__).error(
#ALL_COMPONENTS.append('cli') 'Uncaught exception',
exc_info=(exc_type, exc_value, exc_traceback)
)
sys.excepthook = exception_hook
if __name__ == '__main__': if __name__ == '__main__':
# Setting up root logger
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
arg_parser = argparse.ArgumentParser() arg_parser = argparse.ArgumentParser()
subparsers = arg_parser.add_subparsers(dest='command') subparsers = arg_parser.add_subparsers(dest='command')
parent_parser = argparse.ArgumentParser(add_help=False) parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument('--debug', '-d', action='store_true', default=False, parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
help='More verbose logging') help='More verbose logging')
parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
default=os.path.expanduser('~/uncloud'))
etcd_parser = argparse.ArgumentParser(add_help=False) for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner',
etcd_parser.add_argument('--etcd-host') 'metadata', 'configure', 'cli']:
etcd_parser.add_argument('--etcd-port')
etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
for component in ALL_COMPONENTS:
mod = importlib.import_module('uncloud.{}.main'.format(component)) mod = importlib.import_module('uncloud.{}.main'.format(component))
parser = getattr(mod, 'arg_parser') parser = getattr(mod, 'arg_parser')
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
if component in ETCD_COMPONENTS: args = arg_parser.parse_args()
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) if not args.command:
else:
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
arguments = vars(arg_parser.parse_args())
etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
etcd_arguments = {
'etcd': {
key.replace('etcd_', ''): arguments[key]
for key in etcd_arguments
}
}
if not arguments['command']:
arg_parser.print_help() arg_parser.print_help()
else: else:
# Initializing Settings and resolving otp_credentials
# It is neccessary to resolve_otp_credentials after argument parsing is done because
# previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
# providing the default values for --name, --realm and --seed arguments from the values
# we read from file. But, now we are asking user about where the config file lives. So,
# to providing default value is not possible before parsing arguments. So, we are doing
# it after..
# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
# resolve_otp_credentials(arguments)
name = arguments.pop('command')
mod = importlib.import_module('uncloud.{}.main'.format(name))
main = getattr(mod, 'main')
if arguments['debug']:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
log = logging.getLogger()
# if we start etcd in seperate process with default settings
# i.e inheriting few things from parent process etcd3 module
# errors out, so the following command configure multiprocessing
# module to not inherit anything from parent.
# mp.set_start_method('spawn')
arguments = vars(args)
try: try:
main(arguments) name = arguments.pop('command')
mod = importlib.import_module('uncloud.{}.main'.format(name))
main = getattr(mod, 'main')
main(**arguments)
except UncloudException as err: except UncloudException as err:
log.error(err) logger.error(err)
# except ConnectionFailedError as err:
# log.error('Cannot connect to etcd: {}'.format(err))
except Exception as err: except Exception as err:
log.exception(err) logger.exception(err)

59
uncloud/api/common_fields.py
View file

@ -1,59 +0,0 @@
import os
from uncloud.common.shared import shared
class Optional:
pass
class Field:
def __init__(self, _name, _type, _value=None):
self.name = _name
self.value = _value
self.type = _type
self.__errors = []
def validation(self):
return True
def is_valid(self):
if self.value == KeyError:
self.add_error(
"'{}' field is a required field".format(self.name)
)
else:
if isinstance(self.value, Optional):
pass
elif not isinstance(self.value, self.type):
self.add_error(
"Incorrect Type for '{}' field".format(self.name)
)
else:
self.validation()
if self.__errors:
return False
return True
def get_errors(self):
return self.__errors
def add_error(self, error):
self.__errors.append(error)
class VmUUIDField(Field):
def __init__(self, data):
self.uuid = data.get("uuid", KeyError)
super().__init__("uuid", str, self.uuid)
self.validation = self.vm_uuid_validation
def vm_uuid_validation(self):
r = shared.etcd_client.get(
os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
)
if not r:
self.add_error("VM with uuid {} does not exists".format(self.uuid))

6
uncloud/api/create_image_store.py
View file

@ -4,6 +4,7 @@ import os
from uuid import uuid4 from uuid import uuid4
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
data = { data = {
'is_public': True, 'is_public': True,
@ -13,7 +14,4 @@ data = {
'attributes': {'list': [], 'key': [], 'pool': 'images'}, 'attributes': {'list': [], 'key': [], 'pool': 'images'},
} }
shared.etcd_client.put( shared.etcd_client.put(os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data))
os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
json.dumps(data),
)

110
uncloud/api/helper.py
View file

@ -1,12 +1,11 @@
import binascii import binascii
import ipaddress
import random
import logging import logging
import requests import requests
from pyotp import TOTP from pyotp import TOTP
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -14,25 +13,19 @@ logger = logging.getLogger(__name__)
def check_otp(name, realm, token): def check_otp(name, realm, token):
try: try:
data = { data = {
"auth_name": shared.settings["otp"]["auth_name"], 'auth_name': settings['otp']['auth_name'],
"auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), 'auth_token': TOTP(settings['otp']['auth_seed']).now(),
"auth_realm": shared.settings["otp"]["auth_realm"], 'auth_realm': settings['otp']['auth_realm'],
"name": name, 'name': name,
"realm": realm, 'realm': realm,
"token": token, 'token': token,
} }
except binascii.Error as err: except binascii.Error:
logger.error( logger.error('Cannot compute OTP for seed: {}'.format(settings['otp']['auth_seed']))
"Cannot compute OTP for seed: {}".format(
shared.settings["otp"]["auth_seed"]
)
)
return 400 return 400
else:
response = requests.post( response = requests.post(settings['otp']['verification_controller_url'], json=data)
shared.settings["otp"]["verification_controller_url"], json=data return response.status_code
)
return response.status_code
def resolve_vm_name(name, owner): def resolve_vm_name(name, owner):
@ -42,29 +35,24 @@ def resolve_vm_name(name, owner):
Output: uuid of vm if found otherwise None Output: uuid of vm if found otherwise None
""" """
result = next( result = next(
filter( filter(lambda vm: vm.value['owner'] == owner and vm.value['name'] == name, shared.vm_pool.vms),
lambda vm: vm.value["owner"] == owner None
and vm.value["name"] == name,
shared.vm_pool.vms,
),
None,
) )
if result: if result:
return result.key.split("/")[-1] return result.key.split('/')[-1]
return None return None
def resolve_image_name(name, etcd_client): def resolve_image_name(name):
"""Return image uuid given its name and its store """Return image uuid given its name and its store
* If the provided name is not in correct format * If the provided name is not in correct format
i.e {store_name}:{image_name} return ValueError i.e {store_name}:{image_name} return ValueError
* If no such image found then return KeyError * If no such image found then return KeyError
""" """
seperator = ":" seperator = ':'
# Ensure, user/program passed valid name that is of type string # Ensure, user/program passed valid name that is of type string
try: try:
@ -72,77 +60,35 @@ def resolve_image_name(name, etcd_client):
""" """
Examples, where it would work and where it would raise exception Examples, where it would work and where it would raise exception
"images:alpine" --> ["images", "alpine"] 'images:alpine' --> ['images', 'alpine']
"images" --> ["images"] it would raise Exception as non enough value to unpack 'images' --> ['images'] it would raise Exception as non enough value to unpack
"images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception 'images:alpine:meow' --> ['images', 'alpine', 'meow'] it would raise Exception
as too many values to unpack as too many values to unpack
""" """
store_name, image_name = store_name_and_image_name store_name, image_name = store_name_and_image_name
except Exception: except Exception:
raise ValueError( raise ValueError('Image name not in correct format i.e {store_name}:{image_name}')
"Image name not in correct format i.e {store_name}:{image_name}"
)
images = etcd_client.get_prefix( images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True)
shared.settings["etcd"]["image_prefix"], value_in_json=True
)
# Try to find image with name == image_name and store_name == store_name # Try to find image with name == image_name and store_name == store_name
try: try:
image = next( image = next(
filter( filter(
lambda im: im.value["name"] == image_name lambda im: im.value['name'] == image_name
and im.value["store_name"] == store_name, and im.value['store_name'] == store_name,
images, images,
) )
) )
except StopIteration: except StopIteration:
raise KeyError("No image with name {} found.".format(name)) raise KeyError('No image with name {} found.'.format(name))
else: else:
image_uuid = image.key.split("/")[-1] image_uuid = image.key.split('/')[-1]
return image_uuid return image_uuid
def random_bytes(num=6): def make_return_message(err, status_code=200):
return [random.randrange(256) for _ in range(num)] return {'message': str(err)}, status_code
def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
mac = random_bytes()
if oui:
if type(oui) == str:
oui = [int(chunk) for chunk in oui.split(separator)]
mac = oui + random_bytes(num=6 - len(oui))
else:
if multicast:
mac[0] |= 1 # set bit 0
else:
mac[0] &= ~1 # clear bit 0
if uaa:
mac[0] &= ~(1 << 1) # clear bit 1
else:
mac[0] |= 1 << 1 # set bit 1
return separator.join(byte_fmt % b for b in mac)
def mac2ipv6(mac, prefix):
# only accept MACs separated by a colon
parts = mac.split(":")
# modify parts to match IPv6 value
parts.insert(3, "ff")
parts.insert(4, "fe")
parts[0] = "%x" % (int(parts[0], 16) ^ 2)
# format output
ipv6_parts = [str(0)] * 4
for i in range(0, len(parts), 2):
ipv6_parts.append("".join(parts[i : i + 2]))
lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
prefix = ipaddress.IPv6Address(prefix)
return str(prefix + int(lower_part))

526
uncloud/api/main.py
View file

@ -14,9 +14,13 @@ from uncloud.common.shared import shared
from uncloud.common import counters from uncloud.common import counters
from uncloud.common.vm import VMStatus from uncloud.common.vm import VMStatus
from uncloud.common.host import HostStatus
from uncloud.common.request import RequestEntry, RequestType from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.settings import settings
from uncloud.api import schemas from uncloud.api import schemas
from uncloud.api.helper import generate_mac, mac2ipv6 from uncloud.api.schemas import ValidationException
from uncloud.common.network import generate_mac, mac2ipv6
from uncloud.api.helper import make_return_message
from uncloud import UncloudException from uncloud import UncloudException
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -29,51 +33,52 @@ arg_parser = argparse.ArgumentParser('api', add_help=False)
arg_parser.add_argument('--port', '-p') arg_parser.add_argument('--port', '-p')
@app.errorhandler(Exception) # @app.errorhandler(Exception)
def handle_exception(e): # def handle_exception(e):
app.logger.error(e) # app.logger.error(e)
# pass through HTTP errors #
if isinstance(e, HTTPException): # # pass through HTTP errors
return e # if isinstance(e, HTTPException):
# return e
# now you're handling non-HTTP exceptions only #
return {'message': 'Server Error'}, 500 # # now you're handling non-HTTP exceptions only
# return {'message': 'Server Error'}, 500
class CreateVM(Resource): class CreateVM(Resource):
"""API Request to Handle Creation of VM"""
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.CreateVMSchema(data) try:
if validator.is_valid(): validator = schemas.CreateVMSchema(data)
validator.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
vm_uuid = uuid4().hex vm_uuid = uuid4().hex
vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid)
specs = { specs = {
'cpu': validator.specs['cpu'], 'cpu': validator.specs['cpu'],
'ram': validator.specs['ram'], 'ram': validator.specs['ram'],
'os-ssd': validator.specs['os-ssd'], 'os-ssd': validator.specs['os-ssd'],
'hdd': validator.specs['hdd'], 'hdd': validator.specs['hdd'],
} }
macs = [generate_mac() for _ in range(len(data['network']))] macs = [generate_mac() for _ in range(len(validator.network))]
tap_ids = [ tap_ids = [
counters.increment_etcd_counter( counters.increment_etcd_counter(settings['etcd']['tap_counter'])
shared.etcd_client, shared.settings['etcd']['tap_counter'] for _ in range(len(validator.network))
)
for _ in range(len(data['network']))
] ]
vm_entry = { vm_entry = {
'name': data['vm_name'], 'name': validator.vm_name,
'owner': data['name'], 'owner': validator.name,
'owner_realm': data['realm'], 'owner_realm': validator.realm,
'specs': specs, 'specs': specs,
'hostname': '', 'hostname': '',
'status': VMStatus.stopped, 'status': VMStatus.stopped,
'image_uuid': validator.image_uuid, 'image_uuid': validator.image_uuid,
'log': [], 'log': [],
'vnc_socket': '', 'vnc_socket': '',
'network': list(zip(data['network'], macs, tap_ids)), 'network': list(zip(validator.network, macs, tap_ids)),
'metadata': {'ssh-keys': []}, 'metadata': {'ssh-keys': []},
'in_migration': False, 'in_migration': False,
} }
@ -83,89 +88,79 @@ class CreateVM(Resource):
r = RequestEntry.from_scratch( r = RequestEntry.from_scratch(
type=RequestType.ScheduleVM, type=RequestType.ScheduleVM,
uuid=vm_uuid, uuid=vm_uuid,
request_prefix=shared.settings['etcd']['request_prefix'], request_prefix=settings['etcd']['request_prefix'],
) )
shared.request_pool.put(r) shared.request_pool.put(r)
return make_return_message('VM Creation Queued')
return {'message': 'VM Creation Queued'}, 200
return validator.get_errors(), 400
class VmStatus(Resource): class GetVMStatus(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.VMStatusSchema(data) try:
if validator.is_valid(): validator = schemas.VMStatusSchema(data)
vm = shared.vm_pool.get( validator.is_valid()
join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) except (ValidationException, Exception) as err:
) return make_return_message(err, 400)
else:
vm = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], validator.uuid))
vm_value = vm.value.copy() vm_value = vm.value.copy()
vm_value['ip'] = [] vm_value['ip'] = []
for network_mac_and_tap in vm.network: for network_mac_and_tap in vm.network:
network_name, mac, tap = network_mac_and_tap network_name, mac, tap = network_mac_and_tap
network = shared.etcd_client.get( network = shared.etcd_client.get(
join_path( join_path(settings['etcd']['network_prefix'], validator.name, network_name),
shared.settings['etcd']['network_prefix'],
data['name'],
network_name,
),
value_in_json=True, value_in_json=True,
) )
ipv6_addr = ( ipv6_addr = (network.value.get('ipv6').split('::')[0] + '::')
network.value.get('ipv6').split('::')[0] + '::'
)
vm_value['ip'].append(mac2ipv6(mac, ipv6_addr)) vm_value['ip'].append(mac2ipv6(mac, ipv6_addr))
vm.value = vm_value vm.value = vm_value
return vm.value return vm.value, 200
else:
return validator.get_errors(), 400
class CreateImage(Resource): class CreateImage(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.CreateImageSchema(data) try:
if validator.is_valid(): validator = schemas.CreateImageSchema(data)
file_entry = shared.etcd_client.get( validator.is_valid()
join_path(shared.settings['etcd']['file_prefix'], data['uuid']) except ValidationException as err:
) return make_return_message(err, 400)
file_entry_value = json.loads(file_entry.value) else:
try:
file_entry = shared.etcd_client.get(
join_path(settings['etcd']['file_prefix'], validator.uuid), value_in_json=True
)
except KeyError:
# TODO: Add some message
pass
else:
image_entry_json = {
'status': 'TO_BE_CREATED',
'owner': file_entry.value['owner'],
'filename': file_entry.value['filename'],
'name': validator.name,
'store_name': validator.image_store,
'visibility': 'public',
}
shared.etcd_client.put(
join_path(settings['etcd']['image_prefix'], validator.uuid),
json.dumps(image_entry_json),
)
image_entry_json = { return {'message': 'Image queued for creation.'}, 200
'status': 'TO_BE_CREATED',
'owner': file_entry_value['owner'],
'filename': file_entry_value['filename'],
'name': data['name'],
'store_name': data['image_store'],
'visibility': 'public',
}
shared.etcd_client.put(
join_path(
shared.settings['etcd']['image_prefix'], data['uuid']
),
json.dumps(image_entry_json),
)
return {'message': 'Image queued for creation.'}
return validator.get_errors(), 400
class ListPublicImages(Resource): class ListPublicImages(Resource):
@staticmethod @staticmethod
def get(): def get():
images = shared.etcd_client.get_prefix( images = shared.etcd_client.get_prefix(settings['etcd']['image_prefix'], value_in_json=True)
shared.settings['etcd']['image_prefix'], value_in_json=True
)
r = {'images': []} r = {'images': []}
for image in images: for image in images:
image_key = '{}:{}'.format( image_key = '{}:{}'.format(image.value['store_name'], image.value['name'])
image.value['store_name'], image.value['name'] r['images'].append({'name': image_key, 'status': image.value['status']})
)
r['images'].append(
{'name': image_key, 'status': image.value['status']}
)
return r, 200 return r, 200
@ -173,92 +168,79 @@ class VMAction(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.VmActionSchema(data) try:
validator = schemas.VmActionSchema(data)
if validator.is_valid(): validator.is_valid()
vm_entry = shared.vm_pool.get( except ValidationException as err:
join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) return make_return_message(err, 400)
) else:
action = data['action'] vm_entry = shared.vm_pool.get(join_path(settings['etcd']['vm_prefix'], validator.uuid))
action = validator.action
if action == 'start': if action == 'start':
action = 'schedule' action = 'schedule'
if action == 'delete' and vm_entry.hostname == '': if action == 'delete' and vm_entry.hostname == '':
if shared.storage_handler.is_vm_image_exists( if shared.storage_handler.is_vm_image_exists(vm_entry.uuid):
vm_entry.uuid r_status = shared.storage_handler.delete_vm_image(vm_entry.uuid)
):
r_status = shared.storage_handler.delete_vm_image(
vm_entry.uuid
)
if r_status: if r_status:
shared.etcd_client.client.delete(vm_entry.key) shared.etcd_client.client.delete(vm_entry.key)
return {'message': 'VM successfully deleted'} return make_return_message('VM successfully deleted')
else: else:
logger.error( logger.error('Some Error Occurred while deleting VM')
'Some Error Occurred while deleting VM' return make_return_message('VM deletion unsuccessfull')
)
return {'message': 'VM deletion unsuccessfull'}
else: else:
shared.etcd_client.client.delete(vm_entry.key) shared.etcd_client.client.delete(vm_entry.key)
return {'message': 'VM successfully deleted'} return make_return_message('VM successfully deleted')
r = RequestEntry.from_scratch( r = RequestEntry.from_scratch(
type='{}VM'.format(action.title()), type='{}VM'.format(action.title()),
uuid=data['uuid'], uuid=validator.uuid,
hostname=vm_entry.hostname, hostname=vm_entry.hostname,
request_prefix=shared.settings['etcd']['request_prefix'], request_prefix=settings['etcd']['request_prefix'],
) )
shared.request_pool.put(r) shared.request_pool.put(r)
return ( return make_return_message('VM {} Queued'.format(action.title()))
{'message': 'VM {} Queued'.format(action.title())},
200,
)
else:
return validator.get_errors(), 400
class VMMigration(Resource): class VMMigration(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.VmMigrationSchema(data) try:
validator = schemas.VmMigrationSchema(data)
if validator.is_valid(): validator.is_valid()
vm = shared.vm_pool.get(data['uuid']) except ValidationException as err:
return make_return_message(err), 400
else:
vm = shared.vm_pool.get(validator.uuid)
r = RequestEntry.from_scratch( r = RequestEntry.from_scratch(
type=RequestType.InitVMMigration, type=RequestType.InitVMMigration,
uuid=vm.uuid, uuid=vm.uuid,
hostname=join_path( hostname=join_path(
shared.settings['etcd']['host_prefix'], settings['etcd']['host_prefix'],
validator.destination.value, validator.destination,
), ),
request_prefix=shared.settings['etcd']['request_prefix'], request_prefix=settings['etcd']['request_prefix'],
) )
shared.request_pool.put(r) shared.request_pool.put(r)
return ( return make_return_message('VM Migration Initialization Queued')
{'message': 'VM Migration Initialization Queued'},
200,
)
else:
return validator.get_errors(), 400
class ListUserVM(Resource): class ListUserVM(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.OTPSchema(data) try:
validator = schemas.OTPSchema(data)
if validator.is_valid(): validator.is_valid()
vms = shared.etcd_client.get_prefix( except ValidationException as err:
shared.settings['etcd']['vm_prefix'], value_in_json=True return make_return_message(err, 400)
) else:
vms = shared.etcd_client.get_prefix(settings['etcd']['vm_prefix'], value_in_json=True)
return_vms = [] return_vms = []
user_vms = filter( user_vms = filter(lambda v: v.value['owner'] == validator.name, vms)
lambda v: v.value['owner'] == data['name'], vms
)
for vm in user_vms: for vm in user_vms:
return_vms.append( return_vms.append(
{ {
@ -270,26 +252,22 @@ class ListUserVM(Resource):
'vnc_socket': vm.value.get('vnc_socket', None), 'vnc_socket': vm.value.get('vnc_socket', None),
} }
) )
if return_vms: return make_return_message(return_vms)
return {'message': return_vms}, 200
return {'message': 'No VM found'}, 404
else:
return validator.get_errors(), 400
class ListUserFiles(Resource): class ListUserFiles(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.OTPSchema(data) try:
validator = schemas.OTPSchema(data)
if validator.is_valid(): validator.is_valid()
files = shared.etcd_client.get_prefix( except ValidationException as err:
shared.settings['etcd']['file_prefix'], value_in_json=True return make_return_message(err, 400)
) else:
files = shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
return_files = [] return_files = []
user_files = [f for f in files if f.value['owner'] == data['name']] user_files = [f for f in files if f.value['owner'] == validator.name]
for file in user_files: for file in user_files:
file_uuid = file.key.split('/')[-1] file_uuid = file.key.split('/')[-1]
file = file.value file = file.value
@ -299,33 +277,28 @@ class ListUserFiles(Resource):
file.pop('owner', None) file.pop('owner', None)
return_files.append(file) return_files.append(file)
return {'message': return_files}, 200 return make_return_message(return_files)
else:
return validator.get_errors(), 400
class CreateHost(Resource): class CreateHost(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.CreateHostSchema(data) try:
if validator.is_valid(): validator = schemas.CreateHostSchema(data)
host_key = join_path( validator.is_valid()
shared.settings['etcd']['host_prefix'], uuid4().hex except ValidationException as err:
) return make_return_message(err, 400)
else:
host_key = join_path(settings['etcd']['host_prefix'], uuid4().hex)
host_entry = { host_entry = {
'specs': data['specs'], 'specs': validator.specs,
'hostname': data['hostname'], 'hostname': validator.hostname,
'status': 'DEAD', 'status': HostStatus.dead,
'last_heartbeat': '', 'last_heartbeat': '',
} }
shared.etcd_client.put( shared.etcd_client.put(host_key, host_entry, value_in_json=True)
host_key, host_entry, value_in_json=True return make_return_message('Host Created.')
)
return {'message': 'Host Created'}, 200
return validator.get_errors(), 400
class ListHost(Resource): class ListHost(Resource):
@ -347,200 +320,142 @@ class GetSSHKeys(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.GetSSHSchema(data) try:
if validator.is_valid(): validator = schemas.GetSSHSchema(data)
if not validator.key_name.value: except ValidationException as err:
return make_return_message(err, 400)
# {user_prefix}/{realm}/{name}/key/ else:
etcd_key = join_path( etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm,
shared.settings['etcd']['user_prefix'], validator.name, 'key')
data['realm'], if not validator.key_name:
data['name'], etcd_entry = shared.etcd_client.get_prefix(etcd_key, value_in_json=True)
'key',
)
etcd_entry = shared.etcd_client.get_prefix(
etcd_key, value_in_json=True
)
keys = { keys = {
key.key.split('/')[-1]: key.value key.key.split('/')[-1]: key.value
for key in etcd_entry for key in etcd_entry
} }
return {'keys': keys} return {'keys': keys}
else: else:
etcd_key = join_path(validator.key_name)
# {user_prefix}/{realm}/{name}/key/{key_name} try:
etcd_key = join_path( etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True)
shared.settings['etcd']['user_prefix'], except KeyError:
data['realm'], return make_return_message('No such key found.', 400)
data['name'],
'key',
data['key_name'],
)
etcd_entry = shared.etcd_client.get(
etcd_key, value_in_json=True
)
if etcd_entry:
return {
'keys': {
etcd_entry.key.split('/')[
-1
]: etcd_entry.value
}
}
else: else:
return {'keys': {}} return {
else: 'keys': {etcd_entry.key.split('/')[-1]: etcd_entry.value}
return validator.get_errors(), 400 }
class AddSSHKey(Resource): class AddSSHKey(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.AddSSHSchema(data) try:
if validator.is_valid(): validator = schemas.AddSSHSchema(data)
validator.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
# {user_prefix}/{realm}/{name}/key/{key_name} # {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path( etcd_key = join_path(
shared.settings['etcd']['user_prefix'], settings['etcd']['user_prefix'], validator.realm,
data['realm'], validator.name, 'key', validator.key_name
data['name'],
'key',
data['key_name'],
) )
etcd_entry = shared.etcd_client.get( try:
etcd_key, value_in_json=True shared.etcd_client.get(etcd_key, value_in_json=True)
) except KeyError:
if etcd_entry:
return {
'message': 'Key with name "{}" already exists'.format(
data['key_name']
)
}
else:
# Key Not Found. It implies user' haven't added any key yet. # Key Not Found. It implies user' haven't added any key yet.
shared.etcd_client.put( shared.etcd_client.put(etcd_key, validator.key, value_in_json=True)
etcd_key, data['key'], value_in_json=True return make_return_message('Key added successfully')
) else:
return {'message': 'Key added successfully'} return make_return_message('Key "{}" already exists'.format(validator.key_name))
else:
return validator.get_errors(), 400
class RemoveSSHKey(Resource): class RemoveSSHKey(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.RemoveSSHSchema(data) try:
if validator.is_valid(): validator = schemas.RemoveSSHSchema(data)
validator.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
# {user_prefix}/{realm}/{name}/key/{key_name} # {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path( etcd_key = join_path(settings['etcd']['user_prefix'], validator.realm,
shared.settings['etcd']['user_prefix'], validator.name, 'key', validator.key_name)
data['realm'], try:
data['name'], etcd_entry = shared.etcd_client.get(etcd_key, value_in_json=True)
'key', except KeyError:
data['key_name'], return make_return_message('No Key "{}" exists.'.format(validator.key_name))
)
etcd_entry = shared.etcd_client.get(
etcd_key, value_in_json=True
)
if etcd_entry: if etcd_entry:
shared.etcd_client.client.delete(etcd_key) shared.etcd_client.client.delete(etcd_key)
return {'message': 'Key successfully removed.'} return {'message': 'Key successfully removed.'}
else:
return {
'message': 'No Key with name "{}" Exists at all.'.format(
data['key_name']
)
}
else:
return validator.get_errors(), 400
class CreateNetwork(Resource): class CreateNetwork(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.CreateNetwork(data) try:
validator = schemas.CreateNetwork(data)
if validator.is_valid(): validator.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
network_entry = { network_entry = {
'id': counters.increment_etcd_counter( 'id': counters.increment_etcd_counter(settings['etcd']['vxlan_counter']),
shared.etcd_client, shared.settings['etcd']['vxlan_counter'] 'type': validator.type,
),
'type': data['type'],
} }
if validator.user.value: if validator.user:
try: try:
nb = pynetbox.api( nb = pynetbox.api(url=settings['netbox']['url'], token=settings['netbox']['token'])
url=shared.settings['netbox']['url'], nb_prefix = nb.ipam.prefixes.get(prefix=settings['network']['prefix'])
token=shared.settings['netbox']['token'],
)
nb_prefix = nb.ipam.prefixes.get(
prefix=shared.settings['network']['prefix']
)
prefix = nb_prefix.available_prefixes.create( prefix = nb_prefix.available_prefixes.create(
data={ data={
'prefix_length': int( 'prefix_length': int(settings['network']['prefix_length']),
shared.settings['network']['prefix_length']
),
'description': '{}\'s network "{}"'.format( 'description': '{}\'s network "{}"'.format(
data['name'], data['network_name'] validator.name,
validator.network_name
), ),
'is_pool': True, 'is_pool': True,
} }
) )
except Exception as err: except Exception as err:
app.logger.error(err) app.logger.error(err)
return { return make_return_message('Error occured while creating network.', 400)
'message': 'Error occured while creating network.'
}
else: else:
network_entry['ipv6'] = prefix['prefix'] network_entry['ipv6'] = prefix['prefix']
else: else:
network_entry['ipv6'] = 'fd00::/64' network_entry['ipv6'] = 'fd00::/64'
network_key = join_path( network_key = join_path(settings['etcd']['network_prefix'], validator.name,
shared.settings['etcd']['network_prefix'], validator.network_name)
data['name'], shared.etcd_client.put(network_key, network_entry, value_in_json=True)
data['network_name'], return make_return_message('Network successfully added.')
)
shared.etcd_client.put(
network_key, network_entry, value_in_json=True
)
return {'message': 'Network successfully added.'}
else:
return validator.get_errors(), 400
class ListUserNetwork(Resource): class ListUserNetwork(Resource):
@staticmethod @staticmethod
def post(): def post():
data = request.json data = request.json
validator = schemas.OTPSchema(data) try:
validator = schemas.OTPSchema(data)
if validator.is_valid(): validator.is_valid()
prefix = join_path( except ValidationException as err:
shared.settings['etcd']['network_prefix'], data['name'] return make_return_message(err, 400)
) else:
networks = shared.etcd_client.get_prefix( prefix = join_path(settings['etcd']['network_prefix'], validator.name)
prefix, value_in_json=True networks = shared.etcd_client.get_prefix(prefix, value_in_json=True)
)
user_networks = [] user_networks = []
for net in networks: for net in networks:
net.value['name'] = net.key.split('/')[-1] net.value['name'] = net.key.split('/')[-1]
user_networks.append(net.value) user_networks.append(net.value)
return {'networks': user_networks}, 200 return {'networks': user_networks}, 200
else:
return validator.get_errors(), 400
api.add_resource(CreateVM, '/vm/create') api.add_resource(CreateVM, '/vm/create')
api.add_resource(VmStatus, '/vm/status') api.add_resource(GetVMStatus, '/vm/status')
api.add_resource(VMAction, '/vm/action') api.add_resource(VMAction, '/vm/action')
api.add_resource(VMMigration, '/vm/migrate') api.add_resource(VMMigration, '/vm/migrate')
@ -562,39 +477,12 @@ api.add_resource(ListHost, '/host/list')
api.add_resource(CreateNetwork, '/network/create') api.add_resource(CreateNetwork, '/network/create')
def main(arguments): def main(debug=False, port=None):
debug = arguments['debug']
port = arguments['port']
try:
image_stores = list(
shared.etcd_client.get_prefix(
shared.settings['etcd']['image_store_prefix'], value_in_json=True
)
)
except KeyError:
image_stores = False
# Do not inject default values that might be very wrong
# fail when required, not before
#
# if not image_stores:
# data = {
# 'is_public': True,
# 'type': 'ceph',
# 'name': 'images',
# 'description': 'first ever public image-store',
# 'attributes': {'list': [], 'key': [], 'pool': 'images'},
# }
# shared.etcd_client.put(
# join_path(
# shared.settings['etcd']['image_store_prefix'], uuid4().hex
# ),
# json.dumps(data),
# )
try: try:
app.run(host='::', port=port, debug=debug) app.run(host='::', port=port, debug=debug)
except OSError as e: except OSError as e:
raise UncloudException('Failed to start Flask: {}'.format(e)) raise UncloudException('Failed to start Flask: {}'.format(e))
if __name__ == '__main__':
main()

608
uncloud/api/schemas.py
View file

@ -1,19 +1,3 @@
"""
This module contain classes thats validates and intercept/modify
data coming from uncloud-cli (user)
It was primarily developed as an alternative to argument parser
of Flask_Restful which is going to be deprecated. I also tried
marshmallow for that purpose but it was an overkill (because it
do validation + serialization + deserialization) and little
inflexible for our purpose.
"""
# TODO: Fix error message when user's mentioned VM (referred by name)
# does not exists.
#
# Currently, it says uuid is a required field.
import json import json
import os import os
@ -22,19 +6,54 @@ import bitmath
from uncloud.common.host import HostStatus from uncloud.common.host import HostStatus
from uncloud.common.vm import VMStatus from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared from uncloud.common.shared import shared
from . import helper, logger from uncloud.common.settings import settings
from .common_fields import Field, VmUUIDField from uncloud.api import helper
from .helper import check_otp, resolve_vm_name from uncloud.api.helper import check_otp, resolve_vm_name
class ValidationException(Exception):
"""Validation Error"""
class Field:
def __init__(self, _name, _type, _value=None, validators=None):
if validators is None:
validators = []
assert isinstance(validators, list)
self.name = _name
self.value = _value
self.type = _type
self.validators = validators
def is_valid(self):
if not isinstance(self.value, self.type):
raise ValidationException("Incorrect Type for '{}' field".format(self.name))
for validator in self.validators:
validator()
def __repr__(self):
return self.name
class VmUUIDField(Field):
def __init__(self, data):
self.uuid = data.get('uuid', KeyError)
super().__init__('uuid', str, self.uuid, validators=[self.vm_uuid_validation])
def vm_uuid_validation(self):
try:
shared.etcd_client.get(os.path.join(settings['etcd']['vm_prefix'], self.uuid))
except KeyError:
raise ValidationException('VM with uuid {} does not exists'.format(self.uuid))
class BaseSchema: class BaseSchema:
def __init__(self, data, fields=None): def __init__(self):
_ = data # suppress linter warning self.fields = [getattr(self, field) for field in dir(self) if isinstance(getattr(self, field), Field)]
self.__errors = []
if fields is None:
self.fields = []
else:
self.fields = fields
def validation(self): def validation(self):
# custom validation is optional # custom validation is optional
@ -43,515 +62,340 @@ class BaseSchema:
def is_valid(self): def is_valid(self):
for field in self.fields: for field in self.fields:
field.is_valid() field.is_valid()
self.add_field_errors(field)
for parent in self.__class__.__bases__: for parent in self.__class__.__bases__:
try: parent.validation(self)
parent.validation(self)
except AttributeError:
pass
if not self.__errors:
self.validation()
if self.__errors: self.validation()
return False
return True
def get_errors(self): for field in self.fields:
return {"message": self.__errors} setattr(self, field.name, field.value)
def add_field_errors(self, field: Field):
self.__errors += field.get_errors()
def add_error(self, error): def get(dictionary: dict, key: str, return_default=False, default=None):
self.__errors.append(error) if dictionary is None:
raise ValidationException('No data provided at all.')
try:
value = dictionary[key]
except KeyError:
if return_default:
return default
raise ValidationException("Missing data for '{}' field.".format(key))
else:
return value
class OTPSchema(BaseSchema): class OTPSchema(BaseSchema):
def __init__(self, data: dict, fields=None): def __init__(self, data: dict):
self.name = Field("name", str, data.get("name", KeyError)) self.name = Field('name', str, get(data, 'name'))
self.realm = Field("realm", str, data.get("realm", KeyError)) self.realm = Field('realm', str, get(data, 'realm'))
self.token = Field("token", str, data.get("token", KeyError)) self.token = Field('token', str, get(data, 'token'))
super().__init__()
_fields = [self.name, self.realm, self.token]
if fields:
_fields += fields
super().__init__(data=data, fields=_fields)
def validation(self): def validation(self):
if ( if check_otp(self.name.value, self.realm.value, self.token.value) != 200:
check_otp( raise ValidationException('Wrong Credentials')
self.name.value, self.realm.value, self.token.value
)
!= 200
):
self.add_error("Wrong Credentials")
########################## Image Operations ###############################################
class CreateImageSchema(BaseSchema): class CreateImageSchema(BaseSchema):
def __init__(self, data): def __init__(self, data):
# Fields self.uuid = Field('uuid', str, get(data, 'uuid'), validators=[self.file_uuid_validation])
self.uuid = Field("uuid", str, data.get("uuid", KeyError)) self.name = Field('name', str, get(data, 'name'))
self.name = Field("name", str, data.get("name", KeyError)) self.image_store = Field('image_store', str, get(data, 'image_store'),
self.image_store = Field( validators=[self.image_store_name_validation])
"image_store", str, data.get("image_store", KeyError) super().__init__()
)
# Validations
self.uuid.validation = self.file_uuid_validation
self.image_store.validation = self.image_store_name_validation
# All Fields
fields = [self.uuid, self.name, self.image_store]
super().__init__(data, fields)
def file_uuid_validation(self): def file_uuid_validation(self):
file_entry = shared.etcd_client.get( try:
os.path.join( shared.etcd_client.get(os.path.join(settings['etcd']['file_prefix'], self.uuid.value))
shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value except KeyError:
) raise ValidationException("Image File with uuid '{}' Not Found".format(self.uuid.value))
)
if file_entry is None:
self.add_error(
"Image File with uuid '{}' Not Found".format(
self.uuid.value
)
)
def image_store_name_validation(self): def image_store_name_validation(self):
image_stores = list( image_stores = list(shared.etcd_client.get_prefix(settings['etcd']['image_store_prefix']))
shared.etcd_client.get_prefix( try:
shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] next(filter(lambda s: json.loads(s.value)['name'] == self.image_store.value, image_stores))
) except StopIteration:
) raise ValidationException("Store '{}' does not exists".format(self.image_store.value))
image_store = next(
filter(
lambda s: json.loads(s.value)["name"]
== self.image_store.value,
image_stores,
),
None,
)
if not image_store:
self.add_error(
"Store '{}' does not exists".format(
self.image_store.value
)
)
# Host Operations
class CreateHostSchema(OTPSchema): class CreateHostSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
# Fields self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation])
self.specs = Field("specs", dict, data.get("specs", KeyError)) self.hostname = Field('hostname', str, get(data, 'hostname'))
self.hostname = Field(
"hostname", str, data.get("hostname", KeyError)
)
# Validation super().__init__(data)
self.specs.validation = self.specs_validation
fields = [self.hostname, self.specs]
super().__init__(data=data, fields=fields)
def specs_validation(self): def specs_validation(self):
ALLOWED_BASE = 10 allowed_base = 10
_cpu = self.specs.value.get("cpu", KeyError) _cpu = self.specs.value.get('cpu', KeyError)
_ram = self.specs.value.get("ram", KeyError) _ram = self.specs.value.get('ram', KeyError)
_os_ssd = self.specs.value.get("os-ssd", KeyError) _os_ssd = self.specs.value.get('os-ssd', KeyError)
_hdd = self.specs.value.get("hdd", KeyError) _hdd = self.specs.value.get('hdd', KeyError)
if KeyError in [_cpu, _ram, _os_ssd, _hdd]: if KeyError in [_cpu, _ram, _os_ssd]:
self.add_error( raise ValidationException('You must specify CPU, RAM and OS-SSD in your specs')
"You must specify CPU, RAM and OS-SSD in your specs"
)
return None
try: try:
parsed_ram = bitmath.parse_string_unsafe(_ram) parsed_ram = bitmath.parse_string_unsafe(_ram)
parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd)
if parsed_ram.base != ALLOWED_BASE: if parsed_ram.base != allowed_base:
self.add_error( raise ValidationException('Your specified RAM is not in correct units')
"Your specified RAM is not in correct units"
) if parsed_os_ssd.base != allowed_base:
if parsed_os_ssd.base != ALLOWED_BASE: raise ValidationException('Your specified OS-SSD is not in correct units')
self.add_error(
"Your specified OS-SSD is not in correct units"
)
if _cpu < 1: if _cpu < 1:
self.add_error("CPU must be atleast 1") raise ValidationException('CPU must be atleast 1')
if parsed_ram < bitmath.GB(1): if parsed_ram < bitmath.GB(1):
self.add_error("RAM must be atleast 1 GB") raise ValidationException('RAM must be atleast 1 GB')
if parsed_os_ssd < bitmath.GB(10): if parsed_os_ssd < bitmath.GB(10):
self.add_error("OS-SSD must be atleast 10 GB") raise ValidationException('OS-SSD must be atleast 10 GB')
parsed_hdd = [] parsed_hdd = []
for hdd in _hdd: for hdd in _hdd:
_parsed_hdd = bitmath.parse_string_unsafe(hdd) _parsed_hdd = bitmath.parse_string_unsafe(hdd)
if _parsed_hdd.base != ALLOWED_BASE: if _parsed_hdd.base != allowed_base:
self.add_error( raise ValidationException('Your specified HDD is not in correct units')
"Your specified HDD is not in correct units"
)
break
else: else:
parsed_hdd.append(str(_parsed_hdd)) parsed_hdd.append(str(_parsed_hdd))
except ValueError: except ValueError:
# TODO: Find some good error message raise ValidationException('Specs are not correct.')
self.add_error("Specs are not correct.")
else: else:
if self.get_errors(): self.specs = {
self.specs = { 'cpu': _cpu,
"cpu": _cpu, 'ram': str(parsed_ram),
"ram": str(parsed_ram), 'os-ssd': str(parsed_os_ssd),
"os-ssd": str(parsed_os_ssd), 'hdd': parsed_hdd,
"hdd": parsed_hdd, }
}
def validation(self): def validation(self):
if self.realm.value != "ungleich-admin": if self.realm.value != 'ungleich-admin':
self.add_error( raise ValidationException('Invalid Credentials/Insufficient Permission')
"Invalid Credentials/Insufficient Permission"
)
# VM Operations
class CreateVMSchema(OTPSchema): class CreateVMSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
# Fields self.specs = Field('specs', dict, get(data, 'specs'), validators=[self.specs_validation])
self.specs = Field("specs", dict, data.get("specs", KeyError)) self.vm_name = Field('vm_name', str, get(data, 'vm_name'), validators=[self.vm_name_validation])
self.vm_name = Field( self.image = Field('image', str, get(data, 'image'), validators=[self.image_validation])
"vm_name", str, data.get("vm_name", KeyError) self.network = Field('network', list, get(data, 'network', return_default=True, default=[]),
) validators=[self.network_validation])
self.image = Field("image", str, data.get("image", KeyError)) self.image_uuid = None
self.network = Field(
"network", list, data.get("network", KeyError)
)
# Validation super().__init__(data=data)
self.image.validation = self.image_validation
self.vm_name.validation = self.vm_name_validation
self.specs.validation = self.specs_validation
self.network.validation = self.network_validation
fields = [self.vm_name, self.image, self.specs, self.network]
super().__init__(data=data, fields=fields)
def image_validation(self): def image_validation(self):
try: try:
image_uuid = helper.resolve_image_name( image_uuid = helper.resolve_image_name(self.image.value)
self.image.value, shared.etcd_client except Exception:
) raise ValidationException('No image of name \'{}\' found'.format(self.image.value))
except Exception as e:
logger.exception(
"Cannot resolve image name = %s", self.image.value
)
self.add_error(str(e))
else: else:
self.image_uuid = image_uuid self.image_uuid = image_uuid
def vm_name_validation(self): def vm_name_validation(self):
if resolve_vm_name( if resolve_vm_name(name=self.vm_name.value, owner=self.name.value):
name=self.vm_name.value, owner=self.name.value raise ValidationException("VM with same name '{}' already exists".format(self.vm_name.value))
):
self.add_error(
'VM with same name "{}" already exists'.format(
self.vm_name.value
)
)
def network_validation(self): def network_validation(self):
_network = self.network.value _network = self.network.value
if _network: if _network:
for net in _network: for net in _network:
network = shared.etcd_client.get( try:
os.path.join( shared.etcd_client.get(
shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], os.path.join(settings['etcd']['network_prefix'], self.name.value, net),
self.name.value, value_in_json=True
net,
),
value_in_json=True,
)
if not network:
self.add_error(
"Network with name {} does not exists".format(
net
)
) )
except KeyError:
raise ValidationException('Network with name {} does not exists'.format(net))
def specs_validation(self): def specs_validation(self):
ALLOWED_BASE = 10 allowed_base = 10
_cpu = self.specs.value.get("cpu", KeyError)
_ram = self.specs.value.get("ram", KeyError)
_os_ssd = self.specs.value.get("os-ssd", KeyError)
_hdd = self.specs.value.get("hdd", KeyError)
if KeyError in [_cpu, _ram, _os_ssd, _hdd]:
self.add_error(
"You must specify CPU, RAM and OS-SSD in your specs"
)
return None
try: try:
parsed_ram = bitmath.parse_string_unsafe(_ram) _cpu = get(self.specs.value, 'cpu')
parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) _ram = get(self.specs.value, 'ram')
_os_ssd = get(self.specs.value, 'os-ssd')
if parsed_ram.base != ALLOWED_BASE: _hdd = get(self.specs.value, 'hdd', return_default=True, default=[])
self.add_error( except (KeyError, Exception):
"Your specified RAM is not in correct units" raise ValidationException('You must specify CPU, RAM and OS-SSD in your specs')
)
if parsed_os_ssd.base != ALLOWED_BASE:
self.add_error(
"Your specified OS-SSD is not in correct units"
)
if int(_cpu) < 1:
self.add_error("CPU must be atleast 1")
if parsed_ram < bitmath.GB(1):
self.add_error("RAM must be atleast 1 GB")
if parsed_os_ssd < bitmath.GB(1):
self.add_error("OS-SSD must be atleast 1 GB")
parsed_hdd = []
for hdd in _hdd:
_parsed_hdd = bitmath.parse_string_unsafe(hdd)
if _parsed_hdd.base != ALLOWED_BASE:
self.add_error(
"Your specified HDD is not in correct units"
)
break
else:
parsed_hdd.append(str(_parsed_hdd))
except ValueError:
# TODO: Find some good error message
self.add_error("Specs are not correct.")
else: else:
if self.get_errors(): try:
parsed_ram = bitmath.parse_string_unsafe(_ram)
parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd)
if parsed_ram.base != allowed_base:
raise ValidationException('Your specified RAM is not in correct units')
if parsed_os_ssd.base != allowed_base:
raise ValidationException('Your specified OS-SSD is not in correct units')
if int(_cpu) < 1:
raise ValidationException('CPU must be atleast 1')
if parsed_ram < bitmath.GB(1):
raise ValidationException('RAM must be atleast 1 GB')
if parsed_os_ssd < bitmath.GB(1):
raise ValidationException('OS-SSD must be atleast 1 GB')
parsed_hdd = []
for hdd in _hdd:
_parsed_hdd = bitmath.parse_string_unsafe(hdd)
if _parsed_hdd.base != allowed_base:
raise ValidationException('Your specified HDD is not in correct units')
else:
parsed_hdd.append(str(_parsed_hdd))
except ValueError:
raise ValidationException('Specs are not correct.')
else:
self.specs = { self.specs = {
"cpu": _cpu, 'cpu': _cpu,
"ram": str(parsed_ram), 'ram': str(parsed_ram),
"os-ssd": str(parsed_os_ssd), 'os-ssd': str(parsed_os_ssd),
"hdd": parsed_hdd, 'hdd': parsed_hdd,
} }
class VMStatusSchema(OTPSchema): class VMStatusSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
data["uuid"] = ( data['uuid'] = (
resolve_vm_name( resolve_vm_name(
name=data.get("vm_name", None), name=get(data, 'vm_name', return_default=True),
owner=( owner=(
data.get("in_support_of", None) get(data, 'in_support_of', return_default=True) or
or data.get("name", None) get(data, 'name', return_default=True)
), )
) )
or KeyError or KeyError
) )
self.uuid = VmUUIDField(data) self.uuid = VmUUIDField(data)
fields = [self.uuid] super().__init__(data)
super().__init__(data, fields)
def validation(self): def validation(self):
vm = shared.vm_pool.get(self.uuid.value) vm = shared.vm_pool.get(self.uuid.value)
if not ( if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'):
vm.value["owner"] == self.name.value raise ValidationException('Invalid User')
or self.realm.value == "ungleich-admin"
):
self.add_error("Invalid User")
class VmActionSchema(OTPSchema): class VmActionSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
data["uuid"] = ( data['uuid'] = (
resolve_vm_name( resolve_vm_name(
name=data.get("vm_name", None), name=get(data, 'vm_name', return_default=True),
owner=( owner=(
data.get("in_support_of", None) get(data, 'in_support_of', return_default=True) or
or data.get("name", None) get(data, 'name', return_default=True)
), )
) )
or KeyError or KeyError
) )
self.uuid = VmUUIDField(data) self.uuid = VmUUIDField(data)
self.action = Field("action", str, data.get("action", KeyError)) self.action = Field('action', str, get(data, 'action'), validators=[self.action_validation])
self.action.validation = self.action_validation super().__init__(data=data)
_fields = [self.uuid, self.action]
super().__init__(data=data, fields=_fields)
def action_validation(self): def action_validation(self):
allowed_actions = ["start", "stop", "delete"] allowed_actions = ['start', 'stop', 'delete']
if self.action.value not in allowed_actions: if self.action.value not in allowed_actions:
self.add_error( raise ValidationException('Invalid Action. Allowed Actions are {}'.format(allowed_actions))
"Invalid Action. Allowed Actions are {}".format(
allowed_actions
)
)
def validation(self): def validation(self):
vm = shared.vm_pool.get(self.uuid.value) vm = shared.vm_pool.get(self.uuid.value)
if not ( if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'):
vm.value["owner"] == self.name.value raise ValidationException('Invalid User.')
or self.realm.value == "ungleich-admin"
):
self.add_error("Invalid User")
if ( if self.action.value == 'start' and vm.status == VMStatus.running and vm.hostname != '':
self.action.value == "start" raise ValidationException('VM Already Running')
and vm.status == VMStatus.running
and vm.hostname != ""
):
self.add_error("VM Already Running")
if self.action.value == "stop": if self.action.value == 'stop':
if vm.status == VMStatus.stopped: if vm.status == VMStatus.stopped:
self.add_error("VM Already Stopped") raise ValidationException('VM Already Stopped')
elif vm.status != VMStatus.running: elif vm.status != VMStatus.running:
self.add_error("Cannot stop non-running VM") raise ValidationException('Cannot stop non-running VM')
class VmMigrationSchema(OTPSchema): class VmMigrationSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
data["uuid"] = ( data['uuid'] = (
resolve_vm_name( resolve_vm_name(
name=data.get("vm_name", None), name=get(data, 'vm_name', return_default=True),
owner=( owner=(
data.get("in_support_of", None) get(data, 'in_support_of', return_default=True) or
or data.get("name", None) get(data, 'name', return_default=True)
), )
) ) or KeyError
or KeyError
) )
self.uuid = VmUUIDField(data) self.uuid = VmUUIDField(data)
self.destination = Field( self.destination = Field('destination', str, get(data, 'destination'),
"destination", str, data.get("destination", KeyError) validators=[self.destination_validation])
)
self.destination.validation = self.destination_validation super().__init__(data=data)
fields = [self.destination]
super().__init__(data=data, fields=fields)
def destination_validation(self): def destination_validation(self):
hostname = self.destination.value hostname = self.destination.value
host = next( host = next(filter(lambda h: h.hostname == hostname, shared.host_pool.hosts), None,)
filter(
lambda h: h.hostname == hostname, shared.host_pool.hosts
),
None,
)
if not host: if not host:
self.add_error( raise ValidationException('No Such Host ({}) exists'.format(self.destination.value))
"No Such Host ({}) exists".format(
self.destination.value
)
)
elif host.status != HostStatus.alive: elif host.status != HostStatus.alive:
self.add_error("Destination Host is dead") raise ValidationException('Destination Host is dead')
else: else:
self.destination.value = host.key self.destination.value = host.key
def validation(self): def validation(self):
vm = shared.vm_pool.get(self.uuid.value) vm = shared.vm_pool.get(self.uuid.value)
if not ( if not (vm.value['owner'] == self.name.value or self.realm.value == 'ungleich-admin'):
vm.value["owner"] == self.name.value raise ValidationException('Invalid User')
or self.realm.value == "ungleich-admin"
):
self.add_error("Invalid User")
if vm.status != VMStatus.running: if vm.status != VMStatus.running:
self.add_error("Can't migrate non-running VM") raise ValidationException("Can't migrate non-running VM")
if vm.hostname == os.path.join( if vm.hostname == os.path.join(settings['etcd']['host_prefix'], self.destination.value):
shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value raise ValidationException("Destination host couldn't be same as Source Host")
):
self.add_error(
"Destination host couldn't be same as Source Host"
)
class AddSSHSchema(OTPSchema): class AddSSHSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
self.key_name = Field( self.key_name = Field('key_name', str, get(data, 'key_name'))
"key_name", str, data.get("key_name", KeyError) self.key = Field('key', str, get(data, 'key'))
) super().__init__(data=data)
self.key = Field("key", str, data.get("key_name", KeyError))
fields = [self.key_name, self.key]
super().__init__(data=data, fields=fields)
class RemoveSSHSchema(OTPSchema): class RemoveSSHSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
self.key_name = Field( self.key_name = Field('key_name', str, get(data, 'key_name'))
"key_name", str, data.get("key_name", KeyError) super().__init__(data=data)
)
fields = [self.key_name]
super().__init__(data=data, fields=fields)
class GetSSHSchema(OTPSchema): class GetSSHSchema(OTPSchema):
def __init__(self, data): def __init__(self, data):
self.key_name = Field( self.key_name = Field('key_name', str, get(data, 'key_name', return_default=True))
"key_name", str, data.get("key_name", None) super().__init__(data=data)
)
fields = [self.key_name]
super().__init__(data=data, fields=fields)
class CreateNetwork(OTPSchema): class CreateNetwork(OTPSchema):
def __init__(self, data): def __init__(self, data):
self.network_name = Field("network_name", str, data.get("network_name", KeyError)) self.network_name = Field('network_name', str, get(data, 'name'),
self.type = Field("type", str, data.get("type", KeyError)) validators=[self.network_name_validation])
self.user = Field("user", bool, bool(data.get("user", False))) self.type = Field('type', str, get(data, 'type'), validators=[self.network_type_validation])
self.user = Field('user', bool, bool(get(data, 'user', return_default=True, default=False)))
self.network_name.validation = self.network_name_validation super().__init__(data)
self.type.validation = self.network_type_validation
fields = [self.network_name, self.type, self.user]
super().__init__(data, fields=fields)
def network_name_validation(self): def network_name_validation(self):
key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) key = os.path.join(settings['etcd']['network_prefix'], self.name.value, self.network_name.value)
network = shared.etcd_client.get(key, value_in_json=True) network = shared.etcd_client.get(key, value_in_json=True)
if network: if network:
self.add_error( raise ValidationException('Network with name {} already exists'.format(self.network_name.value))
"Network with name {} already exists".format(
self.network_name.value
)
)
def network_type_validation(self): def network_type_validation(self):
supported_network_types = ["vxlan"] supported_network_types = ['vxlan']
if self.type.value not in supported_network_types: if self.type.value not in supported_network_types:
self.add_error( raise ValidationException('Unsupported Network Type. Supported network types are {}'.format(supported_network_types))
"Unsupported Network Type. Supported network types are {}".format(
supported_network_types
)
)

22
uncloud/cli/helper.py
View file

@ -5,15 +5,15 @@ import binascii
from pyotp import TOTP from pyotp import TOTP
from os.path import join as join_path from os.path import join as join_path
from uncloud.common.shared import shared from uncloud.common.settings import settings
def get_otp_parser(): def get_otp_parser():
otp_parser = argparse.ArgumentParser('otp') otp_parser = argparse.ArgumentParser('otp')
otp_parser.add_argument('--name') otp_parser.add_argument('--name', default=settings['client']['name'])
otp_parser.add_argument('--realm') otp_parser.add_argument('--realm', default=settings['client']['realm'])
otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'],
dest='token', metavar='SEED')
return otp_parser return otp_parser
@ -25,15 +25,11 @@ def load_dump_pretty(content):
def make_request(*args, data=None, request_method=requests.post): def make_request(*args, data=None, request_method=requests.post):
r = request_method(join_path(settings['client']['api_server'], *args), json=data)
try: try:
r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) print(load_dump_pretty(r.content))
except requests.exceptions.RequestException: except Exception:
print('Error occurred while connecting to API server.') print('Error occurred while getting output from api server.')
else:
try:
print(load_dump_pretty(r.content))
except Exception:
print('Error occurred while getting output from api server.')
def get_token(seed): def get_token(seed):

10
uncloud/cli/main.py
View file

@ -12,12 +12,12 @@ for component in ['user', 'host', 'image', 'network', 'vm']:
subparser.add_parser(name=parser.prog, parents=[parser]) subparser.add_parser(name=parser.prog, parents=[parser])
def main(arguments): def main(**kwargs):
if not arguments['subcommand']: if not kwargs['subcommand']:
arg_parser.print_help() arg_parser.print_help()
else: else:
name = arguments.pop('subcommand') name = kwargs.pop('subcommand')
arguments.pop('debug') kwargs.pop('debug')
mod = importlib.import_module('uncloud.cli.{}'.format(name)) mod = importlib.import_module('uncloud.cli.{}'.format(name))
_main = getattr(mod, 'main') _main = getattr(mod, 'main')
_main(**arguments) _main(**kwargs)

26
uncloud/common/cli.py
View file

@ -1,26 +0,0 @@
from uncloud.common.shared import shared
from pyotp import TOTP
def get_token(seed):
if seed is not None:
try:
token = TOTP(seed).now()
except Exception:
raise Exception('Invalid seed')
else:
return token
def resolve_otp_credentials(kwargs):
d = {
'name': shared.settings['client']['name'],
'realm': shared.settings['client']['realm'],
'token': get_token(shared.settings['client']['seed'])
}
for k, v in d.items():
if k in kwargs and kwargs[k] is None:
kwargs.update({k: v})
return d

12
uncloud/common/counters.py
View file

@ -1,8 +1,8 @@
from .etcd_wrapper import Etcd3Wrapper from uncloud.common.shared import shared
def increment_etcd_counter(etcd_client: Etcd3Wrapper, key): def increment_etcd_counter(key):
kv = etcd_client.get(key) kv = shared.etcd_client.get(key)
if kv: if kv:
counter = int(kv.value) counter = int(kv.value)
@ -10,12 +10,12 @@ def increment_etcd_counter(etcd_client: Etcd3Wrapper, key):
else: else:
counter = 1 counter = 1
etcd_client.put(key, str(counter)) shared.etcd_client.put(key, str(counter))
return counter return counter
def get_etcd_counter(etcd_client: Etcd3Wrapper, key): def get_etcd_counter(key):
kv = etcd_client.get(key) kv = shared.etcd_client.get(key)
if kv: if kv:
return int(kv.value) return int(kv.value)
return None return None

49
uncloud/common/etcd_wrapper.py
View file

@ -5,6 +5,7 @@ from functools import wraps
from uncloud import UncloudException from uncloud import UncloudException
from uncloud.common import logger from uncloud.common import logger
from typing import Iterator
class EtcdEntry: class EtcdEntry:
@ -42,14 +43,30 @@ class Etcd3Wrapper:
self.client = etcd3.client(*args, **kwargs) self.client = etcd3.client(*args, **kwargs)
@readable_errors @readable_errors
def get(self, *args, value_in_json=False, **kwargs): def get(self, *args, value_in_json=False, **kwargs) -> EtcdEntry:
"""Get a key/value pair from etcd
:return:
EtcdEntry: if a key/value pair is found in etcd
:raises:
KeyError: If key is not found in etcd
Exception: Different type of exception can be raised depending on
situation
"""
_value, _key = self.client.get(*args, **kwargs) _value, _key = self.client.get(*args, **kwargs)
if _key is None or _value is None: if _key is None or _value is None:
return None raise KeyError
return EtcdEntry(_key, _value, value_in_json=value_in_json) return EtcdEntry(_key, _value, value_in_json=value_in_json)
@readable_errors @readable_errors
def put(self, *args, value_in_json=False, **kwargs): def put(self, *args, value_in_json=False, **kwargs):
"""Put key/value pair in etcd
:return: a response containing a header and the prev_kv
:raises:
Exception: Different type of exception can be raised depending on
situation
"""
_key, _value = args _key, _value = args
if value_in_json: if value_in_json:
_value = json.dumps(_value) _value = json.dumps(_value)
@ -60,16 +77,28 @@ class Etcd3Wrapper:
return self.client.put(_key, _value, **kwargs) return self.client.put(_key, _value, **kwargs)
@readable_errors @readable_errors
def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): def get_prefix(self, *args, value_in_json=False, **kwargs) -> \
Iterator[EtcdEntry]:
event_iterator = self.client.get_prefix(*args, **kwargs) event_iterator = self.client.get_prefix(*args, **kwargs)
for e in event_iterator: for e in event_iterator:
yield EtcdEntry(*e[::-1], value_in_json=value_in_json) yield EtcdEntry(*e[::-1], value_in_json=value_in_json)
@readable_errors @readable_errors
def watch_prefix(self, key, raise_exception=True, value_in_json=False): def watch_prefix(self, key, raise_exception=True, value_in_json=False) -> Iterator[EtcdEntry]:
event_iterator, cancel = self.client.watch_prefix(key) try:
for e in event_iterator: event_iterator, cancel = self.client.watch_prefix(key)
if hasattr(e, '_event'): for e in event_iterator:
e = e._event if hasattr(e, '_event'):
if e.type == e.PUT: e = e._event
yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) if e.type == e.PUT:
yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)
except Exception as err:
if raise_exception:
raise Exception('Exception in etcd_wrapper.get_prefix') from err
else:
logger.exception('Error in etcd_wrapper.watch_prefix')
try:
cancel()
except Exception:
pass
return iter([])

23
uncloud/common/network.py
View file

@ -1,6 +1,7 @@
import subprocess as sp import subprocess as sp
import random import random
import logging import logging
import ipaddress
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -9,9 +10,7 @@ def random_bytes(num=6):
return [random.randrange(256) for _ in range(num)] return [random.randrange(256) for _ in range(num)]
def generate_mac( def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
):
mac = random_bytes() mac = random_bytes()
if oui: if oui:
if type(oui) == str: if type(oui) == str:
@ -68,3 +67,21 @@ def delete_network_interface(iface):
except Exception: except Exception:
logger.exception("Interface %s Deletion failed", iface) logger.exception("Interface %s Deletion failed", iface)
def mac2ipv6(mac, prefix):
# only accept MACs separated by a colon
parts = mac.split(':')
# modify parts to match IPv6 value
parts.insert(3, 'ff')
parts.insert(4, 'fe')
parts[0] = '%x' % (int(parts[0], 16) ^ 2)
# format output
ipv6_parts = [str(0)] * 4
for i in range(0, len(parts), 2):
ipv6_parts.append(''.join(parts[i: i + 2]))
lower_part = ipaddress.IPv6Address(':'.join(ipv6_parts))
prefix = ipaddress.IPv6Address(prefix)
return str(prefix + int(lower_part))

25
uncloud/common/settings.py
View file

@ -8,7 +8,6 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper
from os.path import join as join_path from os.path import join as join_path
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
settings = None
class CustomConfigParser(configparser.RawConfigParser): class CustomConfigParser(configparser.RawConfigParser):
@ -26,8 +25,9 @@ class CustomConfigParser(configparser.RawConfigParser):
class Settings(object): class Settings(object):
def __init__(self, conf_dir, seed_value=None): def __init__(self):
conf_name = 'uncloud.conf' conf_name = 'uncloud.conf'
conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
self.config_file = join_path(conf_dir, conf_name) self.config_file = join_path(conf_dir, conf_name)
# this is used to cache config from etcd for 1 minutes. Without this we # this is used to cache config from etcd for 1 minutes. Without this we
@ -38,19 +38,15 @@ class Settings(object):
self.config_parser.add_section('etcd') self.config_parser.add_section('etcd')
self.config_parser.set('etcd', 'base_prefix', '/') self.config_parser.set('etcd', 'base_prefix', '/')
if os.access(self.config_file, os.R_OK): try:
self.config_parser.read(self.config_file) self.config_parser.read(self.config_file)
else: except Exception as err:
raise FileNotFoundError('Config file %s not found!', self.config_file) logger.error('%s', err)
self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
self.read_internal_values() self.read_internal_values()
if seed_value is None:
seed_value = dict()
self.config_parser.read_dict(seed_value)
def get_etcd_client(self): def get_etcd_client(self):
args = tuple() args = tuple()
try: try:
@ -103,10 +99,12 @@ class Settings(object):
def read_config_file_values(self, config_file): def read_config_file_values(self, config_file):
try: try:
# Trying to read configuration file # Trying to read configuration file
with open(config_file) as config_file_handle: with open(config_file, 'r') as config_file_handle:
self.config_parser.read_file(config_file_handle) self.config_parser.read_file(config_file_handle)
except FileNotFoundError: except FileNotFoundError:
sys.exit('Configuration file {} not found!'.format(config_file)) sys.exit(
'Configuration file {} not found!'.format(config_file)
)
except Exception as err: except Exception as err:
logger.exception(err) logger.exception(err)
sys.exit('Error occurred while reading configuration file') sys.exit('Error occurred while reading configuration file')
@ -132,5 +130,4 @@ class Settings(object):
return self.config_parser[key] return self.config_parser[key]
def get_settings(): settings = Settings()
return settings

22
uncloud/common/shared.py
View file

@ -1,34 +1,34 @@
from uncloud.common.settings import get_settings from uncloud.common.settings import settings
from uncloud.common.vm import VmPool from uncloud.common.vm import VmPool
from uncloud.common.host import HostPool from uncloud.common.host import HostPool
from uncloud.common.request import RequestPool from uncloud.common.request import RequestPool
import uncloud.common.storage_handlers as storage_handlers from uncloud.common.storage_handlers import get_storage_handler
class Shared: class Shared:
@property
def settings(self):
return get_settings()
@property @property
def etcd_client(self): def etcd_client(self):
return self.settings.get_etcd_client() return settings.get_etcd_client()
@property @property
def host_pool(self): def host_pool(self):
return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) return HostPool(
self.etcd_client, settings["etcd"]["host_prefix"]
)
@property @property
def vm_pool(self): def vm_pool(self):
return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"])
@property @property
def request_pool(self): def request_pool(self):
return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) return RequestPool(
self.etcd_client, settings["etcd"]["request_prefix"]
)
@property @property
def storage_handler(self): def storage_handler(self):
return storage_handlers.get_storage_handler() return get_storage_handler()
shared = Shared() shared = Shared()

15
uncloud/common/storage_handlers.py
View file

@ -6,7 +6,8 @@ import stat
from abc import ABC from abc import ABC
from . import logger from . import logger
from os.path import join as join_path from os.path import join as join_path
import uncloud.common.shared as shared
from uncloud.common.settings import settings as config
class ImageStorageHandler(ABC): class ImageStorageHandler(ABC):
@ -192,16 +193,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
def get_storage_handler(): def get_storage_handler():
__storage_backend = shared.shared.settings["storage"]["storage_backend"] __storage_backend = config["storage"]["storage_backend"]
if __storage_backend == "filesystem": if __storage_backend == "filesystem":
return FileSystemBasedImageStorageHandler( return FileSystemBasedImageStorageHandler(
vm_base=shared.shared.settings["storage"]["vm_dir"], vm_base=config["storage"]["vm_dir"],
image_base=shared.shared.settings["storage"]["image_dir"], image_base=config["storage"]["image_dir"],
) )
elif __storage_backend == "ceph": elif __storage_backend == "ceph":
return CEPHBasedImageStorageHandler( return CEPHBasedImageStorageHandler(
vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], vm_base=config["storage"]["ceph_vm_pool"],
image_base=shared.shared.settings["storage"]["ceph_image_pool"], image_base=config["storage"]["ceph_image_pool"],
) )
else: else:
raise Exception("Unknown Image Storage Handler") raise Exception("Unknown Image Storage Handler")

11
uncloud/configure/main.py
View file

@ -1,6 +1,7 @@
import os import os
import argparse import argparse
from uncloud.common.settings import settings
from uncloud.common.shared import shared from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('configure', add_help=False) arg_parser = argparse.ArgumentParser('configure', add_help=False)
@ -39,19 +40,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
def update_config(section, kwargs): def update_config(section, kwargs):
uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
if not uncloud_config: if not uncloud_config:
uncloud_config = {} uncloud_config = {}
else: else:
uncloud_config = uncloud_config.value uncloud_config = uncloud_config.value
uncloud_config[section] = kwargs uncloud_config[section] = kwargs
shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
def main(arguments): def main(**kwargs):
subcommand = arguments['subcommand'] subcommand = kwargs.pop('subcommand')
if not subcommand: if not subcommand:
arg_parser.print_help() arg_parser.print_help()
else: else:
update_config(subcommand, arguments) update_config(subcommand, kwargs)

2
docs/Makefile → uncloud/docs/Makefile
View file

@ -7,7 +7,7 @@ SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build SPHINXBUILD ?= sphinx-build
SOURCEDIR = source/ SOURCEDIR = source/
BUILDDIR = build/ BUILDDIR = build/
DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/ DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/
.PHONY: all build clean .PHONY: all build clean

0
docs/README.md → uncloud/docs/README.md
View file

0
docs/__init__.py → uncloud/docs/__init__.py
View file

0
docs/source/__init__.py → uncloud/docs/source/__init__.py
View file

39
docs/source/admin-guide.rst → uncloud/docs/source/admin-guide
View file

@ -56,13 +56,40 @@ To start host we created earlier, execute the following command
ucloud host ungleich.ch ucloud host ungleich.ch
File & image scanners Create OS Image
-------------------------- ---------------
Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our Create ucloud-init ready OS image (Optional)
uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
ucloud. We can only make images from tracked files. So, we need to track the This step is optional if you just want to test ucloud. However, sooner or later
file by running File Scanner you want to create OS images with ucloud-init to properly
contexualize VMs.
1. Start a VM with OS image on which you want to install ucloud-init
2. Execute the following command on the started VM
.. code-block:: sh
apk add git
git clone https://code.ungleich.ch/ucloud/ucloud-init.git
cd ucloud-init
sh ./install.sh
3. Congratulations. Your image is now ucloud-init ready.
Upload Sample OS Image
~~~~~~~~~~~~~~~~~~~~~~
Execute the following to get the sample OS image file.
.. code-block:: sh
mkdir /var/www/admin
(cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download)
Run File Scanner and Image Scanner
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make
images from tracked files. So, we need to track the file by running File Scanner
.. code-block:: sh .. code-block:: sh

0
docs/source/conf.py → uncloud/docs/source/conf.py
View file

0
docs/source/diagram-code/ucloud → uncloud/docs/source/diagram-code/ucloud
View file

17
uncloud/docs/source/hacking.rst Normal file
View file

@ -0,0 +1,17 @@
Hacking
=======
How to hack on the code.
[ to be done by Balazs:
* make nice
* indent with shell script mode
]
* git clone the repo
* cd to the repo
* Setup your venv: python -m venv venv
* . ./venv/bin/activate # you need the leading dot for sourcing!
* Run ./bin/ucloud-run-reinstall - it should print you an error
message on how to use ucloud

0
docs/source/images/ucloud.svg → uncloud/docs/source/images/ucloud.svg
View file

Side by side Swipe Overlay

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 37 KiB

Before After
Before After

5
docs/source/index.rst → uncloud/docs/source/index.rst
View file

@ -11,13 +11,14 @@ Welcome to ucloud's documentation!
:caption: Contents: :caption: Contents:
introduction introduction
setup-install
vm-images
user-guide user-guide
setup-install
admin-guide admin-guide
user-guide/how-to-create-an-os-image-for-ucloud
troubleshooting troubleshooting
hacking hacking
Indices and tables Indices and tables
================== ==================

0
docs/source/introduction.rst → uncloud/docs/source/introduction.rst
View file

0
docs/source/misc/todo.rst → uncloud/docs/source/misc/todo.rst
View file

0
docs/source/setup-install.rst → uncloud/docs/source/setup-install.rst
View file

0
docs/source/theory/summary.rst → uncloud/docs/source/theory/summary.rst
View file

0
docs/source/troubleshooting.rst → uncloud/docs/source/troubleshooting.rst
View file

0
docs/source/user-guide.rst → uncloud/docs/source/user-guide.rst
View file

0
docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst → uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst
View file

10
uncloud/filescanner/main.py
View file

@ -9,6 +9,7 @@ import bitmath
from uuid import uuid4 from uuid import uuid4
from . import logger from . import logger
from uncloud.common.settings import settings
from uncloud.common.shared import shared from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('filescanner', add_help=False) arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
@ -52,7 +53,7 @@ def track_file(file, base_dir, host):
file_path = file_path.relative_to(owner) file_path = file_path.relative_to(owner)
creation_date = time.ctime(os.stat(file_str).st_ctime) creation_date = time.ctime(os.stat(file_str).st_ctime)
entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
entry_value = { entry_value = {
'filename': str(file_path), 'filename': str(file_path),
'owner': owner, 'owner': owner,
@ -67,9 +68,8 @@ def track_file(file, base_dir, host):
shared.etcd_client.put(entry_key, entry_value, value_in_json=True) shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
def main(arguments): def main(hostname, debug=False):
hostname = arguments['hostname'] base_dir = settings['storage']['file_dir']
base_dir = shared.settings['storage']['file_dir']
# Recursively Get All Files and Folder below BASE_DIR # Recursively Get All Files and Folder below BASE_DIR
files = glob.glob('{}/**'.format(base_dir), recursive=True) files = glob.glob('{}/**'.format(base_dir), recursive=True)
files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
@ -77,7 +77,7 @@ def main(arguments):
# Files that are already tracked # Files that are already tracked
tracked_files = [ tracked_files = [
pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
if f.value['host'] == hostname if f.value['host'] == hostname
] ]
untracked_files = set(files) - set(tracked_files) untracked_files = set(files) - set(tracked_files)

0
uncloud/hack/__init__.py
View file

39
uncloud/hack/config.py
View file

@ -1,39 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
#
# This file is part of uncloud.
#
# uncloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# uncloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
#
#
class Config(object):
def __init__(self, arguments):
""" read arguments dicts as a base """
self.arguments = arguments
# Split them so *etcd_args can be used and we can
# iterate over etcd_hosts
self.etcd_hosts = [ arguments['etcd_host'] ]
self.etcd_args = {
'ca_cert': arguments['etcd_ca_cert'],
'cert_cert': arguments['etcd_cert_cert'],
'cert_key': arguments['etcd_cert_key'],
# 'user': None,
# 'password': None
}
self.etcd_prefix = '/nicohack/'

113
uncloud/hack/db.py
View file

@ -1,113 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
#
# This file is part of uncloud.
#
# uncloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# uncloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
#
#
import etcd3
import json
import logging
from functools import wraps
from uncloud import UncloudException
log = logging.getLogger(__name__)
def readable_errors(func):
@wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except etcd3.exceptions.ConnectionFailedError as e:
raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
except etcd3.exceptions.ConnectionTimeoutError as e:
raise UncloudException('etcd connection timeout. {}'.format(e))
return wrapper
class DB(object):
def __init__(self, config, prefix="/"):
self.config = config
# Root for everything
self.base_prefix= '/nicohack'
# Can be set from outside
self.prefix = prefix
self.connect()
@readable_errors
def connect(self):
self._db_clients = []
for endpoint in self.config.etcd_hosts:
client = etcd3.client(host=endpoint, **self.config.etcd_args)
self._db_clients.append(client)
def realkey(self, key):
return "{}{}/{}".format(self.base_prefix,
self.prefix,
key)
@readable_errors
def get(self, key, as_json=False, **kwargs):
value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
if as_json:
value = json.loads(value)
return value
@readable_errors
def set(self, key, value, as_json=False, **kwargs):
if as_json:
value = json.dumps(value)
# FIXME: iterate over clients in case of failure ?
return self._db_clients[0].put(self.realkey(key), value, **kwargs)
@readable_errors
def increment(self, key, **kwargs):
print(self.realkey(key))
print("prelock")
lock = self._db_clients[0].lock('/nicohack/foo')
print("prelockacq")
lock.acquire()
print("prelockrelease")
lock.release()
with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock:
print("in lock")
pass
# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs))
# self.set(self.realkey(key), str(value + 1), **kwargs)
if __name__ == '__main__':
endpoints = [ "https://etcd1.ungleich.ch:2379",
"https://etcd2.ungleich.ch:2379",
"https://etcd3.ungleich.ch:2379" ]
db = DB(url=endpoints)

3
uncloud/hack/hackcloud/.gitignore vendored
View file

@ -1,3 +0,0 @@
*.iso
radvdpid
foo

1
uncloud/hack/hackcloud/__init__.py
View file

@ -1 +0,0 @@

6
uncloud/hack/hackcloud/etcd-client.sh
View file

@ -1,6 +0,0 @@
#!/bin/sh
etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \
--key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \
--cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \
--endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@"

3
uncloud/hack/hackcloud/ifdown.sh
View file

@ -1,3 +0,0 @@
#!/bin/sh
echo $@

7
uncloud/hack/hackcloud/ifup.sh
View file

@ -1,7 +0,0 @@
#!/bin/sh
dev=$1; shift
# bridge is setup from outside
ip link set dev "$dev" master ${bridge}
ip link set dev "$dev" up

1
uncloud/hack/hackcloud/mac-last
View file

@ -1 +0,0 @@
000000000252

1
uncloud/hack/hackcloud/mac-prefix
View file

@ -1 +0,0 @@
02:00

29
uncloud/hack/hackcloud/net.sh
View file

@ -1,29 +0,0 @@
#!/bin/sh
set -x
netid=100
dev=wlp2s0
dev=wlp0s20f3
#dev=wlan0
ip=2a0a:e5c1:111:888::48/64
vxlandev=vxlan${netid}
bridgedev=br${netid}
ip -6 link add ${vxlandev} type vxlan \
id ${netid} \
dstport 4789 \
group ff05::${netid} \
dev ${dev} \
ttl 5
ip link set ${vxlandev} up
ip link add ${bridgedev} type bridge
ip link set ${bridgedev} up
ip link set ${vxlandev} master ${bridgedev} up
ip addr add ${ip} dev ${bridgedev}

31
uncloud/hack/hackcloud/nftrules
View file

@ -1,31 +0,0 @@
flush ruleset
table bridge filter {
chain prerouting {
type filter hook prerouting priority 0;
policy accept;
ibrname br100 jump br100
}
chain br100 {
# Allow all incoming traffic from outside
iifname vxlan100 accept
# Default blocks: router advertisements, dhcpv6, dhcpv4
icmpv6 type nd-router-advert drop
ip6 version 6 udp sport 547 drop
ip version 4 udp sport 67 drop
jump br100_vmlist
drop
}
chain br100_vmlist {
# VM1
iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept
# VM2
iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept
iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept
}
}

13
uncloud/hack/hackcloud/radvd.conf
View file

@ -1,13 +0,0 @@
interface br100
{
AdvSendAdvert on;
MinRtrAdvInterval 3;
MaxRtrAdvInterval 5;
AdvDefaultLifetime 3600;
prefix 2a0a:e5c1:111:888::/64 {
};
RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; };
DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ;
};

3
uncloud/hack/hackcloud/radvd.sh
View file

@ -1,3 +0,0 @@
#!/bin/sh
radvd -C ./radvd.conf -n -p ./radvdpid

29
uncloud/hack/hackcloud/vm.sh
View file

@ -1,29 +0,0 @@
#!/bin/sh
# if [ $# -ne 1 ]; then
# echo "$0: owner"
# exit 1
# fi
qemu=/usr/bin/qemu-system-x86_64
accel=kvm
#accel=tcg
memory=1024
cores=2
uuid=$(uuidgen)
mac=$(./mac-gen.py)
owner=nico
export bridge=br100
set -x
$qemu -name "uncloud-${uuid}" \
-machine pc,accel=${accel} \
-m ${memory} \
-smp ${cores} \
-uuid ${uuid} \
-drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \
-netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \
-device virtio-net-pci,netdev=netmain,id=net0,mac=${mac}

102
uncloud/hack/mac.py
View file

@ -1,102 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2012 Nico Schottelius (nico-cinv at schottelius.org)
#
# This file is part of cinv.
#
# cinv is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cinv is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cinv. If not, see <http://www.gnu.org/licenses/>.
#
#
import argparse
import logging
import os.path
import os
import re
import json
from uncloud import UncloudException
from uncloud.hack.db import DB
log = logging.getLogger(__name__)
class MAC(object):
def __init__(self, config):
self.config = config
self.no_db = self.config.arguments['no_db']
if not self.no_db:
self.db = DB(config, prefix="/mac")
self.prefix = 0x420000000000
self._number = 0 # Not set by default
@staticmethod
def validate_mac(mac):
if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I):
raise Error("Not a valid mac address: %s" % mac)
def last_used_index(self):
if not self.no_db:
value = self.db.get("last_used_index")
if not value:
self.db.set("last_used_index", "0")
value = self.db.get("last_used_index")
else:
value = "0"
return int(value)
def last_used_mac(self):
return self.int_to_mac(self.prefix + self.last_used_index())
def to_colon_format(self):
b = self._number.to_bytes(6, byteorder="big")
return ':'.join(format(s, '02x') for s in b)
def to_str_format(self):
b = self._number.to_bytes(6, byteorder="big")
return ''.join(format(s, '02x') for s in b)
def create(self):
last_number = self.last_used_index()
if last_number == int('0xffffffff', 16):
raise UncloudException("Exhausted all possible mac addresses - try to free some")
next_number = last_number + 1
self._number = self.prefix + next_number
#next_number_string = "{:012x}".format(next_number)
#next_mac = self.int_to_mac(next_mac_number)
# db_entry = {}
# db_entry['vm_uuid'] = vmuuid
# db_entry['index'] = next_number
# db_entry['mac_address'] = next_mac
# should be one transaction
# self.db.increment("last_used_index")
# self.db.set("used/{}".format(next_mac),
# db_entry, as_json=True)
def __int__(self):
return self._number
def __repr__(self):
return self.to_str_format()
def __str__(self):
return self.to_colon_format()

92
uncloud/hack/main.py
View file

@ -1,92 +0,0 @@
import argparse
import logging
from uncloud.hack.vm import VM
from uncloud.hack.config import Config
from uncloud.hack.mac import MAC
from uncloud.hack.net import VXLANBridge, DNSRA
from uncloud import UncloudException
arg_parser = argparse.ArgumentParser('hack', add_help=False)
#description="Commands that are unfinished - use at own risk")
arg_parser.add_argument('--last-used-mac', action='store_true')
arg_parser.add_argument('--get-new-mac', action='store_true')
arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true')
arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true')
arg_parser.add_argument('--network', help="/64 IPv6 network")
arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0")
arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int)
arg_parser.add_argument('--run-dns-ra', action='store_true',
help="Provide router advertisements and DNS resolution via dnsmasq")
arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true')
arg_parser.add_argument('--create-vm', action='store_true')
arg_parser.add_argument('--destroy-vm', action='store_true')
arg_parser.add_argument('--get-vm-status', action='store_true')
arg_parser.add_argument('--get-vm-vnc', action='store_true')
arg_parser.add_argument('--list-vms', action='store_true')
arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int)
arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int)
arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image")
arg_parser.add_argument('--uuid', help="VM UUID")
arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true')
arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh")
log = logging.getLogger(__name__)
def main(arguments):
config = Config(arguments)
if arguments['create_vm']:
vm = VM(config)
vm.create()
if arguments['destroy_vm']:
vm = VM(config)
vm.stop()
if arguments['get_vm_status']:
vm = VM(config)
vm.status()
if arguments['get_vm_vnc']:
vm = VM(config)
vm.vnc_addr()
if arguments['list_vms']:
vm = VM(config)
vm.list()
if arguments['last_used_mac']:
m = MAC(config)
print(m.last_used_mac())
if arguments['get_new_mac']:
print(MAC(config).get_next())
#if arguments['init_network']:
if arguments['create_vxlan']:
if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']:
raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)")
vb = VXLANBridge(vni=arguments['vni'],
route=arguments['network'],
uplinkdev=arguments['vxlan_uplink_device'],
use_sudo=arguments['use_sudo'])
vb._setup_vxlan()
vb._setup_bridge()
vb._add_vxlan_to_bridge()
vb._route_network()
if arguments['run_dns_ra']:
if not arguments['network'] or not arguments['vni']:
raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)")
dnsra = DNSRA(route=arguments['network'],
vni=arguments['vni'],
use_sudo=arguments['use_sudo'])
dnsra._setup_dnsmasq()

116
uncloud/hack/net.py
View file

@ -1,116 +0,0 @@
import subprocess
import ipaddress
import logging
from uncloud import UncloudException
log = logging.getLogger(__name__)
class VXLANBridge(object):
cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5"
cmd_up_dev = "{sudo}ip link set {dev} up"
cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge"
cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up"
cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}"
cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}"
# VXLAN ids are at maximum 24 bit - use a /104
multicast_network = ipaddress.IPv6Network("ff05::/104")
max_vni = (2**24)-1
def __init__(self,
vni,
uplinkdev,
route=None,
use_sudo=False):
self.config = {}
if vni > self.max_vni:
raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni))
if use_sudo:
self.config['sudo'] = 'sudo '
else:
self.config['sudo'] = ''
self.config['vni_dec'] = vni
self.config['vni_hex'] = "{:x}".format(vni)
self.config['multicast_address'] = self.multicast_network[vni]
self.config['route_network'] = ipaddress.IPv6Network(route)
self.config['route'] = route
self.config['uplinkdev'] = uplinkdev
self.config['vxlandev'] = "vx{}".format(self.config['vni_hex'])
self.config['bridgedev'] = "br{}".format(self.config['vni_hex'])
def setup_networking(self):
pass
def _setup_vxlan(self):
self._execute_cmd(self.cmd_create_vxlan)
self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev'])
def _setup_bridge(self):
self._execute_cmd(self.cmd_create_bridge)
self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev'])
def _route_network(self):
self._execute_cmd(self.cmd_add_route_dev)
def _add_vxlan_to_bridge(self):
self._execute_cmd(self.cmd_add_to_bridge)
def _execute_cmd(self, cmd_string, **kwargs):
cmd = cmd_string.format(**self.config, **kwargs)
log.info("Executing: {}".format(cmd))
subprocess.run(cmd.split())
class ManagementBridge(VXLANBridge):
pass
class DNSRA(object):
# VXLAN ids are at maximum 24 bit
max_vni = (2**24)-1
# Command to start dnsmasq
cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra"
def __init__(self,
vni,
route=None,
use_sudo=False):
self.config = {}
if vni > self.max_vni:
raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni))
if use_sudo:
self.config['sudo'] = 'sudo '
else:
self.config['sudo'] = ''
#TODO: remove if not needed
#self.config['vni_dec'] = vni
self.config['vni_hex'] = "{:x}".format(vni)
# dnsmasq only wants the network without the prefix, therefore, cut it off
self.config['route'] = ipaddress.IPv6Network(route).network_address
self.config['bridgedev'] = "br{}".format(self.config['vni_hex'])
def _setup_dnsmasq(self):
self._execute_cmd(self.cmd_start_dnsmasq)
def _execute_cmd(self, cmd_string, **kwargs):
cmd = cmd_string.format(**self.config, **kwargs)
log.info("Executing: {}".format(cmd))
print("Executing: {}".format(cmd))
subprocess.run(cmd.split())
class Firewall(object):
pass

26
uncloud/hack/uncloud-hack-init-host
View file

@ -1,26 +0,0 @@
id=100
rawdev=eth0
# create vxlan
ip -6 link add vxlan${id} type vxlan \
id ${id} \
dstport 4789 \
group ff05::${id} \
dev ${rawdev} \
ttl 5
ip link set vxlan${id} up
# create bridge
ip link set vxlan${id} up
ip link set br${id} up
# Add vxlan into bridge
ip link set vxlan${id} master br${id}
# useradd -m uncloud
# [18:05] tablett.place10:~# id uncloud
# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu)
# apk add qemu-system-x86_64
# also needs group netdev

25
uncloud/hack/uncloud-run-vm
View file

@ -1,25 +0,0 @@
#!/bin/sh
if [ $# -ne 1 ]; then
echo $0 vmid
exit 1
fi
id=$1; shift
memory=512
macaddress=02:00:b9:cb:70:${id}
netname=net${id}-1
qemu-system-x86_64 \
-name uncloud-${id} \
-accel kvm \
-m ${memory} \
-smp 2,sockets=2,cores=1,threads=1 \
-device virtio-net-pci,netdev=net0,mac=$macaddress \
-netdev tap,id=net0,ifname=${netname},script=no,downscript=no \
-vnc [::]:0
# To be changed:
# -vnc to unix path
# or -spice

136
uncloud/hack/vm.py
View file

@ -1,136 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
#
# This file is part of uncloud.
#
# uncloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# uncloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
# This module is directly called from the hack module, and can be used as follow:
#
# Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix.
# `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2`
#
# List running VMs (returns a list of UUIDs).
# `uncloud hack --hackprefix /tmp/hackcloud --list-vms
#
# Get VM status:
# `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid`
#
# Stop a VM:
# `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid`
# ``
import subprocess
import uuid
import os
import logging
from uncloud.hack.db import DB
from uncloud.hack.mac import MAC
from uncloud.vmm import VMM
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
class VM(object):
def __init__(self, config):
self.config = config
#TODO: Enable etcd lookup
self.no_db = self.config.arguments['no_db']
if not self.no_db:
self.db = DB(self.config, prefix="/vm")
# General CLI arguments.
self.hackprefix = self.config.arguments['hackprefix']
self.uuid = self.config.arguments['uuid']
self.memory = self.config.arguments['memory'] or '1024M'
self.cores = self.config.arguments['cores'] or 1
if self.config.arguments['image']:
self.image = os.path.join(self.hackprefix, self.config.arguments['image'])
else:
self.image = None
# External components.
self.vmm = VMM(vmm_backend=self.hackprefix)
self.mac = MAC(self.config)
# Harcoded & generated values.
self.owner = 'uncoud'
self.image_format='qcow2'
self.accel = 'kvm'
self.threads = 1
self.ifup = os.path.join(self.hackprefix, "ifup.sh")
self.ifdown = os.path.join(self.hackprefix, "ifdown.sh")
self.ifname = "uc{}".format(self.mac.to_str_format())
def get_qemu_args(self):
command = (
"-name {owner}-{name}"
" -machine pc,accel={accel}"
" -drive file={image},format={image_format},if=virtio"
" -device virtio-rng-pci"
" -m {memory} -smp cores={cores},threads={threads}"
" -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}"
" -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}"
).format(
owner=self.owner, name=self.uuid,
accel=self.accel,
image=self.image, image_format=self.image_format,
memory=self.memory, cores=self.cores, threads=self.threads,
ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname,
mac=self.mac
)
return command.split(" ")
def create(self):
# New VM: new UUID, new MAC.
self.uuid = str(uuid.uuid4())
self.mac.create()
qemu_args = self.get_qemu_args()
log.debug("QEMU args passed to VMM: {}".format(qemu_args))
self.vmm.start(
uuid=self.uuid,
migration=False,
*qemu_args
)
def stop(self):
if not self.uuid:
print("Please specific an UUID with the --uuid flag.")
exit(1)
self.vmm.stop(self.uuid)
def status(self):
if not self.uuid:
print("Please specific an UUID with the --uuid flag.")
exit(1)
print(self.vmm.get_status(self.uuid))
def vnc_addr(self):
if not self.uuid:
print("Please specific an UUID with the --uuid flag.")
exit(1)
print(self.vmm.get_vnc(self.uuid))
def list(self):
print(self.vmm.discover())

14
uncloud/host/main.py
View file

@ -6,6 +6,7 @@ from uuid import uuid4
from uncloud.common.request import RequestEntry, RequestType from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.common.vm import VMStatus from uncloud.common.vm import VMStatus
from uncloud.vmm import VMM from uncloud.vmm import VMM
from os.path import join as join_path from os.path import join as join_path
@ -35,7 +36,7 @@ def maintenance(host):
if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
logger.debug('VM {} is running on {}'.format(vm_uuid, host)) logger.debug('VM {} is running on {}'.format(vm_uuid, host))
vm = shared.vm_pool.get( vm = shared.vm_pool.get(
join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) join_path(settings['etcd']['vm_prefix'], vm_uuid)
) )
vm.status = VMStatus.running vm.status = VMStatus.running
vm.vnc_socket = vmm.get_vnc(vm_uuid) vm.vnc_socket = vmm.get_vnc(vm_uuid)
@ -43,15 +44,14 @@ def maintenance(host):
shared.vm_pool.put(vm) shared.vm_pool.put(vm)
def main(arguments): def main(hostname, debug=False):
hostname = arguments['hostname']
host_pool = shared.host_pool host_pool = shared.host_pool
host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
# Does not yet exist, create it # Does not yet exist, create it
if not host: if not host:
host_key = join_path( host_key = join_path(
shared.settings['etcd']['host_prefix'], uuid4().hex settings['etcd']['host_prefix'], uuid4().hex
) )
host_entry = { host_entry = {
'specs': '', 'specs': '',
@ -79,9 +79,9 @@ def main(arguments):
# get prefix until either success or deamon death comes. # get prefix until either success or deamon death comes.
while True: while True:
for events_iterator in [ for events_iterator in [
shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False), raise_exception=False),
shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False) raise_exception=False)
]: ]:
for request_event in events_iterator: for request_event in events_iterator:
@ -94,7 +94,7 @@ def main(arguments):
shared.request_pool.client.client.delete(request_event.key) shared.request_pool.client.client.delete(request_event.key)
vm_entry = shared.etcd_client.get( vm_entry = shared.etcd_client.get(
join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) join_path(settings['etcd']['vm_prefix'], request_event.uuid)
) )
logger.debug('VM hostname: {}'.format(vm_entry.value)) logger.debug('VM hostname: {}'.format(vm_entry.value))

11
uncloud/host/virtualmachine.py
View file

@ -17,6 +17,7 @@ from uncloud.common.network import create_dev, delete_network_interface
from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.common.schemas import VMSchema, NetworkSchema
from uncloud.host import logger from uncloud.host import logger
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.vmm import VMM from uncloud.vmm import VMM
from marshmallow import ValidationError from marshmallow import ValidationError
@ -90,7 +91,7 @@ class VM:
self.vmm.socket_dir, self.uuid self.vmm.socket_dir, self.uuid
), ),
destination_host_key=destination_host_key, # Where source host transfer VM destination_host_key=destination_host_key, # Where source host transfer VM
request_prefix=shared.settings["etcd"]["request_prefix"], request_prefix=settings["etcd"]["request_prefix"],
) )
shared.request_pool.put(r) shared.request_pool.put(r)
else: else:
@ -118,7 +119,7 @@ class VM:
network_name, mac, tap = network_mac_and_tap network_name, mac, tap = network_mac_and_tap
_key = os.path.join( _key = os.path.join(
shared.settings["etcd"]["network_prefix"], settings["etcd"]["network_prefix"],
self.vm["owner"], self.vm["owner"],
network_name, network_name,
) )
@ -132,13 +133,13 @@ class VM:
if network["type"] == "vxlan": if network["type"] == "vxlan":
tap = create_vxlan_br_tap( tap = create_vxlan_br_tap(
_id=network["id"], _id=network["id"],
_dev=shared.settings["network"]["vxlan_phy_dev"], _dev=settings["network"]["vxlan_phy_dev"],
tap_id=tap, tap_id=tap,
ip=network["ipv6"], ip=network["ipv6"],
) )
all_networks = shared.etcd_client.get_prefix( all_networks = shared.etcd_client.get_prefix(
shared.settings["etcd"]["network_prefix"], settings["etcd"]["network_prefix"],
value_in_json=True, value_in_json=True,
) )
@ -228,7 +229,7 @@ class VM:
def resolve_network(network_name, network_owner): def resolve_network(network_name, network_owner):
network = shared.etcd_client.get( network = shared.etcd_client.get(
join_path( join_path(
shared.settings["etcd"]["network_prefix"], settings["etcd"]["network_prefix"],
network_owner, network_owner,
network_name, network_name,
), ),

9
uncloud/imagescanner/main.py
View file

@ -4,6 +4,7 @@ import argparse
import subprocess as sp import subprocess as sp
from os.path import join as join_path from os.path import join as join_path
from uncloud.common.settings import settings
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.imagescanner import logger from uncloud.imagescanner import logger
@ -29,10 +30,10 @@ def qemu_img_type(path):
return qemu_img_info["format"] return qemu_img_info["format"]
def main(arguments): def main(debug=False):
# We want to get images entries that requests images to be created # We want to get images entries that requests images to be created
images = shared.etcd_client.get_prefix( images = shared.etcd_client.get_prefix(
shared.settings["etcd"]["image_prefix"], value_in_json=True settings["etcd"]["image_prefix"], value_in_json=True
) )
images_to_be_created = list( images_to_be_created = list(
filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
@ -45,13 +46,13 @@ def main(arguments):
image_filename = image.value["filename"] image_filename = image.value["filename"]
image_store_name = image.value["store_name"] image_store_name = image.value["store_name"]
image_full_path = join_path( image_full_path = join_path(
shared.settings["storage"]["file_dir"], settings["storage"]["file_dir"],
image_owner, image_owner,
image_filename, image_filename,
) )
image_stores = shared.etcd_client.get_prefix( image_stores = shared.etcd_client.get_prefix(
shared.settings["etcd"]["image_store_prefix"], settings["etcd"]["image_store_prefix"],
value_in_json=True, value_in_json=True,
) )
user_image_store = next( user_image_store = next(

15
uncloud/metadata/main.py
View file

@ -5,6 +5,7 @@ from flask import Flask, request
from flask_restful import Resource, Api from flask_restful import Resource, Api
from werkzeug.exceptions import HTTPException from werkzeug.exceptions import HTTPException
from uncloud.common.settings import settings
from uncloud.common.shared import shared from uncloud.common.shared import shared
app = Flask(__name__) app = Flask(__name__)
@ -12,10 +13,8 @@ api = Api(app)
app.logger.handlers.clear() app.logger.handlers.clear()
DEFAULT_PORT=1234
arg_parser = argparse.ArgumentParser('metadata', add_help=False) arg_parser = argparse.ArgumentParser('metadata', add_help=False)
arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT)) arg_parser.add_argument('--port', '-p', default=80, help='By default bind to port 80')
@app.errorhandler(Exception) @app.errorhandler(Exception)
@ -73,7 +72,7 @@ class Root(Resource):
) )
else: else:
etcd_key = os.path.join( etcd_key = os.path.join(
shared.settings["etcd"]["user_prefix"], settings["etcd"]["user_prefix"],
data.value["owner_realm"], data.value["owner_realm"],
data.value["owner"], data.value["owner"],
"key", "key",
@ -89,7 +88,9 @@ class Root(Resource):
api.add_resource(Root, "/") api.add_resource(Root, "/")
def main(arguments): def main(port=None, debug=False):
port = arguments['port']
debug = arguments['debug']
app.run(debug=debug, host="::", port=port) app.run(debug=debug, host="::", port=port)
if __name__ == "__main__":
main()

3
uncloud/oneshot/__init__.py
View file

@ -1,3 +0,0 @@
import logging
logger = logging.getLogger(__name__)

123
uncloud/oneshot/main.py
View file

@ -1,123 +0,0 @@
import argparse
import os
from pathlib import Path
from uncloud.vmm import VMM
from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap
from . import virtualmachine, logger
###
# Argument parser loaded by scripts/uncloud.
arg_parser = argparse.ArgumentParser('oneshot', add_help=False)
# Actions.
arg_parser.add_argument('--list', action='store_true',
help='list UUID and name of running VMs')
arg_parser.add_argument('--start', nargs=3,
metavar=('IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'),
help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix')
arg_parser.add_argument('--stop', metavar='UUID',
help='stop a VM')
arg_parser.add_argument('--get-status', metavar='UUID',
help='return the status of the VM')
arg_parser.add_argument('--get-vnc', metavar='UUID',
help='return the path of the VNC socket of the VM')
arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK',
help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix')
# Arguments.
arg_parser.add_argument('--workdir', default=Path.home(),
help='Working directory, defaulting to $HOME')
arg_parser.add_argument('--mac',
help='MAC address of the VM to create (--start)')
arg_parser.add_argument('--memory', type=int,
help='Memory (MB) to allocate (--start)')
arg_parser.add_argument('--cores', type=int,
help='Number of cores to allocate (--start)')
arg_parser.add_argument('--threads', type=int,
help='Number of threads to allocate (--start)')
arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'],
help='Format of OS image (--start)')
arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm',
help='QEMU acceleration to use (--start)')
arg_parser.add_argument('--upstream-interface', default='eth0',
help='Name of upstream interface (--start)')
###
# Helpers.
# XXX: check if it is possible to use the type returned by ETCD queries.
class UncloudEntryWrapper:
def __init__(self, value):
self.value = value
def value(self):
return self.value
def status_line(vm):
return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status())
###
# Entrypoint.
def main(arguments):
# Initialize VMM.
workdir = arguments['workdir']
vmm = VMM(vmm_backend=workdir)
# Harcoded debug values.
net_id = 0
# Build VM configuration.
vm_config = {}
vm_options = [
'mac', 'memory', 'cores', 'threads', 'image', 'image_format',
'--upstream_interface', 'upstream_interface', 'network', 'accel'
]
for option in vm_options:
if arguments.get(option):
vm_config[option] = arguments[option]
vm_config['net_id'] = net_id
# Execute requested VM action.
if arguments['reconfigure_radvd']:
# TODO: check that RADVD is available.
prefix = arguments['reconfigure_radvd']
network = UncloudEntryWrapper({
'id': net_id,
'ipv6': prefix
})
# Make use of uncloud.host.virtualmachine for network configuration.
update_radvd_conf([network])
elif arguments['start']:
# Extract from --start positional arguments. Quite fragile.
vm_config['image'] = arguments['start'][0]
vm_config['network'] = arguments['start'][1]
vm_config['upstream_interface'] = arguments['start'][2]
vm_config['tap_interface'] = "uc{}".format(len(vmm.discover()))
vm = virtualmachine.VM(vmm, vm_config)
vm.start()
elif arguments['stop']:
vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']})
vm = virtualmachine.VM(vmm, vm_config)
vm.stop()
elif arguments['get_status']:
vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']})
print(status_line(vm))
elif arguments['get_vnc']:
vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']})
print(vm.get_vnc_addr())
elif arguments['list']:
vms = vmm.discover()
print("Found {} VMs.".format(len(vms)))
for uuid in vms:
vm = virtualmachine.VM(vmm, {'uuid': uuid})
print(status_line(vm))
else:
print('Please specify an action: --start, --stop, --list,\
--get-status, --get-vnc, --reconfigure-radvd')

81
uncloud/oneshot/virtualmachine.py
View file

@ -1,81 +0,0 @@
import uuid
import os
from uncloud.host.virtualmachine import create_vxlan_br_tap
from uncloud.oneshot import logger
class VM(object):
def __init__(self, vmm, config):
self.config = config
self.vmm = vmm
# Extract VM specs/metadata from configuration.
self.name = config.get('name', 'no-name')
self.memory = config.get('memory', 1024)
self.cores = config.get('cores', 1)
self.threads = config.get('threads', 1)
self.image_format = config.get('image_format', 'qcow2')
self.image = config.get('image')
self.uuid = config.get('uuid', str(uuid.uuid4()))
self.mac = config.get('mac')
self.accel = config.get('accel', 'kvm')
self.net_id = config.get('net_id', 0)
self.upstream_interface = config.get('upstream_interface', 'eth0')
self.tap_interface = config.get('tap_interface', 'uc0')
self.network = config.get('network')
def get_qemu_args(self):
command = (
"-uuid {uuid} -name {name} -machine pc,accel={accel}"
" -drive file={image},format={image_format},if=virtio"
" -device virtio-rng-pci"
" -m {memory} -smp cores={cores},threads={threads}"
" -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no"
" -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}"
).format(
uuid=self.uuid, name=self.name, accel=self.accel,
image=self.image, image_format=self.image_format,
memory=self.memory, cores=self.cores, threads=self.threads,
net_id=self.net_id, tap=self.tap_interface, mac=self.mac
)
return command.split(" ")
def start(self):
# Check that VM image is available.
if not os.path.isfile(self.image):
logger.error("Image {} does not exist. Aborting.".format(self.image))
# Create Bridge, VXLAN and tap interface for VM.
create_vxlan_br_tap(
self.net_id, self.upstream_interface, self.tap_interface, self.network
)
# Generate config for and run QEMU.
qemu_args = self.get_qemu_args()
logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args))
self.vmm.start(
uuid=self.uuid,
migration=False,
*qemu_args
)
def stop(self):
self.vmm.stop(self.uuid)
def get_status(self):
return self.vmm.get_status(self.uuid)
def get_vnc_addr(self):
return self.vmm.get_vnc(self.uuid)
def get_uuid(self):
return self.uuid
def get_name(self):
success, json = self.vmm.execute_command(uuid, 'query-name')
if success:
return json['return']['name']
return None

3
uncloud/scheduler/helper.py
View file

@ -7,6 +7,7 @@ from uncloud.common.host import HostStatus
from uncloud.common.request import RequestEntry, RequestType from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.vm import VMStatus from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
def accumulated_specs(vms_specs): def accumulated_specs(vms_specs):
@ -129,7 +130,7 @@ def assign_host(vm):
type=RequestType.StartVM, type=RequestType.StartVM,
uuid=vm.uuid, uuid=vm.uuid,
hostname=vm.hostname, hostname=vm.hostname,
request_prefix=shared.settings["etcd"]["request_prefix"], request_prefix=settings["etcd"]["request_prefix"],
) )
shared.request_pool.put(r) shared.request_pool.put(r)

11
uncloud/scheduler/main.py
View file

@ -6,6 +6,7 @@
import argparse import argparse
from uncloud.common.settings import settings
from uncloud.common.request import RequestEntry, RequestType from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.scheduler import logger from uncloud.scheduler import logger
@ -15,7 +16,7 @@ from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection,
arg_parser = argparse.ArgumentParser('scheduler', add_help=False) arg_parser = argparse.ArgumentParser('scheduler', add_help=False)
def main(arguments): def main(debug=False):
# The below while True is neccessary for gracefully handling leadership transfer and temporary # The below while True is neccessary for gracefully handling leadership transfer and temporary
# unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
# iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
@ -23,9 +24,9 @@ def main(arguments):
# get prefix until either success or deamon death comes. # get prefix until either success or deamon death comes.
while True: while True:
for request_iterator in [ for request_iterator in [
shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False), raise_exception=False),
shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False), raise_exception=False),
]: ]:
for request_event in request_iterator: for request_event in request_iterator:
@ -49,3 +50,7 @@ def main(arguments):
shared.vm_pool.put(vm_entry) shared.vm_pool.put(vm_entry)
logger.info('No Resource Left. Emailing admin....') logger.info('No Resource Left. Emailing admin....')
if __name__ == '__main__':
main()

16
uncloud/vmm/__init__.py
View file

@ -100,9 +100,9 @@ class TransferVM(Process):
class VMM: class VMM:
# Virtual Machine Manager # Virtual Machine Manager
def __init__( def __init__(
self, self,
qemu_path="/usr/bin/qemu-system-x86_64", qemu_path="/usr/bin/qemu-system-x86_64",
vmm_backend=os.path.expanduser("~/uncloud/vmm/"), vmm_backend=os.path.expanduser("~/uncloud/vmm/"),
): ):
self.qemu_path = qemu_path self.qemu_path = qemu_path
self.vmm_backend = vmm_backend self.vmm_backend = vmm_backend
@ -125,7 +125,7 @@ class VMM:
os.makedirs(self.socket_dir, exist_ok=True) os.makedirs(self.socket_dir, exist_ok=True)
def is_running(self, uuid): def is_running(self, uuid):
sock_path = os.path.join(self.socket_dir, uuid) sock_path = os.path.join(self.vmm_backend, uuid)
try: try:
sock = socket.socket(socket.AF_UNIX) sock = socket.socket(socket.AF_UNIX)
sock.connect(sock_path) sock.connect(sock_path)
@ -163,7 +163,7 @@ class VMM:
qmp_arg = ( qmp_arg = (
"-qmp", "-qmp",
"unix:{},server,nowait".format( "unix:{},server,nowait".format(
join_path(self.socket_dir, uuid) join_path(self.vmm_backend, uuid)
), ),
) )
vnc_arg = ( vnc_arg = (
@ -212,7 +212,7 @@ class VMM:
def execute_command(self, uuid, command, **kwargs): def execute_command(self, uuid, command, **kwargs):
# execute_command -> sucess?, output # execute_command -> sucess?, output
try: try:
with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as ( with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as (
sock_handle, sock_handle,
file_handle, file_handle,
): ):
@ -255,8 +255,8 @@ class VMM:
def discover(self): def discover(self):
vms = [ vms = [
uuid uuid
for uuid in os.listdir(self.socket_dir) for uuid in os.listdir(self.vmm_backend)
if not isdir(join_path(self.socket_dir, uuid)) if not isdir(join_path(self.vmm_backend, uuid))
] ]
return vms return vms