uncloud
/
uncloud
2
0
Fork 1
Code Issues 4 Pull Requests Projects Releases Wiki Activity

Merge branch 'master' of code.ungleich.ch:uncloud/uncloud

This commit is contained in:
Nico Schottelius 2020-01-11 00:06:29 +01:00
parent 388127bd11 b9c9a5e0ec
commit 7c9e3d747a
32 changed files with 431 additions and 396 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
conf/uncloud.conf
View File

@ -1,6 +1,7 @@
[etcd]
url = localhost
port = 2379
base_prefix = /
ca_cert
cert_cert
cert_key
@ -9,3 +10,4 @@ cert_key
name = replace_me
realm = replace_me
seed = replace_me
api_server = http://localhost:5000

52
scripts/uncloud
View File

@ -3,10 +3,14 @@ import logging
import sys
import importlib
import argparse
import multiprocessing as mp
from uncloud import UncloudException
from contextlib import suppress
# the components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
ALL_COMPONENTS = ETCD_COMPONENTS.copy()
ALL_COMPONENTS.append('cli')
def exception_hook(exc_type, exc_value, exc_traceback):
@ -27,32 +31,48 @@ if __name__ == '__main__':
subparsers = arg_parser.add_subparsers(dest='command')
parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
parent_parser.add_argument('--debug', '-d',
action='store_true',
default=False,
help='More verbose logging')
parent_parser.add_argument('--conf-dir', '-c',
help='Configuration directory')
for component in ['api', 'scheduler', 'host', 'filescanner', 'imagescanner',
'metadata', 'configure', 'cli']:
etcd_parser = argparse.ArgumentParser(add_help=False)
etcd_parser.add_argument('--etcd-host')
etcd_parser.add_argument('--etcd-port')
etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
for component in ALL_COMPONENTS:
mod = importlib.import_module('uncloud.{}.main'.format(component))
parser = getattr(mod, 'arg_parser')
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
if component in ETCD_COMPONENTS:
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser])
else:
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
args = arg_parser.parse_args()
if not args.command:
arg_parser.print_help()
else:
# if we start etcd in seperate process with default settings
# i.e inheriting few things from parent process etcd3 module
# errors out, so the following command configure multiprocessing
# module to not inherit anything from parent.
mp.set_start_method('spawn')
arguments = vars(args)
name = arguments.pop('command')
mod = importlib.import_module('uncloud.{}.main'.format(name))
main = getattr(mod, 'main')
# If the component requires etcd3, we import it and catch the
# etcd3.exceptions.ConnectionFailedError
if name in ETCD_COMPONENTS:
import etcd3
try:
name = arguments.pop('command')
mod = importlib.import_module('uncloud.{}.main'.format(name))
main = getattr(mod, 'main')
main(**arguments)
main(arguments)
except UncloudException as err:
logger.error(err)
sys.exit(1)
except Exception as err:
logger.exception(err)
sys.exit(1)

4
uncloud/api/common_fields.py
View File

@ -1,7 +1,7 @@
import os
from uncloud.shared import shared
from uncloud.settings import settings
from uncloud.common.shared import shared
from uncloud.common.settings import settings
class Optional:

16
uncloud/api/create_image_store.py
View File

@ -3,18 +3,18 @@ import os
from uuid import uuid4
from uncloud.shared import shared
from uncloud.settings import settings
from uncloud.common.shared import shared
from uncloud.common.settings import settings
data = {
"is_public": True,
"type": "ceph",
"name": "images",
"description": "first ever public image-store",
"attributes": {"list": [], "key": [], "pool": "images"},
'is_public': True,
'type': 'ceph',
'name': 'images',
'description': 'first ever public image-store',
'attributes': {'list': [], 'key': [], 'pool': 'images'},
}
shared.etcd_client.put(
os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex),
os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex),
json.dumps(data),
)

5
uncloud/api/helper.py
View File

@ -1,14 +1,13 @@
import binascii
import ipaddress
import random
import subprocess as sp
import logging
import requests
from pyotp import TOTP
from uncloud.shared import shared
from uncloud.settings import settings
from uncloud.common.shared import shared
from uncloud.common.settings import settings
logger = logging.getLogger(__name__)

44
uncloud/api/main.py
View File

@ -10,11 +10,12 @@ from flask import Flask, request
from flask_restful import Resource, Api
from werkzeug.exceptions import HTTPException
from uncloud.common.shared import shared
from uncloud.common import counters
from uncloud.common.vm import VMStatus
from uncloud.common.request import RequestEntry, RequestType
from uncloud.settings import settings
from uncloud.shared import shared
from uncloud.common.settings import settings
from . import schemas
from .helper import generate_mac, mac2ipv6
from uncloud import UncloudException
@ -41,7 +42,7 @@ def handle_exception(e):
class CreateVM(Resource):
'''API Request to Handle Creation of VM'''
"""API Request to Handle Creation of VM"""
@staticmethod
def post():
@ -59,7 +60,7 @@ class CreateVM(Resource):
macs = [generate_mac() for _ in range(len(data['network']))]
tap_ids = [
counters.increment_etcd_counter(
shared.etcd_client, '/v1/counter/tap'
shared.etcd_client, settings['etcd']['tap_counter']
)
for _ in range(len(data['network']))
]
@ -289,18 +290,16 @@ class ListUserFiles(Resource):
settings['etcd']['file_prefix'], value_in_json=True
)
return_files = []
user_files = list(
filter(
lambda f: f.value['owner'] == data['name'], files
)
)
user_files = [f for f in files if f.value['owner'] == data['name']]
for file in user_files:
return_files.append(
{
'filename': file.value['filename'],
'uuid': file.key.split('/')[-1],
}
)
file_uuid = file.key.split('/')[-1]
file = file.value
file['uuid'] = file_uuid
file.pop('sha512sum', None)
file.pop('owner', None)
return_files.append(file)
return {'message': return_files}, 200
else:
return validator.get_errors(), 400
@ -472,7 +471,7 @@ class CreateNetwork(Resource):
network_entry = {
'id': counters.increment_etcd_counter(
shared.etcd_client, '/v1/counter/vxlan'
shared.etcd_client, settings['etcd']['vxlan_counter']
),
'type': data['type'],
}
@ -564,7 +563,10 @@ api.add_resource(ListHost, '/host/list')
api.add_resource(CreateNetwork, '/network/create')
def main(debug=False, port=None):
def main(arguments):
debug = arguments['debug']
port = arguments['port']
try:
image_stores = list(
shared.etcd_client.get_prefix(
@ -594,12 +596,6 @@ def main(debug=False, port=None):
# )
try:
app.run(host='::',
port=port,
debug=debug)
app.run(host='::', port=port, debug=debug)
except OSError as e:
raise UncloudException('Failed to start Flask: {}'.format(e))
if __name__ == '__main__':
main()

4
uncloud/api/schemas.py
View File

@ -21,8 +21,8 @@ import bitmath
from uncloud.common.host import HostStatus
from uncloud.common.vm import VMStatus
from uncloud.shared import shared
from uncloud.settings import settings
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from . import helper, logger
from .common_fields import Field, VmUUIDField
from .helper import check_otp, resolve_vm_name

19
uncloud/cli/helper.py
View File

@ -5,15 +5,24 @@ import binascii
from pyotp import TOTP
from os.path import join as join_path
from uncloud.settings import settings
from uncloud.common.settings import settings
def get_otp_parser():
otp_parser = argparse.ArgumentParser('otp')
otp_parser.add_argument('--name', default=settings['client']['name'])
otp_parser.add_argument('--realm', default=settings['client']['realm'])
otp_parser.add_argument('--seed', type=get_token, default=settings['client']['seed'],
dest='token', metavar='SEED')
try:
name = settings['client']['name']
realm = settings['client']['realm']
seed = settings['client']['seed']
except Exception:
otp_parser.add_argument('--name', required=True)
otp_parser.add_argument('--realm', required=True)
otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED')
else:
otp_parser.add_argument('--name', default=name)
otp_parser.add_argument('--realm', default=realm)
otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED')
return otp_parser

2
uncloud/cli/image.py
View File

@ -12,7 +12,7 @@ class ImageParser(BaseParser):
p = self.subparser.add_parser('create', **kwargs)
p.add_argument('--name', required=True)
p.add_argument('--uuid', required=True)
p.add_argument('--image-store-name', default='image_store')
p.add_argument('--image-store', required=True, dest='image_store')
def list(self, **kwargs):
self.subparser.add_parser('list', **kwargs)

10
uncloud/cli/main.py
View File

@ -12,12 +12,12 @@ for component in ['user', 'host', 'image', 'network', 'vm']:
subparser.add_parser(name=parser.prog, parents=[parser])
def main(**kwargs):
if not kwargs['subcommand']:
def main(arguments):
if not arguments['subcommand']:
arg_parser.print_help()
else:
name = kwargs.pop('subcommand')
kwargs.pop('debug')
name = arguments.pop('subcommand')
arguments.pop('debug')
mod = importlib.import_module('uncloud.cli.{}'.format(name))
_main = getattr(mod, 'main')
_main(**kwargs)
_main(**arguments)

95
uncloud/common/etcd_wrapper.py
View File

@ -1,24 +1,21 @@
import etcd3
import json
import queue
import copy
from uncloud import UncloudException
from collections import namedtuple
from functools import wraps
from . import logger
PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"])
from uncloud import UncloudException
from uncloud.common import logger
class EtcdEntry:
# key: str
# value: str
def __init__(self, meta, value, value_in_json=False):
self.key = meta.key.decode("utf-8")
self.value = value.decode("utf-8")
def __init__(self, meta_or_key, value, value_in_json=False):
if hasattr(meta_or_key, 'key'):
# if meta has attr 'key' then get it
self.key = meta_or_key.key.decode('utf-8')
else:
# otherwise meta is the 'key'
self.key = meta_or_key
self.value = value.decode('utf-8')
if value_in_json:
self.value = json.loads(self.value)
@ -29,18 +26,12 @@ def readable_errors(func):
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except etcd3.exceptions.ConnectionFailedError as err:
raise UncloudException(
"Cannot connect to etcd: is etcd running as configured in uncloud.conf?"
)
except etcd3.exceptions.ConnectionFailedError:
raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?')
except etcd3.exceptions.ConnectionTimeoutError as err:
raise etcd3.exceptions.ConnectionTimeoutError(
"etcd connection timeout."
) from err
raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err
except Exception:
logger.exception(
"Some etcd error occured. See syslog for details."
)
logger.exception('Some etcd error occured. See syslog for details.')
return wrapper
@ -64,55 +55,21 @@ class Etcd3Wrapper:
_value = json.dumps(_value)
if not isinstance(_key, str):
_key = _key.decode("utf-8")
_key = _key.decode('utf-8')
return self.client.put(_key, _value, **kwargs)
@readable_errors
def get_prefix(self, *args, value_in_json=False, **kwargs):
r = self.client.get_prefix(*args, **kwargs)
for entry in r:
e = EtcdEntry(*entry[::-1], value_in_json=value_in_json)
if e.value:
yield e
def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs):
event_iterator = self.client.get_prefix(*args, **kwargs)
for e in event_iterator:
yield EtcdEntry(*e[::-1], value_in_json=value_in_json)
@readable_errors
def watch_prefix(self, key, timeout=0, value_in_json=False):
timeout_event = EtcdEntry(
PseudoEtcdMeta(key=b"TIMEOUT"),
value=str.encode(
json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"})
),
value_in_json=value_in_json,
)
event_queue = queue.Queue()
def add_event_to_queue(event):
if hasattr(event, "events"):
for e in event.events:
if e.value:
event_queue.put(
EtcdEntry(
e, e.value, value_in_json=value_in_json
)
)
self.client.add_watch_prefix_callback(key, add_event_to_queue)
while True:
try:
while True:
v = event_queue.get(timeout=timeout)
yield v
except queue.Empty:
event_queue.put(copy.deepcopy(timeout_event))
class PsuedoEtcdEntry(EtcdEntry):
def __init__(self, key, value, value_in_json=False):
super().__init__(
PseudoEtcdMeta(key=key.encode("utf-8")),
value,
value_in_json=value_in_json,
)
def watch_prefix(self, key, raise_exception=True, value_in_json=False):
event_iterator, cancel = self.client.watch_prefix(key)
for e in event_iterator:
if hasattr(e, '_event'):
e = e._event
if e.type == e.PUT:
yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)

2
uncloud/common/network.py
View File

@ -1,8 +1,6 @@
import subprocess as sp
import random
import logging
import socket
from contextlib import closing
logger = logging.getLogger(__name__)

11
uncloud/common/request.py
View File

@ -2,8 +2,8 @@ import json
from os.path import join
from uuid import uuid4
from .etcd_wrapper import PsuedoEtcdEntry
from .classes import SpecificEtcdEntryBase
from uncloud.common.etcd_wrapper import EtcdEntry
from uncloud.common.classes import SpecificEtcdEntryBase
class RequestType:
@ -29,11 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase):
@classmethod
def from_scratch(cls, request_prefix, **kwargs):
e = PsuedoEtcdEntry(
join(request_prefix, uuid4().hex),
value=json.dumps(kwargs).encode("utf-8"),
value_in_json=True,
)
e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex),
value=json.dumps(kwargs).encode('utf-8'), value_in_json=True)
return cls(e)

79
uncloud/settings/__init__.py → uncloud/common/settings.py
View File

@ -4,8 +4,8 @@ import sys
import os
from datetime import datetime
from uncloud.common.etcd_wrapper import Etcd3Wrapper
from os.path import join as join_path
logger = logging.getLogger(__name__)
@ -16,7 +16,7 @@ class CustomConfigParser(configparser.RawConfigParser):
result = super().__getitem__(key)
except KeyError as err:
raise KeyError(
"Key '{}' not found in configuration. Make sure you configure uncloud.".format(
'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format(
key
)
) from err
@ -25,40 +25,41 @@ class CustomConfigParser(configparser.RawConfigParser):
class Settings(object):
def __init__(self, config_key="/uncloud/config/"):
conf_name = "uncloud.conf"
conf_dir = os.environ.get(
"UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/")
)
self.config_file = os.path.join(conf_dir, conf_name)
self.config_parser = CustomConfigParser(allow_no_value=True)
self.config_key = config_key
def __init__(self):
conf_name = 'uncloud.conf'
conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
self.config_file = join_path(conf_dir, conf_name)
# this is used to cache config from etcd for 1 minutes. Without this we
# would make a lot of requests to etcd which slows down everything.
self.last_config_update = datetime.fromtimestamp(0)
self.read_internal_values()
self.config_parser = CustomConfigParser(allow_no_value=True)
self.config_parser.add_section('etcd')
self.config_parser.set('etcd', 'base_prefix', '/')
try:
self.config_parser.read(self.config_file)
except Exception as err:
logger.error("%s", err)
logger.error('%s', err)
self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
self.read_internal_values()
def get_etcd_client(self):
args = tuple()
try:
kwargs = {
"host": self.config_parser.get("etcd", "url"),
"port": self.config_parser.get("etcd", "port"),
"ca_cert": self.config_parser.get("etcd", "ca_cert"),
"cert_cert": self.config_parser.get(
"etcd", "cert_cert"
),
"cert_key": self.config_parser.get("etcd", "cert_key"),
'host': self.config_parser.get('etcd', 'url'),
'port': self.config_parser.get('etcd', 'port'),
'ca_cert': self.config_parser.get('etcd', 'ca_cert'),
'cert_cert': self.config_parser.get('etcd', 'cert_cert'),
'cert_key': self.config_parser.get('etcd', 'cert_key'),
}
except configparser.Error as err:
raise configparser.Error(
"{} in config file {}".format(
'{} in config file {}'.format(
err.message, self.config_file
)
) from err
@ -67,8 +68,8 @@ class Settings(object):
wrapper = Etcd3Wrapper(*args, **kwargs)
except Exception as err:
logger.error(
"etcd connection not successfull. Please check your config file."
"\nDetails: %s\netcd connection parameters: %s",
'etcd connection not successfull. Please check your config file.'
'\nDetails: %s\netcd connection parameters: %s',
err,
kwargs,
)
@ -77,17 +78,20 @@ class Settings(object):
return wrapper
def read_internal_values(self):
base_prefix = self['etcd']['base_prefix']
self.config_parser.read_dict(
{
"etcd": {
"file_prefix": "/files/",
"host_prefix": "/hosts/",
"image_prefix": "/images/",
"image_store_prefix": "/imagestore/",
"network_prefix": "/networks/",
"request_prefix": "/requests/",
"user_prefix": "/users/",
"vm_prefix": "/vms/",
'etcd': {
'file_prefix': join_path(base_prefix, 'files/'),
'host_prefix': join_path(base_prefix, 'hosts/'),
'image_prefix': join_path(base_prefix, 'images/'),
'image_store_prefix': join_path(base_prefix, 'imagestore/'),
'network_prefix': join_path(base_prefix, 'networks/'),
'request_prefix': join_path(base_prefix, 'requests/'),
'user_prefix': join_path(base_prefix, 'users/'),
'vm_prefix': join_path(base_prefix, 'vms/'),
'vxlan_counter': join_path(base_prefix, 'counters/vxlan'),
'tap_counter': join_path(base_prefix, 'counters/tap')
}
}
)
@ -95,15 +99,13 @@ class Settings(object):
def read_config_file_values(self, config_file):
try:
# Trying to read configuration file
with open(config_file, "r") as config_file_handle:
with open(config_file) as config_file_handle:
self.config_parser.read_file(config_file_handle)
except FileNotFoundError:
sys.exit(
"Configuration file {} not found!".format(config_file)
)
sys.exit('Configuration file {} not found!'.format(config_file))
except Exception as err:
logger.exception(err)
sys.exit("Error occurred while reading configuration file")
sys.exit('Error occurred while reading configuration file')
def read_values_from_etcd(self):
etcd_client = self.get_etcd_client()
@ -113,7 +115,7 @@ class Settings(object):
self.config_parser.read_dict(config_from_etcd.value)
self.last_config_update = datetime.utcnow()
else:
raise KeyError("Key '{}' not found in etcd. Please configure uncloud.".format(self.config_key))
raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key))
def __getitem__(self, key):
# Allow failing to read from etcd if we have
@ -121,9 +123,8 @@ class Settings(object):
if key not in self.config_parser.sections():
try:
self.read_values_from_etcd()
except KeyError as e:
except KeyError:
pass
return self.config_parser[key]

2
uncloud/shared/__init__.py → uncloud/common/shared.py
View File

@ -1,4 +1,4 @@
from uncloud.settings import settings
from uncloud.common.settings import settings
from uncloud.common.vm import VmPool
from uncloud.common.host import HostPool
from uncloud.common.request import RequestPool

2
uncloud/common/storage_handlers.py
View File

@ -7,7 +7,7 @@ from abc import ABC
from . import logger
from os.path import join as join_path
from uncloud.settings import settings as config
from uncloud.common.settings import settings as config
class ImageStorageHandler(ABC):

12
uncloud/configure/main.py
View File

@ -1,8 +1,8 @@
import os
import argparse
from uncloud.settings import settings
from uncloud.shared import shared
from uncloud.common.settings import settings
from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('configure', add_help=False)
configure_subparsers = arg_parser.add_subparsers(dest='subcommand')
@ -40,18 +40,14 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
def update_config(section, kwargs):
uncloud_config = shared.etcd_client.get(
settings.config_key, value_in_json=True
)
uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
if not uncloud_config:
uncloud_config = {}
else:
uncloud_config = uncloud_config.value
uncloud_config[section] = kwargs
shared.etcd_client.put(
settings.config_key, uncloud_config, value_in_json=True
)
shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
def main(**kwargs):

12
uncloud/docs/README.md Normal file
View File

@ -0,0 +1,12 @@
# uncloud docs
## Requirements
1. Python3
2. Sphinx
## Usage
Run `make build` to build docs.
Run `make clean` to remove build directory.
Run `make publish` to push build dir to https://ungleich.ch/ucloud/

95
uncloud/filescanner/main.py
View File

@ -4,15 +4,16 @@ import pathlib
import subprocess as sp
import time
import argparse
import bitmath
from uuid import uuid4
from . import logger
from uncloud.settings import settings
from uncloud.shared import shared
from uncloud.common.settings import settings
from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
arg_parser.add_argument('--hostname', required=True)
def sha512sum(file: str):
@ -28,66 +29,58 @@ def sha512sum(file: str):
if not isinstance(file, str):
raise TypeError
try:
output = sp.check_output(["sha512sum", file], stderr=sp.PIPE)
output = sp.check_output(['sha512sum', file], stderr=sp.PIPE)
except sp.CalledProcessError as e:
error = e.stderr.decode("utf-8")
if "No such file or directory" in error:
error = e.stderr.decode('utf-8')
if 'No such file or directory' in error:
raise FileNotFoundError from None
else:
output = output.decode("utf-8").strip()
output = output.split(" ")
output = output.decode('utf-8').strip()
output = output.split(' ')
return output[0]
return None
def track_file(file, base_dir):
file_id = uuid4()
def track_file(file, base_dir, host):
file_path = file.relative_to(base_dir)
file_str = str(file)
# Get Username
owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)]
try:
owner = file_path.parts[0]
except IndexError:
pass
else:
file_path = file_path.relative_to(owner)
creation_date = time.ctime(os.stat(file_str).st_ctime)
# Get Creation Date of File
# Here, we are assuming that ctime is creation time
# which is mostly not true.
creation_date = time.ctime(os.stat(file).st_ctime)
entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
entry_value = {
'filename': str(file_path),
'owner': owner,
'sha512sum': sha512sum(file_str),
'creation_date': creation_date,
'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()),
'host': host
}
file_path = pathlib.Path(file).parts[-1]
logger.info('Tracking %s', file_str)
# Create Entry
entry_key = os.path.join(
settings["etcd"]["file_prefix"], str(file_id)
)
entry_value = {
"filename": file_path,
"owner": owner,
"sha512sum": sha512sum(file),
"creation_date": creation_date,
"size": os.path.getsize(file),
}
logger.info("Tracking %s", file)
shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
os.setxattr(file, "user.utracked", b"True")
shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
def main(debug=False):
base_dir = settings["storage"]["file_dir"]
def main(arguments):
hostname = arguments['hostname']
base_dir = settings['storage']['file_dir']
# Recursively Get All Files and Folder below BASE_DIR
files = glob.glob("{}/**".format(base_dir), recursive=True)
files = glob.glob('{}/**'.format(base_dir), recursive=True)
files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
# Retain only Files
files = [file for file in files if os.path.isfile(file)]
untracked_files = []
for file in files:
try:
os.getxattr(file, "user.utracked")
except OSError:
track_file(file, base_dir)
untracked_files.append(file)
if __name__ == "__main__":
main()
# Files that are already tracked
tracked_files = [
pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
if f.value['host'] == hostname
]
untracked_files = set(files) - set(tracked_files)
for file in untracked_files:
track_file(file, base_dir, hostname)

3
uncloud/hack/hackcloud/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
*.iso
radvdpid
foo

9
uncloud/hack/hackcloud/ifup.sh Executable file
View File

@ -0,0 +1,9 @@
#!/bin/sh
echo $@ >> foo
dev=$1; shift
# bridge is setup from outside
ip link set dev "$dev" master ${bridge}
ip link set dev "$dev" up

24
uncloud/hack/hackcloud/net.sh Executable file
View File

@ -0,0 +1,24 @@
#!/bin/sh
netid=100
dev=wlp2s0
dev=wlp0s20f3
dev=wlan0
ip=2a0a:e5c1:111:888::42/64
vxlandev=vxlan${netid}
bridgedev=br${netid}
ip -6 link add ${vxlandev} type vxlan \
id ${netid} \
dstport 4789 \
group ff05::${netid} \
dev ${dev} \
ttl 5
ip link set ${vxlandev} up
ip link add ${bridgedev} type bridge
ip link set ${bridgedev} up
ip addr add ${ip} dev ${bridgedev}

13
uncloud/hack/hackcloud/radvd.conf Normal file
View File

@ -0,0 +1,13 @@
interface br100
{
AdvSendAdvert on;
MinRtrAdvInterval 3;
MaxRtrAdvInterval 5;
AdvDefaultLifetime 3600;
prefix 2a0a:e5c1:111:888::/64 {
};
RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; };
DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ;
};

3
uncloud/hack/hackcloud/radvd.sh Normal file
View File

@ -0,0 +1,3 @@
#!/bin/sh
radvd -C ./radvd.conf -n -p ./radvdpid

48
uncloud/hack/hackcloud/vm.sh Executable file
View File

@ -0,0 +1,48 @@
#!/bin/sh
vmid=$1; shift
qemu=/usr/bin/qemu-system-x86_64
accel=kvm
accel=tcg
memory=1024
cores=2
uuid=732e08c7-84f8-4d43-9571-263db4f80080
export bridge=br100
$qemu -name uc${vmid} \
-machine pc,accel=${accel} \
-m ${memory} \
-smp ${cores} \
-uuid ${uuid} \
-drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \
-netdev tap,id=netmain,script=./ifup.sh \
-device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e
exit 0
-S -object secret,id=masterKey0,format=raw,file=/var/lib/libvirt/qemu/domain-17-one-24992/master-key.aes
-machine pc-i440fx-2.8,accel=kvm,usb=off,dump-guest-core=off
-m 2048
-realtime mlock=off
-smp 1,sockets=1,cores=1,threads=1
-uuid 732e08c7-84f8-4d43-9571-263db4f80080 -no-user-config \
-nodefaults
-chardev socket,id=charmonitor,path=/var/lib/libvirt/qemu/domain-17-one-24992/monitor.sock,server,nowait
-mon chardev=charmonitor,id=monitor,mode=control
-rtc base=utc -no-shutdown
-boot strict=on
-device piix3-usb-uhci,id=usb,bus=pci.0,addr=0x1.0x2
-drive file=rbd:ssd/one-292-24992-0:id=libvirt:auth_supported=cephx\;none:mon_host=ceph1\:6789\;ceph2\:6789\;ceph3\:6789,format=raw,if=none,id=drive-virtio-disk0,cache=none
-device virtio-blk-pci,scsi=off,bus=pci.0,addr=0x4,drive=drive-virtio-disk0,id=virtio-disk0,bootindex=1
-drive file=/var/lib/one//datastores/104/24992/disk.1,format=raw,if=none,id=drive-ide0-0-0,readonly=on
-device ide-cd,bus=ide.0,unit=0,drive=drive-ide0-0-0,id=ide0-0-0
-netdev tap,fd=36,id=hostnet0,vhost=on,vhostfd=38
-device virtio-net-pci,netdev=hostnet0,id=net0,mac=02:00:f0:a9:c4:4e,bus=pci.0,addr=0x3
-vnc [::]:4414 -device cirrus-vga,id=video0,bus=pci.0,addr=0x2 -device virtio-balloon-pci,id=balloon0,bus=pci.0,addr=0x5 -msg timestamp=on

103
uncloud/host/main.py
View File

@ -5,8 +5,8 @@ import time
from uuid import uuid4
from uncloud.common.request import RequestEntry, RequestType
from uncloud.shared import shared
from uncloud.settings import settings
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.common.vm import VMStatus
from uncloud.vmm import VMM
from os.path import join as join_path
@ -33,10 +33,10 @@ def maintenance(host):
vmm = VMM()
running_vms = vmm.discover()
for vm_uuid in running_vms:
if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running":
if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
logger.debug('VM {} is running on {}'.format(vm_uuid, host))
vm = shared.vm_pool.get(
join_path(settings["etcd"]["vm_prefix"], vm_uuid)
join_path(settings['etcd']['vm_prefix'], vm_uuid)
)
vm.status = VMStatus.running
vm.vnc_socket = vmm.get_vnc(vm_uuid)
@ -44,20 +44,21 @@ def maintenance(host):
shared.vm_pool.put(vm)
def main(hostname, debug=False):
def main(arguments):
hostname = arguments['hostname']
host_pool = shared.host_pool
host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
# Does not yet exist, create it
if not host:
host_key = join_path(
settings["etcd"]["host_prefix"], uuid4().hex
settings['etcd']['host_prefix'], uuid4().hex
)
host_entry = {
"specs": "",
"hostname": hostname,
"status": "DEAD",
"last_heartbeat": "",
'specs': '',
'hostname': hostname,
'status': 'DEAD',
'last_heartbeat': '',
}
shared.etcd_client.put(
host_key, host_entry, value_in_json=True
@ -70,54 +71,54 @@ def main(hostname, debug=False):
heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,))
heartbeat_updating_process.start()
except Exception as e:
raise Exception("uncloud-host heartbeat updating mechanism is not working") from e
raise Exception('uncloud-host heartbeat updating mechanism is not working') from e
for events_iterator in [
shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True),
shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True)
]:
for request_event in events_iterator:
request_event = RequestEntry(request_event)
# The below while True is neccessary for gracefully handling leadership transfer and temporary
# unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
# iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
# which ends the loop immediately. So, having it inside infinite loop we try again and again to
# get prefix until either success or deamon death comes.
while True:
for events_iterator in [
shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False)
]:
for request_event in events_iterator:
request_event = RequestEntry(request_event)
if request_event.type == "TIMEOUT":
maintenance(host.key)
elif request_event.hostname == host.key:
logger.debug("VM Request: %s on Host %s", request_event, host.hostname)
shared.request_pool.client.client.delete(request_event.key)
vm_entry = shared.etcd_client.get(
join_path(settings["etcd"]["vm_prefix"], request_event.uuid)
)
logger.debug("VM hostname: {}".format(vm_entry.value))
vm = virtualmachine.VM(vm_entry)
if request_event.type == RequestType.StartVM:
vm.start()
if request_event.hostname == host.key:
logger.debug('VM Request: %s on Host %s', request_event, host.hostname)
elif request_event.type == RequestType.StopVM:
vm.stop()
shared.request_pool.client.client.delete(request_event.key)
vm_entry = shared.etcd_client.get(
join_path(settings['etcd']['vm_prefix'], request_event.uuid)
)
elif request_event.type == RequestType.DeleteVM:
vm.delete()
logger.debug('VM hostname: {}'.format(vm_entry.value))
elif request_event.type == RequestType.InitVMMigration:
vm.start(destination_host_key=host.key)
vm = virtualmachine.VM(vm_entry)
if request_event.type == RequestType.StartVM:
vm.start()
elif request_event.type == RequestType.TransferVM:
destination_host = host_pool.get(request_event.destination_host_key)
if destination_host:
vm.migrate(
destination_host=destination_host.hostname,
destination_sock_path=request_event.destination_sock_path,
)
else:
logger.error("Host %s not found!", request_event.destination_host_key)
elif request_event.type == RequestType.StopVM:
vm.stop()
elif request_event.type == RequestType.DeleteVM:
vm.delete()
if __name__ == "__main__":
argparser = argparse.ArgumentParser()
argparser.add_argument(
"hostname", help="Name of this host. e.g uncloud1.ungleich.ch"
)
args = argparser.parse_args()
mp.set_start_method("spawn")
main(args.hostname)
elif request_event.type == RequestType.InitVMMigration:
vm.start(destination_host_key=host.key)
elif request_event.type == RequestType.TransferVM:
destination_host = host_pool.get(request_event.destination_host_key)
if destination_host:
vm.migrate(
destination_host=destination_host.hostname,
destination_sock_path=request_event.destination_sock_path,
)
else:
logger.error('Host %s not found!', request_event.destination_host_key)

11
uncloud/host/virtualmachine.py
View File

@ -16,8 +16,8 @@ from uncloud.common.vm import VMStatus, declare_stopped
from uncloud.common.network import create_dev, delete_network_interface
from uncloud.common.schemas import VMSchema, NetworkSchema
from uncloud.host import logger
from uncloud.shared import shared
from uncloud.settings import settings
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.vmm import VMM
from marshmallow import ValidationError
@ -42,7 +42,7 @@ class VM:
def get_qemu_args(self):
command = (
"-drive file={file},format=raw,if=virtio,cache=none"
"-drive file={file},format=raw,if=virtio"
" -device virtio-rng-pci"
" -m {memory} -smp cores={cores},threads={threads}"
" -name {owner}_{name}"
@ -153,7 +153,10 @@ class VM:
)
)
return command.split(" ")
if command:
command = command.split(' ')
return command
def delete_network_dev(self):
try:

6
uncloud/imagescanner/main.py
View File

@ -4,8 +4,8 @@ import argparse
import subprocess as sp
from os.path import join as join_path
from uncloud.settings import settings
from uncloud.shared import shared
from uncloud.common.settings import settings
from uncloud.common.shared import shared
from uncloud.imagescanner import logger
@ -30,7 +30,7 @@ def qemu_img_type(path):
return qemu_img_info["format"]
def main(debug=False):
def main(arguments):
# We want to get images entries that requests images to be created
images = shared.etcd_client.get_prefix(
settings["etcd"]["image_prefix"], value_in_json=True

39
uncloud/metadata/main.py
View File

@ -5,8 +5,8 @@ from flask import Flask, request
from flask_restful import Resource, Api
from werkzeug.exceptions import HTTPException
from uncloud.settings import settings
from uncloud.shared import shared
from uncloud.common.settings import settings
from uncloud.common.shared import shared
app = Flask(__name__)
api = Api(app)
@ -84,40 +84,11 @@ class Root(Resource):
data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys
return data.value["metadata"], 200
@staticmethod
def post():
return {"message": "Previous Implementation is deprecated."}
# data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True)
# print(data)
# if data:
# for k in request.json:
# if k not in data.value:
# data.value[k] = request.json[k]
# if k.endswith("-list"):
# data.value[k] = [request.json[k]]
# else:
# if k.endswith("-list"):
# data.value[k].append(request.json[k])
# else:
# data.value[k] = request.json[k]
# etcd_client.put("/v1/metadata/{}".format(request.remote_addr),
# data.value, value_in_json=True)
# else:
# data = {}
# for k in request.json:
# data[k] = request.json[k]
# if k.endswith("-list"):
# data[k] = [request.json[k]]
# etcd_client.put("/v1/metadata/{}".format(request.remote_addr),
# data, value_in_json=True)
api.add_resource(Root, "/")
def main(port=None, debug=False):
def main(arguments):
port = arguments['port']
debug = arguments['debug']
app.run(debug=debug, host="::", port=port)
if __name__ == "__main__":
main()

4
uncloud/scheduler/helper.py
View File

@ -6,8 +6,8 @@ import bitmath
from uncloud.common.host import HostStatus
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.vm import VMStatus
from uncloud.shared import shared
from uncloud.settings import settings
from uncloud.common.shared import shared
from uncloud.common.settings import settings
def accumulated_specs(vms_specs):

80
uncloud/scheduler/main.py
View File

@ -6,59 +6,47 @@
import argparse
from uncloud.common.settings import settings
from uncloud.common.request import RequestEntry, RequestType
from uncloud.shared import shared
from uncloud.settings import settings
from .helper import (dead_host_mitigation, dead_host_detection, assign_host, NoSuitableHostFound)
from . import logger
from uncloud.common.shared import shared
from uncloud.scheduler import logger
from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection,
assign_host, NoSuitableHostFound)
arg_parser = argparse.ArgumentParser('scheduler', add_help=False)
def main(debug=False):
for request_iterator in [
shared.etcd_client.get_prefix(
settings["etcd"]["request_prefix"], value_in_json=True
),
shared.etcd_client.watch_prefix(
settings["etcd"]["request_prefix"],
timeout=5,
value_in_json=True,
),
]:
for request_event in request_iterator:
request_entry = RequestEntry(request_event)
# Never Run time critical mechanism inside timeout
# mechanism because timeout mechanism only comes
# when no other event is happening. It means under
# heavy load there would not be a timeout event.
if request_entry.type == "TIMEOUT":
def main(arguments):
# The below while True is neccessary for gracefully handling leadership transfer and temporary
# unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return
# iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons)
# which ends the loop immediately. So, having it inside infinite loop we try again and again to
# get prefix until either success or deamon death comes.
while True:
for request_iterator in [
shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
]:
for request_event in request_iterator:
dead_host_mitigation(dead_host_detection())
request_entry = RequestEntry(request_event)
# Detect hosts that are dead and set their status
# to "DEAD", and their VMs' status to "KILLED"
dead_hosts = dead_host_detection()
if dead_hosts:
logger.debug("Dead hosts: %s", dead_hosts)
dead_host_mitigation(dead_hosts)
if request_entry.type == RequestType.ScheduleVM:
logger.debug('%s, %s', request_entry.key, request_entry.value)
elif request_entry.type == RequestType.ScheduleVM:
logger.debug("%s, %s", request_entry.key, request_entry.value)
vm_entry = shared.vm_pool.get(request_entry.uuid)
if vm_entry is None:
logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid))
continue
vm_entry = shared.vm_pool.get(request_entry.uuid)
if vm_entry is None:
logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid))
continue
shared.etcd_client.client.delete(request_entry.key) # consume Request
shared.etcd_client.client.delete(request_entry.key) # consume Request
try:
assign_host(vm_entry)
except NoSuitableHostFound:
vm_entry.add_log('Can\'t schedule VM. No Resource Left.')
shared.vm_pool.put(vm_entry)
try:
assign_host(vm_entry)
except NoSuitableHostFound:
vm_entry.add_log("Can't schedule VM. No Resource Left.")
shared.vm_pool.put(vm_entry)
logger.info("No Resource Left. Emailing admin....")
if __name__ == "__main__":
main()
logger.info('No Resource Left. Emailing admin....')

16
uncloud/vmm/__init__.py
View File

@ -190,18 +190,10 @@ class VMM:
err.stderr.decode("utf-8"),
)
else:
with suppress(sp.CalledProcessError):
sp.check_output(
[
"sudo",
"-p",
"Enter password to correct permission for uncloud-vmm's directory",
"chmod",
"-R",
"o=rwx,g=rwx",
self.vmm_backend,
]
)
sp.check_output(
["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory",
"chmod", "-R", "o=rwx,g=rwx", self.vmm_backend]
)
# TODO: Find some good way to check whether the virtual machine is up and
# running without relying on non-guarenteed ways.