fnux
/
uncloud
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge branch 'conf-dir' into 'master' 

Adding conf-dir and etcd-* arguments to command-line-interface

See merge request uncloud/uncloud!2
This commit is contained in:
Ahmed Bilal 2020-01-13 05:57:42 +01:00
parent e6d22a73c5 c3b42aabc6
commit ab65349047
19 changed files with 176 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
scripts/uncloud
View File

@ -3,14 +3,13 @@ import logging
import sys import sys
import importlib import importlib
import argparse import argparse
import os
from etcd3.exceptions import ConnectionFailedError
from uncloud.common import settings
from uncloud import UncloudException from uncloud import UncloudException
from uncloud.common.cli import resolve_otp_credentials
# the components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
ALL_COMPONENTS = ETCD_COMPONENTS.copy()
ALL_COMPONENTS.append('cli')
def exception_hook(exc_type, exc_value, exc_traceback): def exception_hook(exc_type, exc_value, exc_traceback):
@ -22,6 +21,13 @@ def exception_hook(exc_type, exc_value, exc_traceback):
sys.excepthook = exception_hook sys.excepthook = exception_hook
# the components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
ALL_COMPONENTS = ETCD_COMPONENTS.copy()
ALL_COMPONENTS.append('cli')
if __name__ == '__main__': if __name__ == '__main__':
# Setting up root logger # Setting up root logger
logger = logging.getLogger() logger = logging.getLogger()
@ -31,15 +37,13 @@ if __name__ == '__main__':
subparsers = arg_parser.add_subparsers(dest='command') subparsers = arg_parser.add_subparsers(dest='command')
parent_parser = argparse.ArgumentParser(add_help=False) parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument('--debug', '-d', parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
action='store_true',
default=False,
help='More verbose logging') help='More verbose logging')
parent_parser.add_argument('--conf-dir', '-c', parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
help='Configuration directory') default=os.path.expanduser('~/uncloud'))
etcd_parser = argparse.ArgumentParser(add_help=False) etcd_parser = argparse.ArgumentParser(add_help=False)
etcd_parser.add_argument('--etcd-host') etcd_parser.add_argument('--etcd-host', dest='etcd_url')
etcd_parser.add_argument('--etcd-port') etcd_parser.add_argument('--etcd-port')
etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
@ -54,25 +58,36 @@ if __name__ == '__main__':
else: else:
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
args = arg_parser.parse_args() arguments = vars(arg_parser.parse_args())
if not args.command: etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
etcd_arguments = {
'etcd': {
key.replace('etcd_', ''): arguments[key]
for key in etcd_arguments
}
}
if not arguments['command']:
arg_parser.print_help() arg_parser.print_help()
else: else:
arguments = vars(args) # Initializing Settings and resolving otp_credentials
# It is neccessary to resolve_otp_credentials after argument parsing is done because
# previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
# providing the default values for --name, --realm and --seed arguments from the values
# we read from file. But, now we are asking user about where the config file lives. So,
# to providing default value is not possible before parsing arguments. So, we are doing
# it after..
settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
resolve_otp_credentials(arguments)
name = arguments.pop('command') name = arguments.pop('command')
mod = importlib.import_module('uncloud.{}.main'.format(name)) mod = importlib.import_module('uncloud.{}.main'.format(name))
main = getattr(mod, 'main') main = getattr(mod, 'main')
# If the component requires etcd3, we import it and catch the
# etcd3.exceptions.ConnectionFailedError
if name in ETCD_COMPONENTS:
import etcd3
try: try:
main(arguments) main(arguments)
except UncloudException as err: except UncloudException as err:
logger.error(err) logger.error(err)
sys.exit(1) except ConnectionFailedError:
logger.error('Cannot connect to etcd')
except Exception as err: except Exception as err:
logger.exception(err) logger.exception(err)
sys.exit(1)

7
uncloud/api/common_fields.py
View File

@ -1,7 +1,6 @@
import os import os
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
class Optional: class Optional:
@ -54,9 +53,7 @@ class VmUUIDField(Field):
def vm_uuid_validation(self): def vm_uuid_validation(self):
r = shared.etcd_client.get( r = shared.etcd_client.get(
os.path.join(settings["etcd"]["vm_prefix"], self.uuid) os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
) )
if not r: if not r:
self.add_error( self.add_error("VM with uuid {} does not exists".format(self.uuid))
"VM with uuid {} does not exists".format(self.uuid)
)

3
uncloud/api/create_image_store.py
View File

@ -4,7 +4,6 @@ import os
from uuid import uuid4 from uuid import uuid4
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
data = { data = {
'is_public': True, 'is_public': True,
@ -15,6 +14,6 @@ data = {
} }
shared.etcd_client.put( shared.etcd_client.put(
os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
json.dumps(data), json.dumps(data),
) )

18
uncloud/api/helper.py
View File

@ -7,7 +7,6 @@ import requests
from pyotp import TOTP from pyotp import TOTP
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -15,9 +14,9 @@ logger = logging.getLogger(__name__)
def check_otp(name, realm, token): def check_otp(name, realm, token):
try: try:
data = { data = {
"auth_name": settings["otp"]["auth_name"], "auth_name": shared.settings["otp"]["auth_name"],
"auth_token": TOTP(settings["otp"]["auth_seed"]).now(), "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
"auth_realm": settings["otp"]["auth_realm"], "auth_realm": shared.settings["otp"]["auth_realm"],
"name": name, "name": name,
"realm": realm, "realm": realm,
"token": token, "token": token,
@ -25,13 +24,13 @@ def check_otp(name, realm, token):
except binascii.Error as err: except binascii.Error as err:
logger.error( logger.error(
"Cannot compute OTP for seed: {}".format( "Cannot compute OTP for seed: {}".format(
settings["otp"]["auth_seed"] shared.settings["otp"]["auth_seed"]
) )
) )
return 400 return 400
response = requests.post( response = requests.post(
settings["otp"]["verification_controller_url"], json=data shared.settings["otp"]["verification_controller_url"], json=data
) )
return response.status_code return response.status_code
@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client):
) )
images = etcd_client.get_prefix( images = etcd_client.get_prefix(
settings["etcd"]["image_prefix"], value_in_json=True shared.settings["etcd"]["image_prefix"], value_in_json=True
) )
# Try to find image with name == image_name and store_name == store_name # Try to find image with name == image_name and store_name == store_name
@ -111,9 +110,7 @@ def random_bytes(num=6):
return [random.randrange(256) for _ in range(num)] return [random.randrange(256) for _ in range(num)]
def generate_mac( def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
):
mac = random_bytes() mac = random_bytes()
if oui: if oui:
if type(oui) == str: if type(oui) == str:
@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix):
lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
prefix = ipaddress.IPv6Address(prefix) prefix = ipaddress.IPv6Address(prefix)
return str(prefix + int(lower_part)) return str(prefix + int(lower_part))

61
uncloud/api/main.py
View File

@ -15,9 +15,8 @@ from uncloud.common.shared import shared
from uncloud.common import counters from uncloud.common import counters
from uncloud.common.vm import VMStatus from uncloud.common.vm import VMStatus
from uncloud.common.request import RequestEntry, RequestType from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.settings import settings from uncloud.api import schemas
from . import schemas from uncloud.api.helper import generate_mac, mac2ipv6
from .helper import generate_mac, mac2ipv6
from uncloud import UncloudException from uncloud import UncloudException
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -50,7 +49,7 @@ class CreateVM(Resource):
validator = schemas.CreateVMSchema(data) validator = schemas.CreateVMSchema(data)
if validator.is_valid(): if validator.is_valid():
vm_uuid = uuid4().hex vm_uuid = uuid4().hex
vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
specs = { specs = {
'cpu': validator.specs['cpu'], 'cpu': validator.specs['cpu'],
'ram': validator.specs['ram'], 'ram': validator.specs['ram'],
@ -60,7 +59,7 @@ class CreateVM(Resource):
macs = [generate_mac() for _ in range(len(data['network']))] macs = [generate_mac() for _ in range(len(data['network']))]
tap_ids = [ tap_ids = [
counters.increment_etcd_counter( counters.increment_etcd_counter(
shared.etcd_client, settings['etcd']['tap_counter'] shared.etcd_client, shared.settings['etcd']['tap_counter']
) )
for _ in range(len(data['network'])) for _ in range(len(data['network']))
] ]
@ -84,7 +83,7 @@ class CreateVM(Resource):
r = RequestEntry.from_scratch( r = RequestEntry.from_scratch(
type=RequestType.ScheduleVM, type=RequestType.ScheduleVM,
uuid=vm_uuid, uuid=vm_uuid,
request_prefix=settings['etcd']['request_prefix'], request_prefix=shared.settings['etcd']['request_prefix'],
) )
shared.request_pool.put(r) shared.request_pool.put(r)
@ -99,7 +98,7 @@ class VmStatus(Resource):
validator = schemas.VMStatusSchema(data) validator = schemas.VMStatusSchema(data)
if validator.is_valid(): if validator.is_valid():
vm = shared.vm_pool.get( vm = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], data['uuid']) join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
) )
vm_value = vm.value.copy() vm_value = vm.value.copy()
vm_value['ip'] = [] vm_value['ip'] = []
@ -107,7 +106,7 @@ class VmStatus(Resource):
network_name, mac, tap = network_mac_and_tap network_name, mac, tap = network_mac_and_tap
network = shared.etcd_client.get( network = shared.etcd_client.get(
join_path( join_path(
settings['etcd']['network_prefix'], shared.settings['etcd']['network_prefix'],
data['name'], data['name'],
network_name, network_name,
), ),
@ -130,7 +129,7 @@ class CreateImage(Resource):
validator = schemas.CreateImageSchema(data) validator = schemas.CreateImageSchema(data)
if validator.is_valid(): if validator.is_valid():
file_entry = shared.etcd_client.get( file_entry = shared.etcd_client.get(
join_path(settings['etcd']['file_prefix'], data['uuid']) join_path(shared.settings['etcd']['file_prefix'], data['uuid'])
) )
file_entry_value = json.loads(file_entry.value) file_entry_value = json.loads(file_entry.value)
@ -144,7 +143,7 @@ class CreateImage(Resource):
} }
shared.etcd_client.put( shared.etcd_client.put(
join_path( join_path(
settings['etcd']['image_prefix'], data['uuid'] shared.settings['etcd']['image_prefix'], data['uuid']
), ),
json.dumps(image_entry_json), json.dumps(image_entry_json),
) )
@ -157,7 +156,7 @@ class ListPublicImages(Resource):
@staticmethod @staticmethod
def get(): def get():
images = shared.etcd_client.get_prefix( images = shared.etcd_client.get_prefix(
settings['etcd']['image_prefix'], value_in_json=True shared.settings['etcd']['image_prefix'], value_in_json=True
) )
r = {'images': []} r = {'images': []}
for image in images: for image in images:
@ -178,7 +177,7 @@ class VMAction(Resource):
if validator.is_valid(): if validator.is_valid():
vm_entry = shared.vm_pool.get( vm_entry = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], data['uuid']) join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
) )
action = data['action'] action = data['action']
@ -208,7 +207,7 @@ class VMAction(Resource):
type='{}VM'.format(action.title()), type='{}VM'.format(action.title()),
uuid=data['uuid'], uuid=data['uuid'],
hostname=vm_entry.hostname, hostname=vm_entry.hostname,
request_prefix=settings['etcd']['request_prefix'], request_prefix=shared.settings['etcd']['request_prefix'],
) )
shared.request_pool.put(r) shared.request_pool.put(r)
return ( return (
@ -231,10 +230,10 @@ class VMMigration(Resource):
type=RequestType.InitVMMigration, type=RequestType.InitVMMigration,
uuid=vm.uuid, uuid=vm.uuid,
hostname=join_path( hostname=join_path(
settings['etcd']['host_prefix'], shared.settings['etcd']['host_prefix'],
validator.destination.value, validator.destination.value,
), ),
request_prefix=settings['etcd']['request_prefix'], request_prefix=shared.settings['etcd']['request_prefix'],
) )
shared.request_pool.put(r) shared.request_pool.put(r)
@ -254,7 +253,7 @@ class ListUserVM(Resource):
if validator.is_valid(): if validator.is_valid():
vms = shared.etcd_client.get_prefix( vms = shared.etcd_client.get_prefix(
settings['etcd']['vm_prefix'], value_in_json=True shared.settings['etcd']['vm_prefix'], value_in_json=True
) )
return_vms = [] return_vms = []
user_vms = filter( user_vms = filter(
@ -287,7 +286,7 @@ class ListUserFiles(Resource):
if validator.is_valid(): if validator.is_valid():
files = shared.etcd_client.get_prefix( files = shared.etcd_client.get_prefix(
settings['etcd']['file_prefix'], value_in_json=True shared.settings['etcd']['file_prefix'], value_in_json=True
) )
return_files = [] return_files = []
user_files = [f for f in files if f.value['owner'] == data['name']] user_files = [f for f in files if f.value['owner'] == data['name']]
@ -312,7 +311,7 @@ class CreateHost(Resource):
validator = schemas.CreateHostSchema(data) validator = schemas.CreateHostSchema(data)
if validator.is_valid(): if validator.is_valid():
host_key = join_path( host_key = join_path(
settings['etcd']['host_prefix'], uuid4().hex shared.settings['etcd']['host_prefix'], uuid4().hex
) )
host_entry = { host_entry = {
'specs': data['specs'], 'specs': data['specs'],
@ -354,7 +353,7 @@ class GetSSHKeys(Resource):
# {user_prefix}/{realm}/{name}/key/ # {user_prefix}/{realm}/{name}/key/
etcd_key = join_path( etcd_key = join_path(
settings['etcd']['user_prefix'], shared.settings['etcd']['user_prefix'],
data['realm'], data['realm'],
data['name'], data['name'],
'key', 'key',
@ -372,7 +371,7 @@ class GetSSHKeys(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name} # {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path( etcd_key = join_path(
settings['etcd']['user_prefix'], shared.settings['etcd']['user_prefix'],
data['realm'], data['realm'],
data['name'], data['name'],
'key', 'key',
@ -405,7 +404,7 @@ class AddSSHKey(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name} # {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path( etcd_key = join_path(
settings['etcd']['user_prefix'], shared.settings['etcd']['user_prefix'],
data['realm'], data['realm'],
data['name'], data['name'],
'key', 'key',
@ -439,7 +438,7 @@ class RemoveSSHKey(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name} # {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path( etcd_key = join_path(
settings['etcd']['user_prefix'], shared.settings['etcd']['user_prefix'],
data['realm'], data['realm'],
data['name'], data['name'],
'key', 'key',
@ -471,23 +470,23 @@ class CreateNetwork(Resource):
network_entry = { network_entry = {
'id': counters.increment_etcd_counter( 'id': counters.increment_etcd_counter(
shared.etcd_client, settings['etcd']['vxlan_counter'] shared.etcd_client, shared.settings['etcd']['vxlan_counter']
), ),
'type': data['type'], 'type': data['type'],
} }
if validator.user.value: if validator.user.value:
try: try:
nb = pynetbox.api( nb = pynetbox.api(
url=settings['netbox']['url'], url=shared.settings['netbox']['url'],
token=settings['netbox']['token'], token=shared.settings['netbox']['token'],
) )
nb_prefix = nb.ipam.prefixes.get( nb_prefix = nb.ipam.prefixes.get(
prefix=settings['network']['prefix'] prefix=shared.settings['network']['prefix']
) )
prefix = nb_prefix.available_prefixes.create( prefix = nb_prefix.available_prefixes.create(
data={ data={
'prefix_length': int( 'prefix_length': int(
settings['network']['prefix_length'] shared.settings['network']['prefix_length']
), ),
'description': '{}\'s network "{}"'.format( 'description': '{}\'s network "{}"'.format(
data['name'], data['network_name'] data['name'], data['network_name']
@ -506,7 +505,7 @@ class CreateNetwork(Resource):
network_entry['ipv6'] = 'fd00::/64' network_entry['ipv6'] = 'fd00::/64'
network_key = join_path( network_key = join_path(
settings['etcd']['network_prefix'], shared.settings['etcd']['network_prefix'],
data['name'], data['name'],
data['network_name'], data['network_name'],
) )
@ -526,7 +525,7 @@ class ListUserNetwork(Resource):
if validator.is_valid(): if validator.is_valid():
prefix = join_path( prefix = join_path(
settings['etcd']['network_prefix'], data['name'] shared.settings['etcd']['network_prefix'], data['name']
) )
networks = shared.etcd_client.get_prefix( networks = shared.etcd_client.get_prefix(
prefix, value_in_json=True prefix, value_in_json=True
@ -570,7 +569,7 @@ def main(arguments):
try: try:
image_stores = list( image_stores = list(
shared.etcd_client.get_prefix( shared.etcd_client.get_prefix(
settings['etcd']['image_store_prefix'], value_in_json=True shared.settings['etcd']['image_store_prefix'], value_in_json=True
) )
) )
except KeyError: except KeyError:
@ -590,7 +589,7 @@ def main(arguments):
# shared.etcd_client.put( # shared.etcd_client.put(
# join_path( # join_path(
# settings['etcd']['image_store_prefix'], uuid4().hex # shared.settings['etcd']['image_store_prefix'], uuid4().hex
# ), # ),
# json.dumps(data), # json.dumps(data),
# ) # )

13
uncloud/api/schemas.py
View File

@ -22,7 +22,6 @@ import bitmath
from uncloud.common.host import HostStatus from uncloud.common.host import HostStatus
from uncloud.common.vm import VMStatus from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
from . import helper, logger from . import helper, logger
from .common_fields import Field, VmUUIDField from .common_fields import Field, VmUUIDField
from .helper import check_otp, resolve_vm_name from .helper import check_otp, resolve_vm_name
@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema):
def file_uuid_validation(self): def file_uuid_validation(self):
file_entry = shared.etcd_client.get( file_entry = shared.etcd_client.get(
os.path.join( os.path.join(
settings["etcd"]["file_prefix"], self.uuid.value shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
) )
) )
if file_entry is None: if file_entry is None:
@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema):
def image_store_name_validation(self): def image_store_name_validation(self):
image_stores = list( image_stores = list(
shared.etcd_client.get_prefix( shared.etcd_client.get_prefix(
settings["etcd"]["image_store_prefix"] shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
) )
) )
@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema):
for net in _network: for net in _network:
network = shared.etcd_client.get( network = shared.etcd_client.get(
os.path.join( os.path.join(
settings["etcd"]["network_prefix"], shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
self.name.value, self.name.value,
net, net,
), ),
@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema):
self.add_error("Can't migrate non-running VM") self.add_error("Can't migrate non-running VM")
if vm.hostname == os.path.join( if vm.hostname == os.path.join(
settings["etcd"]["host_prefix"], self.destination.value shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
): ):
self.add_error( self.add_error(
"Destination host couldn't be same as Source Host" "Destination host couldn't be same as Source Host"
@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema):
super().__init__(data, fields=fields) super().__init__(data, fields=fields)
def network_name_validation(self): def network_name_validation(self):
print(self.name.value, self.network_name.value) key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
print(key)
network = shared.etcd_client.get(key, value_in_json=True) network = shared.etcd_client.get(key, value_in_json=True)
if network: if network:
self.add_error( self.add_error(

29
uncloud/cli/helper.py
View File

@ -5,23 +5,14 @@ import binascii
from pyotp import TOTP from pyotp import TOTP
from os.path import join as join_path from os.path import join as join_path
from uncloud.common.settings import settings from uncloud.common.shared import shared
def get_otp_parser(): def get_otp_parser():
otp_parser = argparse.ArgumentParser('otp') otp_parser = argparse.ArgumentParser('otp')
try: otp_parser.add_argument('--name')
name = settings['client']['name'] otp_parser.add_argument('--realm')
realm = settings['client']['realm'] otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
seed = settings['client']['seed']
except Exception:
otp_parser.add_argument('--name', required=True)
otp_parser.add_argument('--realm', required=True)
otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED')
else:
otp_parser.add_argument('--name', default=name)
otp_parser.add_argument('--realm', default=realm)
otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED')
return otp_parser return otp_parser
@ -34,11 +25,15 @@ def load_dump_pretty(content):
def make_request(*args, data=None, request_method=requests.post): def make_request(*args, data=None, request_method=requests.post):
r = request_method(join_path(settings['client']['api_server'], *args), json=data)
try: try:
print(load_dump_pretty(r.content)) r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
except Exception: except requests.exceptions.RequestException:
print('Error occurred while getting output from api server.') print('Error occurred while connecting to API server.')
else:
try:
print(load_dump_pretty(r.content))
except Exception:
print('Error occurred while getting output from api server.')
def get_token(seed): def get_token(seed):

26
uncloud/common/cli.py Normal file
View File

@ -0,0 +1,26 @@
from uncloud.common.shared import shared
from pyotp import TOTP
def get_token(seed):
if seed is not None:
try:
token = TOTP(seed).now()
except Exception:
raise Exception('Invalid seed')
else:
return token
def resolve_otp_credentials(kwargs):
d = {
'name': shared.settings['client']['name'],
'realm': shared.settings['client']['realm'],
'token': get_token(shared.settings['client']['seed'])
}
for k, v in d.items():
if k in kwargs and kwargs[k] is None:
kwargs.update({k: v})
return d

19
uncloud/common/settings.py
View File

@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper
from os.path import join as join_path from os.path import join as join_path
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
settings = None
class CustomConfigParser(configparser.RawConfigParser): class CustomConfigParser(configparser.RawConfigParser):
@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser):
class Settings(object): class Settings(object):
def __init__(self): def __init__(self, conf_dir, seed_value=None):
conf_name = 'uncloud.conf' conf_name = 'uncloud.conf'
conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
self.config_file = join_path(conf_dir, conf_name) self.config_file = join_path(conf_dir, conf_name)
# this is used to cache config from etcd for 1 minutes. Without this we # this is used to cache config from etcd for 1 minutes. Without this we
@ -38,15 +38,19 @@ class Settings(object):
self.config_parser.add_section('etcd') self.config_parser.add_section('etcd')
self.config_parser.set('etcd', 'base_prefix', '/') self.config_parser.set('etcd', 'base_prefix', '/')
try: if os.access(self.config_file, os.R_OK):
self.config_parser.read(self.config_file) self.config_parser.read(self.config_file)
except Exception as err: else:
logger.error('%s', err) raise FileNotFoundError('Config file %s not found!', self.config_file)
self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
self.read_internal_values() self.read_internal_values()
if seed_value is None:
seed_value = dict()
self.config_parser.read_dict(seed_value)
def get_etcd_client(self): def get_etcd_client(self):
args = tuple() args = tuple()
try: try:
@ -128,4 +132,5 @@ class Settings(object):
return self.config_parser[key] return self.config_parser[key]
settings = Settings() def get_settings():
return settings

22
uncloud/common/shared.py
View File

@ -1,34 +1,34 @@
from uncloud.common.settings import settings from uncloud.common.settings import get_settings
from uncloud.common.vm import VmPool from uncloud.common.vm import VmPool
from uncloud.common.host import HostPool from uncloud.common.host import HostPool
from uncloud.common.request import RequestPool from uncloud.common.request import RequestPool
from uncloud.common.storage_handlers import get_storage_handler import uncloud.common.storage_handlers as storage_handlers
class Shared: class Shared:
@property
def settings(self):
return get_settings()
@property @property
def etcd_client(self): def etcd_client(self):
return settings.get_etcd_client() return self.settings.get_etcd_client()
@property @property
def host_pool(self): def host_pool(self):
return HostPool( return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
self.etcd_client, settings["etcd"]["host_prefix"]
)
@property @property
def vm_pool(self): def vm_pool(self):
return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
@property @property
def request_pool(self): def request_pool(self):
return RequestPool( return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
self.etcd_client, settings["etcd"]["request_prefix"]
)
@property @property
def storage_handler(self): def storage_handler(self):
return get_storage_handler() return storage_handlers.get_storage_handler()
shared = Shared() shared = Shared()

15
uncloud/common/storage_handlers.py
View File

@ -6,8 +6,7 @@ import stat
from abc import ABC from abc import ABC
from . import logger from . import logger
from os.path import join as join_path from os.path import join as join_path
import uncloud.common.shared as shared
from uncloud.common.settings import settings as config
class ImageStorageHandler(ABC): class ImageStorageHandler(ABC):
@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
def get_storage_handler(): def get_storage_handler():
__storage_backend = config["storage"]["storage_backend"] __storage_backend = shared.shared.settings["storage"]["storage_backend"]
if __storage_backend == "filesystem": if __storage_backend == "filesystem":
return FileSystemBasedImageStorageHandler( return FileSystemBasedImageStorageHandler(
vm_base=config["storage"]["vm_dir"], vm_base=shared.shared.settings["storage"]["vm_dir"],
image_base=config["storage"]["image_dir"], image_base=shared.shared.settings["storage"]["image_dir"],
) )
elif __storage_backend == "ceph": elif __storage_backend == "ceph":
return CEPHBasedImageStorageHandler( return CEPHBasedImageStorageHandler(
vm_base=config["storage"]["ceph_vm_pool"], vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
image_base=config["storage"]["ceph_image_pool"], image_base=shared.shared.settings["storage"]["ceph_image_pool"],
) )
else: else:
raise Exception("Unknown Image Storage Handler") raise Exception("Unknown Image Storage Handler")

11
uncloud/configure/main.py
View File

@ -1,7 +1,6 @@
import os import os
import argparse import argparse
from uncloud.common.settings import settings
from uncloud.common.shared import shared from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('configure', add_help=False) arg_parser = argparse.ArgumentParser('configure', add_help=False)
@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
def update_config(section, kwargs): def update_config(section, kwargs):
uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
if not uncloud_config: if not uncloud_config:
uncloud_config = {} uncloud_config = {}
else: else:
uncloud_config = uncloud_config.value uncloud_config = uncloud_config.value
uncloud_config[section] = kwargs uncloud_config[section] = kwargs
shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
def main(**kwargs): def main(arguments):
subcommand = kwargs.pop('subcommand') subcommand = arguments['subcommand']
if not subcommand: if not subcommand:
arg_parser.print_help() arg_parser.print_help()
else: else:
update_config(subcommand, kwargs) update_config(subcommand, arguments)

7
uncloud/filescanner/main.py
View File

@ -9,7 +9,6 @@ import bitmath
from uuid import uuid4 from uuid import uuid4
from . import logger from . import logger
from uncloud.common.settings import settings
from uncloud.common.shared import shared from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('filescanner', add_help=False) arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
@ -53,7 +52,7 @@ def track_file(file, base_dir, host):
file_path = file_path.relative_to(owner) file_path = file_path.relative_to(owner)
creation_date = time.ctime(os.stat(file_str).st_ctime) creation_date = time.ctime(os.stat(file_str).st_ctime)
entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
entry_value = { entry_value = {
'filename': str(file_path), 'filename': str(file_path),
'owner': owner, 'owner': owner,
@ -70,7 +69,7 @@ def track_file(file, base_dir, host):
def main(arguments): def main(arguments):
hostname = arguments['hostname'] hostname = arguments['hostname']
base_dir = settings['storage']['file_dir'] base_dir = shared.settings['storage']['file_dir']
# Recursively Get All Files and Folder below BASE_DIR # Recursively Get All Files and Folder below BASE_DIR
files = glob.glob('{}/**'.format(base_dir), recursive=True) files = glob.glob('{}/**'.format(base_dir), recursive=True)
files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
@ -78,7 +77,7 @@ def main(arguments):
# Files that are already tracked # Files that are already tracked
tracked_files = [ tracked_files = [
pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
if f.value['host'] == hostname if f.value['host'] == hostname
] ]
untracked_files = set(files) - set(tracked_files) untracked_files = set(files) - set(tracked_files)

11
uncloud/host/main.py
View File

@ -6,7 +6,6 @@ from uuid import uuid4
from uncloud.common.request import RequestEntry, RequestType from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.common.vm import VMStatus from uncloud.common.vm import VMStatus
from uncloud.vmm import VMM from uncloud.vmm import VMM
from os.path import join as join_path from os.path import join as join_path
@ -36,7 +35,7 @@ def maintenance(host):
if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
logger.debug('VM {} is running on {}'.format(vm_uuid, host)) logger.debug('VM {} is running on {}'.format(vm_uuid, host))
vm = shared.vm_pool.get( vm = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], vm_uuid) join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
) )
vm.status = VMStatus.running vm.status = VMStatus.running
vm.vnc_socket = vmm.get_vnc(vm_uuid) vm.vnc_socket = vmm.get_vnc(vm_uuid)
@ -52,7 +51,7 @@ def main(arguments):
# Does not yet exist, create it # Does not yet exist, create it
if not host: if not host:
host_key = join_path( host_key = join_path(
settings['etcd']['host_prefix'], uuid4().hex shared.settings['etcd']['host_prefix'], uuid4().hex
) )
host_entry = { host_entry = {
'specs': '', 'specs': '',
@ -80,9 +79,9 @@ def main(arguments):
# get prefix until either success or deamon death comes. # get prefix until either success or deamon death comes.
while True: while True:
for events_iterator in [ for events_iterator in [
shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False), raise_exception=False),
shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False) raise_exception=False)
]: ]:
for request_event in events_iterator: for request_event in events_iterator:
@ -95,7 +94,7 @@ def main(arguments):
shared.request_pool.client.client.delete(request_event.key) shared.request_pool.client.client.delete(request_event.key)
vm_entry = shared.etcd_client.get( vm_entry = shared.etcd_client.get(
join_path(settings['etcd']['vm_prefix'], request_event.uuid) join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid)
) )
logger.debug('VM hostname: {}'.format(vm_entry.value)) logger.debug('VM hostname: {}'.format(vm_entry.value))

11
uncloud/host/virtualmachine.py
View File

@ -17,7 +17,6 @@ from uncloud.common.network import create_dev, delete_network_interface
from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.common.schemas import VMSchema, NetworkSchema
from uncloud.host import logger from uncloud.host import logger
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.vmm import VMM from uncloud.vmm import VMM
from marshmallow import ValidationError from marshmallow import ValidationError
@ -91,7 +90,7 @@ class VM:
self.vmm.socket_dir, self.uuid self.vmm.socket_dir, self.uuid
), ),
destination_host_key=destination_host_key, # Where source host transfer VM destination_host_key=destination_host_key, # Where source host transfer VM
request_prefix=settings["etcd"]["request_prefix"], request_prefix=shared.settings["etcd"]["request_prefix"],
) )
shared.request_pool.put(r) shared.request_pool.put(r)
else: else:
@ -119,7 +118,7 @@ class VM:
network_name, mac, tap = network_mac_and_tap network_name, mac, tap = network_mac_and_tap
_key = os.path.join( _key = os.path.join(
settings["etcd"]["network_prefix"], shared.settings["etcd"]["network_prefix"],
self.vm["owner"], self.vm["owner"],
network_name, network_name,
) )
@ -133,13 +132,13 @@ class VM:
if network["type"] == "vxlan": if network["type"] == "vxlan":
tap = create_vxlan_br_tap( tap = create_vxlan_br_tap(
_id=network["id"], _id=network["id"],
_dev=settings["network"]["vxlan_phy_dev"], _dev=shared.settings["network"]["vxlan_phy_dev"],
tap_id=tap, tap_id=tap,
ip=network["ipv6"], ip=network["ipv6"],
) )
all_networks = shared.etcd_client.get_prefix( all_networks = shared.etcd_client.get_prefix(
settings["etcd"]["network_prefix"], shared.settings["etcd"]["network_prefix"],
value_in_json=True, value_in_json=True,
) )
@ -229,7 +228,7 @@ class VM:
def resolve_network(network_name, network_owner): def resolve_network(network_name, network_owner):
network = shared.etcd_client.get( network = shared.etcd_client.get(
join_path( join_path(
settings["etcd"]["network_prefix"], shared.settings["etcd"]["network_prefix"],
network_owner, network_owner,
network_name, network_name,
), ),

7
uncloud/imagescanner/main.py
View File

@ -4,7 +4,6 @@ import argparse
import subprocess as sp import subprocess as sp
from os.path import join as join_path from os.path import join as join_path
from uncloud.common.settings import settings
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.imagescanner import logger from uncloud.imagescanner import logger
@ -33,7 +32,7 @@ def qemu_img_type(path):
def main(arguments): def main(arguments):
# We want to get images entries that requests images to be created # We want to get images entries that requests images to be created
images = shared.etcd_client.get_prefix( images = shared.etcd_client.get_prefix(
settings["etcd"]["image_prefix"], value_in_json=True shared.settings["etcd"]["image_prefix"], value_in_json=True
) )
images_to_be_created = list( images_to_be_created = list(
filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
@ -46,13 +45,13 @@ def main(arguments):
image_filename = image.value["filename"] image_filename = image.value["filename"]
image_store_name = image.value["store_name"] image_store_name = image.value["store_name"]
image_full_path = join_path( image_full_path = join_path(
settings["storage"]["file_dir"], shared.settings["storage"]["file_dir"],
image_owner, image_owner,
image_filename, image_filename,
) )
image_stores = shared.etcd_client.get_prefix( image_stores = shared.etcd_client.get_prefix(
settings["etcd"]["image_store_prefix"], shared.settings["etcd"]["image_store_prefix"],
value_in_json=True, value_in_json=True,
) )
user_image_store = next( user_image_store = next(

3
uncloud/metadata/main.py
View File

@ -5,7 +5,6 @@ from flask import Flask, request
from flask_restful import Resource, Api from flask_restful import Resource, Api
from werkzeug.exceptions import HTTPException from werkzeug.exceptions import HTTPException
from uncloud.common.settings import settings
from uncloud.common.shared import shared from uncloud.common.shared import shared
app = Flask(__name__) app = Flask(__name__)
@ -74,7 +73,7 @@ class Root(Resource):
) )
else: else:
etcd_key = os.path.join( etcd_key = os.path.join(
settings["etcd"]["user_prefix"], shared.settings["etcd"]["user_prefix"],
data.value["owner_realm"], data.value["owner_realm"],
data.value["owner"], data.value["owner"],
"key", "key",

3
uncloud/scheduler/helper.py
View File

@ -7,7 +7,6 @@ from uncloud.common.host import HostStatus
from uncloud.common.request import RequestEntry, RequestType from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.vm import VMStatus from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.common.settings import settings
def accumulated_specs(vms_specs): def accumulated_specs(vms_specs):
@ -130,7 +129,7 @@ def assign_host(vm):
type=RequestType.StartVM, type=RequestType.StartVM,
uuid=vm.uuid, uuid=vm.uuid,
hostname=vm.hostname, hostname=vm.hostname,
request_prefix=settings["etcd"]["request_prefix"], request_prefix=shared.settings["etcd"]["request_prefix"],
) )
shared.request_pool.put(r) shared.request_pool.put(r)

5
uncloud/scheduler/main.py
View File

@ -6,7 +6,6 @@
import argparse import argparse
from uncloud.common.settings import settings
from uncloud.common.request import RequestEntry, RequestType from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.shared import shared from uncloud.common.shared import shared
from uncloud.scheduler import logger from uncloud.scheduler import logger
@ -24,9 +23,9 @@ def main(arguments):
# get prefix until either success or deamon death comes. # get prefix until either success or deamon death comes.
while True: while True:
for request_iterator in [ for request_iterator in [
shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False), raise_exception=False),
shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False), raise_exception=False),
]: ]:
for request_event in request_iterator: for request_event in request_iterator: