ahmedbilal/uncloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Added --conf-dir, --etcd-{host,port,ca_cert,cert_cert,cert_key} parameters to cli and settings is now accessbile through uncloud.shared.shared.settings

Browse source
This commit is contained in:
Ahmed Bilal 2020-01-13 05:57:41 +01:00
parent e6d22a73c5
commit c3b42aabc6
19 changed files with 176 additions and 154 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
scripts/uncloud
View file

@ -3,14 +3,13 @@ import logging
import sys
import importlib
import argparse
import os
from etcd3.exceptions import ConnectionFailedError
from uncloud.common import settings
from uncloud import UncloudException
# the components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
ALL_COMPONENTS = ETCD_COMPONENTS.copy()
ALL_COMPONENTS.append('cli')
from uncloud.common.cli import resolve_otp_credentials
def exception_hook(exc_type, exc_value, exc_traceback):
@ -22,6 +21,13 @@ def exception_hook(exc_type, exc_value, exc_traceback):
sys.excepthook = exception_hook
# the components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
ALL_COMPONENTS = ETCD_COMPONENTS.copy()
ALL_COMPONENTS.append('cli')
if __name__ == '__main__':
# Setting up root logger
logger = logging.getLogger()
@ -31,15 +37,13 @@ if __name__ == '__main__':
subparsers = arg_parser.add_subparsers(dest='command')
parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument('--debug', '-d',
action='store_true',
default=False,
parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
help='More verbose logging')
parent_parser.add_argument('--conf-dir', '-c',
help='Configuration directory')
parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
default=os.path.expanduser('~/uncloud'))
etcd_parser = argparse.ArgumentParser(add_help=False)
etcd_parser.add_argument('--etcd-host')
etcd_parser.add_argument('--etcd-host', dest='etcd_url')
etcd_parser.add_argument('--etcd-port')
etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
@ -54,25 +58,36 @@ if __name__ == '__main__':
else:
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
args = arg_parser.parse_args()
if not args.command:
arguments = vars(arg_parser.parse_args())
etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
etcd_arguments = {
'etcd': {
key.replace('etcd_', ''): arguments[key]
for key in etcd_arguments
}
}
if not arguments['command']:
arg_parser.print_help()
else:
arguments = vars(args)
# Initializing Settings and resolving otp_credentials
# It is neccessary to resolve_otp_credentials after argument parsing is done because
# previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
# providing the default values for --name, --realm and --seed arguments from the values
# we read from file. But, now we are asking user about where the config file lives. So,
# to providing default value is not possible before parsing arguments. So, we are doing
# it after..
settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
resolve_otp_credentials(arguments)
name = arguments.pop('command')
mod = importlib.import_module('uncloud.{}.main'.format(name))
main = getattr(mod, 'main')
# If the component requires etcd3, we import it and catch the
# etcd3.exceptions.ConnectionFailedError
if name in ETCD_COMPONENTS:
import etcd3
try:
main(arguments)
except UncloudException as err:
logger.error(err)
sys.exit(1)
except ConnectionFailedError:
logger.error('Cannot connect to etcd')
except Exception as err:
logger.exception(err)
sys.exit(1)

7
uncloud/api/common_fields.py
View file

@ -1,7 +1,6 @@
import os
from uncloud.common.shared import shared
from uncloud.common.settings import settings
class Optional:
@ -54,9 +53,7 @@ class VmUUIDField(Field):
def vm_uuid_validation(self):
r = shared.etcd_client.get(
os.path.join(settings["etcd"]["vm_prefix"], self.uuid)
os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
)
if not r:
self.add_error(
"VM with uuid {} does not exists".format(self.uuid)
)
self.add_error("VM with uuid {} does not exists".format(self.uuid))

3
uncloud/api/create_image_store.py
View file

@ -4,7 +4,6 @@ import os
from uuid import uuid4
from uncloud.common.shared import shared
from uncloud.common.settings import settings
data = {
'is_public': True,
@ -15,6 +14,6 @@ data = {
}
shared.etcd_client.put(
os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex),
os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
json.dumps(data),
)

18
uncloud/api/helper.py
View file

@ -7,7 +7,6 @@ import requests
from pyotp import TOTP
from uncloud.common.shared import shared
from uncloud.common.settings import settings
logger = logging.getLogger(__name__)
@ -15,9 +14,9 @@ logger = logging.getLogger(__name__)
def check_otp(name, realm, token):
try:
data = {
"auth_name": settings["otp"]["auth_name"],
"auth_token": TOTP(settings["otp"]["auth_seed"]).now(),
"auth_realm": settings["otp"]["auth_realm"],
"auth_name": shared.settings["otp"]["auth_name"],
"auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
"auth_realm": shared.settings["otp"]["auth_realm"],
"name": name,
"realm": realm,
"token": token,
@ -25,13 +24,13 @@ def check_otp(name, realm, token):
except binascii.Error as err:
logger.error(
"Cannot compute OTP for seed: {}".format(
settings["otp"]["auth_seed"]
shared.settings["otp"]["auth_seed"]
)
)
return 400
response = requests.post(
settings["otp"]["verification_controller_url"], json=data
shared.settings["otp"]["verification_controller_url"], json=data
)
return response.status_code
@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client):
)
images = etcd_client.get_prefix(
settings["etcd"]["image_prefix"], value_in_json=True
shared.settings["etcd"]["image_prefix"], value_in_json=True
)
# Try to find image with name == image_name and store_name == store_name
@ -111,9 +110,7 @@ def random_bytes(num=6):
return [random.randrange(256) for _ in range(num)]
def generate_mac(
uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
):
def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
mac = random_bytes()
if oui:
if type(oui) == str:
@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix):
lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
prefix = ipaddress.IPv6Address(prefix)
return str(prefix + int(lower_part))

61
uncloud/api/main.py
View file

@ -15,9 +15,8 @@ from uncloud.common.shared import shared
from uncloud.common import counters
from uncloud.common.vm import VMStatus
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.settings import settings
from . import schemas
from .helper import generate_mac, mac2ipv6
from uncloud.api import schemas
from uncloud.api.helper import generate_mac, mac2ipv6
from uncloud import UncloudException
logger = logging.getLogger(__name__)
@ -50,7 +49,7 @@ class CreateVM(Resource):
validator = schemas.CreateVMSchema(data)
if validator.is_valid():
vm_uuid = uuid4().hex
vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid)
vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
specs = {
'cpu': validator.specs['cpu'],
'ram': validator.specs['ram'],
@ -60,7 +59,7 @@ class CreateVM(Resource):
macs = [generate_mac() for _ in range(len(data['network']))]
tap_ids = [
counters.increment_etcd_counter(
shared.etcd_client, settings['etcd']['tap_counter']
shared.etcd_client, shared.settings['etcd']['tap_counter']
)
for _ in range(len(data['network']))
]
@ -84,7 +83,7 @@ class CreateVM(Resource):
r = RequestEntry.from_scratch(
type=RequestType.ScheduleVM,
uuid=vm_uuid,
request_prefix=settings['etcd']['request_prefix'],
request_prefix=shared.settings['etcd']['request_prefix'],
)
shared.request_pool.put(r)
@ -99,7 +98,7 @@ class VmStatus(Resource):
validator = schemas.VMStatusSchema(data)
if validator.is_valid():
vm = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], data['uuid'])
join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
)
vm_value = vm.value.copy()
vm_value['ip'] = []
@ -107,7 +106,7 @@ class VmStatus(Resource):
network_name, mac, tap = network_mac_and_tap
network = shared.etcd_client.get(
join_path(
settings['etcd']['network_prefix'],
shared.settings['etcd']['network_prefix'],
data['name'],
network_name,
),
@ -130,7 +129,7 @@ class CreateImage(Resource):
validator = schemas.CreateImageSchema(data)
if validator.is_valid():
file_entry = shared.etcd_client.get(
join_path(settings['etcd']['file_prefix'], data['uuid'])
join_path(shared.settings['etcd']['file_prefix'], data['uuid'])
)
file_entry_value = json.loads(file_entry.value)
@ -144,7 +143,7 @@ class CreateImage(Resource):
}
shared.etcd_client.put(
join_path(
settings['etcd']['image_prefix'], data['uuid']
shared.settings['etcd']['image_prefix'], data['uuid']
),
json.dumps(image_entry_json),
)
@ -157,7 +156,7 @@ class ListPublicImages(Resource):
@staticmethod
def get():
images = shared.etcd_client.get_prefix(
settings['etcd']['image_prefix'], value_in_json=True
shared.settings['etcd']['image_prefix'], value_in_json=True
)
r = {'images': []}
for image in images:
@ -178,7 +177,7 @@ class VMAction(Resource):
if validator.is_valid():
vm_entry = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], data['uuid'])
join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
)
action = data['action']
@ -208,7 +207,7 @@ class VMAction(Resource):
type='{}VM'.format(action.title()),
uuid=data['uuid'],
hostname=vm_entry.hostname,
request_prefix=settings['etcd']['request_prefix'],
request_prefix=shared.settings['etcd']['request_prefix'],
)
shared.request_pool.put(r)
return (
@ -231,10 +230,10 @@ class VMMigration(Resource):
type=RequestType.InitVMMigration,
uuid=vm.uuid,
hostname=join_path(
settings['etcd']['host_prefix'],
shared.settings['etcd']['host_prefix'],
validator.destination.value,
),
request_prefix=settings['etcd']['request_prefix'],
request_prefix=shared.settings['etcd']['request_prefix'],
)
shared.request_pool.put(r)
@ -254,7 +253,7 @@ class ListUserVM(Resource):
if validator.is_valid():
vms = shared.etcd_client.get_prefix(
settings['etcd']['vm_prefix'], value_in_json=True
shared.settings['etcd']['vm_prefix'], value_in_json=True
)
return_vms = []
user_vms = filter(
@ -287,7 +286,7 @@ class ListUserFiles(Resource):
if validator.is_valid():
files = shared.etcd_client.get_prefix(
settings['etcd']['file_prefix'], value_in_json=True
shared.settings['etcd']['file_prefix'], value_in_json=True
)
return_files = []
user_files = [f for f in files if f.value['owner'] == data['name']]
@ -312,7 +311,7 @@ class CreateHost(Resource):
validator = schemas.CreateHostSchema(data)
if validator.is_valid():
host_key = join_path(
settings['etcd']['host_prefix'], uuid4().hex
shared.settings['etcd']['host_prefix'], uuid4().hex
)
host_entry = {
'specs': data['specs'],
@ -354,7 +353,7 @@ class GetSSHKeys(Resource):
# {user_prefix}/{realm}/{name}/key/
etcd_key = join_path(
settings['etcd']['user_prefix'],
shared.settings['etcd']['user_prefix'],
data['realm'],
data['name'],
'key',
@ -372,7 +371,7 @@ class GetSSHKeys(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path(
settings['etcd']['user_prefix'],
shared.settings['etcd']['user_prefix'],
data['realm'],
data['name'],
'key',
@ -405,7 +404,7 @@ class AddSSHKey(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path(
settings['etcd']['user_prefix'],
shared.settings['etcd']['user_prefix'],
data['realm'],
data['name'],
'key',
@ -439,7 +438,7 @@ class RemoveSSHKey(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path(
settings['etcd']['user_prefix'],
shared.settings['etcd']['user_prefix'],
data['realm'],
data['name'],
'key',
@ -471,23 +470,23 @@ class CreateNetwork(Resource):
network_entry = {
'id': counters.increment_etcd_counter(
shared.etcd_client, settings['etcd']['vxlan_counter']
shared.etcd_client, shared.settings['etcd']['vxlan_counter']
),
'type': data['type'],
}
if validator.user.value:
try:
nb = pynetbox.api(
url=settings['netbox']['url'],
token=settings['netbox']['token'],
url=shared.settings['netbox']['url'],
token=shared.settings['netbox']['token'],
)
nb_prefix = nb.ipam.prefixes.get(
prefix=settings['network']['prefix']
prefix=shared.settings['network']['prefix']
)
prefix = nb_prefix.available_prefixes.create(
data={
'prefix_length': int(
settings['network']['prefix_length']
shared.settings['network']['prefix_length']
),
'description': '{}\'s network "{}"'.format(
data['name'], data['network_name']
@ -506,7 +505,7 @@ class CreateNetwork(Resource):
network_entry['ipv6'] = 'fd00::/64'
network_key = join_path(
settings['etcd']['network_prefix'],
shared.settings['etcd']['network_prefix'],
data['name'],
data['network_name'],
)
@ -526,7 +525,7 @@ class ListUserNetwork(Resource):
if validator.is_valid():
prefix = join_path(
settings['etcd']['network_prefix'], data['name']
shared.settings['etcd']['network_prefix'], data['name']
)
networks = shared.etcd_client.get_prefix(
prefix, value_in_json=True
@ -570,7 +569,7 @@ def main(arguments):
try:
image_stores = list(
shared.etcd_client.get_prefix(
settings['etcd']['image_store_prefix'], value_in_json=True
shared.settings['etcd']['image_store_prefix'], value_in_json=True
)
)
except KeyError:
@ -590,7 +589,7 @@ def main(arguments):
# shared.etcd_client.put(
# join_path(
# settings['etcd']['image_store_prefix'], uuid4().hex
# shared.settings['etcd']['image_store_prefix'], uuid4().hex
# ),
# json.dumps(data),
# )

13
uncloud/api/schemas.py
View file

@ -22,7 +22,6 @@ import bitmath
from uncloud.common.host import HostStatus
from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from . import helper, logger
from .common_fields import Field, VmUUIDField
from .helper import check_otp, resolve_vm_name
@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema):
def file_uuid_validation(self):
file_entry = shared.etcd_client.get(
os.path.join(
settings["etcd"]["file_prefix"], self.uuid.value
shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
)
)
if file_entry is None:
@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema):
def image_store_name_validation(self):
image_stores = list(
shared.etcd_client.get_prefix(
settings["etcd"]["image_store_prefix"]
shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
)
)
@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema):
for net in _network:
network = shared.etcd_client.get(
os.path.join(
settings["etcd"]["network_prefix"],
shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
self.name.value,
net,
),
@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema):
self.add_error("Can't migrate non-running VM")
if vm.hostname == os.path.join(
settings["etcd"]["host_prefix"], self.destination.value
shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
):
self.add_error(
"Destination host couldn't be same as Source Host"
@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema):
super().__init__(data, fields=fields)
def network_name_validation(self):
print(self.name.value, self.network_name.value)
key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
print(key)
key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
network = shared.etcd_client.get(key, value_in_json=True)
if network:
self.add_error(

29
uncloud/cli/helper.py
View file

@ -5,23 +5,14 @@ import binascii
from pyotp import TOTP
from os.path import join as join_path
from uncloud.common.settings import settings
from uncloud.common.shared import shared
def get_otp_parser():
otp_parser = argparse.ArgumentParser('otp')
try:
name = settings['client']['name']
realm = settings['client']['realm']
seed = settings['client']['seed']
except Exception:
otp_parser.add_argument('--name', required=True)
otp_parser.add_argument('--realm', required=True)
otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED')
else:
otp_parser.add_argument('--name', default=name)
otp_parser.add_argument('--realm', default=realm)
otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED')
otp_parser.add_argument('--name')
otp_parser.add_argument('--realm')
otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
return otp_parser
@ -34,11 +25,15 @@ def load_dump_pretty(content):
def make_request(*args, data=None, request_method=requests.post):
r = request_method(join_path(settings['client']['api_server'], *args), json=data)
try:
print(load_dump_pretty(r.content))
except Exception:
print('Error occurred while getting output from api server.')
r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
except requests.exceptions.RequestException:
print('Error occurred while connecting to API server.')
else:
try:
print(load_dump_pretty(r.content))
except Exception:
print('Error occurred while getting output from api server.')
def get_token(seed):

26
uncloud/common/cli.py Normal file
View file

@ -0,0 +1,26 @@
from uncloud.common.shared import shared
from pyotp import TOTP
def get_token(seed):
if seed is not None:
try:
token = TOTP(seed).now()
except Exception:
raise Exception('Invalid seed')
else:
return token
def resolve_otp_credentials(kwargs):
d = {
'name': shared.settings['client']['name'],
'realm': shared.settings['client']['realm'],
'token': get_token(shared.settings['client']['seed'])
}
for k, v in d.items():
if k in kwargs and kwargs[k] is None:
kwargs.update({k: v})
return d

19
uncloud/common/settings.py
View file

@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper
from os.path import join as join_path
logger = logging.getLogger(__name__)
settings = None
class CustomConfigParser(configparser.RawConfigParser):
@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser):
class Settings(object):
def __init__(self):
def __init__(self, conf_dir, seed_value=None):
conf_name = 'uncloud.conf'
conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
self.config_file = join_path(conf_dir, conf_name)
# this is used to cache config from etcd for 1 minutes. Without this we
@ -38,15 +38,19 @@ class Settings(object):
self.config_parser.add_section('etcd')
self.config_parser.set('etcd', 'base_prefix', '/')
try:
if os.access(self.config_file, os.R_OK):
self.config_parser.read(self.config_file)
except Exception as err:
logger.error('%s', err)
else:
raise FileNotFoundError('Config file %s not found!', self.config_file)
self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
self.read_internal_values()
if seed_value is None:
seed_value = dict()
self.config_parser.read_dict(seed_value)
def get_etcd_client(self):
args = tuple()
try:
@ -128,4 +132,5 @@ class Settings(object):
return self.config_parser[key]
settings = Settings()
def get_settings():
return settings

22
uncloud/common/shared.py
View file

@ -1,34 +1,34 @@
from uncloud.common.settings import settings
from uncloud.common.settings import get_settings
from uncloud.common.vm import VmPool
from uncloud.common.host import HostPool
from uncloud.common.request import RequestPool
from uncloud.common.storage_handlers import get_storage_handler
import uncloud.common.storage_handlers as storage_handlers
class Shared:
@property
def settings(self):
return get_settings()
@property
def etcd_client(self):
return settings.get_etcd_client()
return self.settings.get_etcd_client()
@property
def host_pool(self):
return HostPool(
self.etcd_client, settings["etcd"]["host_prefix"]
)
return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
@property
def vm_pool(self):
return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"])
return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
@property
def request_pool(self):
return RequestPool(
self.etcd_client, settings["etcd"]["request_prefix"]
)
return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
@property
def storage_handler(self):
return get_storage_handler()
return storage_handlers.get_storage_handler()
shared = Shared()

15
uncloud/common/storage_handlers.py
View file

@ -6,8 +6,7 @@ import stat
from abc import ABC
from . import logger
from os.path import join as join_path
from uncloud.common.settings import settings as config
import uncloud.common.shared as shared
class ImageStorageHandler(ABC):
@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
def get_storage_handler():
__storage_backend = config["storage"]["storage_backend"]
__storage_backend = shared.shared.settings["storage"]["storage_backend"]
if __storage_backend == "filesystem":
return FileSystemBasedImageStorageHandler(
vm_base=config["storage"]["vm_dir"],
image_base=config["storage"]["image_dir"],
vm_base=shared.shared.settings["storage"]["vm_dir"],
image_base=shared.shared.settings["storage"]["image_dir"],
)
elif __storage_backend == "ceph":
return CEPHBasedImageStorageHandler(
vm_base=config["storage"]["ceph_vm_pool"],
image_base=config["storage"]["ceph_image_pool"],
vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
image_base=shared.shared.settings["storage"]["ceph_image_pool"],
)
else:
raise Exception("Unknown Image Storage Handler")
raise Exception("Unknown Image Storage Handler")

11
uncloud/configure/main.py
View file

@ -1,7 +1,6 @@
import os
import argparse
from uncloud.common.settings import settings
from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('configure', add_help=False)
@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
def update_config(section, kwargs):
uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
if not uncloud_config:
uncloud_config = {}
else:
uncloud_config = uncloud_config.value
uncloud_config[section] = kwargs
shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
def main(**kwargs):
subcommand = kwargs.pop('subcommand')
def main(arguments):
subcommand = arguments['subcommand']
if not subcommand:
arg_parser.print_help()
else:
update_config(subcommand, kwargs)
update_config(subcommand, arguments)

7
uncloud/filescanner/main.py
View file

@ -9,7 +9,6 @@ import bitmath
from uuid import uuid4
from . import logger
from uncloud.common.settings import settings
from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
@ -53,7 +52,7 @@ def track_file(file, base_dir, host):
file_path = file_path.relative_to(owner)
creation_date = time.ctime(os.stat(file_str).st_ctime)
entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
entry_value = {
'filename': str(file_path),
'owner': owner,
@ -70,7 +69,7 @@ def track_file(file, base_dir, host):
def main(arguments):
hostname = arguments['hostname']
base_dir = settings['storage']['file_dir']
base_dir = shared.settings['storage']['file_dir']
# Recursively Get All Files and Folder below BASE_DIR
files = glob.glob('{}/**'.format(base_dir), recursive=True)
files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
@ -78,7 +77,7 @@ def main(arguments):
# Files that are already tracked
tracked_files = [
pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
if f.value['host'] == hostname
]
untracked_files = set(files) - set(tracked_files)

11
uncloud/host/main.py
View file

@ -6,7 +6,6 @@ from uuid import uuid4
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.common.vm import VMStatus
from uncloud.vmm import VMM
from os.path import join as join_path
@ -36,7 +35,7 @@ def maintenance(host):
if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
logger.debug('VM {} is running on {}'.format(vm_uuid, host))
vm = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], vm_uuid)
join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
)
vm.status = VMStatus.running
vm.vnc_socket = vmm.get_vnc(vm_uuid)
@ -52,7 +51,7 @@ def main(arguments):
# Does not yet exist, create it
if not host:
host_key = join_path(
settings['etcd']['host_prefix'], uuid4().hex
shared.settings['etcd']['host_prefix'], uuid4().hex
)
host_entry = {
'specs': '',
@ -80,9 +79,9 @@ def main(arguments):
# get prefix until either success or deamon death comes.
while True:
for events_iterator in [
shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False)
]:
for request_event in events_iterator:
@ -95,7 +94,7 @@ def main(arguments):
shared.request_pool.client.client.delete(request_event.key)
vm_entry = shared.etcd_client.get(
join_path(settings['etcd']['vm_prefix'], request_event.uuid)
join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid)
)
logger.debug('VM hostname: {}'.format(vm_entry.value))

11
uncloud/host/virtualmachine.py
View file

@ -17,7 +17,6 @@ from uncloud.common.network import create_dev, delete_network_interface
from uncloud.common.schemas import VMSchema, NetworkSchema
from uncloud.host import logger
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.vmm import VMM
from marshmallow import ValidationError
@ -91,7 +90,7 @@ class VM:
self.vmm.socket_dir, self.uuid
),
destination_host_key=destination_host_key, # Where source host transfer VM
request_prefix=settings["etcd"]["request_prefix"],
request_prefix=shared.settings["etcd"]["request_prefix"],
)
shared.request_pool.put(r)
else:
@ -119,7 +118,7 @@ class VM:
network_name, mac, tap = network_mac_and_tap
_key = os.path.join(
settings["etcd"]["network_prefix"],
shared.settings["etcd"]["network_prefix"],
self.vm["owner"],
network_name,
)
@ -133,13 +132,13 @@ class VM:
if network["type"] == "vxlan":
tap = create_vxlan_br_tap(
_id=network["id"],
_dev=settings["network"]["vxlan_phy_dev"],
_dev=shared.settings["network"]["vxlan_phy_dev"],
tap_id=tap,
ip=network["ipv6"],
)
all_networks = shared.etcd_client.get_prefix(
settings["etcd"]["network_prefix"],
shared.settings["etcd"]["network_prefix"],
value_in_json=True,
)
@ -229,7 +228,7 @@ class VM:
def resolve_network(network_name, network_owner):
network = shared.etcd_client.get(
join_path(
settings["etcd"]["network_prefix"],
shared.settings["etcd"]["network_prefix"],
network_owner,
network_name,
),

7
uncloud/imagescanner/main.py
View file

@ -4,7 +4,6 @@ import argparse
import subprocess as sp
from os.path import join as join_path
from uncloud.common.settings import settings
from uncloud.common.shared import shared
from uncloud.imagescanner import logger
@ -33,7 +32,7 @@ def qemu_img_type(path):
def main(arguments):
# We want to get images entries that requests images to be created
images = shared.etcd_client.get_prefix(
settings["etcd"]["image_prefix"], value_in_json=True
shared.settings["etcd"]["image_prefix"], value_in_json=True
)
images_to_be_created = list(
filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
@ -46,13 +45,13 @@ def main(arguments):
image_filename = image.value["filename"]
image_store_name = image.value["store_name"]
image_full_path = join_path(
settings["storage"]["file_dir"],
shared.settings["storage"]["file_dir"],
image_owner,
image_filename,
)
image_stores = shared.etcd_client.get_prefix(
settings["etcd"]["image_store_prefix"],
shared.settings["etcd"]["image_store_prefix"],
value_in_json=True,
)
user_image_store = next(

3
uncloud/metadata/main.py
View file

@ -5,7 +5,6 @@ from flask import Flask, request
from flask_restful import Resource, Api
from werkzeug.exceptions import HTTPException
from uncloud.common.settings import settings
from uncloud.common.shared import shared
app = Flask(__name__)
@ -74,7 +73,7 @@ class Root(Resource):
)
else:
etcd_key = os.path.join(
settings["etcd"]["user_prefix"],
shared.settings["etcd"]["user_prefix"],
data.value["owner_realm"],
data.value["owner"],
"key",

3
uncloud/scheduler/helper.py
View file

@ -7,7 +7,6 @@ from uncloud.common.host import HostStatus
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared
from uncloud.common.settings import settings
def accumulated_specs(vms_specs):
@ -130,7 +129,7 @@ def assign_host(vm):
type=RequestType.StartVM,
uuid=vm.uuid,
hostname=vm.hostname,
request_prefix=settings["etcd"]["request_prefix"],
request_prefix=shared.settings["etcd"]["request_prefix"],
)
shared.request_pool.put(r)

5
uncloud/scheduler/main.py
View file

@ -6,7 +6,6 @@
import argparse
from uncloud.common.settings import settings
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.shared import shared
from uncloud.scheduler import logger
@ -24,9 +23,9 @@ def main(arguments):
# get prefix until either success or deamon death comes.
while True:
for request_iterator in [
shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
]:
for request_event in request_iterator: