Commit c3b42aab authored by Ahmed Bilal's avatar Ahmed Bilal 💬

Added --conf-dir, --etcd-{host,port,ca_cert,cert_cert,cert_key} parameters to...

Added --conf-dir, --etcd-{host,port,ca_cert,cert_cert,cert_key} parameters to cli and settings is now accessbile through uncloud.shared.shared.settings
parent e6d22a73
......@@ -3,14 +3,13 @@ import logging
import sys
import importlib
import argparse
import os
from uncloud import UncloudException
# the components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
from etcd3.exceptions import ConnectionFailedError
ALL_COMPONENTS = ETCD_COMPONENTS.copy()
ALL_COMPONENTS.append('cli')
from uncloud.common import settings
from uncloud import UncloudException
from uncloud.common.cli import resolve_otp_credentials
def exception_hook(exc_type, exc_value, exc_traceback):
......@@ -22,6 +21,13 @@ def exception_hook(exc_type, exc_value, exc_traceback):
sys.excepthook = exception_hook
# the components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure']
ALL_COMPONENTS = ETCD_COMPONENTS.copy()
ALL_COMPONENTS.append('cli')
if __name__ == '__main__':
# Setting up root logger
logger = logging.getLogger()
......@@ -31,15 +37,13 @@ if __name__ == '__main__':
subparsers = arg_parser.add_subparsers(dest='command')
parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument('--debug', '-d',
action='store_true',
default=False,
parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
help='More verbose logging')
parent_parser.add_argument('--conf-dir', '-c',
help='Configuration directory')
parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
default=os.path.expanduser('~/uncloud'))
etcd_parser = argparse.ArgumentParser(add_help=False)
etcd_parser.add_argument('--etcd-host')
etcd_parser.add_argument('--etcd-host', dest='etcd_url')
etcd_parser.add_argument('--etcd-port')
etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
......@@ -54,25 +58,36 @@ if __name__ == '__main__':
else:
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
args = arg_parser.parse_args()
if not args.command:
arguments = vars(arg_parser.parse_args())
etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
etcd_arguments = {
'etcd': {
key.replace('etcd_', ''): arguments[key]
for key in etcd_arguments
}
}
if not arguments['command']:
arg_parser.print_help()
else:
arguments = vars(args)
# Initializing Settings and resolving otp_credentials
# It is neccessary to resolve_otp_credentials after argument parsing is done because
# previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
# providing the default values for --name, --realm and --seed arguments from the values
# we read from file. But, now we are asking user about where the config file lives. So,
# to providing default value is not possible before parsing arguments. So, we are doing
# it after..
settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
resolve_otp_credentials(arguments)
name = arguments.pop('command')
mod = importlib.import_module('uncloud.{}.main'.format(name))
main = getattr(mod, 'main')
# If the component requires etcd3, we import it and catch the
# etcd3.exceptions.ConnectionFailedError
if name in ETCD_COMPONENTS:
import etcd3
try:
main(arguments)
except UncloudException as err:
logger.error(err)
sys.exit(1)
except ConnectionFailedError:
logger.error('Cannot connect to etcd')
except Exception as err:
logger.exception(err)
sys.exit(1)
import os
from uncloud.common.shared import shared
from uncloud.common.settings import settings
class Optional:
......@@ -54,9 +53,7 @@ class VmUUIDField(Field):
def vm_uuid_validation(self):
r = shared.etcd_client.get(
os.path.join(settings["etcd"]["vm_prefix"], self.uuid)
os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
)
if not r:
self.add_error(
"VM with uuid {} does not exists".format(self.uuid)
)
self.add_error("VM with uuid {} does not exists".format(self.uuid))
......@@ -4,7 +4,6 @@ import os
from uuid import uuid4
from uncloud.common.shared import shared
from uncloud.common.settings import settings
data = {
'is_public': True,
......@@ -15,6 +14,6 @@ data = {
}
shared.etcd_client.put(
os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex),
os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
json.dumps(data),
)
......@@ -7,7 +7,6 @@ import requests
from pyotp import TOTP
from uncloud.common.shared import shared
from uncloud.common.settings import settings
logger = logging.getLogger(__name__)
......@@ -15,9 +14,9 @@ logger = logging.getLogger(__name__)
def check_otp(name, realm, token):
try:
data = {
"auth_name": settings["otp"]["auth_name"],
"auth_token": TOTP(settings["otp"]["auth_seed"]).now(),
"auth_realm": settings["otp"]["auth_realm"],
"auth_name": shared.settings["otp"]["auth_name"],
"auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
"auth_realm": shared.settings["otp"]["auth_realm"],
"name": name,
"realm": realm,
"token": token,
......@@ -25,13 +24,13 @@ def check_otp(name, realm, token):
except binascii.Error as err:
logger.error(
"Cannot compute OTP for seed: {}".format(
settings["otp"]["auth_seed"]
shared.settings["otp"]["auth_seed"]
)
)
return 400
response = requests.post(
settings["otp"]["verification_controller_url"], json=data
shared.settings["otp"]["verification_controller_url"], json=data
)
return response.status_code
......@@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client):
)
images = etcd_client.get_prefix(
settings["etcd"]["image_prefix"], value_in_json=True
shared.settings["etcd"]["image_prefix"], value_in_json=True
)
# Try to find image with name == image_name and store_name == store_name
......@@ -111,9 +110,7 @@ def random_bytes(num=6):
return [random.randrange(256) for _ in range(num)]
def generate_mac(
uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
):
def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
mac = random_bytes()
if oui:
if type(oui) == str:
......@@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix):
lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
prefix = ipaddress.IPv6Address(prefix)
return str(prefix + int(lower_part))
This diff is collapsed.
......@@ -22,7 +22,6 @@ import bitmath
from uncloud.common.host import HostStatus
from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from . import helper, logger
from .common_fields import Field, VmUUIDField
from .helper import check_otp, resolve_vm_name
......@@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema):
def file_uuid_validation(self):
file_entry = shared.etcd_client.get(
os.path.join(
settings["etcd"]["file_prefix"], self.uuid.value
shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
)
)
if file_entry is None:
......@@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema):
def image_store_name_validation(self):
image_stores = list(
shared.etcd_client.get_prefix(
settings["etcd"]["image_store_prefix"]
shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
)
)
......@@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema):
for net in _network:
network = shared.etcd_client.get(
os.path.join(
settings["etcd"]["network_prefix"],
shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
self.name.value,
net,
),
......@@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema):
self.add_error("Can't migrate non-running VM")
if vm.hostname == os.path.join(
settings["etcd"]["host_prefix"], self.destination.value
shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
):
self.add_error(
"Destination host couldn't be same as Source Host"
......@@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema):
super().__init__(data, fields=fields)
def network_name_validation(self):
print(self.name.value, self.network_name.value)
key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
print(key)
key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
network = shared.etcd_client.get(key, value_in_json=True)
if network:
self.add_error(
......
......@@ -5,23 +5,14 @@ import binascii
from pyotp import TOTP
from os.path import join as join_path
from uncloud.common.settings import settings
from uncloud.common.shared import shared
def get_otp_parser():
otp_parser = argparse.ArgumentParser('otp')
try:
name = settings['client']['name']
realm = settings['client']['realm']
seed = settings['client']['seed']
except Exception:
otp_parser.add_argument('--name', required=True)
otp_parser.add_argument('--realm', required=True)
otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED')
else:
otp_parser.add_argument('--name', default=name)
otp_parser.add_argument('--realm', default=realm)
otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED')
otp_parser.add_argument('--name')
otp_parser.add_argument('--realm')
otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
return otp_parser
......@@ -34,11 +25,15 @@ def load_dump_pretty(content):
def make_request(*args, data=None, request_method=requests.post):
r = request_method(join_path(settings['client']['api_server'], *args), json=data)
try:
print(load_dump_pretty(r.content))
except Exception:
print('Error occurred while getting output from api server.')
r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
except requests.exceptions.RequestException:
print('Error occurred while connecting to API server.')
else:
try:
print(load_dump_pretty(r.content))
except Exception:
print('Error occurred while getting output from api server.')
def get_token(seed):
......
from uncloud.common.shared import shared
from pyotp import TOTP
def get_token(seed):
if seed is not None:
try:
token = TOTP(seed).now()
except Exception:
raise Exception('Invalid seed')
else:
return token
def resolve_otp_credentials(kwargs):
d = {
'name': shared.settings['client']['name'],
'realm': shared.settings['client']['realm'],
'token': get_token(shared.settings['client']['seed'])
}
for k, v in d.items():
if k in kwargs and kwargs[k] is None:
kwargs.update({k: v})
return d
......@@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper
from os.path import join as join_path
logger = logging.getLogger(__name__)
settings = None
class CustomConfigParser(configparser.RawConfigParser):
......@@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser):
class Settings(object):
def __init__(self):
def __init__(self, conf_dir, seed_value=None):
conf_name = 'uncloud.conf'
conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
self.config_file = join_path(conf_dir, conf_name)
# this is used to cache config from etcd for 1 minutes. Without this we
......@@ -38,15 +38,19 @@ class Settings(object):
self.config_parser.add_section('etcd')
self.config_parser.set('etcd', 'base_prefix', '/')
try:
if os.access(self.config_file, os.R_OK):
self.config_parser.read(self.config_file)
except Exception as err:
logger.error('%s', err)
else:
raise FileNotFoundError('Config file %s not found!', self.config_file)
self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
self.read_internal_values()
if seed_value is None:
seed_value = dict()
self.config_parser.read_dict(seed_value)
def get_etcd_client(self):
args = tuple()
try:
......@@ -128,4 +132,5 @@ class Settings(object):
return self.config_parser[key]
settings = Settings()
def get_settings():
return settings
from uncloud.common.settings import settings
from uncloud.common.settings import get_settings
from uncloud.common.vm import VmPool
from uncloud.common.host import HostPool
from uncloud.common.request import RequestPool
from uncloud.common.storage_handlers import get_storage_handler
import uncloud.common.storage_handlers as storage_handlers
class Shared:
@property
def settings(self):
return get_settings()
@property
def etcd_client(self):
return settings.get_etcd_client()
return self.settings.get_etcd_client()
@property
def host_pool(self):
return HostPool(
self.etcd_client, settings["etcd"]["host_prefix"]
)
return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
@property
def vm_pool(self):
return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"])
return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
@property
def request_pool(self):
return RequestPool(
self.etcd_client, settings["etcd"]["request_prefix"]
)
return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
@property
def storage_handler(self):
return get_storage_handler()
return storage_handlers.get_storage_handler()
shared = Shared()
......@@ -6,8 +6,7 @@ import stat
from abc import ABC
from . import logger
from os.path import join as join_path
from uncloud.common.settings import settings as config
import uncloud.common.shared as shared
class ImageStorageHandler(ABC):
......@@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
def get_storage_handler():
__storage_backend = config["storage"]["storage_backend"]
__storage_backend = shared.shared.settings["storage"]["storage_backend"]
if __storage_backend == "filesystem":
return FileSystemBasedImageStorageHandler(
vm_base=config["storage"]["vm_dir"],
image_base=config["storage"]["image_dir"],
vm_base=shared.shared.settings["storage"]["vm_dir"],
image_base=shared.shared.settings["storage"]["image_dir"],
)
elif __storage_backend == "ceph":
return CEPHBasedImageStorageHandler(
vm_base=config["storage"]["ceph_vm_pool"],
image_base=config["storage"]["ceph_image_pool"],
vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
image_base=shared.shared.settings["storage"]["ceph_image_pool"],
)
else:
raise Exception("Unknown Image Storage Handler")
raise Exception("Unknown Image Storage Handler")
\ No newline at end of file
import os
import argparse
from uncloud.common.settings import settings
from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('configure', add_help=False)
......@@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
def update_config(section, kwargs):
uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
if not uncloud_config:
uncloud_config = {}
else:
uncloud_config = uncloud_config.value
uncloud_config[section] = kwargs
shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
def main(**kwargs):
subcommand = kwargs.pop('subcommand')
def main(arguments):
subcommand = arguments['subcommand']
if not subcommand:
arg_parser.print_help()
else:
update_config(subcommand, kwargs)
update_config(subcommand, arguments)
......@@ -9,7 +9,6 @@ import bitmath
from uuid import uuid4
from . import logger
from uncloud.common.settings import settings
from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
......@@ -53,7 +52,7 @@ def track_file(file, base_dir, host):
file_path = file_path.relative_to(owner)
creation_date = time.ctime(os.stat(file_str).st_ctime)
entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
entry_value = {
'filename': str(file_path),
'owner': owner,
......@@ -70,7 +69,7 @@ def track_file(file, base_dir, host):
def main(arguments):
hostname = arguments['hostname']
base_dir = settings['storage']['file_dir']
base_dir = shared.settings['storage']['file_dir']
# Recursively Get All Files and Folder below BASE_DIR
files = glob.glob('{}/**'.format(base_dir), recursive=True)
files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
......@@ -78,7 +77,7 @@ def main(arguments):
# Files that are already tracked
tracked_files = [
pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
if f.value['host'] == hostname
]
untracked_files = set(files) - set(tracked_files)
......
......@@ -6,7 +6,6 @@ from uuid import uuid4
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.common.vm import VMStatus
from uncloud.vmm import VMM
from os.path import join as join_path
......@@ -36,7 +35,7 @@ def maintenance(host):
if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running':
logger.debug('VM {} is running on {}'.format(vm_uuid, host))
vm = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], vm_uuid)
join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
)
vm.status = VMStatus.running
vm.vnc_socket = vmm.get_vnc(vm_uuid)
......@@ -52,7 +51,7 @@ def main(arguments):
# Does not yet exist, create it
if not host:
host_key = join_path(
settings['etcd']['host_prefix'], uuid4().hex
shared.settings['etcd']['host_prefix'], uuid4().hex
)
host_entry = {
'specs': '',
......@@ -80,9 +79,9 @@ def main(arguments):
# get prefix until either success or deamon death comes.
while True:
for events_iterator in [
shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False)
]:
for request_event in events_iterator:
......@@ -95,7 +94,7 @@ def main(arguments):
shared.request_pool.client.client.delete(request_event.key)
vm_entry = shared.etcd_client.get(
join_path(settings['etcd']['vm_prefix'], request_event.uuid)
join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid)
)
logger.debug('VM hostname: {}'.format(vm_entry.value))
......
......@@ -17,7 +17,6 @@ from uncloud.common.network import create_dev, delete_network_interface
from uncloud.common.schemas import VMSchema, NetworkSchema
from uncloud.host import logger
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from uncloud.vmm import VMM
from marshmallow import ValidationError
......@@ -91,7 +90,7 @@ class VM:
self.vmm.socket_dir, self.uuid
),
destination_host_key=destination_host_key, # Where source host transfer VM
request_prefix=settings["etcd"]["request_prefix"],
request_prefix=shared.settings["etcd"]["request_prefix"],
)
shared.request_pool.put(r)
else:
......@@ -119,7 +118,7 @@ class VM:
network_name, mac, tap = network_mac_and_tap
_key = os.path.join(
settings["etcd"]["network_prefix"],
shared.settings["etcd"]["network_prefix"],
self.vm["owner"],
network_name,
)
......@@ -133,13 +132,13 @@ class VM:
if network["type"] == "vxlan":
tap = create_vxlan_br_tap(
_id=network["id"],
_dev=settings["network"]["vxlan_phy_dev"],
_dev=shared.settings["network"]["vxlan_phy_dev"],
tap_id=tap,
ip=network["ipv6"],
)
all_networks = shared.etcd_client.get_prefix(
settings["etcd"]["network_prefix"],
shared.settings["etcd"]["network_prefix"],
value_in_json=True,
)
......@@ -229,7 +228,7 @@ class VM:
def resolve_network(network_name, network_owner):
network = shared.etcd_client.get(
join_path(
settings["etcd"]["network_prefix"],
shared.settings["etcd"]["network_prefix"],
network_owner,
network_name,
),
......
......@@ -4,7 +4,6 @@ import argparse
import subprocess as sp
from os.path import join as join_path
from uncloud.common.settings import settings
from uncloud.common.shared import shared
from uncloud.imagescanner import logger
......@@ -33,7 +32,7 @@ def qemu_img_type(path):
def main(arguments):
# We want to get images entries that requests images to be created
images = shared.etcd_client.get_prefix(
settings["etcd"]["image_prefix"], value_in_json=True
shared.settings["etcd"]["image_prefix"], value_in_json=True
)
images_to_be_created = list(
filter(lambda im: im.value["status"] == "TO_BE_CREATED", images)
......@@ -46,13 +45,13 @@ def main(arguments):
image_filename = image.value["filename"]
image_store_name = image.value["store_name"]
image_full_path = join_path(
settings["storage"]["file_dir"],
shared.settings["storage"]["file_dir"],
image_owner,
image_filename,
)
image_stores = shared.etcd_client.get_prefix(
settings["etcd"]["image_store_prefix"],
shared.settings["etcd"]["image_store_prefix"],
value_in_json=True,
)
user_image_store = next(
......
......@@ -5,7 +5,6 @@ from flask import Flask, request
from flask_restful import Resource, Api
from werkzeug.exceptions import HTTPException
from uncloud.common.settings import settings
from uncloud.common.shared import shared
app = Flask(__name__)
......@@ -74,7 +73,7 @@ class Root(Resource):
)
else:
etcd_key = os.path.join(
settings["etcd"]["user_prefix"],
shared.settings["etcd"]["user_prefix"],
data.value["owner_realm"],
data.value["owner"],
"key",
......
......@@ -7,7 +7,6 @@ from uncloud.common.host import HostStatus
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared
from uncloud.common.settings import settings
def accumulated_specs(vms_specs):
......@@ -130,7 +129,7 @@ def assign_host(vm):
type=RequestType.StartVM,
uuid=vm.uuid,
hostname=vm.hostname,
request_prefix=settings["etcd"]["request_prefix"],
request_prefix=shared.settings["etcd"]["request_prefix"],
)
shared.request_pool.put(r)
......
......@@ -6,7 +6,6 @@
import argparse
from uncloud.common.settings import settings
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.shared import shared
from uncloud.scheduler import logger
......@@ -24,9 +23,9 @@ def main(arguments):
# get prefix until either success or deamon death comes.
while True:
for request_iterator in [
shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True,
shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True,
shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True,
raise_exception=False),
]:
for request_event in request_iterator:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment