fnux
/
uncloud
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Configuration/Setting module added

This commit is contained in:
ahmadbilalkhalid 2019-12-21 14:36:55 +05:00
parent 71279a968f
commit bc58a6ed9c
11 changed files with 217 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

46
conf/ucloud.conf
View File

@ -1,19 +1,3 @@
[otp]
server = https://otp.ungleich.ch/ungleichotp/
verify_endpoint = verify/
auth_name = replace_me
auth_realm = replace_me
auth_seed = replace_me
[network]
prefix_length = 64
prefix = 2001:db8::/48
vxlan_phy_dev = eno1
[netbox]
url = https://replace-me.example.com
token = replace_me
[etcd]
url = localhost
port = 2379
@ -21,33 +5,3 @@ port = 2379
ca_cert
cert_cert
cert_key
file_prefix = /files/
host_prefix = /hosts/
image_prefix = /images/
image_store_prefix = /imagestore/
network_prefix = /networks/
request_prefix = /requests/
user_prefix = /users/
vm_prefix = /vms/
[storage]
#values = filesystem, ceph
backend = filesystem
# if STORAGE_BACKEND = filesystem
vm_dir = /var/lib/ucloud/vms
image_dir = /var/lib/ucloud/images
# if STORAGE_BACKEND = ceph
ceph_vm_pool = ssd
ceph_image_pool = ssd
# Importing uploaded files
file_dir = /var/lib/ucloud/files
# For Migrating VMs over ssh/tcp
[ssh]
username
private_key_path

65
scripts/ucloud
View File

@ -3,37 +3,60 @@
import argparse
import logging
import importlib
import sys
import os
import multiprocessing as mp
import sys
from ucloud.configure.main import update_config, configure_parser
COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata']
def exception_hook(exc_type, exc_value, exc_traceback):
logger.error(
"Uncaught exception",
exc_info=(exc_type, exc_value, exc_traceback)
)
print(exc_type, exc_value)
if __name__ == "__main__":
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG,
format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s',
filename='/var/log/ucloud.log', filemode='a')
logger = logging.getLogger("ucloud")
arg_parser = argparse.ArgumentParser(prog='ucloud',
description='Open Source Cloud Management Software')
arg_parser.add_argument('-c', '--conf-dir', help="Configuration directory")
arg_parser.add_argument('component', choices=COMMANDS)
arg_parser.add_argument('component_args', nargs='*')
sys.excepthook = exception_hook
mp.set_start_method('spawn')
arg_parser = argparse.ArgumentParser()
subparsers = arg_parser.add_subparsers(dest="command")
api_parser = subparsers.add_parser("api")
host_parser = subparsers.add_parser("host")
host_parser.add_argument("--hostname", required=True)
scheduler_parser = subparsers.add_parser("scheduler")
filescanner_parser = subparsers.add_parser("filescanner")
imagescanner_parser = subparsers.add_parser("imagescanner")
metadata_parser = subparsers.add_parser("metadata")
config_parser = subparsers.add_parser("configure")
configure_parser(config_parser)
args = arg_parser.parse_args()
if args.conf_dir:
os.environ['UCLOUD_CONF_DIR'] = args.conf_dir
if not args.command:
arg_parser.print_help()
else:
arguments = vars(args)
try:
name = arguments.pop('command')
mod = importlib.import_module("ucloud.{}.main".format(name))
main = getattr(mod, "main")
main(**arguments)
try:
mp.set_start_method('spawn')
name = args.component
mod = importlib.import_module("ucloud.{}.main".format(name))
main = getattr(mod, "main")
main(*args.component_args)
except Exception as e:
logger.exception(e)
print(e)
except Exception as e:
logger.exception(e)
print(e)

6
ucloud/api/helper.py
View File

@ -29,11 +29,7 @@ def check_otp(name, realm, token):
return 400
response = requests.post(
"{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format(
OTP_SERVER=config['otp']['server'],
OTP_VERIFY_ENDPOINT=config['otp']['verify_endpoint']
),
json=data,
config['otp']['verification_controller_url'], json=data
)
return response.status_code

30
ucloud/config.py
View File

@ -5,32 +5,18 @@ import logging
from ucloud.common.host import HostPool
from ucloud.common.request import RequestPool
from ucloud.common.vm import VmPool
from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler
from ucloud.common.storage_handlers import (FileSystemBasedImageStorageHandler,
CEPHBasedImageStorageHandler)
from ucloud.common.etcd_wrapper import Etcd3Wrapper
from ucloud.settings import Settings
from os.path import join as join_path
log = logging.getLogger('ucloud.config')
logger = logging.getLogger('ucloud.config')
conf_name = 'ucloud.conf'
conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud')
config_file = os.path.join(conf_dir, conf_name)
config = configparser.ConfigParser(allow_no_value=True)
if os.access(config_file, os.R_OK):
config.read(config_file)
else:
log.warning('Configuration file not found - using defaults')
etcd_wrapper_args = ()
etcd_wrapper_kwargs = {
'host': config['etcd']['url'],
'port': config['etcd']['port'],
'ca_cert': config['etcd']['ca_cert'],
'cert_cert': config['etcd']['cert_cert'],
'cert_key': config['etcd']['cert_key']
}
etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
config = Settings()
etcd_client = config.get_etcd_client()
host_pool = HostPool(etcd_client, config['etcd']['host_prefix'])
vm_pool = VmPool(etcd_client, config['etcd']['vm_prefix'])
@ -38,7 +24,7 @@ request_pool = RequestPool(etcd_client, config['etcd']['request_prefix'])
running_vms = []
__storage_backend = config['storage']['backend']
__storage_backend = config['storage']['storage_backend']
if __storage_backend == 'filesystem':
image_storage_handler = FileSystemBasedImageStorageHandler(
vm_base=config['storage']['vm_dir'],

0
ucloud/configure/__init__.py Normal file
View File

67
ucloud/configure/main.py Normal file
View File

@ -0,0 +1,67 @@
import argparse
import sys
import os
from ucloud.settings import Settings
config = Settings()
etcd_client = config.get_etcd_client()
def update_config(section, kwargs):
uncloud_config = etcd_client.get(config.config_key,
value_in_json=True)
if not uncloud_config:
uncloud_config = {}
else:
uncloud_config = uncloud_config.value
uncloud_config[section] = kwargs
etcd_client.put(config.config_key, uncloud_config, value_in_json=True)
def configure_parser(parser):
configure_subparsers = parser.add_subparsers(dest="subcommand")
otp_parser = configure_subparsers.add_parser("otp")
otp_parser.add_argument("--verification-controller-url",
required=True, metavar="URL")
otp_parser.add_argument("--auth-name", required=True,
metavar="OTP-NAME")
otp_parser.add_argument("--auth-realm", required=True,
metavar="OTP-REALM")
otp_parser.add_argument("--auth-seed", required=True,
metavar="OTP-SEED")
network_parser = configure_subparsers.add_parser("network")
network_parser.add_argument("--prefix-length", required=True, type=int)
network_parser.add_argument("--prefix", required=True)
network_parser.add_argument("--vxlan-phy-dev", required=True)
netbox_parser = configure_subparsers.add_parser("netbox")
netbox_parser.add_argument("--url", required=True)
netbox_parser.add_argument("--token", required=True)
ssh_parser = configure_subparsers.add_parser("ssh")
ssh_parser.add_argument('--username', default="root")
ssh_parser.add_argument('--private-key-path',
default=os.path.expanduser("~/.ssh/id_rsa"))
storage_parser = configure_subparsers.add_parser("storage")
storage_parser.add_argument('--file-dir', required=True)
storage_parser_subparsers = storage_parser.add_subparsers(dest="storage_backend")
filesystem_storage_parser = storage_parser_subparsers.add_parser("filesystem")
filesystem_storage_parser.add_argument('--vm-dir', required=True)
filesystem_storage_parser.add_argument('--image-dir', required=True)
ceph_storage_parser = storage_parser_subparsers.add_parser("ceph")
ceph_storage_parser.add_argument('--ceph-vm-pool', required=True)
ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
def main(**kwargs):
subcommand = kwargs.pop('subcommand')
if not subcommand:
pass
else:
update_config(subcommand, kwargs)

1
ucloud/filescanner/main.py
View File

@ -4,6 +4,7 @@ import pathlib
import subprocess as sp
import time
import sys
from uuid import uuid4
from . import logger

10
ucloud/host/main.py
View File

@ -8,17 +8,16 @@ from ucloud.common.etcd_wrapper import Etcd3Wrapper
from ucloud.common.request import RequestEntry, RequestType
from ucloud.config import (vm_pool, request_pool,
etcd_client, running_vms,
etcd_wrapper_args, etcd_wrapper_kwargs,
HostPool, config)
from .helper import find_free_port
from . import virtualmachine
from ucloud.host import logger
from . import virtualmachine, logger
def update_heartbeat(hostname):
"""Update Last HeartBeat Time for :param hostname: in etcd"""
client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
client = config.get_etcd_client()
host_pool = HostPool(client, config['etcd']['host_prefix'])
this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
@ -73,7 +72,7 @@ def maintenance(host):
running_vms.remove(_vm)
def check():
if config['storage']['backend'] == 'filesystem' and \
if config['storage']['storage_backend'] == 'filesystem' and \
not isdir(config['storage']['vm_dir']):
print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned"
@ -81,7 +80,6 @@ def check():
sys.exit(1)
def main(hostname):
check()

10
ucloud/imagescanner/main.py
View File

@ -1,7 +1,9 @@
import json
import os
import subprocess
import sys
from os.path import isdir
from os.path import join as join_path
from ucloud.config import etcd_client, config, image_storage_handler
from ucloud.imagescanner import logger
@ -20,10 +22,10 @@ def qemu_img_type(path):
def check():
""" check whether settings are sane, refuse to start if they aren't """
if config['storage']['backend'] == 'filesystem' and not isdir(config['storage']['image_dir']):
print("You have set STORAGE_BACKEND to filesystem, but "
"{} does not exist. Refusing to start".format(config['storage']['image_dir']))
sys.exit(1)
if config['storage']['storage_backend'] == 'filesystem' and not isdir(config['storage']['image_dir']):
sys.exit("You have set STORAGE_BACKEND to filesystem, but "
"{} does not exist. Refusing to start".format(config['storage']['image_dir'])
)
try:
subprocess.check_output(['which', 'qemu-img'])

2
ucloud/scheduler/main.py
View File

@ -13,8 +13,6 @@ from . import logger
def main():
logger.info("%s SESSION STARTED %s", '*' * 5, '*' * 5)
pending_vms = []
for request_iterator in [

86
ucloud/settings/__init__.py Normal file
View File

@ -0,0 +1,86 @@
import configparser
import logging
import sys
import os
from ucloud.common.etcd_wrapper import Etcd3Wrapper
logger = logging.getLogger(__name__)
class CustomConfigParser(configparser.RawConfigParser):
def __getitem__(self, key):
try:
result = super().__getitem__(key)
except KeyError as err:
raise KeyError("Key '{}' not found in config file"\
.format(key)) from err
else:
return result
class Settings(object):
def __init__(self, config_key='/uncloud/config/'):
conf_name = 'ucloud.conf'
conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud')
config_file = os.path.join(conf_dir, conf_name)
self.config_parser = CustomConfigParser(allow_no_value=True)
self.config_key = config_key
self.read_internal_values()
self.read_config_file_values(config_file)
self.etcd_wrapper_args = tuple()
self.etcd_wrapper_kwargs = {
'host': self.config_parser['etcd']['url'],
'port': self.config_parser['etcd']['port'],
'ca_cert': self.config_parser['etcd']['ca_cert'],
'cert_cert': self.config_parser['etcd']['cert_cert'],
'cert_key': self.config_parser['etcd']['cert_key']
}
def get_etcd_client(self):
args = self.etcd_wrapper_args
kwargs = self.etcd_wrapper_kwargs
return Etcd3Wrapper(*args, **kwargs)
def read_internal_values(self):
self.config_parser.read_dict({
'etcd': {
'file_prefix': '/files/',
'host_prefix': '/hosts/',
'image_prefix': '/images/',
'image_store_prefix': '/imagestore/',
'network_prefix': '/networks/',
'request_prefix': '/requests/',
'user_prefix': '/users/',
'vm_prefix': '/vms/',
}
})
def read_config_file_values(self, config_file):
try:
# Trying to read configuration file
with open(config_file, "r") as config_file_handle:
self.config_parser.read_file(config_file_handle)
except FileNotFoundError:
sys.exit('Configuration file {} not found!'.format(config_file))
except Exception as err:
logger.exception(err)
sys.exit("Error occurred while reading configuration file")
def read_values_from_etcd(self):
etcd_client = self.get_etcd_client()
config_from_etcd = etcd_client.get(self.config_key, value_in_json=True)
if config_from_etcd:
self.config_parser.read_dict(config_from_etcd.value)
else:
return
sys.exit("No settings found in etcd at key {}".format(self.config_key))
def __getitem__(self, key):
self.read_values_from_etcd()
return self.config_parser[key]