forked from uncloud/uncloud
Configuration/Setting module added
This commit is contained in:
parent
71279a968f
commit
bc58a6ed9c
11 changed files with 217 additions and 106 deletions
|
@ -1,19 +1,3 @@
|
|||
[otp]
|
||||
server = https://otp.ungleich.ch/ungleichotp/
|
||||
verify_endpoint = verify/
|
||||
auth_name = replace_me
|
||||
auth_realm = replace_me
|
||||
auth_seed = replace_me
|
||||
|
||||
[network]
|
||||
prefix_length = 64
|
||||
prefix = 2001:db8::/48
|
||||
vxlan_phy_dev = eno1
|
||||
|
||||
[netbox]
|
||||
url = https://replace-me.example.com
|
||||
token = replace_me
|
||||
|
||||
[etcd]
|
||||
url = localhost
|
||||
port = 2379
|
||||
|
@ -21,33 +5,3 @@ port = 2379
|
|||
ca_cert
|
||||
cert_cert
|
||||
cert_key
|
||||
|
||||
file_prefix = /files/
|
||||
host_prefix = /hosts/
|
||||
image_prefix = /images/
|
||||
image_store_prefix = /imagestore/
|
||||
network_prefix = /networks/
|
||||
request_prefix = /requests/
|
||||
user_prefix = /users/
|
||||
vm_prefix = /vms/
|
||||
|
||||
[storage]
|
||||
|
||||
#values = filesystem, ceph
|
||||
backend = filesystem
|
||||
|
||||
# if STORAGE_BACKEND = filesystem
|
||||
vm_dir = /var/lib/ucloud/vms
|
||||
image_dir = /var/lib/ucloud/images
|
||||
|
||||
# if STORAGE_BACKEND = ceph
|
||||
ceph_vm_pool = ssd
|
||||
ceph_image_pool = ssd
|
||||
|
||||
# Importing uploaded files
|
||||
file_dir = /var/lib/ucloud/files
|
||||
|
||||
# For Migrating VMs over ssh/tcp
|
||||
[ssh]
|
||||
username
|
||||
private_key_path
|
|
@ -3,37 +3,60 @@
|
|||
import argparse
|
||||
import logging
|
||||
import importlib
|
||||
import sys
|
||||
import os
|
||||
import multiprocessing as mp
|
||||
import sys
|
||||
|
||||
from ucloud.configure.main import update_config, configure_parser
|
||||
|
||||
|
||||
COMMANDS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata']
|
||||
def exception_hook(exc_type, exc_value, exc_traceback):
|
||||
logger.error(
|
||||
"Uncaught exception",
|
||||
exc_info=(exc_type, exc_value, exc_traceback)
|
||||
)
|
||||
print(exc_type, exc_value)
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.DEBUG,
|
||||
format='%(pathname)s:%(lineno)d -- %(levelname)-8s %(message)s',
|
||||
filename='/var/log/ucloud.log', filemode='a')
|
||||
|
||||
logger = logging.getLogger("ucloud")
|
||||
|
||||
arg_parser = argparse.ArgumentParser(prog='ucloud',
|
||||
description='Open Source Cloud Management Software')
|
||||
arg_parser.add_argument('-c', '--conf-dir', help="Configuration directory")
|
||||
arg_parser.add_argument('component', choices=COMMANDS)
|
||||
arg_parser.add_argument('component_args', nargs='*')
|
||||
sys.excepthook = exception_hook
|
||||
mp.set_start_method('spawn')
|
||||
|
||||
arg_parser = argparse.ArgumentParser()
|
||||
subparsers = arg_parser.add_subparsers(dest="command")
|
||||
|
||||
api_parser = subparsers.add_parser("api")
|
||||
|
||||
host_parser = subparsers.add_parser("host")
|
||||
host_parser.add_argument("--hostname", required=True)
|
||||
|
||||
scheduler_parser = subparsers.add_parser("scheduler")
|
||||
|
||||
filescanner_parser = subparsers.add_parser("filescanner")
|
||||
|
||||
imagescanner_parser = subparsers.add_parser("imagescanner")
|
||||
|
||||
metadata_parser = subparsers.add_parser("metadata")
|
||||
|
||||
config_parser = subparsers.add_parser("configure")
|
||||
configure_parser(config_parser)
|
||||
args = arg_parser.parse_args()
|
||||
|
||||
if args.conf_dir:
|
||||
os.environ['UCLOUD_CONF_DIR'] = args.conf_dir
|
||||
if not args.command:
|
||||
arg_parser.print_help()
|
||||
else:
|
||||
arguments = vars(args)
|
||||
try:
|
||||
name = arguments.pop('command')
|
||||
mod = importlib.import_module("ucloud.{}.main".format(name))
|
||||
main = getattr(mod, "main")
|
||||
main(**arguments)
|
||||
|
||||
try:
|
||||
mp.set_start_method('spawn')
|
||||
name = args.component
|
||||
mod = importlib.import_module("ucloud.{}.main".format(name))
|
||||
main = getattr(mod, "main")
|
||||
main(*args.component_args)
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
print(e)
|
||||
except Exception as e:
|
||||
logger.exception(e)
|
||||
print(e)
|
|
@ -29,11 +29,7 @@ def check_otp(name, realm, token):
|
|||
return 400
|
||||
|
||||
response = requests.post(
|
||||
"{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format(
|
||||
OTP_SERVER=config['otp']['server'],
|
||||
OTP_VERIFY_ENDPOINT=config['otp']['verify_endpoint']
|
||||
),
|
||||
json=data,
|
||||
config['otp']['verification_controller_url'], json=data
|
||||
)
|
||||
return response.status_code
|
||||
|
||||
|
|
|
@ -5,32 +5,18 @@ import logging
|
|||
from ucloud.common.host import HostPool
|
||||
from ucloud.common.request import RequestPool
|
||||
from ucloud.common.vm import VmPool
|
||||
from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler
|
||||
from ucloud.common.storage_handlers import (FileSystemBasedImageStorageHandler,
|
||||
CEPHBasedImageStorageHandler)
|
||||
from ucloud.common.etcd_wrapper import Etcd3Wrapper
|
||||
from ucloud.settings import Settings
|
||||
from os.path import join as join_path
|
||||
|
||||
log = logging.getLogger('ucloud.config')
|
||||
logger = logging.getLogger('ucloud.config')
|
||||
|
||||
conf_name = 'ucloud.conf'
|
||||
conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud')
|
||||
config_file = os.path.join(conf_dir, conf_name)
|
||||
|
||||
config = configparser.ConfigParser(allow_no_value=True)
|
||||
|
||||
if os.access(config_file, os.R_OK):
|
||||
config.read(config_file)
|
||||
else:
|
||||
log.warning('Configuration file not found - using defaults')
|
||||
|
||||
etcd_wrapper_args = ()
|
||||
etcd_wrapper_kwargs = {
|
||||
'host': config['etcd']['url'],
|
||||
'port': config['etcd']['port'],
|
||||
'ca_cert': config['etcd']['ca_cert'],
|
||||
'cert_cert': config['etcd']['cert_cert'],
|
||||
'cert_key': config['etcd']['cert_key']
|
||||
}
|
||||
|
||||
etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
|
||||
config = Settings()
|
||||
etcd_client = config.get_etcd_client()
|
||||
|
||||
host_pool = HostPool(etcd_client, config['etcd']['host_prefix'])
|
||||
vm_pool = VmPool(etcd_client, config['etcd']['vm_prefix'])
|
||||
|
@ -38,7 +24,7 @@ request_pool = RequestPool(etcd_client, config['etcd']['request_prefix'])
|
|||
|
||||
running_vms = []
|
||||
|
||||
__storage_backend = config['storage']['backend']
|
||||
__storage_backend = config['storage']['storage_backend']
|
||||
if __storage_backend == 'filesystem':
|
||||
image_storage_handler = FileSystemBasedImageStorageHandler(
|
||||
vm_base=config['storage']['vm_dir'],
|
||||
|
|
0
ucloud/configure/__init__.py
Normal file
0
ucloud/configure/__init__.py
Normal file
67
ucloud/configure/main.py
Normal file
67
ucloud/configure/main.py
Normal file
|
@ -0,0 +1,67 @@
|
|||
import argparse
|
||||
import sys
|
||||
import os
|
||||
|
||||
from ucloud.settings import Settings
|
||||
|
||||
config = Settings()
|
||||
etcd_client = config.get_etcd_client()
|
||||
|
||||
def update_config(section, kwargs):
|
||||
uncloud_config = etcd_client.get(config.config_key,
|
||||
value_in_json=True)
|
||||
if not uncloud_config:
|
||||
uncloud_config = {}
|
||||
else:
|
||||
uncloud_config = uncloud_config.value
|
||||
|
||||
uncloud_config[section] = kwargs
|
||||
etcd_client.put(config.config_key, uncloud_config, value_in_json=True)
|
||||
|
||||
|
||||
def configure_parser(parser):
|
||||
configure_subparsers = parser.add_subparsers(dest="subcommand")
|
||||
|
||||
otp_parser = configure_subparsers.add_parser("otp")
|
||||
otp_parser.add_argument("--verification-controller-url",
|
||||
required=True, metavar="URL")
|
||||
otp_parser.add_argument("--auth-name", required=True,
|
||||
metavar="OTP-NAME")
|
||||
otp_parser.add_argument("--auth-realm", required=True,
|
||||
metavar="OTP-REALM")
|
||||
otp_parser.add_argument("--auth-seed", required=True,
|
||||
metavar="OTP-SEED")
|
||||
|
||||
network_parser = configure_subparsers.add_parser("network")
|
||||
network_parser.add_argument("--prefix-length", required=True, type=int)
|
||||
network_parser.add_argument("--prefix", required=True)
|
||||
network_parser.add_argument("--vxlan-phy-dev", required=True)
|
||||
|
||||
netbox_parser = configure_subparsers.add_parser("netbox")
|
||||
netbox_parser.add_argument("--url", required=True)
|
||||
netbox_parser.add_argument("--token", required=True)
|
||||
|
||||
ssh_parser = configure_subparsers.add_parser("ssh")
|
||||
ssh_parser.add_argument('--username', default="root")
|
||||
ssh_parser.add_argument('--private-key-path',
|
||||
default=os.path.expanduser("~/.ssh/id_rsa"))
|
||||
|
||||
storage_parser = configure_subparsers.add_parser("storage")
|
||||
storage_parser.add_argument('--file-dir', required=True)
|
||||
storage_parser_subparsers = storage_parser.add_subparsers(dest="storage_backend")
|
||||
|
||||
filesystem_storage_parser = storage_parser_subparsers.add_parser("filesystem")
|
||||
filesystem_storage_parser.add_argument('--vm-dir', required=True)
|
||||
filesystem_storage_parser.add_argument('--image-dir', required=True)
|
||||
|
||||
ceph_storage_parser = storage_parser_subparsers.add_parser("ceph")
|
||||
ceph_storage_parser.add_argument('--ceph-vm-pool', required=True)
|
||||
ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
|
||||
|
||||
|
||||
def main(**kwargs):
|
||||
subcommand = kwargs.pop('subcommand')
|
||||
if not subcommand:
|
||||
pass
|
||||
else:
|
||||
update_config(subcommand, kwargs)
|
|
@ -4,6 +4,7 @@ import pathlib
|
|||
import subprocess as sp
|
||||
import time
|
||||
import sys
|
||||
|
||||
from uuid import uuid4
|
||||
|
||||
from . import logger
|
||||
|
|
|
@ -8,17 +8,16 @@ from ucloud.common.etcd_wrapper import Etcd3Wrapper
|
|||
from ucloud.common.request import RequestEntry, RequestType
|
||||
from ucloud.config import (vm_pool, request_pool,
|
||||
etcd_client, running_vms,
|
||||
etcd_wrapper_args, etcd_wrapper_kwargs,
|
||||
HostPool, config)
|
||||
|
||||
from .helper import find_free_port
|
||||
from . import virtualmachine
|
||||
from ucloud.host import logger
|
||||
from . import virtualmachine, logger
|
||||
|
||||
|
||||
def update_heartbeat(hostname):
|
||||
"""Update Last HeartBeat Time for :param hostname: in etcd"""
|
||||
client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs)
|
||||
|
||||
client = config.get_etcd_client()
|
||||
host_pool = HostPool(client, config['etcd']['host_prefix'])
|
||||
this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
|
||||
|
||||
|
@ -73,7 +72,7 @@ def maintenance(host):
|
|||
running_vms.remove(_vm)
|
||||
|
||||
def check():
|
||||
if config['storage']['backend'] == 'filesystem' and \
|
||||
if config['storage']['storage_backend'] == 'filesystem' and \
|
||||
not isdir(config['storage']['vm_dir']):
|
||||
|
||||
print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned"
|
||||
|
@ -81,7 +80,6 @@ def check():
|
|||
sys.exit(1)
|
||||
|
||||
|
||||
|
||||
def main(hostname):
|
||||
check()
|
||||
|
||||
|
|
|
@ -1,7 +1,9 @@
|
|||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
from os.path import isdir
|
||||
from os.path import join as join_path
|
||||
from ucloud.config import etcd_client, config, image_storage_handler
|
||||
from ucloud.imagescanner import logger
|
||||
|
@ -20,10 +22,10 @@ def qemu_img_type(path):
|
|||
|
||||
def check():
|
||||
""" check whether settings are sane, refuse to start if they aren't """
|
||||
if config['storage']['backend'] == 'filesystem' and not isdir(config['storage']['image_dir']):
|
||||
print("You have set STORAGE_BACKEND to filesystem, but "
|
||||
"{} does not exist. Refusing to start".format(config['storage']['image_dir']))
|
||||
sys.exit(1)
|
||||
if config['storage']['storage_backend'] == 'filesystem' and not isdir(config['storage']['image_dir']):
|
||||
sys.exit("You have set STORAGE_BACKEND to filesystem, but "
|
||||
"{} does not exist. Refusing to start".format(config['storage']['image_dir'])
|
||||
)
|
||||
|
||||
try:
|
||||
subprocess.check_output(['which', 'qemu-img'])
|
||||
|
|
|
@ -13,8 +13,6 @@ from . import logger
|
|||
|
||||
|
||||
def main():
|
||||
logger.info("%s SESSION STARTED %s", '*' * 5, '*' * 5)
|
||||
|
||||
pending_vms = []
|
||||
|
||||
for request_iterator in [
|
||||
|
|
86
ucloud/settings/__init__.py
Normal file
86
ucloud/settings/__init__.py
Normal file
|
@ -0,0 +1,86 @@
|
|||
import configparser
|
||||
import logging
|
||||
import sys
|
||||
import os
|
||||
|
||||
from ucloud.common.etcd_wrapper import Etcd3Wrapper
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class CustomConfigParser(configparser.RawConfigParser):
|
||||
def __getitem__(self, key):
|
||||
try:
|
||||
result = super().__getitem__(key)
|
||||
except KeyError as err:
|
||||
raise KeyError("Key '{}' not found in config file"\
|
||||
.format(key)) from err
|
||||
else:
|
||||
return result
|
||||
|
||||
|
||||
class Settings(object):
|
||||
def __init__(self, config_key='/uncloud/config/'):
|
||||
conf_name = 'ucloud.conf'
|
||||
conf_dir = os.environ.get('UCLOUD_CONF_DIR', '/etc/ucloud')
|
||||
config_file = os.path.join(conf_dir, conf_name)
|
||||
|
||||
self.config_parser = CustomConfigParser(allow_no_value=True)
|
||||
self.config_key = config_key
|
||||
|
||||
self.read_internal_values()
|
||||
self.read_config_file_values(config_file)
|
||||
|
||||
self.etcd_wrapper_args = tuple()
|
||||
self.etcd_wrapper_kwargs = {
|
||||
'host': self.config_parser['etcd']['url'],
|
||||
'port': self.config_parser['etcd']['port'],
|
||||
'ca_cert': self.config_parser['etcd']['ca_cert'],
|
||||
'cert_cert': self.config_parser['etcd']['cert_cert'],
|
||||
'cert_key': self.config_parser['etcd']['cert_key']
|
||||
}
|
||||
|
||||
|
||||
def get_etcd_client(self):
|
||||
args = self.etcd_wrapper_args
|
||||
kwargs = self.etcd_wrapper_kwargs
|
||||
return Etcd3Wrapper(*args, **kwargs)
|
||||
|
||||
def read_internal_values(self):
|
||||
self.config_parser.read_dict({
|
||||
'etcd': {
|
||||
'file_prefix': '/files/',
|
||||
'host_prefix': '/hosts/',
|
||||
'image_prefix': '/images/',
|
||||
'image_store_prefix': '/imagestore/',
|
||||
'network_prefix': '/networks/',
|
||||
'request_prefix': '/requests/',
|
||||
'user_prefix': '/users/',
|
||||
'vm_prefix': '/vms/',
|
||||
}
|
||||
})
|
||||
|
||||
def read_config_file_values(self, config_file):
|
||||
try:
|
||||
# Trying to read configuration file
|
||||
with open(config_file, "r") as config_file_handle:
|
||||
self.config_parser.read_file(config_file_handle)
|
||||
except FileNotFoundError:
|
||||
sys.exit('Configuration file {} not found!'.format(config_file))
|
||||
except Exception as err:
|
||||
logger.exception(err)
|
||||
sys.exit("Error occurred while reading configuration file")
|
||||
|
||||
def read_values_from_etcd(self):
|
||||
etcd_client = self.get_etcd_client()
|
||||
config_from_etcd = etcd_client.get(self.config_key, value_in_json=True)
|
||||
if config_from_etcd:
|
||||
self.config_parser.read_dict(config_from_etcd.value)
|
||||
else:
|
||||
return
|
||||
sys.exit("No settings found in etcd at key {}".format(self.config_key))
|
||||
|
||||
def __getitem__(self, key):
|
||||
self.read_values_from_etcd()
|
||||
return self.config_parser[key]
|
Loading…
Reference in a new issue