Ad capability to add and list hosts

This commit is contained in:
Nico Schottelius 2020-02-09 19:27:24 +01:00
parent a80a279ba5
commit b38c9b6060
5 changed files with 173 additions and 24 deletions

View file

@ -24,6 +24,7 @@ import etcd3
import json import json
import logging import logging
import datetime import datetime
import re
from functools import wraps from functools import wraps
from uncloud import UncloudException from uncloud import UncloudException
@ -108,6 +109,17 @@ class DB(object):
return self._db_clients[0].put(self.realkey(key), value, **kwargs) return self._db_clients[0].put(self.realkey(key), value, **kwargs)
@readable_errors
def list_and_filter(self, key, filter_key=None, filter_regexp=None):
for k,v in self.get_prefix(key, as_json=True):
if filter_key and filter_regexp:
if filter_key in v:
if re.match(filter_regexp, v[filter_key]):
yield v
else:
yield v
@readable_errors @readable_errors
def increment(self, key, **kwargs): def increment(self, key, **kwargs):

75
uncloud/hack/host.py Normal file
View file

@ -0,0 +1,75 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
#
# This file is part of uncloud.
#
# uncloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# uncloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
import uuid
from uncloud.hack.db import DB
from uncloud import UncloudException
class Host(object):
def __init__(self, config, db_entry=None):
self.config = config
self.db = DB(self.config, prefix="/hosts")
if db_entry:
self.db_entry = db_entry
def list_hosts(self, filter_key=None, filter_regexp=None):
""" Return list of all hosts """
for entry in self.db.list_and_filter("", filter_key, filter_regexp):
yield self.__class__(self.config, db_entry=entry)
def cmdline_add_host(self):
""" FIXME: make this a bit smarter and less redundant """
for required_arg in [
'add_vm_host',
'max_cores_per_vm',
'max_cores_total',
'max_memory_in_gb' ]:
if not required_arg in self.config.arguments:
raise UncloudException("Missing argument: {}".format(required_arg))
return self.add_host(
self.config.arguments['add_vm_host'],
self.config.arguments['max_cores_per_vm'],
self.config.arguments['max_cores_total'],
self.config.arguments['max_memory_in_gb'])
def add_host(self,
hostname,
max_cores_per_vm,
max_cores_total,
max_memory_in_gb):
db_entry = {}
db_entry['uuid'] = str(uuid.uuid4())
db_entry['hostname'] = hostname
db_entry['max_cores_per_vm'] = max_cores_per_vm
db_entry['max_cores_total'] = max_cores_total
db_entry['max_memory_in_gb'] = max_memory_in_gb
db_entry["db_version"] = 1
db_entry["log"] = []
self.db.set(db_entry['uuid'], db_entry, as_json=True)
return self.__class__(self.config, db_entry)

View file

@ -6,6 +6,7 @@ import ldap3
from uncloud.hack.vm import VM from uncloud.hack.vm import VM
from uncloud.hack.host import Host
from uncloud.hack.config import Config from uncloud.hack.config import Config
from uncloud.hack.mac import MAC from uncloud.hack.mac import MAC
from uncloud.hack.net import VXLANBridge, DNSRA from uncloud.hack.net import VXLANBridge, DNSRA
@ -64,6 +65,13 @@ arg_parser.add_argument('--mode',
help="Directly manipulate etcd, spawn the API server or behave as a client") help="Directly manipulate etcd, spawn the API server or behave as a client")
arg_parser.add_argument('--add-vm-host', help="Add a host that can run VMs")
arg_parser.add_argument('--list-vm-hosts', action='store_true')
arg_parser.add_argument('--max-cores-per-vm')
arg_parser.add_argument('--max-cores-total')
arg_parser.add_argument('--max-memory-in-gb')
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@ -95,20 +103,28 @@ def order(config):
# create DB entry for VM # create DB entry for VM
vm = VM(config) vm = VM(config)
vm.product.db_entry["owner"] = config.arguments['username'] return vm.product.place_order(owner=config.arguments['username'])
vm.product.place_order()
def main(arguments): def main(arguments):
config = Config(arguments) config = Config(arguments)
if arguments['api']: if arguments['add_vm_host']:
api = API() h = Host(config)
api.run() h.cmdline_add_host()
if arguments['list_vm_hosts']:
h = Host(config)
for host in h.list_hosts(filter_key=arguments['filter_order_key'],
filter_regexp=arguments['filter_order_regexp']):
print("Host {}: {}".format(host.db_entry['uuid'], host.db_entry))
if arguments['order']: if arguments['order']:
order(config) print("Created order: {}".format(order(config)))
if arguments['list_orders']: if arguments['list_orders']:
p = ProductOrder(config) p = ProductOrder(config)

View file

@ -22,6 +22,7 @@ import json
import uuid import uuid
import logging import logging
import re import re
import importlib
from uncloud import UncloudException from uncloud import UncloudException
from uncloud.hack.db import DB, db_logentry from uncloud.hack.db import DB, db_logentry
@ -41,16 +42,9 @@ class ProductOrder(object):
# FIXME: this should return a list of our class! # FIXME: this should return a list of our class!
def list_orders(self, filter_key=None, filter_regexp=None): def list_orders(self, filter_key=None, filter_regexp=None):
"""List all orders with - filtering not yet implemented """ for entry in self.db.list_and_filter("", filter_key, filter_regexp):
yield self.__class__(self.config, db_entry=entry)
for k,v in self.db.get_prefix("", as_json=True):
log.debug("{} {}".format(k,v))
if filter_key and filter_regexp:
if filter_key in v:
if re.match(filter_regexp, v[filter_key]):
yield self.__class__(self.config, db_entry=v)
else:
yield self.__class__(self.config, db_entry=v)
def set_required_values(self): def set_required_values(self):
"""Set values that are required to make the db entry valid""" """Set values that are required to make the db entry valid"""
@ -76,10 +70,15 @@ class ProductOrder(object):
return True return True
def order(self): def order(self):
self.set_required_values()
if not self.db_entry["status"] == "NEW": if not self.db_entry["status"] == "NEW":
raise UncloudException("Cannot re-order same order. Status: {}".format(self.db_entry["status"])) raise UncloudException("Cannot re-order same order. Status: {}".format(self.db_entry["status"]))
self.db.set(self.db_entry["uuid"], self.db_entry, as_json=True)
return self.db_entry["uuid"]
def process_orders(self): def process_orders(self):
"""processing orders can be done stand alone on server side"""
for order in self.list_orders(): for order in self.list_orders():
if order.db_entry["status"] == "NEW": if order.db_entry["status"] == "NEW":
log.info("Handling new order: {}".format(order)) log.info("Handling new order: {}".format(order))
@ -88,12 +87,53 @@ class ProductOrder(object):
if not "log" in order.db_entry: if not "log" in order.db_entry:
order.db_entry['log'] = [] order.db_entry['log'] = []
is_valid = True
# Verify the order entry
for must_attribute in [ "owner", "product" ]: for must_attribute in [ "owner", "product" ]:
if not must_attribute in order.db_entry: if not must_attribute in order.db_entry:
order.db_entry['log'].append(db_logentry("Missing {} entry, rejecting order".format(must_attribute))) message = "Missing {} entry in order, rejecting order".format(must_attribute)
log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message))
order.db_entry['log'].append(db_logentry(message))
order.db_entry['status'] = "REJECTED" order.db_entry['status'] = "REJECTED"
self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True)
is_valid = False
# Rejected the order
if not is_valid:
continue
# Verify the product entry
for must_attribute in [ "python_product_class", "python_product_module" ]:
if not must_attribute in order.db_entry['product']:
message = "Missing {} entry in product of order, rejecting order".format(must_attribute)
log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message))
order.db_entry['log'].append(db_logentry(message))
order.db_entry['status'] = "REJECTED"
self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True)
is_valid = False
# Rejected the order
if not is_valid:
continue
print(order.db_entry["product"]["python_product_class"])
# Create the product
m = importlib.import_module(order.db_entry["product"]["python_product_module"])
c = getattr(m, order.db_entry["product"]["python_product_class"])
product = c(config, db_entry=order.db_entry["product"])
# STOPPED
product.create_product()
order.db_entry['status'] = "SCHEDULED"
self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True)
def __str__(self): def __str__(self):
@ -103,12 +143,15 @@ class Product(object):
def __init__(self, def __init__(self,
config, config,
product_name, product_name,
product_class,
db_entry=None): db_entry=None):
self.config = config self.config = config
self.db = DB(self.config, prefix="/orders") self.db = DB(self.config, prefix="/orders")
self.db_entry = {} self.db_entry = {}
self.db_entry["product_name"] = product_name self.db_entry["product_name"] = product_name
self.db_entry["python_product_class"] = product_class.__qualname__
self.db_entry["python_product_module"] = product_class.__module__
self.db_entry["db_version"] = 1 self.db_entry["db_version"] = 1
self.db_entry["log"] = [] self.db_entry["log"] = []
self.db_entry["features"] = {} self.db_entry["features"] = {}
@ -153,11 +196,11 @@ class Product(object):
for feature in self.db_entry['features']: for feature in self.db_entry['features']:
pass pass
def place_order(self): def place_order(self, owner):
""" Schedule creating the product in etcd """ """ Schedule creating the product in etcd """
order = ProductOrder(self.config, product_entry=self.db_entry) order = ProductOrder(self.config, product_entry=self.db_entry)
order.set_required_values() order.db_entry["owner"] = owner
order.order() return order.order()
def __str__(self): def __str__(self):
return json.dumps(self.db_entry) return json.dumps(self.db_entry)

View file

@ -47,7 +47,7 @@ log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG) log.setLevel(logging.DEBUG)
class VM(object): class VM(object):
def __init__(self, config): def __init__(self, config, db_entry=None):
self.config = config self.config = config
#TODO: Enable etcd lookup #TODO: Enable etcd lookup
@ -55,6 +55,9 @@ class VM(object):
if not self.no_db: if not self.no_db:
self.db = DB(self.config, prefix="/vm") self.db = DB(self.config, prefix="/vm")
if db_entry:
self.db_entry = db_entry
# General CLI arguments. # General CLI arguments.
self.hackprefix = self.config.arguments['hackprefix'] self.hackprefix = self.config.arguments['hackprefix']
self.uuid = self.config.arguments['uuid'] self.uuid = self.config.arguments['uuid']
@ -89,7 +92,8 @@ class VM(object):
self.vm = {} self.vm = {}
self.product = Product(config, product_name="dualstack-vm") self.product = Product(config, product_name="dualstack-vm",
product_class=self.__class__)
self.product.define_feature(name="base", self.product.define_feature(name="base",
one_time_price=0, one_time_price=0,
recurring_price=9, recurring_price=9,
@ -98,8 +102,6 @@ class VM(object):
self.features = [] self.features = []
# self.features.append(self.define_feature(
# self.super().__init__(
def get_qemu_args(self): def get_qemu_args(self):
@ -122,7 +124,8 @@ class VM(object):
return command.split(" ") return command.split(" ")
def create_db_entry(self): def create_product(self):
"""Find a VM host and schedule on it"""
pass pass
def create(self): def create(self):