From b38c9b60606fe6c143fa30b82009e0b23b1ed6bb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 9 Feb 2020 19:27:24 +0100 Subject: [PATCH] Ad capability to add and list hosts --- uncloud/hack/db.py | 12 +++++++ uncloud/hack/host.py | 75 +++++++++++++++++++++++++++++++++++++++++ uncloud/hack/main.py | 28 +++++++++++---- uncloud/hack/product.py | 69 ++++++++++++++++++++++++++++++------- uncloud/hack/vm.py | 13 ++++--- 5 files changed, 173 insertions(+), 24 deletions(-) create mode 100644 uncloud/hack/host.py diff --git a/uncloud/hack/db.py b/uncloud/hack/db.py index a4395de..3d5582e 100644 --- a/uncloud/hack/db.py +++ b/uncloud/hack/db.py @@ -24,6 +24,7 @@ import etcd3 import json import logging import datetime +import re from functools import wraps from uncloud import UncloudException @@ -108,6 +109,17 @@ class DB(object): return self._db_clients[0].put(self.realkey(key), value, **kwargs) + @readable_errors + def list_and_filter(self, key, filter_key=None, filter_regexp=None): + for k,v in self.get_prefix(key, as_json=True): + + if filter_key and filter_regexp: + if filter_key in v: + if re.match(filter_regexp, v[filter_key]): + yield v + else: + yield v + @readable_errors def increment(self, key, **kwargs): diff --git a/uncloud/hack/host.py b/uncloud/hack/host.py new file mode 100644 index 0000000..06ccf98 --- /dev/null +++ b/uncloud/hack/host.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +import uuid + +from uncloud.hack.db import DB +from uncloud import UncloudException + +class Host(object): + def __init__(self, config, db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/hosts") + + if db_entry: + self.db_entry = db_entry + + + def list_hosts(self, filter_key=None, filter_regexp=None): + """ Return list of all hosts """ + for entry in self.db.list_and_filter("", filter_key, filter_regexp): + yield self.__class__(self.config, db_entry=entry) + + def cmdline_add_host(self): + """ FIXME: make this a bit smarter and less redundant """ + + for required_arg in [ + 'add_vm_host', + 'max_cores_per_vm', + 'max_cores_total', + 'max_memory_in_gb' ]: + if not required_arg in self.config.arguments: + raise UncloudException("Missing argument: {}".format(required_arg)) + + return self.add_host( + self.config.arguments['add_vm_host'], + self.config.arguments['max_cores_per_vm'], + self.config.arguments['max_cores_total'], + self.config.arguments['max_memory_in_gb']) + + + def add_host(self, + hostname, + max_cores_per_vm, + max_cores_total, + max_memory_in_gb): + + db_entry = {} + db_entry['uuid'] = str(uuid.uuid4()) + db_entry['hostname'] = hostname + db_entry['max_cores_per_vm'] = max_cores_per_vm + db_entry['max_cores_total'] = max_cores_total + db_entry['max_memory_in_gb'] = max_memory_in_gb + db_entry["db_version"] = 1 + db_entry["log"] = [] + + self.db.set(db_entry['uuid'], db_entry, as_json=True) + + return self.__class__(self.config, db_entry) diff --git a/uncloud/hack/main.py b/uncloud/hack/main.py index c454b03..0ddd8fb 100644 --- a/uncloud/hack/main.py +++ b/uncloud/hack/main.py @@ -6,6 +6,7 @@ import ldap3 from uncloud.hack.vm import VM +from uncloud.hack.host import Host from uncloud.hack.config import Config from uncloud.hack.mac import MAC from uncloud.hack.net import VXLANBridge, DNSRA @@ -64,6 +65,13 @@ arg_parser.add_argument('--mode', help="Directly manipulate etcd, spawn the API server or behave as a client") +arg_parser.add_argument('--add-vm-host', help="Add a host that can run VMs") +arg_parser.add_argument('--list-vm-hosts', action='store_true') + +arg_parser.add_argument('--max-cores-per-vm') +arg_parser.add_argument('--max-cores-total') +arg_parser.add_argument('--max-memory-in-gb') + log = logging.getLogger(__name__) @@ -95,20 +103,28 @@ def order(config): # create DB entry for VM vm = VM(config) - vm.product.db_entry["owner"] = config.arguments['username'] - vm.product.place_order() + return vm.product.place_order(owner=config.arguments['username']) + + def main(arguments): config = Config(arguments) - if arguments['api']: - api = API() - api.run() + if arguments['add_vm_host']: + h = Host(config) + h.cmdline_add_host() + + if arguments['list_vm_hosts']: + h = Host(config) + + for host in h.list_hosts(filter_key=arguments['filter_order_key'], + filter_regexp=arguments['filter_order_regexp']): + print("Host {}: {}".format(host.db_entry['uuid'], host.db_entry)) if arguments['order']: - order(config) + print("Created order: {}".format(order(config))) if arguments['list_orders']: p = ProductOrder(config) diff --git a/uncloud/hack/product.py b/uncloud/hack/product.py index 668b8ea..f979268 100755 --- a/uncloud/hack/product.py +++ b/uncloud/hack/product.py @@ -22,6 +22,7 @@ import json import uuid import logging import re +import importlib from uncloud import UncloudException from uncloud.hack.db import DB, db_logentry @@ -41,16 +42,9 @@ class ProductOrder(object): # FIXME: this should return a list of our class! def list_orders(self, filter_key=None, filter_regexp=None): - """List all orders with - filtering not yet implemented """ + for entry in self.db.list_and_filter("", filter_key, filter_regexp): + yield self.__class__(self.config, db_entry=entry) - for k,v in self.db.get_prefix("", as_json=True): - log.debug("{} {}".format(k,v)) - if filter_key and filter_regexp: - if filter_key in v: - if re.match(filter_regexp, v[filter_key]): - yield self.__class__(self.config, db_entry=v) - else: - yield self.__class__(self.config, db_entry=v) def set_required_values(self): """Set values that are required to make the db entry valid""" @@ -76,10 +70,15 @@ class ProductOrder(object): return True def order(self): + self.set_required_values() if not self.db_entry["status"] == "NEW": raise UncloudException("Cannot re-order same order. Status: {}".format(self.db_entry["status"])) + self.db.set(self.db_entry["uuid"], self.db_entry, as_json=True) + + return self.db_entry["uuid"] def process_orders(self): + """processing orders can be done stand alone on server side""" for order in self.list_orders(): if order.db_entry["status"] == "NEW": log.info("Handling new order: {}".format(order)) @@ -88,12 +87,53 @@ class ProductOrder(object): if not "log" in order.db_entry: order.db_entry['log'] = [] + is_valid = True + # Verify the order entry for must_attribute in [ "owner", "product" ]: if not must_attribute in order.db_entry: - order.db_entry['log'].append(db_logentry("Missing {} entry, rejecting order".format(must_attribute))) + message = "Missing {} entry in order, rejecting order".format(must_attribute) + log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message)) + + order.db_entry['log'].append(db_logentry(message)) order.db_entry['status'] = "REJECTED" self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + is_valid = False + + # Rejected the order + if not is_valid: + continue + + # Verify the product entry + for must_attribute in [ "python_product_class", "python_product_module" ]: + if not must_attribute in order.db_entry['product']: + message = "Missing {} entry in product of order, rejecting order".format(must_attribute) + log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message)) + + order.db_entry['log'].append(db_logentry(message)) + order.db_entry['status'] = "REJECTED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + is_valid = False + + # Rejected the order + if not is_valid: + continue + + print(order.db_entry["product"]["python_product_class"]) + + # Create the product + m = importlib.import_module(order.db_entry["product"]["python_product_module"]) + c = getattr(m, order.db_entry["product"]["python_product_class"]) + + product = c(config, db_entry=order.db_entry["product"]) + + # STOPPED + product.create_product() + + order.db_entry['status'] = "SCHEDULED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + def __str__(self): @@ -103,12 +143,15 @@ class Product(object): def __init__(self, config, product_name, + product_class, db_entry=None): self.config = config self.db = DB(self.config, prefix="/orders") self.db_entry = {} self.db_entry["product_name"] = product_name + self.db_entry["python_product_class"] = product_class.__qualname__ + self.db_entry["python_product_module"] = product_class.__module__ self.db_entry["db_version"] = 1 self.db_entry["log"] = [] self.db_entry["features"] = {} @@ -153,11 +196,11 @@ class Product(object): for feature in self.db_entry['features']: pass - def place_order(self): + def place_order(self, owner): """ Schedule creating the product in etcd """ order = ProductOrder(self.config, product_entry=self.db_entry) - order.set_required_values() - order.order() + order.db_entry["owner"] = owner + return order.order() def __str__(self): return json.dumps(self.db_entry) diff --git a/uncloud/hack/vm.py b/uncloud/hack/vm.py index 6bbe29a..4b0ca14 100755 --- a/uncloud/hack/vm.py +++ b/uncloud/hack/vm.py @@ -47,7 +47,7 @@ log = logging.getLogger(__name__) log.setLevel(logging.DEBUG) class VM(object): - def __init__(self, config): + def __init__(self, config, db_entry=None): self.config = config #TODO: Enable etcd lookup @@ -55,6 +55,9 @@ class VM(object): if not self.no_db: self.db = DB(self.config, prefix="/vm") + if db_entry: + self.db_entry = db_entry + # General CLI arguments. self.hackprefix = self.config.arguments['hackprefix'] self.uuid = self.config.arguments['uuid'] @@ -89,7 +92,8 @@ class VM(object): self.vm = {} - self.product = Product(config, product_name="dualstack-vm") + self.product = Product(config, product_name="dualstack-vm", + product_class=self.__class__) self.product.define_feature(name="base", one_time_price=0, recurring_price=9, @@ -98,8 +102,6 @@ class VM(object): self.features = [] -# self.features.append(self.define_feature( -# self.super().__init__( def get_qemu_args(self): @@ -122,7 +124,8 @@ class VM(object): return command.split(" ") - def create_db_entry(self): + def create_product(self): + """Find a VM host and schedule on it""" pass def create(self):