diff --git a/.gitignore b/.gitignore index 82146fa..ab6a151 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,27 @@ -.idea -.vscode +.idea/ +.vscode/ +__pycache__/ +pay.conf +log.txt +test.py +STRIPE +venv/ -ucloud/docs/build +uncloud/docs/build logs.txt -ucloud.egg-info +uncloud.egg-info # run artefacts default.etcd __pycache__ # build artefacts -ucloud/version.py +uncloud/version.py build/ venv/ dist/ + +*.iso +*.sqlite3 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..e468591 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,18 @@ +stages: + - lint + - test + +run-tests: + stage: test + image: code.ungleich.ch:5050/uncloud/uncloud/uncloud-ci:latest + services: + - postgres:latest + variables: + DATABASE_HOST: postgres + DATABASE_USER: postgres + POSTGRES_HOST_AUTH_METHOD: trust + coverage: /^TOTAL.+?(\d+\%)$/ + script: + - pip install -r requirements.txt + - coverage run --source='.' ./manage.py test + - coverage report diff --git a/README.md b/README.md index 0e32f57..8c53654 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,62 @@ -# ucloud +# Uncloud -Checkout https://ungleich.ch/ucloud/ for the documentation of ucloud. +Cloud management platform, the ungleich way. + + +[![pipeline status](https://code.ungleich.ch/uncloud/uncloud/badges/master/pipeline.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master) +[![coverage report](https://code.ungleich.ch/uncloud/uncloud/badges/master/coverage.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master) + +## Useful commands + +* `./manage.py import-vat-rates path/to/csv` +* `./manage.py make-admin username` + +## Development setup + +Install system dependencies: + +* On Fedora, you will need the following packages: `python3-virtualenv python3-devel openldap-devel gcc chromium` + +NOTE: you will need to configure a LDAP server and credentials for authentication. See `uncloud/settings.py`. + +``` +# Initialize virtualenv. +» virtualenv .venv +Using base prefix '/usr' +New python executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python3 +Also creating executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python +Installing setuptools, pip, wheel... +done. + +# Enter virtualenv. +» source .venv/bin/activate + +# Install dependencies. +» pip install -r requirements.txt +[...] + +# Run migrations. +» ./manage.py migrate +Operations to perform: + Apply all migrations: admin, auth, contenttypes, opennebula, sessions, uncloud_auth, uncloud_net, uncloud_pay, uncloud_service, uncloud_vm +Running migrations: + [...] + +# Run webserver. +» ./manage.py runserver +Watching for file changes with StatReloader +Performing system checks... + +System check identified no issues (0 silenced). +May 07, 2020 - 10:17:08 +Django version 3.0.6, using settings 'uncloud.settings' +Starting development server at http://127.0.0.1:8000/ +Quit the server with CONTROL-C. +``` + +### Note on PGSQL + +If you want to use Postgres: + +* Install on configure PGSQL on your base system. +* OR use a container! `podman run --rm -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -it postgres:latest` diff --git a/archive/issues.org b/archive/issues.org new file mode 100644 index 0000000..840ec3c --- /dev/null +++ b/archive/issues.org @@ -0,0 +1,6 @@ +* Intro + This file lists issues that should be handled, are small and likely + not yet high prio. +* Issues +** TODO Register prefered address in User model +** TODO Allow to specify different recurring periods diff --git a/archive/uncloud_django_based/hacks/abk-hacks.py b/archive/uncloud_django_based/hacks/abk-hacks.py new file mode 100644 index 0000000..abc63d3 --- /dev/null +++ b/archive/uncloud_django_based/hacks/abk-hacks.py @@ -0,0 +1,55 @@ +""" +investigate into a simple python function that maps an ldap user to a vat percentage. Basically you need to +lookup the customer address, check if she is a business/registered tax number and if not apply the local +vat +""" + +import iso3166 +import datetime + +from csv import DictReader + + +def get_vat(street_address, city, postal_code, country, vat_number=None): + vat = { + 'Austria': [ + {'period': '1984-01-01/', 'rate': 0.2}, + {'period': '1976-01-01/1984-01-01', 'rate': 0.18}, + {'period': '1973-01-01/1976-01-01', 'rate': 0.16}, + ] + } + return iso3166.countries.get(country) + + # return iso3166.countries_by_name[country] + + +def main(): + # vat = get_vat( + # street_address='82 Nasheman-e-Iqbal near Wapda Town', + # city='Lahore', + # postal_code=53700, + # country='Pakistan', + # ) + # print(vat) + vat_rates = {} + with open('vat_rates.csv', newline='') as csvfile: + reader = DictReader(csvfile) + for row in reader: + territory_codes = row['territory_codes'].split('\n') + for code in territory_codes: + if code not in vat_rates: + vat_rates[code] = {} + + start_date = row['start_date'] + stop_data = row['stop_date'] + time_period = f'{start_date}|{stop_data}' + r = row.copy() + del r['start_date'] + del r['stop_date'] + del r['territory_codes'] + vat_rates[code][time_period] = r + print(vat_rates) + + +if __name__ == '__main__': + main() diff --git a/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py b/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py new file mode 100644 index 0000000..c0bbaf8 --- /dev/null +++ b/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py @@ -0,0 +1,46 @@ +import importlib +import sys +import os + +from os.path import join as join_path +from xmlrpc.client import ServerProxy as RPCClient + +root = os.path.dirname(os.getcwd()) +sys.path.append(join_path(root, 'uncloud')) +secrets = importlib.import_module('uncloud.secrets') + + +class OpenNebula: + def __init__(self, url, session_string): + self.session_string = session_string + self.client = RPCClient(secrets.OPENNEBULA_URL) + + def create_user(self, username, password, authentication_driver='', group_id=None): + # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-user-allocate + + if group_id is None: + group_id = [] + + return self.client.one.user.allocate( + self.session_string, + username, + password, + authentication_driver, + group_id + ) + + def chmod(self, vm_id, user_id=-1, group_id=-1): + # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-vm-chown + + return self.client.one.vm.chown(self.session_string, vm_id, user_id, group_id) + + +one = OpenNebula(secrets.OPENNEBULA_URL, secrets.OPENNEBULA_USER_PASS) + +# Create User in OpenNebula +# success, response, *_ = one.create_user(username='meow12345', password='hello_world') +# print(success, response) + +# Change owner of a VM +# success, response, *_ = one.chmod(vm_id=25589, user_id=706) +# print(success, response) diff --git a/archive/uncloud_django_based/hacks/command-wrapper.sh b/archive/uncloud_django_based/hacks/command-wrapper.sh new file mode 100644 index 0000000..d6ddd13 --- /dev/null +++ b/archive/uncloud_django_based/hacks/command-wrapper.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +dbhost=$1; shift + +ssh -L5432:localhost:5432 "$dbhost" & + +python manage.py "$@" + + + +# command only needs to be active while manage command is running + +# -T no pseudo terminal + + +# alternatively: commands output shell code + +# ssh uncloud@dbhost "python manage.py --hostname xxx ..." diff --git a/archive/uncloud_django_based/meow-payv1/README.md b/archive/uncloud_django_based/meow-payv1/README.md new file mode 100644 index 0000000..fe6a2a3 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/README.md @@ -0,0 +1,51 @@ +# uncloud-pay + +The generic product/payment system. + +## Installation + +```shell script +pip3 install -r requirements.txt +``` + +## Getting Started + +```shell script +python ucloud_pay.py +``` + +## Usage + +#### 1. Adding of products +```shell script +http --json http://[::]:5000/product/add username=your_username_here password=your_password_here specs:=@ipv6-only-vm.json +``` + +#### 2. Listing of products +```shell script +http --json http://[::]:5000/product/list +``` + +#### 3. Registering user's payment method (credit card for now using Stripe) + +```shell script +http --json http://[::]:5000/user/register_payment card_number=4111111111111111 cvc=123 expiry_year=2020 expiry_month=8 card_holder_name="The test user" username=your_username_here password=your_password_here line1="your_billing_address" city="your_city" country="your_country" +``` + +#### 4. Ordering products + +First of all, user have to buy the membership first. + +```shell script +http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=membership pay=True +``` + +```shell script +http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=ipv6-only-vm cpu=1 ram=1 os-disk-space=10 os=alpine pay=True +``` + +#### 5. Listing users orders + +```shell script +http --json POST http://[::]:5000/order/list username=your_username_here password=your_password_here +``` diff --git a/archive/uncloud_django_based/meow-payv1/config.py b/archive/uncloud_django_based/meow-payv1/config.py new file mode 100644 index 0000000..16804af --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/config.py @@ -0,0 +1,21 @@ +import os + +from ungleich_common.ldap.ldap_manager import LdapManager +from ungleich_common.std.configparser import StrictConfigParser +from ungleich_common.etcd.etcd_wrapper import EtcdWrapper + +config_file = os.environ.get('meow-pay-config-file', default='pay.conf') + +config = StrictConfigParser(allow_no_value=True) +config.read(config_file) + +etcd_client = EtcdWrapper( + host=config.get('etcd', 'host'), port=config.get('etcd', 'port'), + ca_cert=config.get('etcd', 'ca_cert'), cert_key=config.get('etcd', 'cert_key'), + cert_cert=config.get('etcd', 'cert_cert') +) + +ldap_manager = LdapManager( + server=config.get('ldap', 'server'), admin_dn=config.get('ldap', 'admin_dn'), + admin_password=config.get('ldap', 'admin_password') +) diff --git a/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py b/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py new file mode 100644 index 0000000..e6bfb43 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py @@ -0,0 +1,213 @@ +from flask import Flask, request +from flask_restful import Resource, Api +import etcd3 +import json +import logging +from functools import wraps + +from ldaptest import is_valid_ldap_user + +def authenticate(func): + @wraps(func) + def wrapper(*args, **kwargs): + if not getattr(func, 'authenticated', True): + return func(*args, **kwargs) + + # pass in username/password ! + acct = basic_authentication() # custom account lookup function + + if acct: + return func(*args, **kwargs) + + flask_restful.abort(401) + return wrapper + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + + +class DB(object): + def __init__(self, config, prefix="/"): + self.config = config + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + + self.connect() + + @readable_errors + def connect(self): + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) + + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) + + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: + value = json.dumps(value) + + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + +class Membership(Resource): + def __init__(self, config): + self.config = config + + def get(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + return {'message': 'Order successful' }, 200 + + def post(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + return {'message': 'Order 2x successful' }, 200 + + +class Order(Resource): + def __init__(self, config): + self.config = config + + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + +class Product(Resource): + def __init__(self, config): + self.config = config + + self.products = [] + self.products.append( + { "name": "membership-free", + "description": """ +This membership gives you access to the API and includes a VPN +with 1 IPv6 address. +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "a3883466-0012-4d01-80ff-cbf7469957af", + "recurring": True, + "recurring_time_frame": "per_year", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 0 + } + ] + } + ) + self.products.append( + { "name": "membership-standard", + "description": """ +This membership gives you access to the API and includes an IPv6-VPN with +one IPv6 address ("Road warrior") +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "1d85296b-0863-4dd6-a543-a6d5a4fbe4a6", + "recurring": True, + "recurring_time_frame": "per_month", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 5 + } + + ] + } + ) + self.products.append( + { "name": "membership-premium", + "description": """ +This membership gives you access to the API and includes an +IPv6-VPN with a /48 IPv6 network. +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "bfd63fd2-d227-436f-a8b8-600de74dd6ce", + "recurring": True, + "recurring_time_frame": "per_month", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 5 + } + + ] + } + ) + self.products.append( + { "name": "ipv6-vpn-with-/48", + "description": """ +An IPv6 VPN with a /48 network included. +""", + "uuid": "fe5753f8-6fe1-4dc4-9b73-7b803de4c597", + "recurring": True, + "recurring_time_frame": "per_year", + "features": [ + { "name": "vpn", + "price_one_time": 0, + "price_recurring": 120 + } + ] + } + ) + + + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + def get(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + return self.products + + + + + +if __name__ == '__main__': + app = Flask(__name__) + + config = {} + + config['etcd_url']="https://etcd1.ungleich.ch" + config['ldap_url']="ldaps://ldap1.ungleich.ch" + + api = Api(app) + api.add_resource(Order, '/orders', resource_class_args=( config, )) + api.add_resource(Product, '/products', resource_class_args=( config, )) + api.add_resource(Membership, '/membership', resource_class_args=( config, )) + + app.run(host='::', port=5000, debug=True) diff --git a/archive/uncloud_django_based/meow-payv1/helper.py b/archive/uncloud_django_based/meow-payv1/helper.py new file mode 100644 index 0000000..65a5155 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/helper.py @@ -0,0 +1,87 @@ +import logging + +import parsedatetime + +from datetime import datetime +from stripe_utils import StripeUtils + + +def get_plan_id_from_product(product): + plan_id = 'ucloud-v1-' + plan_id += product['name'].strip().replace(' ', '-') + return plan_id + + +def get_pricing(price_in_chf_cents, product_type, recurring_period): + if product_type == 'recurring': + return 'CHF {}/{}'.format(price_in_chf_cents/100, recurring_period) + elif product_type == 'one-time': + return 'CHF {} (One time charge)'.format(price_in_chf_cents/100) + + +def get_user_friendly_product(product_dict): + uf_product = { + 'name': product_dict['name'], + 'description': product_dict['description'], + 'product_id': product_dict['usable-id'], + 'pricing': get_pricing( + product_dict['price'], product_dict['type'], product_dict['recurring_period'] + ) + } + if product_dict['type'] == 'recurring': + uf_product['minimum_subscription_period'] = product_dict['minimum_subscription_period'] + return uf_product + + +def get_token(card_number, cvc, exp_month, exp_year): + stripe_utils = StripeUtils() + token_response = stripe_utils.get_token_from_card( + card_number, cvc, exp_month, exp_year + ) + if token_response['response_object']: + return token_response['response_object'].id + else: + return None + + +def resolve_product(usable_id, etcd_client): + products = etcd_client.get_prefix('/v1/products/', value_in_json=True) + for p in products: + if p.value['usable-id'] == usable_id: + return p.value + return None + + +def calculate_charges(specification, data): + logging.debug('Calculating charges for specs:{} and data:{}'.format(specification, data)) + one_time_charge = 0 + recurring_charge = 0 + for feature_name, feature_detail in specification['features'].items(): + if feature_detail['constant']: + data[feature_name] = 1 + + if feature_detail['unit']['type'] != 'str': + one_time_charge += feature_detail['one_time_fee'] + recurring_charge += ( + feature_detail['price_per_unit_per_period'] * data[feature_name] / + feature_detail['unit']['value'] + ) + return one_time_charge, recurring_charge + + +def is_order_valid(order_timestamp, renewal_period): + """ + Sample Code Usage + + >> current_datetime, status = cal.parse('Now') + >> current_datetime = datetime(*current_datetime[:6]) + + >> print('Is order valid: ', is_order_valid(current_datetime, '1 month')) + >> True + """ + cal = parsedatetime.Calendar() + + renewal_datetime, status = cal.parse(renewal_period) + renewal_datetime = datetime(*renewal_datetime[:6]) + + return order_timestamp <= renewal_datetime diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json new file mode 100644 index 0000000..110027a --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json @@ -0,0 +1,28 @@ +{ + "usable-id": "ipv6-only-django-hosting", + "active": true, + "name": "IPv6 Only Django Hosting", + "description": "Host your Django application on our shiny IPv6 Only VM", + "recurring_period": "month", + "quantity": "inf", + "features": { + "cpu": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 3, + "one_time_fee": 0, + "constant": false + }, + "ram": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 4, + "one_time_fee": 0, + "constant": false + }, + "os-disk-space": { + "unit": {"value": 10, "type":"int"}, + "one_time_fee": 0, + "price_per_unit_per_period": 3.5, + "constant": false + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json new file mode 100644 index 0000000..d07ad6c --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json @@ -0,0 +1,34 @@ +{ + "usable-id": "ipv6-only-vm", + "active": true, + "name": "IPv6 Only VM", + "description": "IPv6 Only VM are accessible to only those having IPv6 for themselves", + "recurring_period": "month", + "quantity": "inf", + "features": { + "cpu": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 3, + "one_time_fee": 0, + "constant": false + }, + "ram": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 4, + "one_time_fee": 0, + "constant": false + }, + "os-disk-space": { + "unit": {"value": 10, "type":"int"}, + "one_time_fee": 0, + "price_per_unit_per_period": 4, + "constant": false + }, + "os": { + "unit": {"value": 1, "type":"str"}, + "one_time_fee": 0, + "price_per_unit_per_period": 0, + "constant": false + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json new file mode 100644 index 0000000..38c6201 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json @@ -0,0 +1,16 @@ +{ + "usable-id": "ipv6-only-vpn", + "active": true, + "name": "IPv6 Only VPN", + "description": "IPv6 VPN enable you to access IPv6 only websites and more", + "recurring_period": "month", + "quantity": "inf", + "features": { + "vpn": { + "unit": {"value": 1, "type": "int"}, + "price_per_unit_per_period": 10, + "one_time_fee": 0, + "constant": true + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6box.json b/archive/uncloud_django_based/meow-payv1/products/ipv6box.json new file mode 100644 index 0000000..eca11f0 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6box.json @@ -0,0 +1,16 @@ +{ + "usable-id": "ipv6-box", + "active": true, + "name": "IPv6 Box", + "description": "A ready-to-go IPv6 Box: it creates a VPN to ungleich and distributes IPv6 addresses to all your computers.", + "recurring_period": "eternity", + "quantity": 4, + "features": { + "ipv6-box": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 0, + "one_time_fee": 250, + "constant": true + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/membership.json b/archive/uncloud_django_based/meow-payv1/products/membership.json new file mode 100644 index 0000000..4003330 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/membership.json @@ -0,0 +1,17 @@ +{ + "usable-id": "membership", + "active": true, + "name": "Membership", + "description": "Membership to use uncloud-pay", + "recurring_period": "month", + "quantity": "inf", + "features": { + "membership": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 5, + "one_time_fee": 0, + "constant": true + } + }, + "max_per_user": "1" +} diff --git a/archive/uncloud_django_based/meow-payv1/requirements.txt b/archive/uncloud_django_based/meow-payv1/requirements.txt new file mode 100644 index 0000000..0b758ca --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/requirements.txt @@ -0,0 +1,7 @@ +stripe +flask +Flask-RESTful +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-etcd&subdirectory=etcd +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-std&subdirectory=std +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-schemas&subdirectory=schemas diff --git a/archive/uncloud_django_based/meow-payv1/sample-pay.conf b/archive/uncloud_django_based/meow-payv1/sample-pay.conf new file mode 100644 index 0000000..5d1fe61 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/sample-pay.conf @@ -0,0 +1,17 @@ +[etcd] +host = 127.0.0.1 +port = 2379 +ca_cert +cert_cert +cert_key + +[stripe] +private_key=stripe_private_key + +[app] +port = 5000 + +[ldap] +server = ldap_server_url +admin_dn = ldap_admin_dn +admin_password = ldap_admin_password diff --git a/archive/uncloud_django_based/meow-payv1/schemas.py b/archive/uncloud_django_based/meow-payv1/schemas.py new file mode 100644 index 0000000..2e3aef7 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/schemas.py @@ -0,0 +1,136 @@ +import logging +import config +import json +import math + +from config import ldap_manager, etcd_client +from helper import resolve_product +from ungleich_common.schemas.schemas import BaseSchema, Field, ValidationException + + +class AddProductSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.add_schema(UserCredentialSchema, data) + self.specs = Field('specs', dict, **self.get(data, 'specs')) + self.update = Field('update', bool, **self.get(data, 'update', return_default=True, default=False)) + + def validation(self): + user = self.objects['user'] + user = json.loads(user.entry_to_json()) + uid, ou, *dc = user['dn'].replace('ou=', '').replace('dc=', '').replace('uid=', '').split(',') + if ou != config.config.get('ldap', 'internal_user_ou', fallback='users'): + raise ValidationException('You do not have access to create product.') + + product = resolve_product(self.specs.value['usable-id'], etcd_client) + if product: + self.objects['product'] = product + + +class AddressSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.line1 = Field('line1', str, **self.get(data, 'line1')) + self.line2 = Field('line2', str, **self.get(data, 'line2', return_default=True)) + self.city = Field('city', str, **self.get(data, 'city')) + self.country = Field('country', str, **self.get(data, 'country')) + self.state = Field('state', str, **self.get(data, 'state', return_default=True)) + self.postal_code = Field('postal_code', str, **self.get(data, 'postal_code', return_default=True)) + + +class UserRegisterPaymentSchema(BaseSchema): + def __init__(self, data): + super().__init__() + + self.add_schema(UserCredentialSchema, data) + self.add_schema(AddressSchema, data, under_field_name='address') + + self.card_number = Field('card_number', str, **self.get(data, 'card_number')) + self.cvc = Field('cvc', str, **self.get(data, 'cvc')) + self.expiry_year = Field('expiry_year', int, **self.get(data, 'expiry_year')) + self.expiry_month = Field('expiry_month', int, **self.get(data, 'expiry_month')) + self.card_holder_name = Field('card_holder_name', str, **self.get(data, 'card_holder_name')) + + +class UserCredentialSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.username = Field('username', str, **self.get(data, 'username')) + self.password = Field('password', str, **self.get(data, 'password')) + + def validation(self): + try: + entry = ldap_manager.is_password_valid(self.username.value, self.password.value, query_key='uid') + except ValueError: + raise ValidationException('No user with \'{}\' username found. You can create account at ' + 'https://account.ungleich.ch'.format(self.username.value)) + except Exception: + raise ValidationException('Invalid username/password.') + else: + self.objects['user'] = entry + + +class ProductOrderSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.product_id = Field( + 'product_id', str, **self.get(data, 'product_id'), validators=[self.product_id_validation] + ) + self.pay_consent = Field('pay', bool, **self.get(data, 'pay', return_default=True, default=False)) + self.add_schema(UserCredentialSchema, data) + + def product_id_validation(self): + product = resolve_product(self.product_id.value, etcd_client) + if product: + product['quantity'] = float(product['quantity']) + self.product_id.value = product['uuid'] + self.objects['product'] = product + logging.debug('Got product {}'.format(product)) + + if not product['active']: + raise ValidationException('Product is not active at the moment.') + + if product['quantity'] <= 0: + raise ValidationException('Out of stock.') + else: + raise ValidationException('No such product exists.') + + def validation(self): + username = self.objects['user'].uid + customer_previous_orders = etcd_client.get_prefix('/v1/user/{}'.format(username), value_in_json=True) + customer_previous_orders = [o.value for o in customer_previous_orders] + membership = next(filter(lambda o: o['product'] == 'membership', customer_previous_orders), None) + if membership is None and self.objects['product']['usable-id'] != 'membership': + raise ValidationException('Please buy membership first to use this facility') + max_quantity_user_can_order = float(self.objects['product'].get('max_per_user', math.inf)) + previous_order_of_same_product = [ + o for o in customer_previous_orders if o['product'] == self.objects['product']['usable-id'] + ] + if len(previous_order_of_same_product) >= max_quantity_user_can_order: + raise ValidationException( + 'You cannot buy {} more than {} times'.format( + self.objects['product']['name'], int(max_quantity_user_can_order) + ) + ) + + +class OrderListSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.add_schema(UserCredentialSchema, data) + + +def make_return_message(err, status_code=200): + logging.debug('message: {}'.format(str(err))) + return {'message': str(err)}, status_code + + +def create_schema(specification, data): + fields = {} + for feature_name, feature_detail in specification['features'].items(): + if not feature_detail['constant']: + fields[feature_name] = Field( + feature_name, eval(feature_detail['unit']['type']), **BaseSchema.get(data, feature_name) + ) + + return type('{}Schema'.format(specification['name']), (BaseSchema,), fields) diff --git a/archive/uncloud_django_based/meow-payv1/stripe_hack.py b/archive/uncloud_django_based/meow-payv1/stripe_hack.py new file mode 100644 index 0000000..f436c62 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/stripe_hack.py @@ -0,0 +1,7 @@ +import stripe_utils +import os + + +if __name__ == '__main__': + s = stripe_utils.StripeUtils(os.environ['STRIPE_PRIVATE_KEY']) + print(s.get_stripe_customer_from_email('coder.purple+2002@gmail.com')) diff --git a/archive/uncloud_django_based/meow-payv1/stripe_utils.py b/archive/uncloud_django_based/meow-payv1/stripe_utils.py new file mode 100644 index 0000000..6a2cd29 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/stripe_utils.py @@ -0,0 +1,491 @@ +import re +import stripe +import stripe.error +import logging + +from config import etcd_client as client, config as config + +stripe.api_key = config.get('stripe', 'private_key') + + +def handle_stripe_error(f): + def handle_problems(*args, **kwargs): + response = { + 'paid': False, + 'response_object': None, + 'error': None + } + + common_message = "Currently it's not possible to make payments." + try: + response_object = f(*args, **kwargs) + response = { + 'response_object': response_object, + 'error': None + } + return response + except stripe.error.CardError as e: + # Since it's a decline, stripe.error.CardError will be caught + body = e.json_body + err = body['error'] + response.update({'error': err['message']}) + logging.error(str(e)) + return response + except stripe.error.RateLimitError: + response.update( + {'error': "Too many requests made to the API too quickly"}) + return response + except stripe.error.InvalidRequestError as e: + logging.error(str(e)) + response.update({'error': "Invalid parameters"}) + return response + except stripe.error.AuthenticationError as e: + # Authentication with Stripe's API failed + # (maybe you changed API keys recently) + logging.error(str(e)) + response.update({'error': common_message}) + return response + except stripe.error.APIConnectionError as e: + logging.error(str(e)) + response.update({'error': common_message}) + return response + except stripe.error.StripeError as e: + # maybe send email + logging.error(str(e)) + response.update({'error': common_message}) + return response + except Exception as e: + # maybe send email + logging.error(str(e)) + response.update({'error': common_message}) + return response + + return handle_problems + + +class StripeUtils(object): + CURRENCY = 'chf' + INTERVAL = 'month' + SUCCEEDED_STATUS = 'succeeded' + STRIPE_PLAN_ALREADY_EXISTS = 'Plan already exists' + STRIPE_NO_SUCH_PLAN = 'No such plan' + PLAN_EXISTS_ERROR_MSG = 'Plan {} exists already.\nCreating a local StripePlan now.' + PLAN_DOES_NOT_EXIST_ERROR_MSG = 'Plan {} does not exist.' + + def __init__(self, private_key): + self.stripe = stripe + stripe.api_key = private_key + + @handle_stripe_error + def card_exists(self, customer, cc_number, exp_month, exp_year, cvc): + token_obj = stripe.Token.create( + card={ + 'number': cc_number, + 'exp_month': exp_month, + 'exp_year': exp_year, + 'cvc': cvc, + }, + ) + cards = stripe.Customer.list_sources( + customer, + limit=20, + object='card' + ) + + for card in cards.data: + if (card.fingerprint == token_obj.card.fingerprint and + int(card.exp_month) == int(exp_month) and int(card.exp_year) == int(exp_year)): + return True + return False + + @staticmethod + def get_stripe_customer_from_email(email): + customer = stripe.Customer.list(limit=1, email=email) + return customer.data[0] if len(customer.data) == 1 else None + + @staticmethod + def update_customer_token(customer, token): + customer.source = token + customer.save() + + @handle_stripe_error + def get_token_from_card(self, cc_number, cvc, expiry_month, expiry_year): + token_obj = stripe.Token.create( + card={ + 'number': cc_number, + 'exp_month': expiry_month, + 'exp_year': expiry_year, + 'cvc': cvc, + }, + ) + return token_obj + + @handle_stripe_error + def associate_customer_card(self, stripe_customer_id, token, + set_as_default=False): + customer = stripe.Customer.retrieve(stripe_customer_id) + card = customer.sources.create(source=token) + if set_as_default: + customer.default_source = card.id + customer.save() + return True + + @handle_stripe_error + def dissociate_customer_card(self, stripe_customer_id, card_id): + customer = stripe.Customer.retrieve(stripe_customer_id) + card = customer.sources.retrieve(card_id) + card.delete() + + @handle_stripe_error + def update_customer_card(self, customer_id, token): + customer = stripe.Customer.retrieve(customer_id) + current_card_token = customer.default_source + customer.sources.retrieve(current_card_token).delete() + customer.source = token + customer.save() + credit_card_raw_data = customer.sources.data.pop() + new_card_data = { + 'last4': credit_card_raw_data.last4, + 'brand': credit_card_raw_data.brand + } + return new_card_data + + @handle_stripe_error + def get_card_details(self, customer_id): + customer = stripe.Customer.retrieve(customer_id) + credit_card_raw_data = customer.sources.data.pop() + card_details = { + 'last4': credit_card_raw_data.last4, + 'brand': credit_card_raw_data.brand, + 'exp_month': credit_card_raw_data.exp_month, + 'exp_year': credit_card_raw_data.exp_year, + 'fingerprint': credit_card_raw_data.fingerprint, + 'card_id': credit_card_raw_data.id + } + return card_details + + @handle_stripe_error + def get_all_invoices(self, customer_id, created_gt): + return_list = [] + has_more_invoices = True + starting_after = False + while has_more_invoices: + if starting_after: + invoices = stripe.Invoice.list( + limit=10, customer=customer_id, created={'gt': created_gt}, + starting_after=starting_after + ) + else: + invoices = stripe.Invoice.list( + limit=10, customer=customer_id, created={'gt': created_gt} + ) + has_more_invoices = invoices.has_more + for invoice in invoices.data: + sub_ids = [] + for line in invoice.lines.data: + if line.type == 'subscription': + sub_ids.append(line.id) + elif line.type == 'invoiceitem': + sub_ids.append(line.subscription) + else: + sub_ids.append('') + invoice_details = { + 'created': invoice.created, + 'receipt_number': invoice.receipt_number, + 'invoice_number': invoice.number, + 'paid_at': invoice.status_transitions.paid_at if invoice.paid else 0, + 'period_start': invoice.period_start, + 'period_end': invoice.period_end, + 'billing_reason': invoice.billing_reason, + 'discount': invoice.discount.coupon.amount_off if invoice.discount else 0, + 'total': invoice.total, + # to see how many line items we have in this invoice and + # then later check if we have more than 1 + 'lines_data_count': len(invoice.lines.data) if invoice.lines.data is not None else 0, + 'invoice_id': invoice.id, + 'lines_meta_data_csv': ','.join( + [line.metadata.VM_ID if hasattr(line.metadata, 'VM_ID') else '' for line in invoice.lines.data] + ), + 'subscription_ids_csv': ','.join(sub_ids), + 'line_items': invoice.lines.data + } + starting_after = invoice.id + return_list.append(invoice_details) + return return_list + + @handle_stripe_error + def get_cards_details_from_token(self, token): + stripe_token = stripe.Token.retrieve(token) + card_details = { + 'last4': stripe_token.card.last4, + 'brand': stripe_token.card.brand, + 'exp_month': stripe_token.card.exp_month, + 'exp_year': stripe_token.card.exp_year, + 'fingerprint': stripe_token.card.fingerprint, + 'card_id': stripe_token.card.id + } + return card_details + + def check_customer(self, stripe_cus_api_id, user, token): + try: + customer = stripe.Customer.retrieve(stripe_cus_api_id) + except stripe.error.InvalidRequestError: + customer = self.create_customer(token, user.email, user.name) + user.stripecustomer.stripe_id = customer.get( + 'response_object').get('id') + user.stripecustomer.save() + if type(customer) is dict: + customer = customer['response_object'] + return customer + + @handle_stripe_error + def get_customer(self, stripe_api_cus_id): + customer = stripe.Customer.retrieve(stripe_api_cus_id) + # data = customer.get('response_object') + return customer + + @handle_stripe_error + def create_customer(self, token, email, name=None, address=None): + if name is None or name.strip() == "": + name = email + customer = self.stripe.Customer.create( + source=token, + description=name, + email=email, + address=address + ) + return customer + + @handle_stripe_error + def make_charge(self, amount=None, customer=None): + _amount = float(amount) + amount = int(_amount * 100) # stripe amount unit, in cents + charge = self.stripe.Charge.create( + amount=amount, # in cents + currency=self.CURRENCY, + customer=customer + ) + return charge + + @staticmethod + def _get_all_stripe_plans(): + all_stripe_plans = client.get("/v1/stripe_plans") + all_stripe_plans_set = set() + if all_stripe_plans: + all_stripe_plans_obj = all_stripe_plans.value + if all_stripe_plans_obj and len(all_stripe_plans_obj['plans']) > 0: + all_stripe_plans_set = set(all_stripe_plans_obj["plans"]) + return all_stripe_plans_set + + @staticmethod + def _save_all_stripe_plans(stripe_plans): + client.put("/v1/stripe_plans", {"plans": list(stripe_plans)}) + + @handle_stripe_error + def get_or_create_stripe_plan(self, product_name, amount, stripe_plan_id, + interval=INTERVAL): + """ + This function checks if a StripePlan with the given + stripe_plan_id already exists. If it exists then the function + returns this object otherwise it creates a new StripePlan and + returns the new object. + + :param amount: The amount in CHF cents + :param product_name: The name of the Stripe plan (product) to be created. + :param stripe_plan_id: The id of the Stripe plan to be + created. Use get_stripe_plan_id_string function to + obtain the name of the plan to be created + :param interval: The interval for subscription {month, year}. Defaults + to month if not provided + :return: The StripePlan object if it exists else creates a + Plan object in Stripe and a local StripePlan and + returns it. Returns None in case of Stripe error + """ + _amount = float(amount) + amount = int(_amount * 100) # stripe amount unit, in cents + all_stripe_plans = self._get_all_stripe_plans() + if stripe_plan_id in all_stripe_plans: + logging.debug("{} plan exists in db.".format(stripe_plan_id)) + else: + logging.debug(("{} plan DOES NOT exist in db. " + "Creating").format(stripe_plan_id)) + try: + plan_obj = self.stripe.Plan.retrieve(id=stripe_plan_id) + logging.debug("{} plan exists in Stripe".format(stripe_plan_id)) + all_stripe_plans.add(stripe_plan_id) + except stripe.error.InvalidRequestError as e: + if "No such plan" in str(e): + logging.debug("Plan {} does not exist in Stripe, Creating") + plan_obj = self.stripe.Plan.create( + amount=amount, + product={'name': product_name}, + interval=interval, + currency=self.CURRENCY, + id=stripe_plan_id) + logging.debug(self.PLAN_EXISTS_ERROR_MSG.format(stripe_plan_id)) + all_stripe_plans.add(stripe_plan_id) + self._save_all_stripe_plans(all_stripe_plans) + return stripe_plan_id + + @handle_stripe_error + def delete_stripe_plan(self, stripe_plan_id): + """ + Deletes the Plan in Stripe and also deletes the local db copy + of the plan if it exists + + :param stripe_plan_id: The stripe plan id that needs to be + deleted + :return: True if the plan was deleted successfully from + Stripe, False otherwise. + """ + return_value = False + try: + plan = self.stripe.Plan.retrieve(stripe_plan_id) + plan.delete() + return_value = True + all_stripe_plans = self._get_all_stripe_plans() + all_stripe_plans.remove(stripe_plan_id) + self._save_all_stripe_plans(all_stripe_plans) + except stripe.error.InvalidRequestError as e: + if self.STRIPE_NO_SUCH_PLAN in str(e): + logging.debug( + self.PLAN_DOES_NOT_EXIST_ERROR_MSG.format(stripe_plan_id)) + return return_value + + @handle_stripe_error + def subscribe_customer_to_plan(self, customer, plans, trial_end=None): + """ + Subscribes the given customer to the list of given plans + + :param customer: The stripe customer identifier + :param plans: A list of stripe plans. + :param trial_end: An integer representing when the Stripe subscription + is supposed to end + Ref: https://stripe.com/docs/api/python#create_subscription-items + e.g. + plans = [ + { + "plan": "dcl-v1-cpu-2-ram-5gb-ssd-10gb", + }, + ] + :return: The subscription StripeObject + """ + + subscription_result = self.stripe.Subscription.create( + customer=customer, items=plans, trial_end=trial_end + ) + return subscription_result + + @handle_stripe_error + def set_subscription_metadata(self, subscription_id, metadata): + subscription = stripe.Subscription.retrieve(subscription_id) + subscription.metadata = metadata + subscription.save() + + @handle_stripe_error + def unsubscribe_customer(self, subscription_id): + """ + Cancels a given subscription + + :param subscription_id: The Stripe subscription id string + :return: + """ + sub = stripe.Subscription.retrieve(subscription_id) + return sub.delete() + + @handle_stripe_error + def make_payment(self, customer, amount, token): + charge = self.stripe.Charge.create( + amount=amount, # in cents + currency=self.CURRENCY, + customer=customer + ) + return charge + + @staticmethod + def get_stripe_plan_id(cpu, ram, ssd, version, app='dcl', hdd=None, + price=None): + """ + Returns the Stripe plan id string of the form + `dcl-v1-cpu-2-ram-5gb-ssd-10gb` based on the input parameters + + :param cpu: The number of cores + :param ram: The size of the RAM in GB + :param ssd: The size of ssd storage in GB + :param hdd: The size of hdd storage in GB + :param version: The version of the Stripe plans + :param app: The application to which the stripe plan belongs + to. By default it is 'dcl' + :param price: The price for this plan + :return: A string of the form `dcl-v1-cpu-2-ram-5gb-ssd-10gb` + """ + dcl_plan_string = 'cpu-{cpu}-ram-{ram}gb-ssd-{ssd}gb'.format(cpu=cpu, + ram=ram, + ssd=ssd) + if hdd is not None: + dcl_plan_string = '{dcl_plan_string}-hdd-{hdd}gb'.format( + dcl_plan_string=dcl_plan_string, hdd=hdd) + stripe_plan_id_string = '{app}-v{version}-{plan}'.format( + app=app, + version=version, + plan=dcl_plan_string + ) + if price is not None: + stripe_plan_id_string_with_price = '{}-{}chf'.format( + stripe_plan_id_string, + round(price, 2) + ) + return stripe_plan_id_string_with_price + else: + return stripe_plan_id_string + + @staticmethod + def get_vm_config_from_stripe_id(stripe_id): + """ + Given a string like "dcl-v1-cpu-2-ram-5gb-ssd-10gb" return different + configuration params as a dict + + :param stripe_id|str + :return: dict + """ + pattern = re.compile(r'^dcl-v(\d+)-cpu-(\d+)-ram-(\d+\.?\d*)gb-ssd-(\d+)gb-?(\d*\.?\d*)(chf)?$') + match_res = pattern.match(stripe_id) + if match_res is not None: + price = None + try: + price = match_res.group(5) + except IndexError: + logging.debug("Did not find price in {}".format(stripe_id)) + return { + 'version': match_res.group(1), + 'cores': match_res.group(2), + 'ram': match_res.group(3), + 'ssd': match_res.group(4), + 'price': price + } + + @staticmethod + def get_stripe_plan_name(cpu, memory, disk_size, price): + """ + Returns the Stripe plan name + :return: + """ + return "{cpu} Cores, {memory} GB RAM, {disk_size} GB SSD, " \ + "{price} CHF".format( + cpu=cpu, + memory=memory, + disk_size=disk_size, + price=round(price, 2) + ) + + @handle_stripe_error + def set_subscription_meta_data(self, subscription_id, meta_data): + """ + Adds VM metadata to a subscription + :param subscription_id: Stripe identifier for the subscription + :param meta_data: A dict of meta data to be added + :return: + """ + subscription = stripe.Subscription.retrieve(subscription_id) + subscription.metadata = meta_data + subscription.save() diff --git a/archive/uncloud_django_based/meow-payv1/ucloud_pay.py b/archive/uncloud_django_based/meow-payv1/ucloud_pay.py new file mode 100644 index 0000000..dbc0d2c --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/ucloud_pay.py @@ -0,0 +1,338 @@ +import logging + +from datetime import datetime +from uuid import uuid4 + +from flask import Flask, request +from flask_restful import Resource, Api +from werkzeug.exceptions import HTTPException +from config import etcd_client as client, config as config +from stripe_utils import StripeUtils +from schemas import ( + make_return_message, ValidationException, UserRegisterPaymentSchema, + AddProductSchema, ProductOrderSchema, OrderListSchema, create_schema +) +from helper import get_plan_id_from_product, calculate_charges + + +class ListProducts(Resource): + @staticmethod + def get(): + products = client.get_prefix('/v1/products/') + products = [ + product + for product in [p.value for p in products] + if product['active'] + ] + prod_dict = {} + for p in products: + prod_dict[p['usable-id']] = { + 'name': p['name'], + 'description': p['description'], + } + logger.debug('Products = {}'.format(prod_dict)) + return prod_dict, 200 + +class AddProduct(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + logger.debug('Got data: {}'.format(str(data))) + validator = AddProductSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + previous_product = cleaned_values.get('product', None) + if previous_product: + if not cleaned_values['update']: + return make_return_message('Product already exists. Pass --update to update the product.') + else: + product_uuid = previous_product.pop('uuid') + else: + product_uuid = uuid4().hex + + product_value = cleaned_values['specs'] + + product_key = '/v1/products/{}'.format(product_uuid) + product_value['uuid'] = product_uuid + + logger.debug('Adding product data: {}'.format(str(product_value))) + client.put(product_key, product_value) + if not previous_product: + return make_return_message('Product created.') + else: + return make_return_message('Product updated.') + +################################################################################ +# Nico-ok-marker + + +class UserRegisterPayment(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + logger.debug('Got data: {}'.format(str(data))) + validator = UserRegisterPaymentSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + last4 = data['card_number'].strip()[-4:] + + stripe_utils = StripeUtils() + + # Does customer already exist ? + stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail']) + + # Does customer already exist ? + if stripe_customer is not None: + logger.debug('Customer {}-{} exists already'.format( + cleaned_values['username'], cleaned_values['user']['mail']) + ) + + # Check if the card already exists + ce_response = stripe_utils.card_exists( + stripe_customer.id, cc_number=data['card_number'], + exp_month=int(data['expiry_month']), + exp_year=int(data['expiry_year']), + cvc=data['cvc']) + + if ce_response['response_object']: + message = 'The given card ending in {} exists already.'.format(last4) + return make_return_message(message, 400) + + elif ce_response['response_object'] is False: + # Associate card with user + logger.debug('Adding card ending in {}'.format(last4)) + token_response = stripe_utils.get_token_from_card( + data['card_number'], data['cvc'], data['expiry_month'], + data['expiry_year'] + ) + if token_response['response_object']: + logger.debug('Token {}'.format(token_response['response_object'].id)) + resp = stripe_utils.associate_customer_card( + stripe_customer.id, token_response['response_object'].id + ) + if resp['response_object']: + return make_return_message( + 'Card ending in {} registered as your payment source'.format(last4) + ) + else: + return make_return_message('Error with payment gateway. Contact support', 400) + else: + return make_return_message('Error: {}'.format(ce_response['error']), 400) + else: + # Stripe customer does not exist, create a new one + logger.debug( + 'Customer {} does not exist, creating new'.format(cleaned_values['user']['mail']) + ) + token_response = stripe_utils.get_token_from_card( + cleaned_values['card_number'], cleaned_values['cvc'], + cleaned_values['expiry_month'], cleaned_values['expiry_year'] + ) + if token_response['response_object']: + logger.debug('Token {}'.format(token_response['response_object'].id)) + + # Create stripe customer + stripe_customer_resp = stripe_utils.create_customer( + name=cleaned_values['card_holder_name'], + token=token_response['response_object'].id, + email=cleaned_values['user']['mail'], + address=cleaned_values['address'] + ) + stripe_customer = stripe_customer_resp['response_object'] + + if stripe_customer: + logger.debug('Created stripe customer {}'.format(stripe_customer.id)) + return make_return_message( + 'Card ending in {} registered as your payment source'.format(last4) + ) + else: + return make_return_message('Error with card. Contact support', 400) + else: + return make_return_message('Error with payment gateway. Contact support', 400) + + +class ProductOrder(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + validator = ProductOrderSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + stripe_utils = StripeUtils() + + product = cleaned_values['product'] + + # Check the user has a payment source added + stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail']) + + if not stripe_customer or len(stripe_customer.sources) == 0: + return make_return_message('Please register your payment method first.', 400) + + try: + product_schema = create_schema(product, data) + product_schema = product_schema() + product_schema.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + transformed_data = product_schema.get_cleaned_values() + logger.debug('Tranformed data: {}'.format(transformed_data)) + one_time_charge, recurring_charge = calculate_charges(product, transformed_data) + recurring_charge = int(recurring_charge) + + if not cleaned_values['pay']: + return make_return_message( + 'You would be charged {} CHF one time and {} CHF every {}. ' + 'Add --pay to command to order.'.format( + one_time_charge, recurring_charge, product['recurring_period'] + ) + ) + + with client.client.lock('product-order') as _: + # Initiate a one-time/subscription based on product type + if recurring_charge > 0: + logger.debug('Product {} is recurring payment'.format(product['name'])) + plan_id = get_plan_id_from_product(product) + res = stripe_utils.get_or_create_stripe_plan( + product_name=product['name'], + stripe_plan_id=plan_id, amount=recurring_charge, + interval=product['recurring_period'], + ) + if res['response_object']: + logger.debug('Obtained plan {}'.format(plan_id)) + subscription_res = stripe_utils.subscribe_customer_to_plan( + stripe_customer.id, + [{'plan': plan_id}] + ) + subscription_obj = subscription_res['response_object'] + if subscription_obj is None or subscription_obj.status != 'active': + return make_return_message( + 'Error subscribing to plan. Detail: {}'.format(subscription_res['error']), 400 + ) + else: + order_obj = { + 'order-id': uuid4().hex, + 'ordered-at': datetime.now().isoformat(), + 'product': product['usable-id'], + 'one-time-price': one_time_charge, + 'recurring-price': recurring_charge, + 'recurring-period': product['recurring_period'] + } + client.put( + '/v1/user/{}/orders/{}'.format( + cleaned_values['username'], order_obj['order-id'] + ), order_obj + ) + product['quantity'] -= 1 + client.put('/v1/products/{}'.format(product['uuid']), product) + + return { + 'message': 'Order Successful.', + **order_obj + } + else: + logger.error('Could not create plan {}'.format(plan_id)) + return make_return_message('Something wrong happened. Contact administrator', 400) + + elif recurring_charge == 0 and one_time_charge > 0: + logger.debug('Product {} is one-time payment'.format(product['name'])) + charge_response = stripe_utils.make_charge( + amount=one_time_charge, + customer=stripe_customer.id + ) + stripe_onetime_charge = charge_response.get('response_object') + + # Check if the payment was approved + if not stripe_onetime_charge: + msg = charge_response.get('error') + return make_return_message('Error subscribing to plan. Details: {}'.format(msg), 400) + + order_obj = { + 'order-id': uuid4().hex, + 'ordered-at': datetime.now().isoformat(), + 'product': product['usable-id'], + 'one-time-price': one_time_charge, + } + client.put( + '/v1/user/{}/orders/{}'.format(cleaned_values['username'], order_obj['order-id']), + order_obj + ) + product['quantity'] -= 1 + client.put('/v1/products/{}'.format(product['uuid']), product) + + return {'message': 'Order successful', **order_obj}, 200 + + +class OrderList(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + validator = OrderListSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + orders = client.get_prefix('/v1/user/{}/orders'.format(cleaned_values['username'])) + orders_dict = { + order.value['order-id']: { + **order.value + } + for order in orders + } + logger.debug('Orders = {}'.format(orders_dict)) + return {'orders': orders_dict}, 200 + + +if __name__ == '__main__': + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + log_formater = logging.Formatter('[%(filename)s:%(lineno)d] %(message)s') + + stream_logger = logging.StreamHandler() + stream_logger.setFormatter(log_formater) + + # file_logger = logging.FileHandler('log.txt') + # file_logger.setLevel(logging.DEBUG) + # file_logger.setFormatter(log_formater) + + logger.addHandler(stream_logger) + # logger.addHandler(file_logger) + + app = Flask(__name__) + + api = Api(app) + api.add_resource(ListProducts, '/product/list') + api.add_resource(AddProduct, '/product/add') + api.add_resource(ProductOrder, '/product/order') + api.add_resource(UserRegisterPayment, '/user/register_payment') + api.add_resource(OrderList, '/order/list') + + app.run(host='::', port=config.get('app', 'port', fallback=5000), debug=True) + + + @app.errorhandler(Exception) + def handle_exception(e): + app.logger.error(e) + # pass through HTTP errors + if isinstance(e, HTTPException): + return e + + # now you're handling non-HTTP exceptions only + return {'message': 'Server Error'}, 500 diff --git a/archive/uncloud_django_based/notes-abk.md b/archive/uncloud_django_based/notes-abk.md new file mode 100644 index 0000000..6d5c223 --- /dev/null +++ b/archive/uncloud_django_based/notes-abk.md @@ -0,0 +1,11 @@ +## TODO 2020-02-22 + +* ~~move the current rest api to /opennebula~~ +* ~~make the /opennebula api only accessible by an admin account~~ +* ~~create a new filtered api on /vm/list that~~ + * ~~a) requires authentication~~ + * ~~b) only shows the VMs of the current user~~ +* ~~the new api should not contain all details, but: cpus (as read by the vcpu field), ram, ips, disks~~ +* ~~also make a (random) uuid the primary key for VMs - everything in this uncloud hack will use uuids as the id~~ +* ~~still expose the opennebula id as opennebula_id~~ +* ~~note put all secrets/configs into uncloud.secrets - I added a sample file into the repo~~ diff --git a/archive/uncloud_django_based/notes-nico.org b/archive/uncloud_django_based/notes-nico.org new file mode 100644 index 0000000..811fbff --- /dev/null +++ b/archive/uncloud_django_based/notes-nico.org @@ -0,0 +1,102 @@ +* snapshot feature +** product: vm-snapshot +** flow +*** list all my VMs +**** get the uuid of the VM I want to take a snapshot of +*** request a snapshot +``` +vmuuid=$(http nicocustomer +http -a nicocustomer:xxx http://uncloud.ch/vm/create_snapshot uuid= +password=... +``` +** backend realisation +*** list snapshots + - have them in the DB + - create an entry on create +*** creating snapshots + - vm sync / fsync? + - rbd snapshot + - host/cluster mapping? + - need image(s) + +* steps +** DONE authenticate via ldap + CLOSED: [2020-02-20 Thu 19:05] +** DONE Make classes / views require authentication + CLOSED: [2020-02-20 Thu 19:05] +** TODO register credit card +*** TODO find out what saving with us +*** Info +**** should not be fully saved in the DB +**** model needs to be a bit different +* Decide where to save sensitive data +** stripe access key, etc. +* python requirements (nicohack202002) + django djangorestframework django-auth-ldap stripe +* os package requirements (alpine) + openldap-dev +* VPN case +** put on /orders with uuid +** register cc +* CC +** TODO check whether we can register or not at stripe +* membership +** required for "smaller" / "shorter" products + +* TODO Membership missing +* Flows to be implemented - see https://redmine.ungleich.ch/issues/7609 +** Membership +*** 5 CHF +** Django Hosting +*** One time payment 35 CHF +*** Monthly payment depends on VM size +*** Parameters: same as IPv6 only VM +** IPv6 VPN +*** Parameters: none +*** Is for free if the customer has an active VM +** IPv6 only VM +*** Parameters: cores, ram, os_disk_size, OS +* Django rest framework +** viewset: .list and .create +** view: .get .post +* TODO register CC +* DONE list products + CLOSED: [2020-02-24 Mon 20:15] +* An ungleich account - can be registered for free on + https://account.ungleich.ch +* httpie installed (provides the http command) + +## Get a membership + + +## Registering a payment method + +To be able to pay for the membership, you will need to register a +credit card or apply for payment on bill (TO BE IMPLEMENTED). + +### Register credit card + +``` +http POST https://api.ungleich.ch/membership \ + username=nico password=yourpassword \ + cc_number=.. \ + cc_ + +``` + + + +### Request payment via bill + + + + +## Create the membership + + +``` +http POST https://api.ungleich.ch/membership username=nico password=yourpassword + +``` + +## List available products diff --git a/archive/uncloud_django_based/plan.org b/archive/uncloud_django_based/plan.org new file mode 100644 index 0000000..9f172c2 --- /dev/null +++ b/archive/uncloud_django_based/plan.org @@ -0,0 +1,6 @@ +* TODO register CC +* TODO list products +* ahmed +** schemas +*** field: is_valid? - used by schemas +*** definition of a "schema" diff --git a/archive/uncloud_django_based/uncloud/.gitignore b/archive/uncloud_django_based/uncloud/.gitignore new file mode 100644 index 0000000..71202e1 --- /dev/null +++ b/archive/uncloud_django_based/uncloud/.gitignore @@ -0,0 +1,4 @@ +db.sqlite3 +uncloud/secrets.py +debug.log +uncloud/local_settings.py \ No newline at end of file diff --git a/archive/uncloud_etcd_based/bin/gen-version b/archive/uncloud_etcd_based/bin/gen-version new file mode 100755 index 0000000..06c3e22 --- /dev/null +++ b/archive/uncloud_etcd_based/bin/gen-version @@ -0,0 +1,29 @@ +#!/bin/sh +# -*- coding: utf-8 -*- +# +# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + + +# Wrapper for real script to allow execution from checkout +dir=${0%/*} + +# Ensure version is present - the bundled/shipped version contains a static version, +# the git version contains a dynamic version +printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py diff --git a/bin/ucloud b/archive/uncloud_etcd_based/bin/uncloud similarity index 90% rename from bin/ucloud rename to archive/uncloud_etcd_based/bin/uncloud index e178413..1c572d5 100755 --- a/bin/ucloud +++ b/archive/uncloud_etcd_based/bin/uncloud @@ -25,9 +25,9 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../ucloud/version.py +${dir}/gen-version libdir=$(cd "${dir}/../" && pwd -P) export PYTHONPATH="${libdir}" -"$dir/../scripts/ucloud" "$@" +"$dir/../scripts/uncloud" "$@" diff --git a/archive/uncloud_etcd_based/bin/uncloud-run-reinstall b/archive/uncloud_etcd_based/bin/uncloud-run-reinstall new file mode 100755 index 0000000..b211613 --- /dev/null +++ b/archive/uncloud_etcd_based/bin/uncloud-run-reinstall @@ -0,0 +1,29 @@ +#!/bin/sh +# -*- coding: utf-8 -*- +# +# 2012-2019 Nico Schottelius (nico-ucloud at schottelius.org) +# +# This file is part of ucloud. +# +# ucloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ucloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ucloud. If not, see . +# +# + +# Wrapper for real script to allow execution from checkout +dir=${0%/*} + +${dir}/gen-version; +pip uninstall -y uncloud >/dev/null +python setup.py install >/dev/null +${dir}/uncloud "$@" diff --git a/archive/uncloud_etcd_based/conf/uncloud.conf b/archive/uncloud_etcd_based/conf/uncloud.conf new file mode 100644 index 0000000..6a1b500 --- /dev/null +++ b/archive/uncloud_etcd_based/conf/uncloud.conf @@ -0,0 +1,13 @@ +[etcd] +url = localhost +port = 2379 +base_prefix = / +ca_cert +cert_cert +cert_key + +[client] +name = replace_me +realm = replace_me +seed = replace_me +api_server = http://localhost:5000 \ No newline at end of file diff --git a/ucloud/docs/Makefile b/archive/uncloud_etcd_based/docs/Makefile similarity index 93% rename from ucloud/docs/Makefile rename to archive/uncloud_etcd_based/docs/Makefile index 5e7ea85..246b56c 100644 --- a/ucloud/docs/Makefile +++ b/archive/uncloud_etcd_based/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source/ BUILDDIR = build/ -DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/ +DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/ .PHONY: all build clean diff --git a/archive/uncloud_etcd_based/docs/README.md b/archive/uncloud_etcd_based/docs/README.md new file mode 100644 index 0000000..a5afbaa --- /dev/null +++ b/archive/uncloud_etcd_based/docs/README.md @@ -0,0 +1,12 @@ +# uncloud docs + +## Requirements +1. Python3 +2. Sphinx + +## Usage +Run `make build` to build docs. + +Run `make clean` to remove build directory. + +Run `make publish` to push build dir to https://ungleich.ch/ucloud/ \ No newline at end of file diff --git a/ucloud/__init__.py b/archive/uncloud_etcd_based/docs/__init__.py similarity index 100% rename from ucloud/__init__.py rename to archive/uncloud_etcd_based/docs/__init__.py diff --git a/ucloud/docs/__init__.py b/archive/uncloud_etcd_based/docs/source/__init__.py similarity index 100% rename from ucloud/docs/__init__.py rename to archive/uncloud_etcd_based/docs/source/__init__.py diff --git a/ucloud/docs/source/admin-guide b/archive/uncloud_etcd_based/docs/source/admin-guide.rst similarity index 72% rename from ucloud/docs/source/admin-guide rename to archive/uncloud_etcd_based/docs/source/admin-guide.rst index ec6597d..b62808d 100644 --- a/ucloud/docs/source/admin-guide +++ b/archive/uncloud_etcd_based/docs/source/admin-guide.rst @@ -56,40 +56,13 @@ To start host we created earlier, execute the following command ucloud host ungleich.ch -Create OS Image ---------------- +File & image scanners +-------------------------- -Create ucloud-init ready OS image (Optional) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This step is optional if you just want to test ucloud. However, sooner or later -you want to create OS images with ucloud-init to properly -contexualize VMs. - -1. Start a VM with OS image on which you want to install ucloud-init -2. Execute the following command on the started VM - - .. code-block:: sh - - apk add git - git clone https://code.ungleich.ch/ucloud/ucloud-init.git - cd ucloud-init - sh ./install.sh -3. Congratulations. Your image is now ucloud-init ready. - - -Upload Sample OS Image -~~~~~~~~~~~~~~~~~~~~~~ -Execute the following to get the sample OS image file. - -.. code-block:: sh - - mkdir /var/www/admin - (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download) - -Run File Scanner and Image Scanner -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make -images from tracked files. So, we need to track the file by running File Scanner +Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our +uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by +ucloud. We can only make images from tracked files. So, we need to track the +file by running File Scanner .. code-block:: sh diff --git a/ucloud/docs/source/conf.py b/archive/uncloud_etcd_based/docs/source/conf.py similarity index 90% rename from ucloud/docs/source/conf.py rename to archive/uncloud_etcd_based/docs/source/conf.py index 9b133f9..c8138a7 100644 --- a/ucloud/docs/source/conf.py +++ b/archive/uncloud_etcd_based/docs/source/conf.py @@ -17,9 +17,9 @@ # -- Project information ----------------------------------------------------- -project = 'ucloud' -copyright = '2019, ungleich' -author = 'ungleich' +project = "uncloud" +copyright = "2019, ungleich" +author = "ungleich" # -- General configuration --------------------------------------------------- @@ -27,12 +27,12 @@ author = 'ungleich' # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. extensions = [ - 'sphinx.ext.autodoc', - 'sphinx_rtd_theme', + "sphinx.ext.autodoc", + "sphinx_rtd_theme", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. @@ -50,4 +50,4 @@ html_theme = "sphinx_rtd_theme" # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] diff --git a/ucloud/docs/source/diagram-code/ucloud b/archive/uncloud_etcd_based/docs/source/diagram-code/ucloud similarity index 100% rename from ucloud/docs/source/diagram-code/ucloud rename to archive/uncloud_etcd_based/docs/source/diagram-code/ucloud diff --git a/archive/uncloud_etcd_based/docs/source/hacking.rst b/archive/uncloud_etcd_based/docs/source/hacking.rst new file mode 100644 index 0000000..1c750d6 --- /dev/null +++ b/archive/uncloud_etcd_based/docs/source/hacking.rst @@ -0,0 +1,36 @@ +Hacking +======= +Using uncloud in hacking (aka development) mode. + + +Get the code +------------ +.. code-block:: sh + :linenos: + + git clone https://code.ungleich.ch/uncloud/uncloud.git + + + +Install python requirements +--------------------------- +You need to have python3 installed. + +.. code-block:: sh + :linenos: + + cd uncloud! + python -m venv venv + . ./venv/bin/activate + ./bin/uncloud-run-reinstall + + + +Install os requirements +----------------------- +Install the following software packages: **dnsmasq**. + +If you already have a working IPv6 SLAAC and DNS setup, +this step can be skipped. + +Note that you need at least one /64 IPv6 network to run uncloud. diff --git a/ucloud/docs/source/images/ucloud.svg b/archive/uncloud_etcd_based/docs/source/images/ucloud.svg similarity index 100% rename from ucloud/docs/source/images/ucloud.svg rename to archive/uncloud_etcd_based/docs/source/images/ucloud.svg diff --git a/ucloud/docs/source/index.rst b/archive/uncloud_etcd_based/docs/source/index.rst similarity index 90% rename from ucloud/docs/source/index.rst rename to archive/uncloud_etcd_based/docs/source/index.rst index 879ac32..fad1f88 100644 --- a/ucloud/docs/source/index.rst +++ b/archive/uncloud_etcd_based/docs/source/index.rst @@ -11,12 +11,12 @@ Welcome to ucloud's documentation! :caption: Contents: introduction - user-guide setup-install + vm-images + user-guide admin-guide - user-guide/how-to-create-an-os-image-for-ucloud troubleshooting - + hacking Indices and tables ================== diff --git a/ucloud/docs/source/introduction.rst b/archive/uncloud_etcd_based/docs/source/introduction.rst similarity index 100% rename from ucloud/docs/source/introduction.rst rename to archive/uncloud_etcd_based/docs/source/introduction.rst diff --git a/ucloud/docs/source/misc/todo.rst b/archive/uncloud_etcd_based/docs/source/misc/todo.rst similarity index 100% rename from ucloud/docs/source/misc/todo.rst rename to archive/uncloud_etcd_based/docs/source/misc/todo.rst diff --git a/ucloud/docs/source/setup-install.rst b/archive/uncloud_etcd_based/docs/source/setup-install.rst similarity index 100% rename from ucloud/docs/source/setup-install.rst rename to archive/uncloud_etcd_based/docs/source/setup-install.rst diff --git a/ucloud/docs/source/theory/summary.rst b/archive/uncloud_etcd_based/docs/source/theory/summary.rst similarity index 100% rename from ucloud/docs/source/theory/summary.rst rename to archive/uncloud_etcd_based/docs/source/theory/summary.rst diff --git a/ucloud/docs/source/troubleshooting.rst b/archive/uncloud_etcd_based/docs/source/troubleshooting.rst similarity index 100% rename from ucloud/docs/source/troubleshooting.rst rename to archive/uncloud_etcd_based/docs/source/troubleshooting.rst diff --git a/ucloud/docs/source/user-guide.rst b/archive/uncloud_etcd_based/docs/source/user-guide.rst similarity index 100% rename from ucloud/docs/source/user-guide.rst rename to archive/uncloud_etcd_based/docs/source/user-guide.rst diff --git a/ucloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst b/archive/uncloud_etcd_based/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst similarity index 100% rename from ucloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst rename to archive/uncloud_etcd_based/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst diff --git a/archive/uncloud_etcd_based/docs/source/vm-images.rst b/archive/uncloud_etcd_based/docs/source/vm-images.rst new file mode 100644 index 0000000..4b2758a --- /dev/null +++ b/archive/uncloud_etcd_based/docs/source/vm-images.rst @@ -0,0 +1,66 @@ +VM images +================================== + +Overview +--------- + +ucloud tries to be least invasise towards VMs and only require +strictly necessary changes for running in a virtualised +environment. This includes configurations for: + +* Configuring the network +* Managing access via ssh keys +* Resizing the attached disk(s) + +Upstream images +--------------- + +The 'official' uncloud images are defined in the `uncloud/images +`_ repository. + +How to make you own Uncloud images +---------------------------------- + +.. note:: + It is fairly easy to create your own images for uncloud, as the common + operations (which are detailed below) can be automatically handled by the + `uncloud/uncloud-init `_ tool. + +Network configuration +~~~~~~~~~~~~~~~~~~~~~ +All VMs in ucloud are required to support IPv6. The primary network +configuration is always done using SLAAC. A VM thus needs only to be +configured to + +* accept router advertisements on all network interfaces +* use the router advertisements to configure the network interfaces +* accept the DNS entries from the router advertisements + + +Configuring SSH keys +~~~~~~~~~~~~~~~~~~~~ + +To be able to access the VM, ucloud support provisioning SSH keys. + +To accept ssh keys in your VM, request the URL +*http://metadata/ssh_keys*. Add the content to the appropriate user's +**authorized_keys** file. Below you find sample code to accomplish +this task: + +.. code-block:: sh + + tmp=$(mktemp) + curl -s http://metadata/ssk_keys > "$tmp" + touch ~/.ssh/authorized_keys # ensure it exists + cat ~/.ssh/authorized_keys >> "$tmp" + sort "$tmp" | uniq > ~/.ssh/authorized_keys + + +Disk resize +~~~~~~~~~~~ +In virtualised environments, the disk sizes might grow. The operating +system should detect disks that are bigger than the existing partition +table and resize accordingly. This task is os specific. + +ucloud does not support shrinking disks due to the complexity and +intra OS dependencies. diff --git a/archive/uncloud_etcd_based/scripts/uncloud b/archive/uncloud_etcd_based/scripts/uncloud new file mode 100755 index 0000000..9517b01 --- /dev/null +++ b/archive/uncloud_etcd_based/scripts/uncloud @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +import logging +import sys +import importlib +import argparse +import os + +from etcd3.exceptions import ConnectionFailedError + +from uncloud.common import settings +from uncloud import UncloudException +from uncloud.common.cli import resolve_otp_credentials + +# Components that use etcd +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', + 'imagescanner', 'metadata', 'configure', 'hack'] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('oneshot') +#ALL_COMPONENTS.append('cli') + + +if __name__ == '__main__': + arg_parser = argparse.ArgumentParser() + subparsers = arg_parser.add_subparsers(dest='command') + + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument('--debug', '-d', action='store_true', default=False, + help='More verbose logging') + parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory', + default=os.path.expanduser('~/uncloud')) + + etcd_parser = argparse.ArgumentParser(add_help=False) + etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-port') + etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') + etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') + etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') + + for component in ALL_COMPONENTS: + mod = importlib.import_module('uncloud.{}.main'.format(component)) + parser = getattr(mod, 'arg_parser') + + if component in ETCD_COMPONENTS: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) + else: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) + + arguments = vars(arg_parser.parse_args()) + etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value] + etcd_arguments = { + 'etcd': { + key.replace('etcd_', ''): arguments[key] + for key in etcd_arguments + } + } + if not arguments['command']: + arg_parser.print_help() + else: + # Initializing Settings and resolving otp_credentials + # It is neccessary to resolve_otp_credentials after argument parsing is done because + # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and + # providing the default values for --name, --realm and --seed arguments from the values + # we read from file. But, now we are asking user about where the config file lives. So, + # to providing default value is not possible before parsing arguments. So, we are doing + # it after.. +# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) +# resolve_otp_credentials(arguments) + + name = arguments.pop('command') + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') + + if arguments['debug']: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + log = logging.getLogger() + + try: + main(arguments) + except UncloudException as err: + log.error(err) + sys.exit(1) +# except ConnectionFailedError as err: +# log.error('Cannot connect to etcd: {}'.format(err)) + except Exception as err: + log.exception(err) diff --git a/archive/uncloud_etcd_based/setup.py b/archive/uncloud_etcd_based/setup.py new file mode 100644 index 0000000..f5e0718 --- /dev/null +++ b/archive/uncloud_etcd_based/setup.py @@ -0,0 +1,51 @@ +import os + +from setuptools import setup, find_packages + +with open("README.md", "r") as fh: + long_description = fh.read() + +try: + import uncloud.version + + version = uncloud.version.VERSION +except: + import subprocess + + c = subprocess.check_output(["git", "describe"]) + version = c.decode("utf-8").strip() + + +setup( + name="uncloud", + version=version, + description="uncloud cloud management", + url="https://code.ungleich.ch/uncloud/uncloud", + long_description=long_description, + long_description_content_type="text/markdown", + classifiers=[ + "Development Status :: 3 - Alpha", + "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", + "Programming Language :: Python :: 3", + ], + author="ungleich", + author_email="technik@ungleich.ch", + packages=find_packages(), + install_requires=[ + "requests", + "Flask>=1.1.1", + "flask-restful", + "bitmath", + "pyotp", + "pynetbox", + "colorama", + "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3", + "marshmallow", + "ldap3" + ], + scripts=["scripts/uncloud"], + data_files=[ + (os.path.expanduser("~/uncloud/"), ["conf/uncloud.conf"]) + ], + zip_safe=False, +) diff --git a/ucloud/docs/source/__init__.py b/archive/uncloud_etcd_based/test/__init__.py similarity index 100% rename from ucloud/docs/source/__init__.py rename to archive/uncloud_etcd_based/test/__init__.py diff --git a/archive/uncloud_etcd_based/test/test_mac_local.py b/archive/uncloud_etcd_based/test/test_mac_local.py new file mode 100644 index 0000000..3a4ac3a --- /dev/null +++ b/archive/uncloud_etcd_based/test/test_mac_local.py @@ -0,0 +1,37 @@ +import unittest +from unittest.mock import Mock + +from uncloud.hack.mac import MAC +from uncloud import UncloudException + +class TestMacLocal(unittest.TestCase): + def setUp(self): + self.config = Mock() + self.config.arguments = {"no_db":True} + self.mac = MAC(self.config) + self.mac.create() + + def testMacInt(self): + self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong first MAC index") + + def testMacRepr(self): + self.assertEqual(self.mac.__repr__(), '420000000001', "wrong first MAC index") + + def testMacStr(self): + self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong first MAC index") + + def testValidationRaise(self): + with self.assertRaises(UncloudException): + self.mac.validate_mac("2") + + def testValidation(self): + self.assertTrue(self.mac.validate_mac("42:00:00:00:00:01"), "Validation of a given MAC not working properly") + + def testNextMAC(self): + self.mac.create() + self.assertEqual(self.mac.__repr__(), '420000000001', "wrong second MAC index") + self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong second MAC index") + self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong second MAC index") + +if __name__ == '__main__': + unittest.main() diff --git a/archive/uncloud_etcd_based/uncloud/__init__.py b/archive/uncloud_etcd_based/uncloud/__init__.py new file mode 100644 index 0000000..2920f47 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/__init__.py @@ -0,0 +1,2 @@ +class UncloudException(Exception): + pass diff --git a/ucloud/api/README.md b/archive/uncloud_etcd_based/uncloud/api/README.md similarity index 100% rename from ucloud/api/README.md rename to archive/uncloud_etcd_based/uncloud/api/README.md diff --git a/ucloud/api/__init__.py b/archive/uncloud_etcd_based/uncloud/api/__init__.py similarity index 100% rename from ucloud/api/__init__.py rename to archive/uncloud_etcd_based/uncloud/api/__init__.py diff --git a/ucloud/api/common_fields.py b/archive/uncloud_etcd_based/uncloud/api/common_fields.py similarity index 72% rename from ucloud/api/common_fields.py rename to archive/uncloud_etcd_based/uncloud/api/common_fields.py index e9903ac..ba9fb37 100755 --- a/ucloud/api/common_fields.py +++ b/archive/uncloud_etcd_based/uncloud/api/common_fields.py @@ -1,6 +1,6 @@ import os -from ucloud.config import etcd_client, env_vars +from uncloud.common.shared import shared class Optional: @@ -19,12 +19,16 @@ class Field: def is_valid(self): if self.value == KeyError: - self.add_error("'{}' field is a required field".format(self.name)) + self.add_error( + "'{}' field is a required field".format(self.name) + ) else: if isinstance(self.value, Optional): pass elif not isinstance(self.value, self.type): - self.add_error("Incorrect Type for '{}' field".format(self.name)) + self.add_error( + "Incorrect Type for '{}' field".format(self.name) + ) else: self.validation() @@ -48,6 +52,8 @@ class VmUUIDField(Field): self.validation = self.vm_uuid_validation def vm_uuid_validation(self): - r = etcd_client.get(os.path.join(env_vars.get('VM_PREFIX'), self.uuid)) + r = shared.etcd_client.get( + os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid) + ) if not r: self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/archive/uncloud_etcd_based/uncloud/api/create_image_store.py b/archive/uncloud_etcd_based/uncloud/api/create_image_store.py new file mode 100755 index 0000000..90e0f92 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/api/create_image_store.py @@ -0,0 +1,19 @@ +import json +import os + +from uuid import uuid4 + +from uncloud.common.shared import shared + +data = { + 'is_public': True, + 'type': 'ceph', + 'name': 'images', + 'description': 'first ever public image-store', + 'attributes': {'list': [], 'key': [], 'pool': 'images'}, +} + +shared.etcd_client.put( + os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex), + json.dumps(data), +) diff --git a/ucloud/api/helper.py b/archive/uncloud_etcd_based/uncloud/api/helper.py similarity index 55% rename from ucloud/api/helper.py rename to archive/uncloud_etcd_based/uncloud/api/helper.py index 63d2f90..8ceb3a6 100755 --- a/ucloud/api/helper.py +++ b/archive/uncloud_etcd_based/uncloud/api/helper.py @@ -1,48 +1,51 @@ import binascii import ipaddress import random -import subprocess as sp - +import logging import requests from pyotp import TOTP -from ucloud.config import vm_pool, env_vars +from uncloud.common.shared import shared + +logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": env_vars.get("AUTH_NAME"), - "auth_token": TOTP(env_vars.get("AUTH_SEED")).now(), - "auth_realm": env_vars.get("AUTH_REALM"), + "auth_name": shared.settings["otp"]["auth_name"], + "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), + "auth_realm": shared.settings["otp"]["auth_realm"], "name": name, "realm": realm, "token": token, } - except binascii.Error: + except binascii.Error as err: + logger.error( + "Cannot compute OTP for seed: {}".format( + shared.settings["otp"]["auth_seed"] + ) + ) return 400 response = requests.post( - "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format( - OTP_SERVER=env_vars.get("OTP_SERVER", ""), - OTP_VERIFY_ENDPOINT=env_vars.get("OTP_VERIFY_ENDPOINT", "verify/"), - ), - json=data, + shared.settings["otp"]["verification_controller_url"], json=data ) return response.status_code def resolve_vm_name(name, owner): """Return UUID of Virtual Machine of name == name and owner == owner - + Input: name of vm, owner of vm. Output: uuid of vm if found otherwise None """ result = next( filter( - lambda vm: vm.value["owner"] == owner and vm.value["name"] == name, - vm_pool.vms, + lambda vm: vm.value["owner"] == owner + and vm.value["name"] == name, + shared.vm_pool.vms, ), None, ) @@ -54,7 +57,7 @@ def resolve_vm_name(name, owner): def resolve_image_name(name, etcd_client): """Return image uuid given its name and its store - + * If the provided name is not in correct format i.e {store_name}:{image_name} return ValueError * If no such image found then return KeyError @@ -70,26 +73,35 @@ def resolve_image_name(name, etcd_client): """ Examples, where it would work and where it would raise exception "images:alpine" --> ["images", "alpine"] - + "images" --> ["images"] it would raise Exception as non enough value to unpack - + "images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception as too many values to unpack """ store_name, image_name = store_name_and_image_name except Exception: - raise ValueError("Image name not in correct format i.e {store_name}:{image_name}") + raise ValueError( + "Image name not in correct format i.e {store_name}:{image_name}" + ) - images = etcd_client.get_prefix(env_vars.get('IMAGE_PREFIX'), value_in_json=True) + images = etcd_client.get_prefix( + shared.settings["etcd"]["image_prefix"], value_in_json=True + ) # Try to find image with name == image_name and store_name == store_name try: - image = next(filter(lambda im: im.value['name'] == image_name - and im.value['store_name'] == store_name, images)) + image = next( + filter( + lambda im: im.value["name"] == image_name + and im.value["store_name"] == store_name, + images, + ) + ) except StopIteration: raise KeyError("No image with name {} found.".format(name)) else: - image_uuid = image.key.split('/')[-1] + image_uuid = image.key.split("/")[-1] return image_uuid @@ -98,7 +110,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -116,36 +128,6 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt=' return separator.join(byte_fmt % b for b in mac) -def get_ip_addr(mac_address, device): - """Return IP address of a device provided its mac address / link local address - and the device with which it is connected. - - For Example, if we call get_ip_addr(mac_address="52:54:00:12:34:56", device="br0") - the following two scenarios can happen - 1. It would return None if we can't be able to find device whose mac_address is equal - to the arg:mac_address or the mentioned arg:device does not exists or the ip address - we found is local. - 2. It would return ip_address of device whose mac_address is equal to arg:mac_address - and is connected/neighbor of arg:device - """ - try: - output = sp.check_output(['ip', '-6', 'neigh', 'show', 'dev', device], stderr=sp.PIPE) - except sp.CalledProcessError: - return None - else: - result = [] - output = output.strip().decode("utf-8") - output = output.split("\n") - for entry in output: - entry = entry.split() - if entry: - ip = ipaddress.ip_address(entry[0]) - mac = entry[2] - if ip.is_global and mac_address == mac: - result.append(ip) - return result - - def mac2ipv6(mac, prefix): # only accept MACs separated by a colon parts = mac.split(":") @@ -158,8 +140,9 @@ def mac2ipv6(mac, prefix): # format output ipv6_parts = [str(0)] * 4 for i in range(0, len(parts), 2): - ipv6_parts.append("".join(parts[i:i + 2])) + ipv6_parts.append("".join(parts[i : i + 2])) lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) prefix = ipaddress.IPv6Address(prefix) return str(prefix + int(lower_part)) + diff --git a/archive/uncloud_etcd_based/uncloud/api/main.py b/archive/uncloud_etcd_based/uncloud/api/main.py new file mode 100644 index 0000000..73e8e21 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/api/main.py @@ -0,0 +1,600 @@ +import json +import pynetbox +import logging +import argparse + +from uuid import uuid4 +from os.path import join as join_path + +from flask import Flask, request +from flask_restful import Resource, Api +from werkzeug.exceptions import HTTPException + +from uncloud.common.shared import shared + +from uncloud.common import counters +from uncloud.common.vm import VMStatus +from uncloud.common.request import RequestEntry, RequestType +from uncloud.api import schemas +from uncloud.api.helper import generate_mac, mac2ipv6 +from uncloud import UncloudException + +logger = logging.getLogger(__name__) + +app = Flask(__name__) +api = Api(app) +app.logger.handlers.clear() + +arg_parser = argparse.ArgumentParser('api', add_help=False) +arg_parser.add_argument('--port', '-p') + + +@app.errorhandler(Exception) +def handle_exception(e): + app.logger.error(e) + # pass through HTTP errors + if isinstance(e, HTTPException): + return e + + # now you're handling non-HTTP exceptions only + return {'message': 'Server Error'}, 500 + + +class CreateVM(Resource): + """API Request to Handle Creation of VM""" + + @staticmethod + def post(): + data = request.json + validator = schemas.CreateVMSchema(data) + if validator.is_valid(): + vm_uuid = uuid4().hex + vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) + specs = { + 'cpu': validator.specs['cpu'], + 'ram': validator.specs['ram'], + 'os-ssd': validator.specs['os-ssd'], + 'hdd': validator.specs['hdd'], + } + macs = [generate_mac() for _ in range(len(data['network']))] + tap_ids = [ + counters.increment_etcd_counter( + shared.etcd_client, shared.settings['etcd']['tap_counter'] + ) + for _ in range(len(data['network'])) + ] + vm_entry = { + 'name': data['vm_name'], + 'owner': data['name'], + 'owner_realm': data['realm'], + 'specs': specs, + 'hostname': '', + 'status': VMStatus.stopped, + 'image_uuid': validator.image_uuid, + 'log': [], + 'vnc_socket': '', + 'network': list(zip(data['network'], macs, tap_ids)), + 'metadata': {'ssh-keys': []}, + 'in_migration': False, + } + shared.etcd_client.put(vm_key, vm_entry, value_in_json=True) + + # Create ScheduleVM Request + r = RequestEntry.from_scratch( + type=RequestType.ScheduleVM, + uuid=vm_uuid, + request_prefix=shared.settings['etcd']['request_prefix'], + ) + shared.request_pool.put(r) + + return {'message': 'VM Creation Queued'}, 200 + return validator.get_errors(), 400 + + +class VmStatus(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.VMStatusSchema(data) + if validator.is_valid(): + vm = shared.vm_pool.get( + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) + ) + vm_value = vm.value.copy() + vm_value['ip'] = [] + for network_mac_and_tap in vm.network: + network_name, mac, tap = network_mac_and_tap + network = shared.etcd_client.get( + join_path( + shared.settings['etcd']['network_prefix'], + data['name'], + network_name, + ), + value_in_json=True, + ) + ipv6_addr = ( + network.value.get('ipv6').split('::')[0] + '::' + ) + vm_value['ip'].append(mac2ipv6(mac, ipv6_addr)) + vm.value = vm_value + return vm.value + else: + return validator.get_errors(), 400 + + +class CreateImage(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.CreateImageSchema(data) + if validator.is_valid(): + file_entry = shared.etcd_client.get( + join_path(shared.settings['etcd']['file_prefix'], data['uuid']) + ) + file_entry_value = json.loads(file_entry.value) + + image_entry_json = { + 'status': 'TO_BE_CREATED', + 'owner': file_entry_value['owner'], + 'filename': file_entry_value['filename'], + 'name': data['name'], + 'store_name': data['image_store'], + 'visibility': 'public', + } + shared.etcd_client.put( + join_path( + shared.settings['etcd']['image_prefix'], data['uuid'] + ), + json.dumps(image_entry_json), + ) + + return {'message': 'Image queued for creation.'} + return validator.get_errors(), 400 + + +class ListPublicImages(Resource): + @staticmethod + def get(): + images = shared.etcd_client.get_prefix( + shared.settings['etcd']['image_prefix'], value_in_json=True + ) + r = {'images': []} + for image in images: + image_key = '{}:{}'.format( + image.value['store_name'], image.value['name'] + ) + r['images'].append( + {'name': image_key, 'status': image.value['status']} + ) + return r, 200 + + +class VMAction(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.VmActionSchema(data) + + if validator.is_valid(): + vm_entry = shared.vm_pool.get( + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) + ) + action = data['action'] + + if action == 'start': + action = 'schedule' + + if action == 'delete' and vm_entry.hostname == '': + if shared.storage_handler.is_vm_image_exists( + vm_entry.uuid + ): + r_status = shared.storage_handler.delete_vm_image( + vm_entry.uuid + ) + if r_status: + shared.etcd_client.client.delete(vm_entry.key) + return {'message': 'VM successfully deleted'} + else: + logger.error( + 'Some Error Occurred while deleting VM' + ) + return {'message': 'VM deletion unsuccessfull'} + else: + shared.etcd_client.client.delete(vm_entry.key) + return {'message': 'VM successfully deleted'} + + r = RequestEntry.from_scratch( + type='{}VM'.format(action.title()), + uuid=data['uuid'], + hostname=vm_entry.hostname, + request_prefix=shared.settings['etcd']['request_prefix'], + ) + shared.request_pool.put(r) + return ( + {'message': 'VM {} Queued'.format(action.title())}, + 200, + ) + else: + return validator.get_errors(), 400 + + +class VMMigration(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.VmMigrationSchema(data) + + if validator.is_valid(): + vm = shared.vm_pool.get(data['uuid']) + r = RequestEntry.from_scratch( + type=RequestType.InitVMMigration, + uuid=vm.uuid, + hostname=join_path( + shared.settings['etcd']['host_prefix'], + validator.destination.value, + ), + request_prefix=shared.settings['etcd']['request_prefix'], + ) + + shared.request_pool.put(r) + return ( + {'message': 'VM Migration Initialization Queued'}, + 200, + ) + else: + return validator.get_errors(), 400 + + +class ListUserVM(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.OTPSchema(data) + + if validator.is_valid(): + vms = shared.etcd_client.get_prefix( + shared.settings['etcd']['vm_prefix'], value_in_json=True + ) + return_vms = [] + user_vms = filter( + lambda v: v.value['owner'] == data['name'], vms + ) + for vm in user_vms: + return_vms.append( + { + 'name': vm.value['name'], + 'vm_uuid': vm.key.split('/')[-1], + 'specs': vm.value['specs'], + 'status': vm.value['status'], + 'hostname': vm.value['hostname'], + 'vnc_socket': vm.value.get('vnc_socket', None), + } + ) + if return_vms: + return {'message': return_vms}, 200 + return {'message': 'No VM found'}, 404 + + else: + return validator.get_errors(), 400 + + +class ListUserFiles(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.OTPSchema(data) + + if validator.is_valid(): + files = shared.etcd_client.get_prefix( + shared.settings['etcd']['file_prefix'], value_in_json=True + ) + return_files = [] + user_files = [f for f in files if f.value['owner'] == data['name']] + for file in user_files: + file_uuid = file.key.split('/')[-1] + file = file.value + file['uuid'] = file_uuid + + file.pop('sha512sum', None) + file.pop('owner', None) + + return_files.append(file) + return {'message': return_files}, 200 + else: + return validator.get_errors(), 400 + + +class CreateHost(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.CreateHostSchema(data) + if validator.is_valid(): + host_key = join_path( + shared.settings['etcd']['host_prefix'], uuid4().hex + ) + host_entry = { + 'specs': data['specs'], + 'hostname': data['hostname'], + 'status': 'DEAD', + 'last_heartbeat': '', + } + shared.etcd_client.put( + host_key, host_entry, value_in_json=True + ) + + return {'message': 'Host Created'}, 200 + + return validator.get_errors(), 400 + + +class ListHost(Resource): + @staticmethod + def get(): + hosts = shared.host_pool.hosts + r = { + host.key: { + 'status': host.status, + 'specs': host.specs, + 'hostname': host.hostname, + } + for host in hosts + } + return r, 200 + + +class GetSSHKeys(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.GetSSHSchema(data) + if validator.is_valid(): + if not validator.key_name.value: + + # {user_prefix}/{realm}/{name}/key/ + etcd_key = join_path( + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + ) + etcd_entry = shared.etcd_client.get_prefix( + etcd_key, value_in_json=True + ) + + keys = { + key.key.split('/')[-1]: key.value + for key in etcd_entry + } + return {'keys': keys} + else: + + # {user_prefix}/{realm}/{name}/key/{key_name} + etcd_key = join_path( + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], + ) + etcd_entry = shared.etcd_client.get( + etcd_key, value_in_json=True + ) + + if etcd_entry: + return { + 'keys': { + etcd_entry.key.split('/')[ + -1 + ]: etcd_entry.value + } + } + else: + return {'keys': {}} + else: + return validator.get_errors(), 400 + + +class AddSSHKey(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.AddSSHSchema(data) + if validator.is_valid(): + + # {user_prefix}/{realm}/{name}/key/{key_name} + etcd_key = join_path( + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], + ) + etcd_entry = shared.etcd_client.get( + etcd_key, value_in_json=True + ) + if etcd_entry: + return { + 'message': 'Key with name "{}" already exists'.format( + data['key_name'] + ) + } + else: + # Key Not Found. It implies user' haven't added any key yet. + shared.etcd_client.put( + etcd_key, data['key'], value_in_json=True + ) + return {'message': 'Key added successfully'} + else: + return validator.get_errors(), 400 + + +class RemoveSSHKey(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.RemoveSSHSchema(data) + if validator.is_valid(): + + # {user_prefix}/{realm}/{name}/key/{key_name} + etcd_key = join_path( + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], + ) + etcd_entry = shared.etcd_client.get( + etcd_key, value_in_json=True + ) + if etcd_entry: + shared.etcd_client.client.delete(etcd_key) + return {'message': 'Key successfully removed.'} + else: + return { + 'message': 'No Key with name "{}" Exists at all.'.format( + data['key_name'] + ) + } + else: + return validator.get_errors(), 400 + + +class CreateNetwork(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.CreateNetwork(data) + + if validator.is_valid(): + + network_entry = { + 'id': counters.increment_etcd_counter( + shared.etcd_client, shared.settings['etcd']['vxlan_counter'] + ), + 'type': data['type'], + } + if validator.user.value: + try: + nb = pynetbox.api( + url=shared.settings['netbox']['url'], + token=shared.settings['netbox']['token'], + ) + nb_prefix = nb.ipam.prefixes.get( + prefix=shared.settings['network']['prefix'] + ) + prefix = nb_prefix.available_prefixes.create( + data={ + 'prefix_length': int( + shared.settings['network']['prefix_length'] + ), + 'description': '{}\'s network "{}"'.format( + data['name'], data['network_name'] + ), + 'is_pool': True, + } + ) + except Exception as err: + app.logger.error(err) + return { + 'message': 'Error occured while creating network.' + } + else: + network_entry['ipv6'] = prefix['prefix'] + else: + network_entry['ipv6'] = 'fd00::/64' + + network_key = join_path( + shared.settings['etcd']['network_prefix'], + data['name'], + data['network_name'], + ) + shared.etcd_client.put( + network_key, network_entry, value_in_json=True + ) + return {'message': 'Network successfully added.'} + else: + return validator.get_errors(), 400 + + +class ListUserNetwork(Resource): + @staticmethod + def post(): + data = request.json + validator = schemas.OTPSchema(data) + + if validator.is_valid(): + prefix = join_path( + shared.settings['etcd']['network_prefix'], data['name'] + ) + networks = shared.etcd_client.get_prefix( + prefix, value_in_json=True + ) + user_networks = [] + for net in networks: + net.value['name'] = net.key.split('/')[-1] + user_networks.append(net.value) + return {'networks': user_networks}, 200 + else: + return validator.get_errors(), 400 + + +api.add_resource(CreateVM, '/vm/create') +api.add_resource(VmStatus, '/vm/status') + +api.add_resource(VMAction, '/vm/action') +api.add_resource(VMMigration, '/vm/migrate') + +api.add_resource(CreateImage, '/image/create') +api.add_resource(ListPublicImages, '/image/list-public') + +api.add_resource(ListUserVM, '/user/vms') +api.add_resource(ListUserFiles, '/user/files') +api.add_resource(ListUserNetwork, '/user/networks') + +api.add_resource(AddSSHKey, '/user/add-ssh') +api.add_resource(RemoveSSHKey, '/user/remove-ssh') +api.add_resource(GetSSHKeys, '/user/get-ssh') + +api.add_resource(CreateHost, '/host/create') +api.add_resource(ListHost, '/host/list') + +api.add_resource(CreateNetwork, '/network/create') + + +def main(arguments): + debug = arguments['debug'] + port = arguments['port'] + + try: + image_stores = list( + shared.etcd_client.get_prefix( + shared.settings['etcd']['image_store_prefix'], value_in_json=True + ) + ) + except KeyError: + image_stores = False + + # Do not inject default values that might be very wrong + # fail when required, not before + # + # if not image_stores: + # data = { + # 'is_public': True, + # 'type': 'ceph', + # 'name': 'images', + # 'description': 'first ever public image-store', + # 'attributes': {'list': [], 'key': [], 'pool': 'images'}, + # } + + # shared.etcd_client.put( + # join_path( + # shared.settings['etcd']['image_store_prefix'], uuid4().hex + # ), + # json.dumps(data), + # ) + + try: + app.run(host='::', port=port, debug=debug) + except OSError as e: + raise UncloudException('Failed to start Flask: {}'.format(e)) diff --git a/ucloud/api/schemas.py b/archive/uncloud_etcd_based/uncloud/api/schemas.py similarity index 58% rename from ucloud/api/schemas.py rename to archive/uncloud_etcd_based/uncloud/api/schemas.py index c4f60ca..87f20c9 100755 --- a/ucloud/api/schemas.py +++ b/archive/uncloud_etcd_based/uncloud/api/schemas.py @@ -1,6 +1,6 @@ """ This module contain classes thats validates and intercept/modify -data coming from ucloud-cli (user) +data coming from uncloud-cli (user) It was primarily developed as an alternative to argument parser of Flask_Restful which is going to be deprecated. I also tried @@ -19,10 +19,10 @@ import os import bitmath -from ucloud.common.host import HostStatus -from ucloud.common.vm import VMStatus -from ucloud.config import etcd_client, env_vars, vm_pool, host_pool -from . import helper +from uncloud.common.host import HostStatus +from uncloud.common.vm import VMStatus +from uncloud.common.shared import shared +from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -79,7 +79,12 @@ class OTPSchema(BaseSchema): super().__init__(data=data, fields=_fields) def validation(self): - if check_otp(self.name.value, self.realm.value, self.token.value) != 200: + if ( + check_otp( + self.name.value, self.realm.value, self.token.value + ) + != 200 + ): self.add_error("Wrong Credentials") @@ -91,7 +96,9 @@ class CreateImageSchema(BaseSchema): # Fields self.uuid = Field("uuid", str, data.get("uuid", KeyError)) self.name = Field("name", str, data.get("name", KeyError)) - self.image_store = Field("image_store", str, data.get("image_store", KeyError)) + self.image_store = Field( + "image_store", str, data.get("image_store", KeyError) + ) # Validations self.uuid.validation = self.file_uuid_validation @@ -102,34 +109,51 @@ class CreateImageSchema(BaseSchema): super().__init__(data, fields) def file_uuid_validation(self): - file_entry = etcd_client.get(os.path.join(env_vars.get('FILE_PREFIX'), self.uuid.value)) + file_entry = shared.etcd_client.get( + os.path.join( + shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value + ) + ) if file_entry is None: self.add_error( - "Image File with uuid '{}' Not Found".format(self.uuid.value) + "Image File with uuid '{}' Not Found".format( + self.uuid.value + ) ) def image_store_name_validation(self): - image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'))) + image_stores = list( + shared.etcd_client.get_prefix( + shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] + ) + ) image_store = next( filter( - lambda s: json.loads(s.value)["name"] == self.image_store.value, + lambda s: json.loads(s.value)["name"] + == self.image_store.value, image_stores, ), None, ) if not image_store: - self.add_error("Store '{}' does not exists".format(self.image_store.value)) + self.add_error( + "Store '{}' does not exists".format( + self.image_store.value + ) + ) # Host Operations + class CreateHostSchema(OTPSchema): def __init__(self, data): - self.parsed_specs = {} # Fields self.specs = Field("specs", dict, data.get("specs", KeyError)) - self.hostname = Field("hostname", str, data.get("hostname", KeyError)) + self.hostname = Field( + "hostname", str, data.get("hostname", KeyError) + ) # Validation self.specs.validation = self.specs_validation @@ -141,22 +165,28 @@ class CreateHostSchema(OTPSchema): def specs_validation(self): ALLOWED_BASE = 10 - _cpu = self.specs.value.get('cpu', KeyError) - _ram = self.specs.value.get('ram', KeyError) - _os_ssd = self.specs.value.get('os-ssd', KeyError) - _hdd = self.specs.value.get('hdd', KeyError) + _cpu = self.specs.value.get("cpu", KeyError) + _ram = self.specs.value.get("ram", KeyError) + _os_ssd = self.specs.value.get("os-ssd", KeyError) + _hdd = self.specs.value.get("hdd", KeyError) if KeyError in [_cpu, _ram, _os_ssd, _hdd]: - self.add_error("You must specify CPU, RAM and OS-SSD in your specs") + self.add_error( + "You must specify CPU, RAM and OS-SSD in your specs" + ) return None try: parsed_ram = bitmath.parse_string_unsafe(_ram) parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) if parsed_ram.base != ALLOWED_BASE: - self.add_error("Your specified RAM is not in correct units") + self.add_error( + "Your specified RAM is not in correct units" + ) if parsed_os_ssd.base != ALLOWED_BASE: - self.add_error("Your specified OS-SSD is not in correct units") + self.add_error( + "Your specified OS-SSD is not in correct units" + ) if _cpu < 1: self.add_error("CPU must be atleast 1") @@ -171,7 +201,9 @@ class CreateHostSchema(OTPSchema): for hdd in _hdd: _parsed_hdd = bitmath.parse_string_unsafe(hdd) if _parsed_hdd.base != ALLOWED_BASE: - self.add_error("Your specified HDD is not in correct units") + self.add_error( + "Your specified HDD is not in correct units" + ) break else: parsed_hdd.append(str(_parsed_hdd)) @@ -182,15 +214,17 @@ class CreateHostSchema(OTPSchema): else: if self.get_errors(): self.specs = { - 'cpu': _cpu, - 'ram': str(parsed_ram), - 'os-ssd': str(parsed_os_ssd), - 'hdd': parsed_hdd + "cpu": _cpu, + "ram": str(parsed_ram), + "os-ssd": str(parsed_os_ssd), + "hdd": parsed_hdd, } def validation(self): if self.realm.value != "ungleich-admin": - self.add_error("Invalid Credentials/Insufficient Permission") + self.add_error( + "Invalid Credentials/Insufficient Permission" + ) # VM Operations @@ -198,13 +232,15 @@ class CreateHostSchema(OTPSchema): class CreateVMSchema(OTPSchema): def __init__(self, data): - self.parsed_specs = {} - # Fields self.specs = Field("specs", dict, data.get("specs", KeyError)) - self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError)) + self.vm_name = Field( + "vm_name", str, data.get("vm_name", KeyError) + ) self.image = Field("image", str, data.get("image", KeyError)) - self.network = Field("network", list, data.get("network", KeyError)) + self.network = Field( + "network", list, data.get("network", KeyError) + ) # Validation self.image.validation = self.image_validation @@ -218,16 +254,25 @@ class CreateVMSchema(OTPSchema): def image_validation(self): try: - image_uuid = helper.resolve_image_name(self.image.value, etcd_client) + image_uuid = helper.resolve_image_name( + self.image.value, shared.etcd_client + ) except Exception as e: + logger.exception( + "Cannot resolve image name = %s", self.image.value + ) self.add_error(str(e)) else: self.image_uuid = image_uuid def vm_name_validation(self): - if resolve_vm_name(name=self.vm_name.value, owner=self.name.value): + if resolve_vm_name( + name=self.vm_name.value, owner=self.name.value + ): self.add_error( - 'VM with same name "{}" already exists'.format(self.vm_name.value) + 'VM with same name "{}" already exists'.format( + self.vm_name.value + ) ) def network_validation(self): @@ -235,34 +280,48 @@ class CreateVMSchema(OTPSchema): if _network: for net in _network: - network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'), - self.name.value, - net), value_in_json=True) + network = shared.etcd_client.get( + os.path.join( + shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], + self.name.value, + net, + ), + value_in_json=True, + ) if not network: - self.add_error("Network with name {} does not exists" \ - .format(net)) + self.add_error( + "Network with name {} does not exists".format( + net + ) + ) def specs_validation(self): ALLOWED_BASE = 10 - _cpu = self.specs.value.get('cpu', KeyError) - _ram = self.specs.value.get('ram', KeyError) - _os_ssd = self.specs.value.get('os-ssd', KeyError) - _hdd = self.specs.value.get('hdd', KeyError) + _cpu = self.specs.value.get("cpu", KeyError) + _ram = self.specs.value.get("ram", KeyError) + _os_ssd = self.specs.value.get("os-ssd", KeyError) + _hdd = self.specs.value.get("hdd", KeyError) if KeyError in [_cpu, _ram, _os_ssd, _hdd]: - self.add_error("You must specify CPU, RAM and OS-SSD in your specs") + self.add_error( + "You must specify CPU, RAM and OS-SSD in your specs" + ) return None try: parsed_ram = bitmath.parse_string_unsafe(_ram) parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd) if parsed_ram.base != ALLOWED_BASE: - self.add_error("Your specified RAM is not in correct units") + self.add_error( + "Your specified RAM is not in correct units" + ) if parsed_os_ssd.base != ALLOWED_BASE: - self.add_error("Your specified OS-SSD is not in correct units") + self.add_error( + "Your specified OS-SSD is not in correct units" + ) - if _cpu < 1: + if int(_cpu) < 1: self.add_error("CPU must be atleast 1") if parsed_ram < bitmath.GB(1): @@ -275,7 +334,9 @@ class CreateVMSchema(OTPSchema): for hdd in _hdd: _parsed_hdd = bitmath.parse_string_unsafe(hdd) if _parsed_hdd.base != ALLOWED_BASE: - self.add_error("Your specified HDD is not in correct units") + self.add_error( + "Your specified HDD is not in correct units" + ) break else: parsed_hdd.append(str(_parsed_hdd)) @@ -286,21 +347,24 @@ class CreateVMSchema(OTPSchema): else: if self.get_errors(): self.specs = { - 'cpu': _cpu, - 'ram': str(parsed_ram), - 'os-ssd': str(parsed_os_ssd), - 'hdd': parsed_hdd + "cpu": _cpu, + "ram": str(parsed_ram), + "os-ssd": str(parsed_os_ssd), + "hdd": parsed_hdd, } class VMStatusSchema(OTPSchema): def __init__(self, data): data["uuid"] = ( - resolve_vm_name( - name=data.get("vm_name", None), - owner=(data.get("in_support_of", None) or data.get("name", None)), - ) - or KeyError + resolve_vm_name( + name=data.get("vm_name", None), + owner=( + data.get("in_support_of", None) + or data.get("name", None) + ), + ) + or KeyError ) self.uuid = VmUUIDField(data) @@ -309,9 +373,10 @@ class VMStatusSchema(OTPSchema): super().__init__(data, fields) def validation(self): - vm = vm_pool.get(self.uuid.value) + vm = shared.vm_pool.get(self.uuid.value) if not ( - vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + vm.value["owner"] == self.name.value + or self.realm.value == "ungleich-admin" ): self.add_error("Invalid User") @@ -319,11 +384,14 @@ class VMStatusSchema(OTPSchema): class VmActionSchema(OTPSchema): def __init__(self, data): data["uuid"] = ( - resolve_vm_name( - name=data.get("vm_name", None), - owner=(data.get("in_support_of", None) or data.get("name", None)), - ) - or KeyError + resolve_vm_name( + name=data.get("vm_name", None), + owner=( + data.get("in_support_of", None) + or data.get("name", None) + ), + ) + or KeyError ) self.uuid = VmUUIDField(data) self.action = Field("action", str, data.get("action", KeyError)) @@ -338,20 +406,23 @@ class VmActionSchema(OTPSchema): allowed_actions = ["start", "stop", "delete"] if self.action.value not in allowed_actions: self.add_error( - "Invalid Action. Allowed Actions are {}".format(allowed_actions) + "Invalid Action. Allowed Actions are {}".format( + allowed_actions + ) ) def validation(self): - vm = vm_pool.get(self.uuid.value) + vm = shared.vm_pool.get(self.uuid.value) if not ( - vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + vm.value["owner"] == self.name.value + or self.realm.value == "ungleich-admin" ): self.add_error("Invalid User") if ( - self.action.value == "start" - and vm.status == VMStatus.running - and vm.hostname != "" + self.action.value == "start" + and vm.status == VMStatus.running + and vm.hostname != "" ): self.add_error("VM Already Running") @@ -365,15 +436,20 @@ class VmActionSchema(OTPSchema): class VmMigrationSchema(OTPSchema): def __init__(self, data): data["uuid"] = ( - resolve_vm_name( - name=data.get("vm_name", None), - owner=(data.get("in_support_of", None) or data.get("name", None)), - ) - or KeyError + resolve_vm_name( + name=data.get("vm_name", None), + owner=( + data.get("in_support_of", None) + or data.get("name", None) + ), + ) + or KeyError ) self.uuid = VmUUIDField(data) - self.destination = Field("destination", str, data.get("destination", KeyError)) + self.destination = Field( + "destination", str, data.get("destination", KeyError) + ) self.destination.validation = self.destination_validation @@ -382,31 +458,47 @@ class VmMigrationSchema(OTPSchema): def destination_validation(self): hostname = self.destination.value - host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + host = next( + filter( + lambda h: h.hostname == hostname, shared.host_pool.hosts + ), + None, + ) if not host: - self.add_error("No Such Host ({}) exists".format(self.destination.value)) + self.add_error( + "No Such Host ({}) exists".format( + self.destination.value + ) + ) elif host.status != HostStatus.alive: self.add_error("Destination Host is dead") else: self.destination.value = host.key def validation(self): - vm = vm_pool.get(self.uuid.value) + vm = shared.vm_pool.get(self.uuid.value) if not ( - vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin" + vm.value["owner"] == self.name.value + or self.realm.value == "ungleich-admin" ): self.add_error("Invalid User") if vm.status != VMStatus.running: self.add_error("Can't migrate non-running VM") - if vm.hostname == os.path.join(env_vars.get('HOST_PREFIX'), self.destination.value): - self.add_error("Destination host couldn't be same as Source Host") + if vm.hostname == os.path.join( + shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value + ): + self.add_error( + "Destination host couldn't be same as Source Host" + ) class AddSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field("key_name", str, data.get("key_name", KeyError)) + self.key_name = Field( + "key_name", str, data.get("key_name", KeyError) + ) self.key = Field("key", str, data.get("key_name", KeyError)) fields = [self.key_name, self.key] @@ -415,7 +507,9 @@ class AddSSHSchema(OTPSchema): class RemoveSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field("key_name", str, data.get("key_name", KeyError)) + self.key_name = Field( + "key_name", str, data.get("key_name", KeyError) + ) fields = [self.key_name] super().__init__(data=data, fields=fields) @@ -423,7 +517,9 @@ class RemoveSSHSchema(OTPSchema): class GetSSHSchema(OTPSchema): def __init__(self, data): - self.key_name = Field("key_name", str, data.get("key_name", None)) + self.key_name = Field( + "key_name", str, data.get("key_name", None) + ) fields = [self.key_name] super().__init__(data=data, fields=fields) @@ -442,15 +538,20 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'), - self.name.value, - self.network_name.value), - value_in_json=True) + key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) + network = shared.etcd_client.get(key, value_in_json=True) if network: - self.add_error("Network with name {} already exists" \ - .format(self.network_name.value)) + self.add_error( + "Network with name {} already exists".format( + self.network_name.value + ) + ) def network_type_validation(self): supported_network_types = ["vxlan"] if self.type.value not in supported_network_types: - self.add_error("Unsupported Network Type. Supported network types are {}".format(supported_network_types)) + self.add_error( + "Unsupported Network Type. Supported network types are {}".format( + supported_network_types + ) + ) diff --git a/ucloud/metadata/__init__.py b/archive/uncloud_etcd_based/uncloud/cli/__init__.py similarity index 100% rename from ucloud/metadata/__init__.py rename to archive/uncloud_etcd_based/uncloud/cli/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/cli/helper.py b/archive/uncloud_etcd_based/uncloud/cli/helper.py new file mode 100644 index 0000000..51a4355 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/helper.py @@ -0,0 +1,46 @@ +import requests +import json +import argparse +import binascii + +from pyotp import TOTP +from os.path import join as join_path +from uncloud.common.shared import shared + + +def get_otp_parser(): + otp_parser = argparse.ArgumentParser('otp') + otp_parser.add_argument('--name') + otp_parser.add_argument('--realm') + otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') + + return otp_parser + + +def load_dump_pretty(content): + if isinstance(content, bytes): + content = content.decode('utf-8') + parsed = json.loads(content) + return json.dumps(parsed, indent=4, sort_keys=True) + + +def make_request(*args, data=None, request_method=requests.post): + try: + r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) + except requests.exceptions.RequestException: + print('Error occurred while connecting to API server.') + else: + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except binascii.Error: + raise argparse.ArgumentTypeError('Invalid seed') + else: + return token diff --git a/archive/uncloud_etcd_based/uncloud/cli/host.py b/archive/uncloud_etcd_based/uncloud/cli/host.py new file mode 100644 index 0000000..e912567 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/host.py @@ -0,0 +1,45 @@ +import requests + +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class HostParser(BaseParser): + def __init__(self): + super().__init__('host') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs) + p.add_argument('--hostname', required=True) + p.add_argument('--cpu', required=True, type=int) + p.add_argument('--ram', required=True) + p.add_argument('--os-ssd', required=True) + p.add_argument('--hdd', default=list()) + + def list(self, **kwargs): + self.subparser.add_parser('list', **kwargs) + + +parser = HostParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('host_subcommand') + if not subcommand: + arg_parser.print_help() + else: + request_method = requests.post + data = None + if subcommand == 'create': + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + data = kwargs + elif subcommand == 'list': + request_method = requests.get + + make_request('host', subcommand, data=data, request_method=request_method) diff --git a/archive/uncloud_etcd_based/uncloud/cli/image.py b/archive/uncloud_etcd_based/uncloud/cli/image.py new file mode 100644 index 0000000..2f59c32 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/image.py @@ -0,0 +1,38 @@ +import requests + +from uncloud.cli.helper import make_request +from uncloud.common.parser import BaseParser + + +class ImageParser(BaseParser): + def __init__(self): + super().__init__('image') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', **kwargs) + p.add_argument('--name', required=True) + p.add_argument('--uuid', required=True) + p.add_argument('--image-store', required=True, dest='image_store') + + def list(self, **kwargs): + self.subparser.add_parser('list', **kwargs) + + +parser = ImageParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('image_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = None + request_method = requests.post + if subcommand == 'list': + subcommand = 'list-public' + request_method = requests.get + elif subcommand == 'create': + data = kwargs + + make_request('image', subcommand, data=data, request_method=request_method) diff --git a/archive/uncloud_etcd_based/uncloud/cli/main.py b/archive/uncloud_etcd_based/uncloud/cli/main.py new file mode 100644 index 0000000..9a42497 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import argparse +import importlib + +arg_parser = argparse.ArgumentParser('cli', add_help=False) +subparser = arg_parser.add_subparsers(dest='subcommand') + +for component in ['user', 'host', 'image', 'network', 'vm']: + module = importlib.import_module('uncloud.cli.{}'.format(component)) + parser = getattr(module, 'arg_parser') + subparser.add_parser(name=parser.prog, parents=[parser]) + + +def main(arguments): + if not arguments['subcommand']: + arg_parser.print_help() + else: + name = arguments.pop('subcommand') + arguments.pop('debug') + mod = importlib.import_module('uncloud.cli.{}'.format(name)) + _main = getattr(mod, 'main') + _main(**arguments) diff --git a/archive/uncloud_etcd_based/uncloud/cli/network.py b/archive/uncloud_etcd_based/uncloud/cli/network.py new file mode 100644 index 0000000..55798bf --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/network.py @@ -0,0 +1,32 @@ +import requests + +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class NetworkParser(BaseParser): + def __init__(self): + super().__init__('network') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs) + p.add_argument('--network-name', required=True) + p.add_argument('--network-type', required=True, dest='type') + p.add_argument('--user', action='store_true') + + +parser = NetworkParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('network_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = None + request_method = requests.post + if subcommand == 'create': + data = kwargs + + make_request('network', subcommand, data=data, request_method=request_method) diff --git a/archive/uncloud_etcd_based/uncloud/cli/user.py b/archive/uncloud_etcd_based/uncloud/cli/user.py new file mode 100755 index 0000000..3a4cc4e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/user.py @@ -0,0 +1,41 @@ +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class UserParser(BaseParser): + def __init__(self): + super().__init__('user') + + def files(self, **kwargs): + self.subparser.add_parser('files', parents=[get_otp_parser()], **kwargs) + + def vms(self, **kwargs): + self.subparser.add_parser('vms', parents=[get_otp_parser()], **kwargs) + + def networks(self, **kwargs): + self.subparser.add_parser('networks', parents=[get_otp_parser()], **kwargs) + + def add_ssh(self, **kwargs): + p = self.subparser.add_parser('add-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', required=True) + p.add_argument('--key', required=True) + + def get_ssh(self, **kwargs): + p = self.subparser.add_parser('get-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', default='') + + def remove_ssh(self, **kwargs): + p = self.subparser.add_parser('remove-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', required=True) + + +parser = UserParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('user_subcommand') + if not subcommand: + arg_parser.print_help() + else: + make_request('user', subcommand, data=kwargs) diff --git a/archive/uncloud_etcd_based/uncloud/cli/vm.py b/archive/uncloud_etcd_based/uncloud/cli/vm.py new file mode 100644 index 0000000..396530e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/vm.py @@ -0,0 +1,62 @@ +from uncloud.common.parser import BaseParser +from uncloud.cli.helper import make_request, get_otp_parser + + +class VMParser(BaseParser): + def __init__(self): + super().__init__('vm') + + def start(self, **args): + p = self.subparser.add_parser('start', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def stop(self, **args): + p = self.subparser.add_parser('stop', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def status(self, **args): + p = self.subparser.add_parser('status', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def delete(self, **args): + p = self.subparser.add_parser('delete', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def migrate(self, **args): + p = self.subparser.add_parser('migrate', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + p.add_argument('--destination', required=True) + + def create(self, **args): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **args) + p.add_argument('--cpu', required=True) + p.add_argument('--ram', required=True) + p.add_argument('--os-ssd', required=True) + p.add_argument('--hdd', action='append', default=list()) + p.add_argument('--image', required=True) + p.add_argument('--network', action='append', default=[]) + p.add_argument('--vm-name', required=True) + + +parser = VMParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('vm_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = kwargs + endpoint = subcommand + if subcommand in ['start', 'stop', 'delete']: + endpoint = 'action' + data['action'] = subcommand + elif subcommand == 'create': + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + make_request('vm', endpoint, data=data) diff --git a/ucloud/network/__init__.py b/archive/uncloud_etcd_based/uncloud/client/__init__.py similarity index 100% rename from ucloud/network/__init__.py rename to archive/uncloud_etcd_based/uncloud/client/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/client/main.py b/archive/uncloud_etcd_based/uncloud/client/main.py new file mode 100644 index 0000000..062308c --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/client/main.py @@ -0,0 +1,23 @@ +import argparse +import etcd3 +from uncloud.common.etcd_wrapper import Etcd3Wrapper + +arg_parser = argparse.ArgumentParser('client', add_help=False) +arg_parser.add_argument('--dump-etcd-contents-prefix', help="Dump contents below the given prefix") + +def dump_etcd_contents(prefix): + etcd = Etcd3Wrapper() + for k,v in etcd.get_prefix_raw(prefix): + k = k.decode('utf-8') + v = v.decode('utf-8') + print("{} = {}".format(k,v)) +# print("{} = {}".format(k,v)) + +# for k,v in etcd.get_prefix(prefix): +# + print("done") + + +def main(arguments): + if 'dump_etcd_contents_prefix' in arguments: + dump_etcd_contents(prefix=arguments['dump_etcd_contents_prefix']) diff --git a/ucloud/common/__init__.py b/archive/uncloud_etcd_based/uncloud/common/__init__.py similarity index 100% rename from ucloud/common/__init__.py rename to archive/uncloud_etcd_based/uncloud/common/__init__.py diff --git a/ucloud/common/classes.py b/archive/uncloud_etcd_based/uncloud/common/classes.py similarity index 93% rename from ucloud/common/classes.py rename to archive/uncloud_etcd_based/uncloud/common/classes.py index 2eae809..29dffd4 100644 --- a/ucloud/common/classes.py +++ b/archive/uncloud_etcd_based/uncloud/common/classes.py @@ -1,4 +1,4 @@ -from etcd3_wrapper import EtcdEntry +from .etcd_wrapper import EtcdEntry class SpecificEtcdEntryBase: diff --git a/archive/uncloud_etcd_based/uncloud/common/cli.py b/archive/uncloud_etcd_based/uncloud/common/cli.py new file mode 100644 index 0000000..3d3c248 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/cli.py @@ -0,0 +1,26 @@ +from uncloud.common.shared import shared +from pyotp import TOTP + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except Exception: + raise Exception('Invalid seed') + else: + return token + + +def resolve_otp_credentials(kwargs): + d = { + 'name': shared.settings['client']['name'], + 'realm': shared.settings['client']['realm'], + 'token': get_token(shared.settings['client']['seed']) + } + + for k, v in d.items(): + if k in kwargs and kwargs[k] is None: + kwargs.update({k: v}) + + return d diff --git a/ucloud/common/counters.py b/archive/uncloud_etcd_based/uncloud/common/counters.py similarity index 91% rename from ucloud/common/counters.py rename to archive/uncloud_etcd_based/uncloud/common/counters.py index 066a870..2d4a8e9 100644 --- a/ucloud/common/counters.py +++ b/archive/uncloud_etcd_based/uncloud/common/counters.py @@ -1,4 +1,4 @@ -from etcd3_wrapper import Etcd3Wrapper +from .etcd_wrapper import Etcd3Wrapper def increment_etcd_counter(etcd_client: Etcd3Wrapper, key): diff --git a/archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py b/archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py new file mode 100644 index 0000000..38471ab --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py @@ -0,0 +1,75 @@ +import etcd3 +import json + +from functools import wraps + +from uncloud import UncloudException +from uncloud.common import logger + + +class EtcdEntry: + def __init__(self, meta_or_key, value, value_in_json=False): + if hasattr(meta_or_key, 'key'): + # if meta has attr 'key' then get it + self.key = meta_or_key.key.decode('utf-8') + else: + # otherwise meta is the 'key' + self.key = meta_or_key + self.value = value.decode('utf-8') + + if value_in_json: + self.value = json.loads(self.value) + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError: + raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') + except etcd3.exceptions.ConnectionTimeoutError as err: + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err + except Exception: + logger.exception('Some etcd error occured. See syslog for details.') + + return wrapper + + +class Etcd3Wrapper: + @readable_errors + def __init__(self, *args, **kwargs): + self.client = etcd3.client(*args, **kwargs) + + @readable_errors + def get(self, *args, value_in_json=False, **kwargs): + _value, _key = self.client.get(*args, **kwargs) + if _key is None or _value is None: + return None + return EtcdEntry(_key, _value, value_in_json=value_in_json) + + @readable_errors + def put(self, *args, value_in_json=False, **kwargs): + _key, _value = args + if value_in_json: + _value = json.dumps(_value) + + if not isinstance(_key, str): + _key = _key.decode('utf-8') + + return self.client.put(_key, _value, **kwargs) + + @readable_errors + def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) + + @readable_errors + def watch_prefix(self, key, raise_exception=True, value_in_json=False): + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = e._event + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) diff --git a/ucloud/common/host.py b/archive/uncloud_etcd_based/uncloud/common/host.py similarity index 85% rename from ucloud/common/host.py rename to archive/uncloud_etcd_based/uncloud/common/host.py index ccbf7a8..f7bb7d5 100644 --- a/ucloud/common/host.py +++ b/archive/uncloud_etcd_based/uncloud/common/host.py @@ -7,7 +7,7 @@ from .classes import SpecificEtcdEntryBase class HostStatus: - """Possible Statuses of ucloud host.""" + """Possible Statuses of uncloud host.""" alive = "ALIVE" dead = "DEAD" @@ -26,11 +26,13 @@ class HostEntry(SpecificEtcdEntryBase): def update_heartbeat(self): self.status = HostStatus.alive - self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S") + self.last_heartbeat = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") def is_alive(self): - last_heartbeat = datetime.strptime(self.last_heartbeat, "%Y-%m-%d %H:%M:%S") - delta = datetime.now() - last_heartbeat + last_heartbeat = datetime.strptime( + self.last_heartbeat, "%Y-%m-%d %H:%M:%S" + ) + delta = datetime.utcnow() - last_heartbeat if delta.total_seconds() > 60: return False return True diff --git a/archive/uncloud_etcd_based/uncloud/common/network.py b/archive/uncloud_etcd_based/uncloud/common/network.py new file mode 100644 index 0000000..32f6951 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/network.py @@ -0,0 +1,70 @@ +import subprocess as sp +import random +import logging + +logger = logging.getLogger(__name__) + + +def random_bytes(num=6): + return [random.randrange(256) for _ in range(num)] + + +def generate_mac( + uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" +): + mac = random_bytes() + if oui: + if type(oui) == str: + oui = [int(chunk) for chunk in oui.split(separator)] + mac = oui + random_bytes(num=6 - len(oui)) + else: + if multicast: + mac[0] |= 1 # set bit 0 + else: + mac[0] &= ~1 # clear bit 0 + if uaa: + mac[0] &= ~(1 << 1) # clear bit 1 + else: + mac[0] |= 1 << 1 # set bit 1 + return separator.join(byte_fmt % b for b in mac) + + +def create_dev(script, _id, dev, ip=None): + command = [ + "sudo", + "-p", + "Enter password to create network devices for vm: ", + script, + str(_id), + dev, + ] + if ip: + command.append(ip) + try: + output = sp.check_output(command, stderr=sp.PIPE) + except Exception: + logger.exception("Creation of interface %s failed.", dev) + return None + else: + return output.decode("utf-8").strip() + + +def delete_network_interface(iface): + try: + sp.check_output( + [ + "sudo", + "-p", + "Enter password to remove {} network device: ".format( + iface + ), + "ip", + "link", + "del", + iface, + ], + stderr=sp.PIPE, + ) + except Exception: + logger.exception("Interface %s Deletion failed", iface) + diff --git a/archive/uncloud_etcd_based/uncloud/common/parser.py b/archive/uncloud_etcd_based/uncloud/common/parser.py new file mode 100644 index 0000000..576f0e7 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/parser.py @@ -0,0 +1,13 @@ +import argparse + + +class BaseParser: + def __init__(self, command): + self.arg_parser = argparse.ArgumentParser(command, add_help=False) + self.subparser = self.arg_parser.add_subparsers(dest='{}_subcommand'.format(command)) + self.common_args = {'add_help': False} + + methods = [attr for attr in dir(self) if not attr.startswith('__') + and type(getattr(self, attr)).__name__ == 'method'] + for method in methods: + getattr(self, method)(**self.common_args) diff --git a/ucloud/common/request.py b/archive/uncloud_etcd_based/uncloud/common/request.py similarity index 74% rename from ucloud/common/request.py rename to archive/uncloud_etcd_based/uncloud/common/request.py index cadac80..cb0add5 100644 --- a/ucloud/common/request.py +++ b/archive/uncloud_etcd_based/uncloud/common/request.py @@ -2,9 +2,8 @@ import json from os.path import join from uuid import uuid4 -from etcd3_wrapper.etcd3_wrapper import PsuedoEtcdEntry - -from .classes import SpecificEtcdEntryBase +from uncloud.common.etcd_wrapper import EtcdEntry +from uncloud.common.classes import SpecificEtcdEntryBase class RequestType: @@ -18,8 +17,9 @@ class RequestType: class RequestEntry(SpecificEtcdEntryBase): - def __init__(self, e): + self.destination_sock_path = None + self.destination_host_key = None self.type = None # type: str self.migration = None # type: bool self.destination = None # type: str @@ -29,8 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase): @classmethod def from_scratch(cls, request_prefix, **kwargs): - e = PsuedoEtcdEntry(join(request_prefix, uuid4().hex), - value=json.dumps(kwargs).encode("utf-8"), value_in_json=True) + e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex), + value=json.dumps(kwargs).encode('utf-8'), value_in_json=True) return cls(e) diff --git a/archive/uncloud_etcd_based/uncloud/common/schemas.py b/archive/uncloud_etcd_based/uncloud/common/schemas.py new file mode 100644 index 0000000..04978a5 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/schemas.py @@ -0,0 +1,41 @@ +import bitmath + +from marshmallow import fields, Schema + + +class StorageUnit(fields.Field): + def _serialize(self, value, attr, obj, **kwargs): + return str(value) + + def _deserialize(self, value, attr, data, **kwargs): + return bitmath.parse_string_unsafe(value) + + +class SpecsSchema(Schema): + cpu = fields.Int() + ram = StorageUnit() + os_ssd = StorageUnit(data_key="os-ssd", attribute="os-ssd") + hdd = fields.List(StorageUnit()) + + +class VMSchema(Schema): + name = fields.Str() + owner = fields.Str() + owner_realm = fields.Str() + specs = fields.Nested(SpecsSchema) + status = fields.Str() + log = fields.List(fields.Str()) + vnc_socket = fields.Str() + image_uuid = fields.Str() + hostname = fields.Str() + metadata = fields.Dict() + network = fields.List( + fields.Tuple((fields.Str(), fields.Str(), fields.Int())) + ) + in_migration = fields.Bool() + + +class NetworkSchema(Schema): + _id = fields.Int(data_key="id", attribute="id") + _type = fields.Str(data_key="type", attribute="type") + ipv6 = fields.Str() diff --git a/archive/uncloud_etcd_based/uncloud/common/settings.py b/archive/uncloud_etcd_based/uncloud/common/settings.py new file mode 100644 index 0000000..8503f42 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/settings.py @@ -0,0 +1,136 @@ +import configparser +import logging +import sys +import os + +from datetime import datetime +from uncloud.common.etcd_wrapper import Etcd3Wrapper +from os.path import join as join_path + +logger = logging.getLogger(__name__) +settings = None + + +class CustomConfigParser(configparser.RawConfigParser): + def __getitem__(self, key): + try: + result = super().__getitem__(key) + except KeyError as err: + raise KeyError( + 'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format( + key + ) + ) from err + else: + return result + + +class Settings(object): + def __init__(self, conf_dir, seed_value=None): + conf_name = 'uncloud.conf' + self.config_file = join_path(conf_dir, conf_name) + + # this is used to cache config from etcd for 1 minutes. Without this we + # would make a lot of requests to etcd which slows down everything. + self.last_config_update = datetime.fromtimestamp(0) + + self.config_parser = CustomConfigParser(allow_no_value=True) + self.config_parser.add_section('etcd') + self.config_parser.set('etcd', 'base_prefix', '/') + + if os.access(self.config_file, os.R_OK): + self.config_parser.read(self.config_file) + else: + raise FileNotFoundError('Config file %s not found!', self.config_file) + self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') + + self.read_internal_values() + + if seed_value is None: + seed_value = dict() + + self.config_parser.read_dict(seed_value) + + def get_etcd_client(self): + args = tuple() + try: + kwargs = { + 'host': self.config_parser.get('etcd', 'url'), + 'port': self.config_parser.get('etcd', 'port'), + 'ca_cert': self.config_parser.get('etcd', 'ca_cert'), + 'cert_cert': self.config_parser.get('etcd', 'cert_cert'), + 'cert_key': self.config_parser.get('etcd', 'cert_key'), + } + except configparser.Error as err: + raise configparser.Error( + '{} in config file {}'.format( + err.message, self.config_file + ) + ) from err + else: + try: + wrapper = Etcd3Wrapper(*args, **kwargs) + except Exception as err: + logger.error( + 'etcd connection not successfull. Please check your config file.' + '\nDetails: %s\netcd connection parameters: %s', + err, + kwargs, + ) + sys.exit(1) + else: + return wrapper + + def read_internal_values(self): + base_prefix = self['etcd']['base_prefix'] + self.config_parser.read_dict( + { + 'etcd': { + 'file_prefix': join_path(base_prefix, 'files/'), + 'host_prefix': join_path(base_prefix, 'hosts/'), + 'image_prefix': join_path(base_prefix, 'images/'), + 'image_store_prefix': join_path(base_prefix, 'imagestore/'), + 'network_prefix': join_path(base_prefix, 'networks/'), + 'request_prefix': join_path(base_prefix, 'requests/'), + 'user_prefix': join_path(base_prefix, 'users/'), + 'vm_prefix': join_path(base_prefix, 'vms/'), + 'vxlan_counter': join_path(base_prefix, 'counters/vxlan'), + 'tap_counter': join_path(base_prefix, 'counters/tap') + } + } + ) + + def read_config_file_values(self, config_file): + try: + # Trying to read configuration file + with open(config_file) as config_file_handle: + self.config_parser.read_file(config_file_handle) + except FileNotFoundError: + sys.exit('Configuration file {} not found!'.format(config_file)) + except Exception as err: + logger.exception(err) + sys.exit('Error occurred while reading configuration file') + + def read_values_from_etcd(self): + etcd_client = self.get_etcd_client() + if (datetime.utcnow() - self.last_config_update).total_seconds() > 60: + config_from_etcd = etcd_client.get(self.config_key, value_in_json=True) + if config_from_etcd: + self.config_parser.read_dict(config_from_etcd.value) + self.last_config_update = datetime.utcnow() + else: + raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key)) + + def __getitem__(self, key): + # Allow failing to read from etcd if we have + # it locally + if key not in self.config_parser.sections(): + try: + self.read_values_from_etcd() + except KeyError: + pass + return self.config_parser[key] + + +def get_settings(): + return settings diff --git a/archive/uncloud_etcd_based/uncloud/common/shared.py b/archive/uncloud_etcd_based/uncloud/common/shared.py new file mode 100644 index 0000000..aea7cbc --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/shared.py @@ -0,0 +1,34 @@ +from uncloud.common.settings import get_settings +from uncloud.common.vm import VmPool +from uncloud.common.host import HostPool +from uncloud.common.request import RequestPool +import uncloud.common.storage_handlers as storage_handlers + + +class Shared: + @property + def settings(self): + return get_settings() + + @property + def etcd_client(self): + return self.settings.get_etcd_client() + + @property + def host_pool(self): + return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) + + @property + def vm_pool(self): + return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) + + @property + def request_pool(self): + return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) + + @property + def storage_handler(self): + return storage_handlers.get_storage_handler() + + +shared = Shared() diff --git a/ucloud/common/storage_handlers.py b/archive/uncloud_etcd_based/uncloud/common/storage_handlers.py similarity index 63% rename from ucloud/common/storage_handlers.py rename to archive/uncloud_etcd_based/uncloud/common/storage_handlers.py index 8b1097a..58c2dc2 100644 --- a/ucloud/common/storage_handlers.py +++ b/archive/uncloud_etcd_based/uncloud/common/storage_handlers.py @@ -6,17 +6,20 @@ import stat from abc import ABC from . import logger from os.path import join as join_path +import uncloud.common.shared as shared class ImageStorageHandler(ABC): + handler_name = "base" + def __init__(self, image_base, vm_base): self.image_base = image_base self.vm_base = vm_base def import_image(self, image_src, image_dest, protect=False): """Put an image at the destination - :param src: An Image file - :param dest: A path where :param src: is to be put. + :param image_src: An Image file + :param image_dest: A path where :param src: is to be put. :param protect: If protect is true then the dest is protect (readonly etc) The obj must exist on filesystem. """ @@ -26,8 +29,8 @@ class ImageStorageHandler(ABC): def make_vm_image(self, image_path, path): """Copy image from src to dest - :param src: A path - :param dest: A path + :param image_path: A path + :param path: A path src and destination must be on same storage system i.e both on file system or both on CEPH etc. """ @@ -43,14 +46,17 @@ class ImageStorageHandler(ABC): def delete_vm_image(self, path): raise NotImplementedError() - def execute_command(self, command, report=True): + def execute_command(self, command, report=True, error_origin=None): + if not error_origin: + error_origin = self.handler_name + command = list(map(str, command)) try: - output = sp.check_output(command, stderr=sp.PIPE) - except Exception as e: + sp.check_output(command, stderr=sp.PIPE) + except sp.CalledProcessError as e: + _stderr = e.stderr.decode("utf-8").strip() if report: - print(e) - logger.exception(e) + logger.exception("%s:- %s", error_origin, _stderr) return False return True @@ -65,12 +71,16 @@ class ImageStorageHandler(ABC): class FileSystemBasedImageStorageHandler(ImageStorageHandler): + handler_name = "Filesystem" + def import_image(self, src, dest, protect=False): dest = join_path(self.image_base, dest) try: shutil.copy(src, dest) if protect: - os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + os.chmod( + dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH + ) except Exception as e: logger.exception(e) return False @@ -80,7 +90,7 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler): src = join_path(self.image_base, src) dest = join_path(self.vm_base, dest) try: - shutil.copy(src, dest) + shutil.copyfile(src, dest) except Exception as e: logger.exception(e) return False @@ -88,7 +98,14 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler): def resize_vm_image(self, path, size): path = join_path(self.vm_base, path) - command = ["qemu-img", "resize", "-f", "raw", path, "{}M".format(size)] + command = [ + "qemu-img", + "resize", + "-f", + "raw", + path, + "{}M".format(size), + ] if self.execute_command(command): return True else: @@ -117,17 +134,33 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler): class CEPHBasedImageStorageHandler(ImageStorageHandler): + handler_name = "Ceph" + def import_image(self, src, dest, protect=False): dest = join_path(self.image_base, dest) - command = ["rbd", "import", src, dest] + import_command = ["rbd", "import", src, dest] + commands = [import_command] if protect: - snap_create_command = ["rbd", "snap", "create", "{}@protected".format(dest)] - snap_protect_command = ["rbd", "snap", "protect", "{}@protected".format(dest)] + snap_create_command = [ + "rbd", + "snap", + "create", + "{}@protected".format(dest), + ] + snap_protect_command = [ + "rbd", + "snap", + "protect", + "{}@protected".format(dest), + ] + commands.append(snap_create_command) + commands.append(snap_protect_command) - return self.execute_command(command) and self.execute_command(snap_create_command) and\ - self.execute_command(snap_protect_command) + result = True + for command in commands: + result = result and self.execute_command(command) - return self.execute_command(command) + return result def make_vm_image(self, src, dest): src = join_path(self.image_base, src) @@ -156,3 +189,19 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): path = join_path(self.vm_base, path) command = ["rbd", "info", path] return self.execute_command(command, report=False) + + +def get_storage_handler(): + __storage_backend = shared.shared.settings["storage"]["storage_backend"] + if __storage_backend == "filesystem": + return FileSystemBasedImageStorageHandler( + vm_base=shared.shared.settings["storage"]["vm_dir"], + image_base=shared.shared.settings["storage"]["image_dir"], + ) + elif __storage_backend == "ceph": + return CEPHBasedImageStorageHandler( + vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], + image_base=shared.shared.settings["storage"]["ceph_image_pool"], + ) + else: + raise Exception("Unknown Image Storage Handler") \ No newline at end of file diff --git a/ucloud/common/vm.py b/archive/uncloud_etcd_based/uncloud/common/vm.py similarity index 92% rename from ucloud/common/vm.py rename to archive/uncloud_etcd_based/uncloud/common/vm.py index 0fb5cea..d11046d 100644 --- a/ucloud/common/vm.py +++ b/archive/uncloud_etcd_based/uncloud/common/vm.py @@ -12,8 +12,13 @@ class VMStatus: error = "ERROR" # An error occurred that cannot be resolved automatically -class VMEntry(SpecificEtcdEntryBase): +def declare_stopped(vm): + vm["hostname"] = "" + vm["in_migration"] = False + vm["status"] = VMStatus.stopped + +class VMEntry(SpecificEtcdEntryBase): def __init__(self, e): self.owner = None # type: str self.specs = None # type: dict @@ -42,7 +47,9 @@ class VMEntry(SpecificEtcdEntryBase): def add_log(self, msg): self.log = self.log[:5] - self.log.append("{} - {}".format(datetime.now().isoformat(), msg)) + self.log.append( + "{} - {}".format(datetime.now().isoformat(), msg) + ) class VmPool: diff --git a/ucloud/scheduler/tests/__init__.py b/archive/uncloud_etcd_based/uncloud/configure/__init__.py similarity index 100% rename from ucloud/scheduler/tests/__init__.py rename to archive/uncloud_etcd_based/uncloud/configure/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/configure/main.py b/archive/uncloud_etcd_based/uncloud/configure/main.py new file mode 100644 index 0000000..87f5752 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/configure/main.py @@ -0,0 +1,57 @@ +import os +import argparse + +from uncloud.common.shared import shared + +arg_parser = argparse.ArgumentParser('configure', add_help=False) +configure_subparsers = arg_parser.add_subparsers(dest='subcommand') + +otp_parser = configure_subparsers.add_parser('otp') +otp_parser.add_argument('--verification-controller-url', required=True, metavar='URL') +otp_parser.add_argument('--auth-name', required=True, metavar='OTP-NAME') +otp_parser.add_argument('--auth-realm', required=True, metavar='OTP-REALM') +otp_parser.add_argument('--auth-seed', required=True, metavar='OTP-SEED') + +network_parser = configure_subparsers.add_parser('network') +network_parser.add_argument('--prefix-length', required=True, type=int) +network_parser.add_argument('--prefix', required=True) +network_parser.add_argument('--vxlan-phy-dev', required=True) + +netbox_parser = configure_subparsers.add_parser('netbox') +netbox_parser.add_argument('--url', required=True) +netbox_parser.add_argument('--token', required=True) + +ssh_parser = configure_subparsers.add_parser('ssh') +ssh_parser.add_argument('--username', default='root') +ssh_parser.add_argument('--private-key-path', default=os.path.expanduser('~/.ssh/id_rsa'),) + +storage_parser = configure_subparsers.add_parser('storage') +storage_parser.add_argument('--file-dir', required=True) +storage_parser_subparsers = storage_parser.add_subparsers(dest='storage_backend') + +filesystem_storage_parser = storage_parser_subparsers.add_parser('filesystem') +filesystem_storage_parser.add_argument('--vm-dir', required=True) +filesystem_storage_parser.add_argument('--image-dir', required=True) + +ceph_storage_parser = storage_parser_subparsers.add_parser('ceph') +ceph_storage_parser.add_argument('--ceph-vm-pool', required=True) +ceph_storage_parser.add_argument('--ceph-image-pool', required=True) + + +def update_config(section, kwargs): + uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) + if not uncloud_config: + uncloud_config = {} + else: + uncloud_config = uncloud_config.value + + uncloud_config[section] = kwargs + shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) + + +def main(arguments): + subcommand = arguments['subcommand'] + if not subcommand: + arg_parser.print_help() + else: + update_config(subcommand, arguments) diff --git a/ucloud/filescanner/__init__.py b/archive/uncloud_etcd_based/uncloud/filescanner/__init__.py similarity index 100% rename from ucloud/filescanner/__init__.py rename to archive/uncloud_etcd_based/uncloud/filescanner/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/filescanner/main.py b/archive/uncloud_etcd_based/uncloud/filescanner/main.py new file mode 100755 index 0000000..046f915 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/filescanner/main.py @@ -0,0 +1,85 @@ +import glob +import os +import pathlib +import subprocess as sp +import time +import argparse +import bitmath + +from uuid import uuid4 + +from . import logger +from uncloud.common.shared import shared + +arg_parser = argparse.ArgumentParser('filescanner', add_help=False) +arg_parser.add_argument('--hostname', required=True) + + +def sha512sum(file: str): + """Use sha512sum utility to compute sha512 sum of arg:file + + IF arg:file does not exists: + raise FileNotFoundError exception + ELSE IF sum successfully computer: + return computed sha512 sum + ELSE: + return None + """ + if not isinstance(file, str): + raise TypeError + try: + output = sp.check_output(['sha512sum', file], stderr=sp.PIPE) + except sp.CalledProcessError as e: + error = e.stderr.decode('utf-8') + if 'No such file or directory' in error: + raise FileNotFoundError from None + else: + output = output.decode('utf-8').strip() + output = output.split(' ') + return output[0] + return None + + +def track_file(file, base_dir, host): + file_path = file.relative_to(base_dir) + file_str = str(file) + # Get Username + try: + owner = file_path.parts[0] + except IndexError: + pass + else: + file_path = file_path.relative_to(owner) + creation_date = time.ctime(os.stat(file_str).st_ctime) + + entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) + entry_value = { + 'filename': str(file_path), + 'owner': owner, + 'sha512sum': sha512sum(file_str), + 'creation_date': creation_date, + 'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()), + 'host': host + } + + logger.info('Tracking %s', file_str) + + shared.etcd_client.put(entry_key, entry_value, value_in_json=True) + + +def main(arguments): + hostname = arguments['hostname'] + base_dir = shared.settings['storage']['file_dir'] + # Recursively Get All Files and Folder below BASE_DIR + files = glob.glob('{}/**'.format(base_dir), recursive=True) + files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] + + # Files that are already tracked + tracked_files = [ + pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) + for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) + if f.value['host'] == hostname + ] + untracked_files = set(files) - set(tracked_files) + for file in untracked_files: + track_file(file, base_dir, hostname) diff --git a/ucloud/hack/README.org b/archive/uncloud_etcd_based/uncloud/hack/README.org similarity index 100% rename from ucloud/hack/README.org rename to archive/uncloud_etcd_based/uncloud/hack/README.org diff --git a/archive/uncloud_etcd_based/uncloud/hack/__init__.py b/archive/uncloud_etcd_based/uncloud/hack/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/__init__.py @@ -0,0 +1 @@ + diff --git a/ucloud/hack/conf.d/ucloud-host b/archive/uncloud_etcd_based/uncloud/hack/conf.d/ucloud-host similarity index 100% rename from ucloud/hack/conf.d/ucloud-host rename to archive/uncloud_etcd_based/uncloud/hack/conf.d/ucloud-host diff --git a/archive/uncloud_etcd_based/uncloud/hack/config.py b/archive/uncloud_etcd_based/uncloud/hack/config.py new file mode 100644 index 0000000..7e2655d --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/config.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +class Config(object): + def __init__(self, arguments): + """ read arguments dicts as a base """ + + self.arguments = arguments + + # Split them so *etcd_args can be used and we can + # iterate over etcd_hosts + self.etcd_hosts = [ arguments['etcd_host'] ] + self.etcd_args = { + 'ca_cert': arguments['etcd_ca_cert'], + 'cert_cert': arguments['etcd_cert_cert'], + 'cert_key': arguments['etcd_cert_key'], +# 'user': None, +# 'password': None + } + self.etcd_prefix = '/nicohack/' diff --git a/archive/uncloud_etcd_based/uncloud/hack/db.py b/archive/uncloud_etcd_based/uncloud/hack/db.py new file mode 100644 index 0000000..3d5582e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/db.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +import etcd3 +import json +import logging +import datetime +import re + +from functools import wraps +from uncloud import UncloudException + +log = logging.getLogger(__name__) + +def db_logentry(message): + timestamp = datetime.datetime.now() + return { + "timestamp": str(timestamp), + "message": message + } + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + + +class DB(object): + def __init__(self, config, prefix="/"): + self.config = config + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + + try: + self.connect() + except FileNotFoundError as e: + raise UncloudException("Is the path to the etcd certs correct? {}".format(e)) + + @readable_errors + def connect(self): + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) + + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) + + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + @readable_errors + def get_prefix(self, key, as_json=False, **kwargs): + for value, meta in self._db_clients[0].get_prefix(self.realkey(key), **kwargs): + k = meta.key.decode("utf-8") + value = value.decode("utf-8") + if as_json: + value = json.loads(value) + + yield (k, value) + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: + value = json.dumps(value) + + log.debug("Setting {} = {}".format(self.realkey(key), value)) + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + + @readable_errors + def list_and_filter(self, key, filter_key=None, filter_regexp=None): + for k,v in self.get_prefix(key, as_json=True): + + if filter_key and filter_regexp: + if filter_key in v: + if re.match(filter_regexp, v[filter_key]): + yield v + else: + yield v + + + @readable_errors + def increment(self, key, **kwargs): + print(self.realkey(key)) + + + print("prelock") + lock = self._db_clients[0].lock('/nicohack/foo') + print("prelockacq") + lock.acquire() + print("prelockrelease") + lock.release() + + with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock: + print("in lock") + pass + +# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs)) +# self.set(self.realkey(key), str(value + 1), **kwargs) + + +if __name__ == '__main__': + endpoints = [ "https://etcd1.ungleich.ch:2379", + "https://etcd2.ungleich.ch:2379", + "https://etcd3.ungleich.ch:2379" ] + + db = DB(url=endpoints) diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore new file mode 100644 index 0000000..0ad647b --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore @@ -0,0 +1,3 @@ +*.iso +radvdpid +foo diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py @@ -0,0 +1 @@ + diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh new file mode 100644 index 0000000..ab102a5 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \ + --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \ + --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \ + --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@" diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh new file mode 100755 index 0000000..5753099 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo $@ diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh new file mode 100755 index 0000000..e0a3ca0 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +dev=$1; shift + +# bridge is setup from outside +ip link set dev "$dev" master ${bridge} +ip link set dev "$dev" up diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last new file mode 100644 index 0000000..8c5f254 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last @@ -0,0 +1 @@ +000000000252 diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix new file mode 100644 index 0000000..5084a2f --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix @@ -0,0 +1 @@ +02:00 diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/net.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/net.sh new file mode 100755 index 0000000..4e2bfa1 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/net.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -x + +netid=100 +dev=wlp2s0 +dev=wlp0s20f3 +#dev=wlan0 + +ip=2a0a:e5c1:111:888::48/64 +vxlandev=vxlan${netid} +bridgedev=br${netid} + +ip -6 link add ${vxlandev} type vxlan \ + id ${netid} \ + dstport 4789 \ + group ff05::${netid} \ + dev ${dev} \ + ttl 5 + +ip link set ${vxlandev} up + + +ip link add ${bridgedev} type bridge +ip link set ${bridgedev} up + +ip link set ${vxlandev} master ${bridgedev} up + +ip addr add ${ip} dev ${bridgedev} diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules new file mode 100644 index 0000000..636c63d --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules @@ -0,0 +1,31 @@ +flush ruleset + +table bridge filter { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; + + ibrname br100 jump br100 + } + + chain br100 { + # Allow all incoming traffic from outside + iifname vxlan100 accept + + # Default blocks: router advertisements, dhcpv6, dhcpv4 + icmpv6 type nd-router-advert drop + ip6 version 6 udp sport 547 drop + ip version 4 udp sport 67 drop + + jump br100_vmlist + drop + } + chain br100_vmlist { + # VM1 + iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept + + # VM2 + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept + } +} diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules-v2 b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules-v2 new file mode 100644 index 0000000..b6d4cf3 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules-v2 @@ -0,0 +1,64 @@ +flush ruleset + +table bridge filter { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; + + ibrname br100 jump netpublic + } + + chain netpublic { + iifname vxlan100 jump from_uncloud + + # Default blocks: router advertisements, dhcpv6, dhcpv4 + icmpv6 type nd-router-advert drop + ip6 version 6 udp sport 547 drop + ip version 4 udp sport 67 drop + + # Individual blocks +# iifname tap1 jump vm1 + } + + chain vm1 { + ether saddr != 02:00:f0:a9:c4:4e drop + ip6 saddr != 2a0a:e5c1:111:888:0:f0ff:fea9:c44e drop + } + + chain from_uncloud { + accept + } +} + +# table ip6 filter { +# chain forward { +# type filter hook forward priority 0; + +# # policy drop; + +# ct state established,related accept; + +# } + +# } + +# table ip filter { +# chain input { +# type filter hook input priority filter; policy drop; +# iif "lo" accept +# icmp type { echo-reply, destination-unreachable, source-quench, redirect, echo-request, router-advertisement, router-solicitation, time-exceeded, parameter-problem, timestamp-request, timestamp-reply, info-request, info-reply, address-mask-request, address-mask-reply } accept +# ct state established,related accept +# tcp dport { 22 } accept +# log prefix "firewall-ipv4: " +# udp sport 67 drop +# } + +# chain forward { +# type filter hook forward priority filter; policy drop; +# log prefix "firewall-ipv4: " +# } + +# chain output { +# type filter hook output priority filter; policy accept; +# } +# } diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.conf b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.conf new file mode 100644 index 0000000..3d8ce4d --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.conf @@ -0,0 +1,13 @@ +interface br100 +{ + AdvSendAdvert on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 5; + AdvDefaultLifetime 3600; + + prefix 2a0a:e5c1:111:888::/64 { + }; + + RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; }; + DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ; +}; diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.sh new file mode 100644 index 0000000..9d0e7d1 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +radvd -C ./radvd.conf -n -p ./radvdpid diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm-2.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm-2.sh new file mode 100755 index 0000000..af9dec7 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm-2.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +vmid=$1; shift + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +#accel=tcg + +memory=1024 +cores=2 +uuid=732e08c7-84f8-4d43-9571-263db4f80080 + +export bridge=br100 + +$qemu -name uc${vmid} \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh new file mode 100755 index 0000000..dd9be84 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# if [ $# -ne 1 ]; then +# echo "$0: owner" +# exit 1 +# fi + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +#accel=tcg + +memory=1024 +cores=2 +uuid=$(uuidgen) +mac=$(./mac-gen.py) +owner=nico + +export bridge=br100 + +set -x +$qemu -name "uncloud-${uuid}" \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac} diff --git a/archive/uncloud_etcd_based/uncloud/hack/host.py b/archive/uncloud_etcd_based/uncloud/hack/host.py new file mode 100644 index 0000000..06ccf98 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/host.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +import uuid + +from uncloud.hack.db import DB +from uncloud import UncloudException + +class Host(object): + def __init__(self, config, db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/hosts") + + if db_entry: + self.db_entry = db_entry + + + def list_hosts(self, filter_key=None, filter_regexp=None): + """ Return list of all hosts """ + for entry in self.db.list_and_filter("", filter_key, filter_regexp): + yield self.__class__(self.config, db_entry=entry) + + def cmdline_add_host(self): + """ FIXME: make this a bit smarter and less redundant """ + + for required_arg in [ + 'add_vm_host', + 'max_cores_per_vm', + 'max_cores_total', + 'max_memory_in_gb' ]: + if not required_arg in self.config.arguments: + raise UncloudException("Missing argument: {}".format(required_arg)) + + return self.add_host( + self.config.arguments['add_vm_host'], + self.config.arguments['max_cores_per_vm'], + self.config.arguments['max_cores_total'], + self.config.arguments['max_memory_in_gb']) + + + def add_host(self, + hostname, + max_cores_per_vm, + max_cores_total, + max_memory_in_gb): + + db_entry = {} + db_entry['uuid'] = str(uuid.uuid4()) + db_entry['hostname'] = hostname + db_entry['max_cores_per_vm'] = max_cores_per_vm + db_entry['max_cores_total'] = max_cores_total + db_entry['max_memory_in_gb'] = max_memory_in_gb + db_entry["db_version"] = 1 + db_entry["log"] = [] + + self.db.set(db_entry['uuid'], db_entry, as_json=True) + + return self.__class__(self.config, db_entry) diff --git a/archive/uncloud_etcd_based/uncloud/hack/mac.py b/archive/uncloud_etcd_based/uncloud/hack/mac.py new file mode 100755 index 0000000..e35cd9f --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/mac.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2012 Nico Schottelius (nico-cinv at schottelius.org) +# +# This file is part of cinv. +# +# cinv is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cinv is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cinv. If not, see . +# +# + +import argparse +import logging +import os.path +import os +import re +import json + +from uncloud import UncloudException +from uncloud.hack.db import DB + +log = logging.getLogger(__name__) + + +class MAC(object): + def __init__(self, config): + self.config = config + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(config, prefix="/mac") + + self.prefix = 0x420000000000 + self._number = 0 # Not set by default + + @staticmethod + def validate_mac(mac): + if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): + raise UncloudException("Not a valid mac address: %s" % mac) + else: + return True + + def last_used_index(self): + if not self.no_db: + value = self.db.get("last_used_index") + if not value: + self.db.set("last_used_index", "0") + value = self.db.get("last_used_index") + + else: + value = "0" + + return int(value) + + def last_used_mac(self): + return self.int_to_mac(self.prefix + self.last_used_index()) + + def to_colon_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ':'.join(format(s, '02x') for s in b) + + def to_str_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ''.join(format(s, '02x') for s in b) + + def create(self): + last_number = self.last_used_index() + + if last_number == int('0xffffffff', 16): + raise UncloudException("Exhausted all possible mac addresses - try to free some") + + next_number = last_number + 1 + self._number = self.prefix + next_number + + #next_number_string = "{:012x}".format(next_number) + #next_mac = self.int_to_mac(next_mac_number) + # db_entry = {} + # db_entry['vm_uuid'] = vmuuid + # db_entry['index'] = next_number + # db_entry['mac_address'] = next_mac + + # should be one transaction + # self.db.increment("last_used_index") + # self.db.set("used/{}".format(next_mac), + # db_entry, as_json=True) + + def __int__(self): + return self._number + + def __repr__(self): + return self.to_str_format() + + def __str__(self): + return self.to_colon_format() diff --git a/archive/uncloud_etcd_based/uncloud/hack/main.py b/archive/uncloud_etcd_based/uncloud/hack/main.py new file mode 100644 index 0000000..0ddd8fb --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/main.py @@ -0,0 +1,186 @@ +import argparse +import logging +import re + +import ldap3 + + +from uncloud.hack.vm import VM +from uncloud.hack.host import Host +from uncloud.hack.config import Config +from uncloud.hack.mac import MAC +from uncloud.hack.net import VXLANBridge, DNSRA + +from uncloud import UncloudException +from uncloud.hack.product import ProductOrder + +arg_parser = argparse.ArgumentParser('hack', add_help=False) + #description="Commands that are unfinished - use at own risk") +arg_parser.add_argument('--last-used-mac', action='store_true') +arg_parser.add_argument('--get-new-mac', action='store_true') + +arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true') +arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true') +arg_parser.add_argument('--network', help="/64 IPv6 network") +arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0") +arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) +arg_parser.add_argument('--run-dns-ra', action='store_true', + help="Provide router advertisements and DNS resolution via dnsmasq") +arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') + +arg_parser.add_argument('--create-vm', action='store_true') +arg_parser.add_argument('--destroy-vm', action='store_true') +arg_parser.add_argument('--get-vm-status', action='store_true') +arg_parser.add_argument('--get-vm-vnc', action='store_true') +arg_parser.add_argument('--list-vms', action='store_true') +arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int, default=2) +arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int, default=1) +arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") + +arg_parser.add_argument('--image-format', help="Image format: qcow2 or raw", choices=['raw', 'qcow2']) +arg_parser.add_argument('--uuid', help="VM UUID") + +arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') +arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") + +# order based commands => later to be shifted below "order" +arg_parser.add_argument('--order', action='store_true') +arg_parser.add_argument('--list-orders', help="List all orders", action='store_true') +arg_parser.add_argument('--filter-order-key', help="Which key to filter on") +arg_parser.add_argument('--filter-order-regexp', help="Which regexp the value should match") + +arg_parser.add_argument('--process-orders', help="Process all (pending) orders", action='store_true') + +arg_parser.add_argument('--product', choices=["dualstack-vm"]) +arg_parser.add_argument('--os-image-name', help="Name of OS image (successor to --image)") +arg_parser.add_argument('--os-image-size', help="Size of OS image in GB", type=int, default=10) + +arg_parser.add_argument('--username') +arg_parser.add_argument('--password') + +arg_parser.add_argument('--api', help="Run the API") +arg_parser.add_argument('--mode', + choices=["direct", "api", "client"], + default="client", + help="Directly manipulate etcd, spawn the API server or behave as a client") + + +arg_parser.add_argument('--add-vm-host', help="Add a host that can run VMs") +arg_parser.add_argument('--list-vm-hosts', action='store_true') + +arg_parser.add_argument('--max-cores-per-vm') +arg_parser.add_argument('--max-cores-total') +arg_parser.add_argument('--max-memory-in-gb') + + +log = logging.getLogger(__name__) + +def authenticate(username, password, totp_token=None): + server = ldap3.Server("ldaps://ldap1.ungleich.ch") + dn = "uid={},ou=customer,dc=ungleich,dc=ch".format(username) + + log.debug("LDAP: connecting to {} as {}".format(server, dn)) + + try: + conn = ldap3.Connection(server, dn, password, auto_bind=True) + except ldap3.core.exceptions.LDAPBindError as e: + raise UncloudException("Credentials not verified by LDAP server: {}".format(e)) + + + +def order(config): + for required_arg in [ 'product', 'username', 'password' ]: + if not config.arguments[required_arg]: + raise UncloudException("Missing required argument: {}".format(required_arg)) + + if config.arguments['product'] == 'dualstack-vm': + for required_arg in [ 'cores', 'memory', 'os_image_name', 'os_image_size' ]: + if not config.arguments[required_arg]: + raise UncloudException("Missing required argument: {}".format(required_arg)) + + log.debug(config.arguments) + authenticate(config.arguments['username'], config.arguments['password']) + + # create DB entry for VM + vm = VM(config) + return vm.product.place_order(owner=config.arguments['username']) + + + + + +def main(arguments): + config = Config(arguments) + + if arguments['add_vm_host']: + h = Host(config) + h.cmdline_add_host() + + if arguments['list_vm_hosts']: + h = Host(config) + + for host in h.list_hosts(filter_key=arguments['filter_order_key'], + filter_regexp=arguments['filter_order_regexp']): + print("Host {}: {}".format(host.db_entry['uuid'], host.db_entry)) + + if arguments['order']: + print("Created order: {}".format(order(config))) + + if arguments['list_orders']: + p = ProductOrder(config) + for product_order in p.list_orders(filter_key=arguments['filter_order_key'], + filter_regexp=arguments['filter_order_regexp']): + print("Order {}: {}".format(product_order.db_entry['uuid'], product_order.db_entry)) + + if arguments['process_orders']: + p = ProductOrder(config) + p.process_orders() + + if arguments['create_vm']: + vm = VM(config) + vm.create() + + if arguments['destroy_vm']: + vm = VM(config) + vm.stop() + + if arguments['get_vm_status']: + vm = VM(config) + vm.status() + + if arguments['get_vm_vnc']: + vm = VM(config) + vm.vnc_addr() + + if arguments['list_vms']: + vm = VM(config) + vm.list() + + if arguments['last_used_mac']: + m = MAC(config) + print(m.last_used_mac()) + + if arguments['get_new_mac']: + print(MAC(config).get_next()) + + #if arguments['init_network']: + if arguments['create_vxlan']: + if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: + raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + vb = VXLANBridge(vni=arguments['vni'], + route=arguments['network'], + uplinkdev=arguments['vxlan_uplink_device'], + use_sudo=arguments['use_sudo']) + vb._setup_vxlan() + vb._setup_bridge() + vb._add_vxlan_to_bridge() + vb._route_network() + + if arguments['run_dns_ra']: + if not arguments['network'] or not arguments['vni']: + raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + + dnsra = DNSRA(route=arguments['network'], + vni=arguments['vni'], + use_sudo=arguments['use_sudo']) + dnsra._setup_dnsmasq() diff --git a/archive/uncloud_etcd_based/uncloud/hack/net.py b/archive/uncloud_etcd_based/uncloud/hack/net.py new file mode 100644 index 0000000..4887e04 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/net.py @@ -0,0 +1,116 @@ +import subprocess +import ipaddress +import logging + + +from uncloud import UncloudException + +log = logging.getLogger(__name__) + + +class VXLANBridge(object): + cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" + cmd_up_dev = "{sudo}ip link set {dev} up" + cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge" + cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up" + cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}" + cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}" + + # VXLAN ids are at maximum 24 bit - use a /104 + multicast_network = ipaddress.IPv6Network("ff05::/104") + max_vni = (2**24)-1 + + def __init__(self, + vni, + uplinkdev, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' + + self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + self.config['multicast_address'] = self.multicast_network[vni] + + self.config['route_network'] = ipaddress.IPv6Network(route) + self.config['route'] = route + + self.config['uplinkdev'] = uplinkdev + self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + + def setup_networking(self): + pass + + def _setup_vxlan(self): + self._execute_cmd(self.cmd_create_vxlan) + self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev']) + + def _setup_bridge(self): + self._execute_cmd(self.cmd_create_bridge) + self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev']) + + def _route_network(self): + self._execute_cmd(self.cmd_add_route_dev) + + def _add_vxlan_to_bridge(self): + self._execute_cmd(self.cmd_add_to_bridge) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + +class ManagementBridge(VXLANBridge): + pass + + +class DNSRA(object): + # VXLAN ids are at maximum 24 bit + max_vni = (2**24)-1 + + + # Command to start dnsmasq + cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra --no-daemon" + + def __init__(self, + vni, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' + + #TODO: remove if not needed + #self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + + # dnsmasq only wants the network without the prefix, therefore, cut it off + self.config['route'] = ipaddress.IPv6Network(route).network_address + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + def _setup_dnsmasq(self): + self._execute_cmd(self.cmd_start_dnsmasq) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + print("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + +class Firewall(object): + pass diff --git a/ucloud/hack/nftables.conf b/archive/uncloud_etcd_based/uncloud/hack/nftables.conf similarity index 100% rename from ucloud/hack/nftables.conf rename to archive/uncloud_etcd_based/uncloud/hack/nftables.conf diff --git a/archive/uncloud_etcd_based/uncloud/hack/product.py b/archive/uncloud_etcd_based/uncloud/hack/product.py new file mode 100755 index 0000000..f979268 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/product.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +import json +import uuid +import logging +import re +import importlib + +from uncloud import UncloudException +from uncloud.hack.db import DB, db_logentry + +log = logging.getLogger(__name__) + +class ProductOrder(object): + def __init__(self, config, product_entry=None, db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/orders") + self.db_entry = {} + self.db_entry["product"] = product_entry + + # Overwrite if we are loading an existing product order + if db_entry: + self.db_entry = db_entry + + # FIXME: this should return a list of our class! + def list_orders(self, filter_key=None, filter_regexp=None): + for entry in self.db.list_and_filter("", filter_key, filter_regexp): + yield self.__class__(self.config, db_entry=entry) + + + def set_required_values(self): + """Set values that are required to make the db entry valid""" + if not "uuid" in self.db_entry: + self.db_entry["uuid"] = str(uuid.uuid4()) + if not "status" in self.db_entry: + self.db_entry["status"] = "NEW" + if not "owner" in self.db_entry: + self.db_entry["owner"] = "UNKNOWN" + if not "log" in self.db_entry: + self.db_entry["log"] = [] + if not "db_version" in self.db_entry: + self.db_entry["db_version"] = 1 + + def validate_status(self): + if "status" in self.db_entry: + if self.db_entry["status"] in [ "NEW", + "SCHEDULED", + "CREATED_ACTIVE", + "CANCELLED", + "REJECTED" ]: + return False + return True + + def order(self): + self.set_required_values() + if not self.db_entry["status"] == "NEW": + raise UncloudException("Cannot re-order same order. Status: {}".format(self.db_entry["status"])) + self.db.set(self.db_entry["uuid"], self.db_entry, as_json=True) + + return self.db_entry["uuid"] + + def process_orders(self): + """processing orders can be done stand alone on server side""" + for order in self.list_orders(): + if order.db_entry["status"] == "NEW": + log.info("Handling new order: {}".format(order)) + + # FIXME: these all should be a transactions! -> fix concurrent access! ! + if not "log" in order.db_entry: + order.db_entry['log'] = [] + + is_valid = True + # Verify the order entry + for must_attribute in [ "owner", "product" ]: + if not must_attribute in order.db_entry: + message = "Missing {} entry in order, rejecting order".format(must_attribute) + log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message)) + + order.db_entry['log'].append(db_logentry(message)) + order.db_entry['status'] = "REJECTED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + is_valid = False + + # Rejected the order + if not is_valid: + continue + + # Verify the product entry + for must_attribute in [ "python_product_class", "python_product_module" ]: + if not must_attribute in order.db_entry['product']: + message = "Missing {} entry in product of order, rejecting order".format(must_attribute) + log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message)) + + order.db_entry['log'].append(db_logentry(message)) + order.db_entry['status'] = "REJECTED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + is_valid = False + + # Rejected the order + if not is_valid: + continue + + print(order.db_entry["product"]["python_product_class"]) + + # Create the product + m = importlib.import_module(order.db_entry["product"]["python_product_module"]) + c = getattr(m, order.db_entry["product"]["python_product_class"]) + + product = c(config, db_entry=order.db_entry["product"]) + + # STOPPED + product.create_product() + + order.db_entry['status'] = "SCHEDULED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + + + def __str__(self): + return str(self.db_entry) + +class Product(object): + def __init__(self, + config, + product_name, + product_class, + db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/orders") + + self.db_entry = {} + self.db_entry["product_name"] = product_name + self.db_entry["python_product_class"] = product_class.__qualname__ + self.db_entry["python_product_module"] = product_class.__module__ + self.db_entry["db_version"] = 1 + self.db_entry["log"] = [] + self.db_entry["features"] = {} + + # Existing product? Read in db_entry + if db_entry: + self.db_entry = db_entry + + self.valid_periods = [ "per_year", "per_month", "per_week", + "per_day", "per_hour", + "per_minute", "per_second" ] + + def define_feature(self, + name, + one_time_price, + recurring_price, + recurring_period, + minimum_period): + + self.db_entry['features'][name] = {} + self.db_entry['features'][name]['one_time_price'] = one_time_price + self.db_entry['features'][name]['recurring_price'] = recurring_price + + if not recurring_period in self.valid_periods: + raise UncloudException("Invalid recurring period: {}".format(recurring_period)) + + self.db_entry['features'][name]['recurring_period'] = recurring_period + + if not minimum_period in self.valid_periods: + raise UncloudException("Invalid recurring period: {}".format(recurring_period)) + + recurring_index = self.valid_periods.index(recurring_period) + minimum_index = self.valid_periods.index(minimum_period) + + if minimum_index < recurring_index: + raise UncloudException("Minimum period for product '{}' feature '{}' must be shorter or equal than/as recurring period: {} > {}".format(self.db_entry['product_name'], name, minimum_period, recurring_period)) + + self.db_entry['features'][name]['minimum_period'] = minimum_period + + + def validate_product(self): + for feature in self.db_entry['features']: + pass + + def place_order(self, owner): + """ Schedule creating the product in etcd """ + order = ProductOrder(self.config, product_entry=self.db_entry) + order.db_entry["owner"] = owner + return order.order() + + def __str__(self): + return json.dumps(self.db_entry) diff --git a/ucloud/hack/rc-scripts/ucloud-api b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-api similarity index 100% rename from ucloud/hack/rc-scripts/ucloud-api rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-api diff --git a/ucloud/hack/rc-scripts/ucloud-host b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-host similarity index 100% rename from ucloud/hack/rc-scripts/ucloud-host rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-host diff --git a/ucloud/hack/rc-scripts/ucloud-metadata b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-metadata similarity index 100% rename from ucloud/hack/rc-scripts/ucloud-metadata rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-metadata diff --git a/ucloud/hack/rc-scripts/ucloud-scheduler b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-scheduler similarity index 100% rename from ucloud/hack/rc-scripts/ucloud-scheduler rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-scheduler diff --git a/archive/uncloud_etcd_based/uncloud/hack/uncloud-hack-init-host b/archive/uncloud_etcd_based/uncloud/hack/uncloud-hack-init-host new file mode 100644 index 0000000..787ff80 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/uncloud-hack-init-host @@ -0,0 +1,26 @@ +id=100 +rawdev=eth0 + +# create vxlan +ip -6 link add vxlan${id} type vxlan \ + id ${id} \ + dstport 4789 \ + group ff05::${id} \ + dev ${rawdev} \ + ttl 5 + +ip link set vxlan${id} up + +# create bridge +ip link set vxlan${id} up +ip link set br${id} up + +# Add vxlan into bridge +ip link set vxlan${id} master br${id} + + +# useradd -m uncloud +# [18:05] tablett.place10:~# id uncloud +# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu) +# apk add qemu-system-x86_64 +# also needs group netdev diff --git a/archive/uncloud_etcd_based/uncloud/hack/uncloud-run-vm b/archive/uncloud_etcd_based/uncloud/hack/uncloud-run-vm new file mode 100644 index 0000000..33e5860 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/uncloud-run-vm @@ -0,0 +1,25 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo $0 vmid + exit 1 +fi + +id=$1; shift + +memory=512 +macaddress=02:00:b9:cb:70:${id} +netname=net${id}-1 + +qemu-system-x86_64 \ + -name uncloud-${id} \ + -accel kvm \ + -m ${memory} \ + -smp 2,sockets=2,cores=1,threads=1 \ + -device virtio-net-pci,netdev=net0,mac=$macaddress \ + -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \ + -vnc [::]:0 + +# To be changed: +# -vnc to unix path +# or -spice diff --git a/archive/uncloud_etcd_based/uncloud/hack/vm.py b/archive/uncloud_etcd_based/uncloud/hack/vm.py new file mode 100755 index 0000000..4b0ca14 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/vm.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +# This module is directly called from the hack module, and can be used as follow: +# +# Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix. +# `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2` +# +# List running VMs (returns a list of UUIDs). +# `uncloud hack --hackprefix /tmp/hackcloud --list-vms +# +# Get VM status: +# `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid` +# +# Stop a VM: +# `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` +# `` + +import subprocess +import uuid +import os +import logging + +from uncloud.hack.db import DB +from uncloud.hack.mac import MAC +from uncloud.vmm import VMM +from uncloud.hack.product import Product + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +class VM(object): + def __init__(self, config, db_entry=None): + self.config = config + + #TODO: Enable etcd lookup + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(self.config, prefix="/vm") + + if db_entry: + self.db_entry = db_entry + + # General CLI arguments. + self.hackprefix = self.config.arguments['hackprefix'] + self.uuid = self.config.arguments['uuid'] + self.memory = self.config.arguments['memory'] or '1024M' + self.cores = self.config.arguments['cores'] or 1 + + if self.config.arguments['image']: + self.image = os.path.join(self.hackprefix, self.config.arguments['image']) + else: + self.image = None + + if self.config.arguments['image_format']: + self.image_format=self.config.arguments['image_format'] + else: + self.image_format='qcow2' + + # External components. + + # This one is broken: + # TypeError: expected str, bytes or os.PathLike object, not NoneType + # Fix before re-enabling + # self.vmm = VMM(vmm_backend=self.hackprefix) + self.mac = MAC(self.config) + + # Harcoded & generated values. + self.owner = 'uncloud' + self.accel = 'kvm' + self.threads = 1 + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + self.ifname = "uc{}".format(self.mac.to_str_format()) + + self.vm = {} + + self.product = Product(config, product_name="dualstack-vm", + product_class=self.__class__) + self.product.define_feature(name="base", + one_time_price=0, + recurring_price=9, + recurring_period="per_month", + minimum_period="per_hour") + + + self.features = [] + + + def get_qemu_args(self): + command = ( + "-name {owner}-{name}" + " -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}" + " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" + ).format( + owner=self.owner, name=self.uuid, + accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname, + mac=self.mac + ) + + return command.split(" ") + + def create_product(self): + """Find a VM host and schedule on it""" + pass + + def create(self): + # New VM: new UUID, new MAC. + self.uuid = str(uuid.uuid4()) + self.mac=MAC(self.config) + self.mac.create() + + qemu_args = self.get_qemu_args() + log.debug("QEMU args passed to VMM: {}".format(qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + + self.mac.create() + self.vm['mac'] = self.mac + self.vm['ifname'] = "uc{}".format(self.mac.__repr__()) + + # FIXME: TODO: turn this into a string and THEN + # .split() it later -- easier for using .format() + #self.vm['commandline'] = [ "{}".format(self.sudo), + self.vm['commandline'] = "{sudo}{qemu} -name uncloud-{uuid} -machine pc,accel={accel} -m {memory} -smp {cores} -uuid {uuid} -drive file={os_image},media=cdrom -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname} -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" +# self.vm['commandline'] = [ "{}".format(self.sudo), +# "{}".format(self.qemu), +# "-name", "uncloud-{}".format(self.vm['uuid']), +# "-machine", "pc,accel={}".format(self.accel), +# "-m", "{}".format(self.vm['memory']), +# "-smp", "{}".format(self.vm['cores']), +# "-uuid", "{}".format(self.vm['uuid']), +# "-drive", "file={},media=cdrom".format(self.vm['os_image']), +# "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), +# "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.vm['mac']) +# ] + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.vm, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + + def stop(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + self.vmm.stop(self.uuid) + + def status(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_status(self.uuid)) + + def vnc_addr(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_vnc(self.uuid)) + + def list(self): + print(self.vmm.discover()) diff --git a/ucloud/host/__init__.py b/archive/uncloud_etcd_based/uncloud/host/__init__.py similarity index 100% rename from ucloud/host/__init__.py rename to archive/uncloud_etcd_based/uncloud/host/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/host/main.py b/archive/uncloud_etcd_based/uncloud/host/main.py new file mode 100755 index 0000000..f680991 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/host/main.py @@ -0,0 +1,123 @@ +import argparse +import multiprocessing as mp +import time + +from uuid import uuid4 + +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.shared import shared +from uncloud.common.vm import VMStatus +from uncloud.vmm import VMM +from os.path import join as join_path + +from . import virtualmachine, logger + +arg_parser = argparse.ArgumentParser('host', add_help=False) +arg_parser.add_argument('--hostname', required=True) + + +def update_heartbeat(hostname): + """Update Last HeartBeat Time for :param hostname: in etcd""" + host_pool = shared.host_pool + this_host = next( + filter(lambda h: h.hostname == hostname, host_pool.hosts), None + ) + while True: + this_host.update_heartbeat() + host_pool.put(this_host) + time.sleep(10) + + +def maintenance(host): + vmm = VMM() + running_vms = vmm.discover() + for vm_uuid in running_vms: + if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': + logger.debug('VM {} is running on {}'.format(vm_uuid, host)) + vm = shared.vm_pool.get( + join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) + ) + vm.status = VMStatus.running + vm.vnc_socket = vmm.get_vnc(vm_uuid) + vm.hostname = host + shared.vm_pool.put(vm) + + +def main(arguments): + hostname = arguments['hostname'] + host_pool = shared.host_pool + host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + + # Does not yet exist, create it + if not host: + host_key = join_path( + shared.settings['etcd']['host_prefix'], uuid4().hex + ) + host_entry = { + 'specs': '', + 'hostname': hostname, + 'status': 'DEAD', + 'last_heartbeat': '', + } + shared.etcd_client.put( + host_key, host_entry, value_in_json=True + ) + + # update, get ourselves now for sure + host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + + try: + heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) + heartbeat_updating_process.start() + except Exception as e: + raise Exception('uncloud-host heartbeat updating mechanism is not working') from e + + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for events_iterator in [ + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False) + ]: + for request_event in events_iterator: + request_event = RequestEntry(request_event) + + maintenance(host.key) + + if request_event.hostname == host.key: + logger.debug('VM Request: %s on Host %s', request_event, host.hostname) + + shared.request_pool.client.client.delete(request_event.key) + vm_entry = shared.etcd_client.get( + join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) + ) + + logger.debug('VM hostname: {}'.format(vm_entry.value)) + + vm = virtualmachine.VM(vm_entry) + if request_event.type == RequestType.StartVM: + vm.start() + + elif request_event.type == RequestType.StopVM: + vm.stop() + + elif request_event.type == RequestType.DeleteVM: + vm.delete() + + elif request_event.type == RequestType.InitVMMigration: + vm.start(destination_host_key=host.key) + + elif request_event.type == RequestType.TransferVM: + destination_host = host_pool.get(request_event.destination_host_key) + if destination_host: + vm.migrate( + destination_host=destination_host.hostname, + destination_sock_path=request_event.destination_sock_path, + ) + else: + logger.error('Host %s not found!', request_event.destination_host_key) diff --git a/archive/uncloud_etcd_based/uncloud/host/virtualmachine.py b/archive/uncloud_etcd_based/uncloud/host/virtualmachine.py new file mode 100755 index 0000000..a592efc --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/host/virtualmachine.py @@ -0,0 +1,303 @@ +# QEMU Manual +# https://qemu.weilnetz.de/doc/qemu-doc.html + +# For QEMU Monitor Protocol Commands Information, See +# https://qemu.weilnetz.de/doc/qemu-doc.html#pcsys_005fmonitor + +import os +import subprocess as sp +import ipaddress + +from string import Template +from os.path import join as join_path + +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.vm import VMStatus, declare_stopped +from uncloud.common.network import create_dev, delete_network_interface +from uncloud.common.schemas import VMSchema, NetworkSchema +from uncloud.host import logger +from uncloud.common.shared import shared +from uncloud.vmm import VMM + +from marshmallow import ValidationError + + +class VM: + def __init__(self, vm_entry): + self.schema = VMSchema() + self.vmm = VMM() + self.key = vm_entry.key + try: + self.vm = self.schema.loads(vm_entry.value) + except ValidationError: + logger.exception( + "Couldn't validate VM Entry", vm_entry.value + ) + self.vm = None + else: + self.uuid = vm_entry.key.split("/")[-1] + self.host_key = self.vm["hostname"] + logger.debug('VM Hostname {}'.format(self.host_key)) + + def get_qemu_args(self): + command = ( + "-drive file={file},format=raw,if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -name {owner}_{name}" + ).format( + owner=self.vm["owner"], + name=self.vm["name"], + memory=int(self.vm["specs"]["ram"].to_MB()), + cores=self.vm["specs"]["cpu"], + threads=1, + file=shared.storage_handler.qemu_path_string(self.uuid), + ) + + return command.split(" ") + + def start(self, destination_host_key=None): + migration = False + if destination_host_key: + migration = True + + self.create() + try: + network_args = self.create_network_dev() + except Exception as err: + declare_stopped(self.vm) + self.vm["log"].append("Cannot Setup Network Properly") + logger.error("Cannot Setup Network Properly for vm %s", self.uuid, exc_info=err) + else: + self.vmm.start( + uuid=self.uuid, + migration=migration, + *self.get_qemu_args(), + *network_args + ) + + status = self.vmm.get_status(self.uuid) + logger.debug('VM {} status is {}'.format(self.uuid, status)) + if status == "running": + self.vm["status"] = VMStatus.running + self.vm["vnc_socket"] = self.vmm.get_vnc(self.uuid) + elif status == "inmigrate": + r = RequestEntry.from_scratch( + type=RequestType.TransferVM, # Transfer VM + hostname=self.host_key, # Which VM should get this request. It is source host + uuid=self.uuid, # uuid of VM + destination_sock_path=join_path( + self.vmm.socket_dir, self.uuid + ), + destination_host_key=destination_host_key, # Where source host transfer VM + request_prefix=shared.settings["etcd"]["request_prefix"], + ) + shared.request_pool.put(r) + else: + self.stop() + declare_stopped(self.vm) + logger.debug('VM {} has hostname {}'.format(self.uuid, self.vm['hostname'])) + self.sync() + + def stop(self): + self.vmm.stop(self.uuid) + self.delete_network_dev() + declare_stopped(self.vm) + self.sync() + + def migrate(self, destination_host, destination_sock_path): + self.vmm.transfer( + src_uuid=self.uuid, + destination_sock_path=destination_sock_path, + host=destination_host, + ) + + def create_network_dev(self): + command = "" + for network_mac_and_tap in self.vm["network"]: + network_name, mac, tap = network_mac_and_tap + + _key = os.path.join( + shared.settings["etcd"]["network_prefix"], + self.vm["owner"], + network_name, + ) + network = shared.etcd_client.get(_key, value_in_json=True) + network_schema = NetworkSchema() + try: + network = network_schema.load(network.value) + except ValidationError: + continue + + if network["type"] == "vxlan": + tap = create_vxlan_br_tap( + _id=network["id"], + _dev=shared.settings["network"]["vxlan_phy_dev"], + tap_id=tap, + ip=network["ipv6"], + ) + + all_networks = shared.etcd_client.get_prefix( + shared.settings["etcd"]["network_prefix"], + value_in_json=True, + ) + + if ipaddress.ip_network(network["ipv6"]).is_global: + update_radvd_conf(all_networks) + + command += ( + "-netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}".format( + tap=tap, net_id=network["id"], mac=mac + ) + ) + + if command: + command = command.split(' ') + + return command + + def delete_network_dev(self): + try: + for network in self.vm["network"]: + network_name = network[0] + _ = network[1] # tap_mac + tap_id = network[2] + + delete_network_interface("tap{}".format(tap_id)) + + owners_vms = shared.vm_pool.by_owner(self.vm["owner"]) + owners_running_vms = shared.vm_pool.by_status( + VMStatus.running, _vms=owners_vms + ) + + networks = map( + lambda n: n[0], + map(lambda vm: vm.network, owners_running_vms), + ) + networks_in_use_by_user_vms = [vm[0] for vm in networks] + if network_name not in networks_in_use_by_user_vms: + network_entry = resolve_network( + network[0], self.vm["owner"] + ) + if network_entry: + network_type = network_entry.value["type"] + network_id = network_entry.value["id"] + if network_type == "vxlan": + delete_network_interface( + "br{}".format(network_id) + ) + delete_network_interface( + "vxlan{}".format(network_id) + ) + except Exception: + logger.exception("Exception in network interface deletion") + + def create(self): + if shared.storage_handler.is_vm_image_exists(self.uuid): + # File Already exists. No Problem Continue + logger.debug("Image for vm %s exists", self.uuid) + else: + if shared.storage_handler.make_vm_image( + src=self.vm["image_uuid"], dest=self.uuid + ): + if not shared.storage_handler.resize_vm_image( + path=self.uuid, + size=int(self.vm["specs"]["os-ssd"].to_MB()), + ): + self.vm["status"] = VMStatus.error + else: + logger.info("New VM Created") + + def sync(self): + shared.etcd_client.put( + self.key, self.schema.dump(self.vm), value_in_json=True + ) + + def delete(self): + self.stop() + + if shared.storage_handler.is_vm_image_exists(self.uuid): + r_status = shared.storage_handler.delete_vm_image(self.uuid) + if r_status: + shared.etcd_client.client.delete(self.key) + else: + shared.etcd_client.client.delete(self.key) + + +def resolve_network(network_name, network_owner): + network = shared.etcd_client.get( + join_path( + shared.settings["etcd"]["network_prefix"], + network_owner, + network_name, + ), + value_in_json=True, + ) + return network + + +def create_vxlan_br_tap(_id, _dev, tap_id, ip=None): + network_script_base = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "network" + ) + vxlan = create_dev( + script=os.path.join(network_script_base, "create-vxlan.sh"), + _id=_id, + dev=_dev, + ) + if vxlan: + bridge = create_dev( + script=os.path.join( + network_script_base, "create-bridge.sh" + ), + _id=_id, + dev=vxlan, + ip=ip, + ) + if bridge: + tap = create_dev( + script=os.path.join( + network_script_base, "create-tap.sh" + ), + _id=str(tap_id), + dev=bridge, + ) + if tap: + return tap + + +def update_radvd_conf(all_networks): + network_script_base = os.path.join( + os.path.dirname(os.path.dirname(__file__)), "network" + ) + + networks = { + net.value["ipv6"]: net.value["id"] + for net in all_networks + if net.value.get("ipv6") + and ipaddress.ip_network(net.value.get("ipv6")).is_global + } + radvd_template = open( + os.path.join(network_script_base, "radvd-template.conf"), "r" + ).read() + radvd_template = Template(radvd_template) + + content = [ + radvd_template.safe_substitute( + bridge="br{}".format(networks[net]), prefix=net + ) + for net in networks + if networks.get(net) + ] + with open("/etc/radvd.conf", "w") as radvd_conf: + radvd_conf.writelines(content) + try: + sp.check_output(["systemctl", "restart", "radvd"]) + except sp.CalledProcessError: + try: + sp.check_output(["service", "radvd", "restart"]) + except sp.CalledProcessError as err: + raise err.__class__( + "Cannot start/restart radvd service", err.cmd + ) from err diff --git a/ucloud/imagescanner/__init__.py b/archive/uncloud_etcd_based/uncloud/imagescanner/__init__.py similarity index 100% rename from ucloud/imagescanner/__init__.py rename to archive/uncloud_etcd_based/uncloud/imagescanner/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/imagescanner/main.py b/archive/uncloud_etcd_based/uncloud/imagescanner/main.py new file mode 100755 index 0000000..ee9da2e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/imagescanner/main.py @@ -0,0 +1,121 @@ +import json +import os +import argparse +import subprocess as sp + +from os.path import join as join_path +from uncloud.common.shared import shared +from uncloud.imagescanner import logger + + +arg_parser = argparse.ArgumentParser('imagescanner', add_help=False) + + +def qemu_img_type(path): + qemu_img_info_command = [ + "qemu-img", + "info", + "--output", + "json", + path, + ] + try: + qemu_img_info = sp.check_output(qemu_img_info_command) + except Exception as e: + logger.exception(e) + return None + else: + qemu_img_info = json.loads(qemu_img_info.decode("utf-8")) + return qemu_img_info["format"] + + +def main(arguments): + # We want to get images entries that requests images to be created + images = shared.etcd_client.get_prefix( + shared.settings["etcd"]["image_prefix"], value_in_json=True + ) + images_to_be_created = list( + filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) + ) + + for image in images_to_be_created: + try: + image_uuid = image.key.split("/")[-1] + image_owner = image.value["owner"] + image_filename = image.value["filename"] + image_store_name = image.value["store_name"] + image_full_path = join_path( + shared.settings["storage"]["file_dir"], + image_owner, + image_filename, + ) + + image_stores = shared.etcd_client.get_prefix( + shared.settings["etcd"]["image_store_prefix"], + value_in_json=True, + ) + user_image_store = next( + filter( + lambda s, store_name=image_store_name: s.value[ + "name" + ] + == store_name, + image_stores, + ) + ) + + image_store_pool = user_image_store.value["attributes"][ + "pool" + ] + + except Exception as e: + logger.exception(e) + else: + # At least our basic data is available + qemu_img_convert_command = [ + "qemu-img", + "convert", + "-f", + "qcow2", + "-O", + "raw", + image_full_path, + "image.raw", + ] + + if qemu_img_type(image_full_path) == "qcow2": + try: + # Convert .qcow2 to .raw + sp.check_output(qemu_img_convert_command,) + + except sp.CalledProcessError: + logger.exception( + "Image convertion from .qcow2 to .raw failed." + ) + else: + # Import and Protect + r_status = shared.storage_handler.import_image( + src="image.raw", dest=image_uuid, protect=True + ) + if r_status: + # Everything is successfully done + image.value["status"] = "CREATED" + shared.etcd_client.put( + image.key, json.dumps(image.value) + ) + finally: + try: + os.remove("image.raw") + except Exception: + pass + + else: + # The user provided image is either not found or of invalid format + image.value["status"] = "INVALID_IMAGE" + shared.etcd_client.put( + image.key, json.dumps(image.value) + ) + + +if __name__ == "__main__": + main() diff --git a/archive/uncloud_etcd_based/uncloud/metadata/__init__.py b/archive/uncloud_etcd_based/uncloud/metadata/__init__.py new file mode 100644 index 0000000..eea436a --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/metadata/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/archive/uncloud_etcd_based/uncloud/metadata/main.py b/archive/uncloud_etcd_based/uncloud/metadata/main.py new file mode 100644 index 0000000..374260e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/metadata/main.py @@ -0,0 +1,95 @@ +import os +import argparse + +from flask import Flask, request +from flask_restful import Resource, Api +from werkzeug.exceptions import HTTPException + +from uncloud.common.shared import shared + +app = Flask(__name__) +api = Api(app) + +app.logger.handlers.clear() + +DEFAULT_PORT=1234 + +arg_parser = argparse.ArgumentParser('metadata', add_help=False) +arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT)) + + +@app.errorhandler(Exception) +def handle_exception(e): + app.logger.error(e) + # pass through HTTP errors + if isinstance(e, HTTPException): + return e + + # now you're handling non-HTTP exceptions only + return {"message": "Server Error"}, 500 + + +def get_vm_entry(mac_addr): + return next( + filter( + lambda vm: mac_addr in list(zip(*vm.network))[1], + shared.vm_pool.vms, + ), + None, + ) + + +# https://stackoverflow.com/questions/37140846/how-to-convert-ipv6-link-local-address-to-mac-address-in-python +def ipv62mac(ipv6): + # remove subnet info if given + subnet_index = ipv6.find("/") + if subnet_index != -1: + ipv6 = ipv6[:subnet_index] + + ipv6_parts = ipv6.split(":") + mac_parts = list() + for ipv6_part in ipv6_parts[-4:]: + while len(ipv6_part) < 4: + ipv6_part = "0" + ipv6_part + mac_parts.append(ipv6_part[:2]) + mac_parts.append(ipv6_part[-2:]) + + # modify parts to match MAC value + mac_parts[0] = "%02x" % (int(mac_parts[0], 16) ^ 2) + del mac_parts[4] + del mac_parts[3] + return ":".join(mac_parts) + + +class Root(Resource): + @staticmethod + def get(): + data = get_vm_entry(ipv62mac(request.remote_addr)) + + if not data: + return ( + {"message": "Metadata for such VM does not exists."}, + 404, + ) + else: + etcd_key = os.path.join( + shared.settings["etcd"]["user_prefix"], + data.value["owner_realm"], + data.value["owner"], + "key", + ) + etcd_entry = shared.etcd_client.get_prefix( + etcd_key, value_in_json=True + ) + user_personal_ssh_keys = [key.value for key in etcd_entry] + data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys + return data.value["metadata"], 200 + + +api.add_resource(Root, "/") + + +def main(arguments): + port = arguments['port'] + debug = arguments['debug'] + app.run(debug=debug, host="::", port=port) diff --git a/ucloud/network/README b/archive/uncloud_etcd_based/uncloud/network/README similarity index 100% rename from ucloud/network/README rename to archive/uncloud_etcd_based/uncloud/network/README diff --git a/archive/uncloud_etcd_based/uncloud/network/__init__.py b/archive/uncloud_etcd_based/uncloud/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ucloud/network/create-bridge.sh b/archive/uncloud_etcd_based/uncloud/network/create-bridge.sh similarity index 100% rename from ucloud/network/create-bridge.sh rename to archive/uncloud_etcd_based/uncloud/network/create-bridge.sh diff --git a/ucloud/network/create-tap.sh b/archive/uncloud_etcd_based/uncloud/network/create-tap.sh similarity index 100% rename from ucloud/network/create-tap.sh rename to archive/uncloud_etcd_based/uncloud/network/create-tap.sh diff --git a/ucloud/network/create-vxlan.sh b/archive/uncloud_etcd_based/uncloud/network/create-vxlan.sh similarity index 100% rename from ucloud/network/create-vxlan.sh rename to archive/uncloud_etcd_based/uncloud/network/create-vxlan.sh diff --git a/ucloud/network/radvd-template.conf b/archive/uncloud_etcd_based/uncloud/network/radvd-template.conf similarity index 100% rename from ucloud/network/radvd-template.conf rename to archive/uncloud_etcd_based/uncloud/network/radvd-template.conf diff --git a/archive/uncloud_etcd_based/uncloud/oneshot/__init__.py b/archive/uncloud_etcd_based/uncloud/oneshot/__init__.py new file mode 100644 index 0000000..eea436a --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/oneshot/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/archive/uncloud_etcd_based/uncloud/oneshot/main.py b/archive/uncloud_etcd_based/uncloud/oneshot/main.py new file mode 100644 index 0000000..dbb3b32 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/oneshot/main.py @@ -0,0 +1,123 @@ +import argparse +import os + + +from pathlib import Path +from uncloud.vmm import VMM +from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap + +from . import virtualmachine, logger + +### +# Argument parser loaded by scripts/uncloud. +arg_parser = argparse.ArgumentParser('oneshot', add_help=False) + +# Actions. +arg_parser.add_argument('--list', action='store_true', + help='list UUID and name of running VMs') +arg_parser.add_argument('--start', nargs=4, + metavar=('NAME', 'IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'), + help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix') +arg_parser.add_argument('--stop', metavar='UUID', + help='stop a VM') +arg_parser.add_argument('--get-status', metavar='UUID', + help='return the status of the VM') +arg_parser.add_argument('--get-vnc', metavar='UUID', + help='return the path of the VNC socket of the VM') +arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK', + help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix') + +# Arguments. +arg_parser.add_argument('--workdir', default=Path.home(), + help='Working directory, defaulting to $HOME') +arg_parser.add_argument('--mac', + help='MAC address of the VM to create (--start)') +arg_parser.add_argument('--memory', type=int, + help='Memory (MB) to allocate (--start)') +arg_parser.add_argument('--cores', type=int, + help='Number of cores to allocate (--start)') +arg_parser.add_argument('--threads', type=int, + help='Number of threads to allocate (--start)') +arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], + help='Format of OS image (--start)') +arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm', + help='QEMU acceleration to use (--start)') +arg_parser.add_argument('--upstream-interface', default='eth0', + help='Name of upstream interface (--start)') + +### +# Helpers. + +# XXX: check if it is possible to use the type returned by ETCD queries. +class UncloudEntryWrapper: + def __init__(self, value): + self.value = value + + def value(self): + return self.value + +def status_line(vm): + return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status()) + +### +# Entrypoint. + +def main(arguments): + # Initialize VMM. + workdir = arguments['workdir'] + vmm = VMM(vmm_backend=workdir) + + # Harcoded debug values. + net_id = 0 + + # Build VM configuration. + vm_config = {} + vm_options = [ + 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', + '--upstream_interface', 'upstream_interface', 'network', 'accel' + ] + for option in vm_options: + if arguments.get(option): + vm_config[option] = arguments[option] + + vm_config['net_id'] = net_id + + # Execute requested VM action. + if arguments['reconfigure_radvd']: + # TODO: check that RADVD is available. + prefix = arguments['reconfigure_radvd'] + network = UncloudEntryWrapper({ + 'id': net_id, + 'ipv6': prefix + }) + + # Make use of uncloud.host.virtualmachine for network configuration. + update_radvd_conf([network]) + elif arguments['start']: + # Extract from --start positional arguments. Quite fragile. + vm_config['name'] = arguments['start'][0] + vm_config['image'] = arguments['start'][1] + vm_config['network'] = arguments['start'][2] + vm_config['upstream_interface'] = arguments['start'][3] + + vm_config['tap_interface'] = "uc{}".format(len(vmm.discover())) + vm = virtualmachine.VM(vmm, vm_config) + vm.start() + elif arguments['stop']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']}) + vm.stop() + elif arguments['get_status']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']}) + print(status_line(vm)) + elif arguments['get_vnc']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']}) + print(vm.get_vnc_addr()) + elif arguments['list']: + vms = vmm.discover() + print("Found {} VMs.".format(len(vms))) + for uuid in vms: + vm = virtualmachine.VM(vmm, {'uuid': uuid}) + print(status_line(vm)) + else: + print('Please specify an action: --start, --stop, --list,\ +--get-status, --get-vnc, --reconfigure-radvd') diff --git a/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py b/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py new file mode 100644 index 0000000..5749bee --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py @@ -0,0 +1,81 @@ +import uuid +import os + +from uncloud.host.virtualmachine import create_vxlan_br_tap +from uncloud.oneshot import logger + +class VM(object): + def __init__(self, vmm, config): + self.config = config + self.vmm = vmm + + # Extract VM specs/metadata from configuration. + self.name = config.get('name', 'no-name') + self.memory = config.get('memory', 1024) + self.cores = config.get('cores', 1) + self.threads = config.get('threads', 1) + self.image_format = config.get('image_format', 'qcow2') + self.image = config.get('image') + self.uuid = config.get('uuid', str(uuid.uuid4())) + self.mac = config.get('mac') + self.accel = config.get('accel', 'kvm') + + self.net_id = config.get('net_id', 0) + self.upstream_interface = config.get('upstream_interface', 'eth0') + self.tap_interface = config.get('tap_interface', 'uc0') + self.network = config.get('network') + + def get_qemu_args(self): + command = ( + "-uuid {uuid} -name {name} -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" + ).format( + uuid=self.uuid, name=self.name, accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + net_id=self.net_id, tap=self.tap_interface, mac=self.mac + ) + + return command.split(" ") + + def start(self): + # Check that VM image is available. + if not os.path.isfile(self.image): + logger.error("Image {} does not exist. Aborting.".format(self.image)) + + # Create Bridge, VXLAN and tap interface for VM. + create_vxlan_br_tap( + self.net_id, self.upstream_interface, self.tap_interface, self.network + ) + + # Generate config for and run QEMU. + qemu_args = self.get_qemu_args() + logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + self.vmm.stop(self.uuid) + + def get_status(self): + return self.vmm.get_status(self.uuid) + + def get_vnc_addr(self): + return self.vmm.get_vnc(self.uuid) + + def get_uuid(self): + return self.uuid + + def get_name(self): + success, json = self.vmm.execute_command(self.uuid, 'query-name') + if success: + return json['return']['name'] + + return None diff --git a/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py b/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py new file mode 100644 index 0000000..eea436a --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/ucloud/scheduler/helper.py b/archive/uncloud_etcd_based/uncloud/scheduler/helper.py similarity index 50% rename from ucloud/scheduler/helper.py rename to archive/uncloud_etcd_based/uncloud/scheduler/helper.py index ba577d6..79db322 100755 --- a/ucloud/scheduler/helper.py +++ b/archive/uncloud_etcd_based/uncloud/scheduler/helper.py @@ -3,10 +3,10 @@ from functools import reduce import bitmath -from ucloud.common.host import HostStatus -from ucloud.common.request import RequestEntry, RequestType -from ucloud.common.vm import VMStatus -from ucloud.config import vm_pool, host_pool, request_pool, env_vars +from uncloud.common.host import HostStatus +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.vm import VMStatus +from uncloud.common.shared import shared def accumulated_specs(vms_specs): @@ -23,17 +23,35 @@ def remaining_resources(host_specs, vms_specs): for component in _vms_specs: if isinstance(_vms_specs[component], str): - _vms_specs[component] = int(bitmath.parse_string_unsafe(_vms_specs[component]).to_MB()) + _vms_specs[component] = int( + bitmath.parse_string_unsafe( + _vms_specs[component] + ).to_MB() + ) elif isinstance(_vms_specs[component], list): - _vms_specs[component] = map(lambda x: int(bitmath.parse_string_unsafe(x).to_MB()), _vms_specs[component]) - _vms_specs[component] = reduce(lambda x, y: x + y, _vms_specs[component], 0) + _vms_specs[component] = map( + lambda x: int(bitmath.parse_string_unsafe(x).to_MB()), + _vms_specs[component], + ) + _vms_specs[component] = reduce( + lambda x, y: x + y, _vms_specs[component], 0 + ) for component in _remaining: if isinstance(_remaining[component], str): - _remaining[component] = int(bitmath.parse_string_unsafe(_remaining[component]).to_MB()) + _remaining[component] = int( + bitmath.parse_string_unsafe( + _remaining[component] + ).to_MB() + ) elif isinstance(_remaining[component], list): - _remaining[component] = map(lambda x: int(bitmath.parse_string_unsafe(x).to_MB()), _remaining[component]) - _remaining[component] = reduce(lambda x, y: x + y, _remaining[component], 0) + _remaining[component] = map( + lambda x: int(bitmath.parse_string_unsafe(x).to_MB()), + _remaining[component], + ) + _remaining[component] = reduce( + lambda x, y: x + y, _remaining[component], 0 + ) _remaining.subtract(_vms_specs) @@ -46,23 +64,27 @@ class NoSuitableHostFound(Exception): def get_suitable_host(vm_specs, hosts=None): if hosts is None: - hosts = host_pool.by_status(HostStatus.alive) + hosts = shared.host_pool.by_status(HostStatus.alive) for host in hosts: # Filter them by host_name - vms = vm_pool.by_host(host.key) + vms = shared.vm_pool.by_host(host.key) # Filter them by status - vms = vm_pool.by_status(VMStatus.running, vms) + vms = shared.vm_pool.by_status(VMStatus.running, vms) running_vms_specs = [vm.specs for vm in vms] # Accumulate all of their combined specs - running_vms_accumulated_specs = accumulated_specs(running_vms_specs) + running_vms_accumulated_specs = accumulated_specs( + running_vms_specs + ) # Find out remaining resources after # host_specs - already running vm_specs - remaining = remaining_resources(host.specs, running_vms_accumulated_specs) + remaining = remaining_resources( + host.specs, running_vms_accumulated_specs + ) # Find out remaining - new_vm_specs remaining = remaining_resources(remaining, vm_specs) @@ -75,7 +97,7 @@ def get_suitable_host(vm_specs, hosts=None): def dead_host_detection(): # Bring out your dead! - Monty Python and the Holy Grail - hosts = host_pool.by_status(HostStatus.alive) + hosts = shared.host_pool.by_status(HostStatus.alive) dead_hosts_keys = [] for host in hosts: @@ -89,25 +111,27 @@ def dead_host_detection(): def dead_host_mitigation(dead_hosts_keys): for host_key in dead_hosts_keys: - host = host_pool.get(host_key) + host = shared.host_pool.get(host_key) host.declare_dead() - vms_hosted_on_dead_host = vm_pool.by_host(host_key) + vms_hosted_on_dead_host = shared.vm_pool.by_host(host_key) for vm in vms_hosted_on_dead_host: - vm.declare_killed() - vm_pool.put(vm) - host_pool.put(host) + vm.status = "UNKNOWN" + shared.vm_pool.put(vm) + shared.host_pool.put(host) def assign_host(vm): vm.hostname = get_suitable_host(vm.specs) - vm_pool.put(vm) + shared.vm_pool.put(vm) - r = RequestEntry.from_scratch(type=RequestType.StartVM, - uuid=vm.uuid, - hostname=vm.hostname, - request_prefix=env_vars.get("REQUEST_PREFIX")) - request_pool.put(r) + r = RequestEntry.from_scratch( + type=RequestType.StartVM, + uuid=vm.uuid, + hostname=vm.hostname, + request_prefix=shared.settings["etcd"]["request_prefix"], + ) + shared.request_pool.put(r) vm.log.append("VM scheduled for starting") return vm.hostname diff --git a/archive/uncloud_etcd_based/uncloud/scheduler/main.py b/archive/uncloud_etcd_based/uncloud/scheduler/main.py new file mode 100755 index 0000000..38c07bf --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/scheduler/main.py @@ -0,0 +1,51 @@ +# TODO +# 1. send an email to an email address defined by env['admin-email'] +# if resources are finished +# 2. Introduce a status endpoint of the scheduler - +# maybe expose a prometheus compatible output + +import argparse + +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.shared import shared +from uncloud.scheduler import logger +from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection, + assign_host, NoSuitableHostFound) + +arg_parser = argparse.ArgumentParser('scheduler', add_help=False) + + +def main(arguments): + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for request_iterator in [ + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + ]: + for request_event in request_iterator: + dead_host_mitigation(dead_host_detection()) + request_entry = RequestEntry(request_event) + + if request_entry.type == RequestType.ScheduleVM: + logger.debug('%s, %s', request_entry.key, request_entry.value) + + vm_entry = shared.vm_pool.get(request_entry.uuid) + if vm_entry is None: + logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid)) + continue + + shared.etcd_client.client.delete(request_entry.key) # consume Request + + try: + assign_host(vm_entry) + except NoSuitableHostFound: + vm_entry.add_log('Can\'t schedule VM. No Resource Left.') + shared.vm_pool.put(vm_entry) + + logger.info('No Resource Left. Emailing admin....') diff --git a/archive/uncloud_etcd_based/uncloud/scheduler/tests/__init__.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/ucloud/scheduler/tests/test_basics.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/test_basics.py similarity index 83% rename from ucloud/scheduler/tests/test_basics.py rename to archive/uncloud_etcd_based/uncloud/scheduler/tests/test_basics.py index 92b3a83..defeb23 100755 --- a/ucloud/scheduler/tests/test_basics.py +++ b/archive/uncloud_etcd_based/uncloud/scheduler/tests/test_basics.py @@ -15,7 +15,7 @@ from main import ( main, ) -from ucloud.config import etcd_client +from uncloud.config import etcd_client class TestFunctions(unittest.TestCase): @@ -70,9 +70,15 @@ class TestFunctions(unittest.TestCase): "last_heartbeat": datetime.utcnow().isoformat(), } with self.client.client.lock("lock"): - self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True) - self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True) - self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True) + self.client.put( + f"{self.host_prefix}/1", host1, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/2", host2, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/3", host3, value_in_json=True + ) def create_vms(self): vm1 = json.dumps( @@ -146,15 +152,17 @@ class TestFunctions(unittest.TestCase): {"cpu": 8, "ram": 32}, ] self.assertEqual( - accumulated_specs(vms), {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10} + accumulated_specs(vms), + {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10}, ) def test_remaining_resources(self): host_specs = {"ssd": 10, "cpu": 16, "ram": 48, "hdd": 10} vms_specs = {"ssd": 10, "cpu": 32, "ram": 12, "hdd": 0} resultant_specs = {"ssd": 0, "cpu": -16, "ram": 36, "hdd": 10} - self.assertEqual(remaining_resources(host_specs, vms_specs), - resultant_specs) + self.assertEqual( + remaining_resources(host_specs, vms_specs), resultant_specs + ) def test_vmpool(self): self.p.join(1) @@ -167,7 +175,12 @@ class TestFunctions(unittest.TestCase): f"{self.vm_prefix}/1", { "owner": "meow", - "specs": {"cpu": 4, "ram": 8, "hdd": 100, "sdd": 256}, + "specs": { + "cpu": 4, + "ram": 8, + "hdd": 100, + "sdd": 256, + }, "hostname": f"{self.host_prefix}/3", "status": "SCHEDULED_DEPLOY", }, @@ -182,7 +195,12 @@ class TestFunctions(unittest.TestCase): f"{self.vm_prefix}/7", { "owner": "meow", - "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "specs": { + "cpu": 10, + "ram": 22, + "hdd": 146, + "sdd": 0, + }, "hostname": "", "status": "REQUESTED_NEW", }, @@ -197,7 +215,12 @@ class TestFunctions(unittest.TestCase): f"{self.vm_prefix}/7", { "owner": "meow", - "specs": {"cpu": 10, "ram": 22, "hdd": 146, "sdd": 0}, + "specs": { + "cpu": 10, + "ram": 22, + "hdd": 146, + "sdd": 0, + }, "hostname": "", "status": "REQUESTED_NEW", }, diff --git a/ucloud/scheduler/tests/test_dead_host_mechanism.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/test_dead_host_mechanism.py similarity index 70% rename from ucloud/scheduler/tests/test_dead_host_mechanism.py rename to archive/uncloud_etcd_based/uncloud/scheduler/tests/test_dead_host_mechanism.py index 0b403ef..466b9ee 100755 --- a/ucloud/scheduler/tests/test_dead_host_mechanism.py +++ b/archive/uncloud_etcd_based/uncloud/scheduler/tests/test_dead_host_mechanism.py @@ -6,11 +6,7 @@ from os.path import dirname BASE_DIR = dirname(dirname(__file__)) sys.path.insert(0, BASE_DIR) -from main import ( - dead_host_detection, - dead_host_mitigation, - config -) +from main import dead_host_detection, dead_host_mitigation, config class TestDeadHostMechanism(unittest.TestCase): @@ -52,13 +48,23 @@ class TestDeadHostMechanism(unittest.TestCase): "last_heartbeat": datetime(2011, 1, 1).isoformat(), } with self.client.client.lock("lock"): - self.client.put(f"{self.host_prefix}/1", host1, value_in_json=True) - self.client.put(f"{self.host_prefix}/2", host2, value_in_json=True) - self.client.put(f"{self.host_prefix}/3", host3, value_in_json=True) - self.client.put(f"{self.host_prefix}/4", host4, value_in_json=True) + self.client.put( + f"{self.host_prefix}/1", host1, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/2", host2, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/3", host3, value_in_json=True + ) + self.client.put( + f"{self.host_prefix}/4", host4, value_in_json=True + ) def test_dead_host_detection(self): - hosts = self.client.get_prefix(self.host_prefix, value_in_json=True) + hosts = self.client.get_prefix( + self.host_prefix, value_in_json=True + ) deads = dead_host_detection(hosts) self.assertEqual(deads, ["/test/host/2", "/test/host/3"]) return deads @@ -66,7 +72,9 @@ class TestDeadHostMechanism(unittest.TestCase): def test_dead_host_mitigation(self): deads = self.test_dead_host_detection() dead_host_mitigation(self.client, deads) - hosts = self.client.get_prefix(self.host_prefix, value_in_json=True) + hosts = self.client.get_prefix( + self.host_prefix, value_in_json=True + ) deads = dead_host_detection(hosts) self.assertEqual(deads, []) diff --git a/archive/uncloud_etcd_based/uncloud/version.py b/archive/uncloud_etcd_based/uncloud/version.py new file mode 100644 index 0000000..ccf3980 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/version.py @@ -0,0 +1 @@ +VERSION = "0.0.5-30-ge91fd9e" diff --git a/archive/uncloud_etcd_based/uncloud/vmm/__init__.py b/archive/uncloud_etcd_based/uncloud/vmm/__init__.py new file mode 100644 index 0000000..6db61eb --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/vmm/__init__.py @@ -0,0 +1,284 @@ +import os +import subprocess as sp +import logging +import socket +import json +import tempfile +import time + +from contextlib import suppress +from multiprocessing import Process +from os.path import join as join_path +from os.path import isdir + +logger = logging.getLogger(__name__) + + +class VMQMPHandles: + def __init__(self, path): + self.path = path + self.sock = socket.socket(socket.AF_UNIX) + self.file = self.sock.makefile() + + def __enter__(self): + self.sock.connect(self.path) + + # eat qmp greetings + self.file.readline() + + # init qmp + self.sock.sendall(b'{ "execute": "qmp_capabilities" }') + self.file.readline() + + return self.sock, self.file + + def __exit__(self, exc_type, exc_val, exc_tb): + self.file.close() + self.sock.close() + + if exc_type: + logger.error( + "Couldn't get handle for VM.", exc_type, exc_val, exc_tb + ) + raise exc_type("Couldn't get handle for VM.") from exc_type + + +class TransferVM(Process): + def __init__(self, src_uuid, dest_sock_path, host, socket_dir): + self.src_uuid = src_uuid + self.host = host + self.src_sock_path = os.path.join(socket_dir, self.src_uuid) + self.dest_sock_path = dest_sock_path + + super().__init__() + + def run(self): + with suppress(FileNotFoundError): + os.remove(self.src_sock_path) + + command = [ + "ssh", + "-nNT", + "-L", + "{}:{}".format(self.src_sock_path, self.dest_sock_path), + "root@{}".format(self.host), + ] + + try: + p = sp.Popen(command) + except Exception as e: + logger.error( + "Couldn' forward unix socks over ssh.", exc_info=e + ) + else: + time.sleep(2) + vmm = VMM() + logger.debug("Executing: ssh forwarding command: %s", command) + vmm.execute_command( + self.src_uuid, + command="migrate", + arguments={"uri": "unix:{}".format(self.src_sock_path)}, + ) + + while p.poll() is None: + success, output = vmm.execute_command(self.src_uuid, command="query-migrate") + if success: + status = output["return"]["status"] + logger.info('Migration Status: {}'.format(status)) + if status == "completed": + vmm.stop(self.src_uuid) + return + elif status in ['failed', 'cancelled']: + return + else: + logger.error("Couldn't be able to query VM {} that was in migration".format(self.src_uuid)) + return + + time.sleep(2) + + +class VMM: + # Virtual Machine Manager + def __init__( + self, + qemu_path="/usr/bin/qemu-system-x86_64", + vmm_backend=os.path.expanduser("~/uncloud/vmm/"), + ): + self.qemu_path = qemu_path + self.vmm_backend = vmm_backend + self.socket_dir = os.path.join(self.vmm_backend, "sock") + + if not os.path.isdir(self.vmm_backend): + logger.info( + "{} does not exists. Creating it...".format( + self.vmm_backend + ) + ) + os.makedirs(self.vmm_backend, exist_ok=True) + + if not os.path.isdir(self.socket_dir): + logger.info( + "{} does not exists. Creating it...".format( + self.socket_dir + ) + ) + os.makedirs(self.socket_dir, exist_ok=True) + + def is_running(self, uuid): + sock_path = os.path.join(self.socket_dir, uuid) + try: + sock = socket.socket(socket.AF_UNIX) + sock.connect(sock_path) + recv = sock.recv(4096) + except Exception as err: + # unix sock doesn't exists or it is closed + logger.debug( + "VM {} sock either don' exists or it is closed. It mean VM is stopped.".format( + uuid + ), + exc_info=err, + ) + else: + # if we receive greetings from qmp it mean VM is running + if len(recv) > 0: + return True + + with suppress(FileNotFoundError): + os.remove(sock_path) + + return False + + def start(self, *args, uuid, migration=False): + # start --> sucess? + migration_args = () + if migration: + migration_args = ( + "-incoming", + "unix:{}".format(os.path.join(self.socket_dir, uuid)), + ) + + if self.is_running(uuid): + logger.warning("Cannot start VM. It is already running.") + else: + qmp_arg = ( + "-qmp", + "unix:{},server,nowait".format( + join_path(self.socket_dir, uuid) + ), + ) + vnc_arg = ( + "-vnc", + "unix:{}".format(tempfile.NamedTemporaryFile().name), + ) + + command = [ + "sudo", + "-p", + "Enter password to start VM {}: ".format(uuid), + self.qemu_path, + *args, + *qmp_arg, + *migration_args, + *vnc_arg, + "-daemonize", + ] + try: + sp.check_output(command, stderr=sp.PIPE) + except sp.CalledProcessError as err: + logger.exception( + "Error occurred while starting VM.\nDetail %s", + err.stderr.decode("utf-8"), + ) + else: + sp.check_output( + ["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory", + "chmod", "-R", "o=rwx,g=rwx", self.vmm_backend] + ) + + # TODO: Find some good way to check whether the virtual machine is up and + # running without relying on non-guarenteed ways. + for _ in range(10): + time.sleep(2) + status = self.get_status(uuid) + if status in ["running", "inmigrate"]: + return status + logger.warning( + "Timeout on VM's status. Shutting down VM %s", uuid + ) + self.stop(uuid) + # TODO: What should we do more. VM can still continue to run in background. + # If we have pid of vm we can kill it using OS. + + def execute_command(self, uuid, command, **kwargs): + # execute_command -> sucess?, output + try: + with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as ( + sock_handle, + file_handle, + ): + command_to_execute = {"execute": command, **kwargs} + sock_handle.sendall( + json.dumps(command_to_execute).encode("utf-8") + ) + output = file_handle.readline() + except Exception: + logger.exception( + "Error occurred while executing command and getting valid output from qmp" + ) + else: + try: + output = json.loads(output) + except Exception: + logger.exception( + "QMP Output isn't valid JSON. %s", output + ) + else: + return "return" in output, output + return False, None + + def stop(self, uuid): + success, output = self.execute_command( + command="quit", uuid=uuid + ) + return success + + def get_status(self, uuid): + success, output = self.execute_command( + command="query-status", uuid=uuid + ) + if success: + return output["return"]["status"] + else: + # TODO: Think about this for a little more + return "STOPPED" + + def discover(self): + vms = [ + uuid + for uuid in os.listdir(self.socket_dir) + if not isdir(join_path(self.socket_dir, uuid)) + ] + return vms + + def get_vnc(self, uuid): + success, output = self.execute_command( + uuid, command="query-vnc" + ) + if success: + return output["return"]["service"] + return None + + def transfer(self, src_uuid, destination_sock_path, host): + p = TransferVM( + src_uuid, + destination_sock_path, + socket_dir=self.socket_dir, + host=host, + ) + p.start() + + # TODO: the following method should clean things that went wrong + # e.g If VM migration fails or didn't start for long time + # i.e 15 minutes we should stop the waiting VM. + def maintenace(self): + pass diff --git a/bin/make-migrations-from-scratch.sh b/bin/make-migrations-from-scratch.sh new file mode 100644 index 0000000..8baccfa --- /dev/null +++ b/bin/make-migrations-from-scratch.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# For undoing/redoing everything +# Needed in special cases and needs to be avoided as soon as +# uncloud.version >= 1 +for a in */migrations; do rm ${a}/*.py; done +for a in */migrations; do python manage.py makemigrations ${a%%/migrations}; done diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 0000000..b51a70d --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1,2 @@ +*.pdf +*.tex diff --git a/doc/README-billing.org b/doc/README-billing.org new file mode 100644 index 0000000..50b26fa --- /dev/null +++ b/doc/README-billing.org @@ -0,0 +1,85 @@ +* How to handle billing in general +** Manual test flow / setting up bills + - Needs orders + - +** Orders + - Orders are the heart of uncloud billing + - Have a starting date + - Have an ending date + - Orders are immutable + - Can usually not be cancelled / cancellation is not a refund + - Customer/user commits on a certain period -> gets discount + based on it + - Can be upgraded + - Create a new order + - We link the new order to the old order and say this one + replaces it + - If the price of the new order is HIGHER than the OLD order, + then we charge the difference until the end of the order period + - In the next billing run we set the OLD order to not to bill anymore + - And only the NEW order will be billed afterwards + - Can be downgraded in the next period (but not for this period) + - We create a new order, same as for upgrade + - The new order starts directly after the OLD order + - As the amount is LOWER than the OLD order, no additional charge is done + during this order period + - We might need to have an activate datetime + - When to implement this + - Order periods can be +*** Statuses + - CREATING/PREPARING + - INACTIVE (?) + - TO_BILL + - NOT_TO_BILL: we use this to accelerate queries to the DB +*** Updating status of orders + - If has succeeding order and billing date is last month -> set inactive +** Bills + - Are always for a month + - Can be preliminary +*** Which orders to include + - Not the cancelled ones / not active ones +** Flows / Approach +*** Finding all orders for a bill + - Get all orders, state != NOT_TO_BILL; for each order do: + - is it a one time order? + - has it a bill assigned? + - yes: set to NOT_TO_BILL + - no: + - get_or_create_bill_for_this_month + - assign bill to this order + - set to NOT_TO_BILL + - is it a recurring order? + - if it has a REPLACING order: + - + - First of month + - Last of month +*** Handling replacement of orders + - The OLD order will appear in the month that it was cancelled on + the bill + - The OLD order needs to be set to NOT_TO_BILL after it was billed + the last time + - The NEW order will be added pro rata if the amount is higher in + the same month + - The NEW order will be used next month +**** Disabling the old order + - On billing run + - If order.replacement_order (naming!) is set + - if the order.replacement_order starts during THIS_MONTH + - add order to bill + - if NOT: + - the order was already replaced in a previous billing period + - set the order to NOT_TO_BILL +**** Billing the new order + - If order.previous_order +*** Handling multiple times a recurring order + - For each recurring order check the order.period + - Find out when it was billed last + - lookup latest bill + - Calculate how many times it has been used until 2359, last day + of month + - For preliminary bill: until datetime.now() + - Call the bill_end_datetime + - Getting duration: bill_end_datetime - order.last_billed + - Amount in seconds; duration_in_seconds + - Divide duration_in_seconds by order.period; amount_used: + - If >= 1: add amount_used * order.recurring_amount to bill diff --git a/doc/README-how-to-configure-remote-uncloud-clients.org b/doc/README-how-to-configure-remote-uncloud-clients.org new file mode 100644 index 0000000..b48886b --- /dev/null +++ b/doc/README-how-to-configure-remote-uncloud-clients.org @@ -0,0 +1,28 @@ +* What is a remote uncloud client? +** Systems that configure themselves for the use with uncloud +** Examples are VMHosts, VPN Servers, cdist control server, etc. +* Which access do these clients need? +** They need read / write access to the database +* Possible methods +** Overview +| | pros | cons | +| SSL based | Once setup, can access all django parts natively, locally | X.509 infrastructure | +| SSH -L tunnel | All nodes can use [::1]:5432 | SSH setup can be fragile | +| ssh djangohost manage.py | All DB ops locally | Code is only executed on django host | +| https + token | Rest alike / consistent access | Code is only executed on django host | +| from_django | Everything is on the django host | main host can become bottleneck | +** remote vs. local Django code execution + - If manage.py is executed locally (= on the client), it can + check/modify local configs + - However local execution requires a pyvenv + packages + db access + - Local execution also *could* make use of postgresql notify for + triggering actions (which is quite neat) + - Remote execution (= on the primary django host) can acess the db + via unix socket + - However remote execution cannot check local state +** from_django + - might reuse existing methods like celery + - reduces the amount of things to be installed on the client to + almost zero + - follows the opennebula model + - has a single point of failurebin diff --git a/doc/uncloud-manual-2020-08-01.org b/doc/uncloud-manual-2020-08-01.org new file mode 100644 index 0000000..5c6a9f7 --- /dev/null +++ b/doc/uncloud-manual-2020-08-01.org @@ -0,0 +1,382 @@ +* Bootstrap / Installation +** Pre-requisites by operating system +*** General + To run uncloud you need: + - ldap development libraries + - libxml2-dev libxslt-dev + - gcc / libc headers: for compiling things + - python3-dev + - wireguard: wg (for checking keys) +*** Alpine + #+BEGIN_SRC sh +apk add openldap-dev postgresql-dev libxml2-dev libxslt-dev gcc python3-dev musl-dev wireguard-tools-wg +#+END_SRC +*** Debian/Devuan: + #+BEGIN_SRC sh +apt install postgresql-server-dev-all +#+END_SRC +** Creating a virtual environment / installing python requirements +*** Virtual env + To separate uncloud requirements, you can use a python virtual + env as follows: + #+BEGIN_SRC sh +python3 -m venv venv +. ./venv/bin/activate +#+END_SRC + Then install the requirements + #+BEGIN_SRC sh +pip install -r requirements.txt +#+END_SRC +** Setting up the the database +*** Install the database service + The database can run on the same host as uncloud, but can also run + a different server. Consult the usual postgresql documentation for + a secure configuration. +**** Alpine + #+BEGIN_SRC sh +apk add postgresql-server +rc-update add postgresql +rc-service postgresql start` +#+END_SRC + +**** Debian/Devuan: + #+BEGIN_SRC sh + apt install postgresql + #+END_SRC +*** Create the database + Due to the use of the JSONField, postgresql is required. + To get started, + create a database and have it owned by the user that runs uncloud + (usually "uncloud"): + + #+BEGIN_SRC sh +bridge:~# su - postgres +bridge:~$ psql +postgres=# create role uncloud login; +postgres=# create database uncloud owner nico; +#+END_SRC +*** Creating the schema + #+BEGIN_SRC sh +python manage.py migrate +#+END_SRC + +** Bootstrap + - Login via a user so that the user object gets created + - Run the following (replace nicocustomer with the username) + #+BEGIN_SRC sh + python manage.py bootstrap-user --username nicocustomer + #+END_SRC + +** Initialise the database + While it is not strictly required to add default values to the + database, it might significantly reduce the starting time with + uncloud. + + To add the default database values run: + + #+BEGIN_SRC shell + # Add local objects + python manage.py db-add-defaults + + # Import VAT rates + python manage.py import-vat-rates + #+END_SRC + +* Testing / CLI Access + Access via the commandline (CLI) can be done using curl or + httpie. In our examples we will use httpie. +** Checkout out the API + #+BEGIN_SRC sh + http localhost:8000/api/ + #+END_SRC +** Authenticate via ldap user in password store + #+BEGIN_SRC sh + http --auth nicocustomer:$(pass ldap/nicocustomer) localhost:8000/api/ + #+END_SRC +* Database +** uncloud clients access the data base from a variety of outside hosts +** So the postgresql data base needs to be remotely accessible +** Instead of exposing the tcp socket, we make postgresql bind to localhost via IPv6 +*** ::1, port 5432 +** Then we remotely connect to the database server with ssh tunneling +*** ssh -L5432:localhost:5432 uncloud-database-host +** Configuring your database for SSH based remote access +*** host all all ::1/128 trust + +* URLs + - api/ - the rest API +* uncloud Products +** Product features + - Dependencies on other products + - Minimum parameters (min cpu, min ram, etc). + - Can also realise the dcl vm + - dualstack vm = VM + IPv4 + SSD + - Need to have a non-misguiding name for the "bare VM" + - Should support network boot (?) + +** VPN +*** How to add a new VPN Host +**** Install wireguard to the host +**** Install uncloud to the host +**** Add `python manage.py vpn --hostname fqdn-of-this-host` to the crontab +**** Use the CLI to configure one or more VPN Networks for this host +*** Example of adding a VPN host at ungleich +**** Create a new dual stack alpine VM +**** Add it to DNS as vpn-XXX.ungleich.ch +**** Route a /40 network to its IPv6 address +**** Install wireguard on it +**** TODO [#C] Enable wireguard on boot +**** TODO [#C] Create a new VPNPool on uncloud with +***** the network address (selecting from our existing pool) +***** the network size (/...) +***** the vpn host that provides the network (selecting the created VM) +***** the wireguard private key of the vpn host (using wg genkey) +***** http command + ``` + http -a nicoschottelius:$(pass + ungleich.ch/nico.schottelius@ungleich.ch) + http://localhost:8000/admin/vpnpool/ network=2a0a:e5c1:200:: \ + network_size=40 subnetwork_size=48 + vpn_hostname=vpn-2a0ae5c1200.ungleich.ch + wireguard_private_key=... + ``` +*** Example http commands / REST calls +**** creating a new vpn pool + http -a nicoschottelius:$(pass + ungleich.ch/nico.schottelius@ungleich.ch) + http://localhost:8000/admin/vpnpool/ network_size=40 + subnetwork_size=48 network=2a0a:e5c1:200:: + vpn_hostname=vpn-2a0ae5c1200.ungleich.ch wireguard_private_key=$(wg + genkey) +**** Creating a new vpn network +*** Creating a VPN pool + + #+BEGIN_SRC sh +http -a uncloudadmin:$(pass uncloudadmin) https://localhost:8000/v1/admin/vpnpool/ \ + network=2a0a:e5c1:200:: network_size=40 subnetwork_size=48 \ + vpn_hostname=vpn-2a0ae5c1200.ungleich.ch wireguard_private_key=$(wg genkey) + #+END_SRC + +This will create the VPNPool 2a0a:e5c1:200::/40 from which /48 +networks will be used for clients. + +VPNPools can only be managed by staff. + +*** Managing VPNNetworks + +To request a network as a client, use the following call: + + #+BEGIN_SRC sh + http -a user:$(pass user) https://localhost:8000/v1/net/vpn/ \ + network_size=48 \ + wireguard_public_key=$(wg genkey | tee privatekey | wg pubkey) +``` + +VPNNetworks can be managed by all authenticated users. + +* Developer Handbook + The following section describe decisions / architecture of + uncloud. These chapters are intended to be read by developers. +** This Documentation + This documentation is written in org-mode. To compile it to + html/pdf, just open emacs and press *C-c C-e l p*. +** Models +*** Bill + Bills are summarising usage in a specific timeframe. Bills usually + spawn one month. +*** BillRecord + Bill records are used to model the usage of one order during the + timeframe. +*** Order + Orders register the intent of a user to buy something. They might + refer to a product. (???) + Order register the one time price and the recurring price. These + fields should be treated as immutable. If they need to be modified, + a new order that replaces the current order should be created. +**** Replacing orders + If an order is updated, a new order is created and points to the + old order. The old order stops one second before the new order + starts. + + If a order has been replaced can be seen by its replaced_by count: + #+BEGIN_SRC sh + >>> Order.objects.get(id=1).replaced_by.count() + 1 + #+END_SRC + +*** Product and Product Children + - A product describes something a user can buy + - A product inherits from the uncloud_pay.models.Product model to + get basic attributes +** Identifiers +*** Problem description + Identifiers can be integers, strings or other objects. They should + be unique. +*** Approach 1: integers + Integers are somewhat easy to remember, but also include + predictable growth, which might allow access to guessed hacking + (obivously proper permissions should prevent this). +*** Approach 2: random uuids + UUIDs are 128 bit integers. Python supports uuid.uuid4() for random + uuids. +*** Approach 3: IPv6 addresses + uncloud heavily depends on IPv6 in the first place. uncloud could + use a /48 to identify all objects. Objects that have IPv6 addresses + on their own, don't need to draw from the system /48. +**** Possible Subnetworks + Assuming uncloud uses a /48 to represent all resources. + + | Network | Name | Description | + |-----------------+-----------------+----------------------------------------------| + | 2001:db8::/48 | uncloud network | All identifiers drawn from here | + | 2001:db8:1::/64 | VM network | Every VM has an IPv6 address in this network | + | 2001:db8:2::/64 | Bill network | Every bill has an IPv6 address | + | 2001:db8:3::/64 | Order network | Every order has an IPv6 address | + | 2001:db8:5::/64 | Product network | Every product (?) has an IPv6 address | + | 2001:db8:4::/64 | Disk network | Every disk is identified | + +**** Tests + [15:47:37] black3.place6:~# rbd create -s 10G ssd/2a0a:e5c0:1::8 + +*** Decision + We use integers, because they are easy. + +** Distributing/Dispatching/Orchestrating +*** Variant 1: using cdist + - The uncloud server can git commit things + - The uncloud server loads cdist and configures the server + - Advantages + - Fully integrated into normal flow + - Disadvantage + - web frontend has access to more data than it needs + - On compromise of the machine, more data leaks + - Some cdist usual delay +*** Variant 2: via celery + - The uncloud server dispatches via celery + - Every decentral node also runs celery/connects to the broker + - Summary brokers: + - If local only celery -> good to use redis - Broker + - If remote: probably better to use rabbitmq + - redis + - simpler + - rabbitmq + - more versatile + - made for remote connections + - quorom queues would be nice, but not clear if supported + - https://github.com/celery/py-amqp/issues/302 + - https://github.com/celery/celery/issues/6067 + - Cannot be installed on alpine Linux at the moment + - Advantage + - Very python / django integrated + - Rather instant + - Disadvantages + - Every decentral node needs to have the uncloud code available + - Decentral nodes *might* need to access the database + - Tasks can probably be written to work without that + (i.e. only strings/bytes) + +**** log/tests + (venv) [19:54] vpn-2a0ae5c1200:~/uncloud$ celery -A uncloud -b redis://bridge.place7.ungleich.ch worker -n worker1@%h --logfile ~/celery.log - +Q vpn-2a0ae5c1200.ungleich.ch + + +*** Variant 3: dedicated cdist instance via message broker + - A separate VM/machine + - Has Checkout of ~/.cdist + - Has cdist checkout + - Tiny API for management + - Not directly web accessible + - "cdist" queue + +** Milestones :uncloud: +*** 1.1 (cleanup 1) +**** TODO [#C] Unify ValidationError, FieldError - define proper Exception + - What do we use for model errors +*** 1.0 (initial release) +**** TODO [#C] Initial Generic product support + - Product +***** TODO [#C] Recurring product support +****** TODO [#C] Support replacing orders for updates +****** DONE [#A] Finish split of bill creation + CLOSED: [2020-09-11 Fri 23:19] +****** TODO [#C] Test the new functions in the Order class +****** Define the correct order replacement logic + Assumption: + - recurringperiods are 30days +******* Case 1: downgrading + - User commits to 10 CHF for 30 days + - Wants to downgrade after 15 days to 5 CHF product + - Expected result: + - order 1: 10 CHF until +30days + - order 2: 5 CHF starting 30days + 1s + - Sum of the two orders is 15 CHF + - Question is + - when is the VM shutdown? + - a) instantly + - b) at the end of the cycle + - best solution + - user can choose between a ... b any time +******* Duration + - You cannot cancel the duration + - You can upgrade and with that cancel the duration + - The idea of a duration is that you commit for it + - If you want to commit lower (daily basis for instance) you + have higher per period prices +******* Case X + - User has VM with 2 Core / 2 GB RAM + - User modifies with to 1 core / 3 GB RAM + - We treat it as down/upgrade independent of the modifications + +******* Case 2: upgrading after 1 day + - committed for 30 days + - upgrade after 1 day + - so first order will be charged for 1/30ths + +******* Case 2: upgrading + - User commits to 10 CHF for 30 days + - Wants to upgrade after 15 days to 20 CHF product + - Order 1 : 1 VM with 2 Core / 2 GB / 10 SSD -- 10 CHF + - 30days period, stopped after 15, so quantity is 0.5 = 5 CHF + - Order 2 : 1 VM with 2 Core / 6 GB / 10 SSD -- 20 CHF + - after 15 days + - VM is upgraded instantly + - Expected result: + - order 1: 10 CHF until +15days = 0.5 units = 5 CHF + - order 2: 20 CHF starting 15days + 1s ... +30 days after + the 15 days -> 45 days = 1 unit = 20 CHF + - Total on bill: 25 CHF + +******* Case 2: upgrading + - User commits to 10 CHF for 30 days + - Wants to upgrade after 15 days to 20 CHF product + - Expected result: + - order 1: 10 CHF until +30days = 1 units = 10 CHF + + - order 2: 20 CHF starting 15days + 1s = 1 unit = 20 CHF + - Total on bill: 30 CHF + + +****** TODO [#C] Note: ending date not set if replaced by default (implicit!) + - Should the new order modify the old order on save()? +****** DONE Fix totally wrong bill dates in our test case + CLOSED: [2020-09-09 Wed 01:00] + - 2020 used instead of 2019 + - Was due to existing test data ... +***** DONE Bill logic is still wrong + CLOSED: [2020-11-05 Thu 18:58] + - Bill starting_date is the date of the first order + - However first encountered order does not have to be the + earliest in the bill! + - Bills should not have a duration + - Bills should only have a (unique) issue date + - We charge based on bill_records + - Last time charged issue date of the bill OR earliest date + after that + - Every bill generation checks all (relevant) orders + - add a flag "not_for_billing" or "closed" + - query on that flag + - verify it every time + + +***** TODO Generating bill for admins/staff + - diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..b050590 --- /dev/null +++ b/manage.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/models.dot b/models.dot new file mode 100644 index 0000000..0adfba8 --- /dev/null +++ b/models.dot @@ -0,0 +1,1482 @@ +digraph model_graph { + // Dotfile by Django-Extensions graph_models + // Created: 2020-03-17 12:30 + // Cli Options: -a + + fontname = "Roboto" + fontsize = 8 + splines = true + + node [ + fontname = "Roboto" + fontsize = 8 + shape = "plaintext" + ] + + edge [ + fontname = "Roboto" + fontsize = 8 + ] + + // Labels + + + django_contrib_admin_models_LogEntry [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + LogEntry +
+ id + + AutoField +
+ content_type + + ForeignKey (id) +
+ user + + ForeignKey (id) +
+ action_flag + + PositiveSmallIntegerField +
+ action_time + + DateTimeField +
+ change_message + + TextField +
+ object_id + + TextField +
+ object_repr + + CharField +
+ >] + + + + + django_contrib_auth_models_Permission [label=< + + + + + + + + + + + + + + + + + + + +
+ + Permission +
+ id + + AutoField +
+ content_type + + ForeignKey (id) +
+ codename + + CharField +
+ name + + CharField +
+ >] + + django_contrib_auth_models_Group [label=< + + + + + + + + + + + +
+ + Group +
+ id + + AutoField +
+ name + + CharField +
+ >] + + + + + django_contrib_contenttypes_models_ContentType [label=< + + + + + + + + + + + + + + + +
+ + ContentType +
+ id + + AutoField +
+ app_label + + CharField +
+ model + + CharField +
+ >] + + + + + django_contrib_sessions_base_session_AbstractBaseSession [label=< + + + + + + + + + + + +
+ + AbstractBaseSession +
+ expire_date + + DateTimeField +
+ session_data + + TextField +
+ >] + + django_contrib_sessions_models_Session [label=< + + + + + + + + + + + + + + + +
+ + Session
<AbstractBaseSession> +
+ session_key + + CharField +
+ expire_date + + DateTimeField +
+ session_data + + TextField +
+ >] + + + + + uncloud_pay_models_StripeCustomer [label=< + + + + + + + + + + + +
+ + StripeCustomer +
+ owner + + OneToOneField (id) +
+ stripe_id + + CharField +
+ >] + + uncloud_pay_models_Payment [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + Payment +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ amount + + DecimalField +
+ source + + CharField +
+ timestamp + + DateTimeField +
+ >] + + uncloud_pay_models_PaymentMethod [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + PaymentMethod +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ description + + TextField +
+ primary + + BooleanField +
+ source + + CharField +
+ stripe_card_id + + CharField +
+ >] + + uncloud_pay_models_Bill [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Bill +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ creation_date + + DateTimeField +
+ due_date + + DateField +
+ ending_date + + DateTimeField +
+ starting_date + + DateTimeField +
+ valid + + BooleanField +
+ >] + + uncloud_pay_models_Order [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Order +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ creation_date + + DateTimeField +
+ ending_date + + DateTimeField +
+ recurring_period + + CharField +
+ starting_date + + DateTimeField +
+ >] + + uncloud_pay_models_OrderRecord [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + OrderRecord +
+ id + + AutoField +
+ order + + ForeignKey (uuid) +
+ description + + TextField +
+ one_time_price + + DecimalField +
+ recurring_price + + DecimalField +
+ >] + + + + + django_contrib_auth_models_AbstractUser [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + AbstractUser
<AbstractBaseUser,PermissionsMixin> +
+ date_joined + + DateTimeField +
+ email + + EmailField +
+ first_name + + CharField +
+ is_active + + BooleanField +
+ is_staff + + BooleanField +
+ is_superuser + + BooleanField +
+ last_login + + DateTimeField +
+ last_name + + CharField +
+ password + + CharField +
+ username + + CharField +
+ >] + + uncloud_auth_models_User [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + User
<AbstractUser> +
+ id + + AutoField +
+ date_joined + + DateTimeField +
+ email + + EmailField +
+ first_name + + CharField +
+ is_active + + BooleanField +
+ is_staff + + BooleanField +
+ is_superuser + + BooleanField +
+ last_login + + DateTimeField +
+ last_name + + CharField +
+ password + + CharField +
+ username + + CharField +
+ >] + + + + + uncloud_pay_models_Product [label=< + + + + + + + + + + + + + + + +
+ + Product +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ status + + CharField +
+ >] + + uncloud_vm_models_VMHost [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMHost +
+ uuid + + UUIDField +
+ hostname + + CharField +
+ physical_cores + + IntegerField +
+ status + + CharField +
+ usable_cores + + IntegerField +
+ usable_ram_in_gb + + FloatField +
+ >] + + uncloud_vm_models_VMProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vmhost + + ForeignKey (uuid) +
+ cores + + IntegerField +
+ name + + CharField +
+ ram_in_gb + + FloatField +
+ status + + CharField +
+ vmid + + IntegerField +
+ >] + + uncloud_vm_models_VMWithOSProduct [label=< + + + + + + + +
+ + VMWithOSProduct +
+ vmproduct_ptr + + OneToOneField (uuid) +
+ >] + + uncloud_vm_models_VMDiskImageProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMDiskImageProduct +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ image_source + + CharField +
+ image_source_type + + CharField +
+ import_url + + URLField +
+ is_os_image + + BooleanField +
+ is_public + + BooleanField +
+ name + + CharField +
+ size_in_gb + + FloatField +
+ status + + CharField +
+ storage_class + + CharField +
+ >] + + uncloud_vm_models_VMDiskProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMDiskProduct +
+ uuid + + UUIDField +
+ image + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ size_in_gb + + FloatField +
+ >] + + uncloud_vm_models_VMNetworkCard [label=< + + + + + + + + + + + + + + + + + + + +
+ + VMNetworkCard +
+ id + + AutoField +
+ vm + + ForeignKey (uuid) +
+ ip_address + + GenericIPAddressField +
+ mac_address + + BigIntegerField +
+ >] + + uncloud_vm_models_VMSnapshotProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMSnapshotProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ gb_hdd + + FloatField +
+ gb_ssd + + FloatField +
+ status + + CharField +
+ >] + + + + + uncloud_pay_models_Product [label=< + + + + + + + + + + + + + + + +
+ + Product +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ status + + CharField +
+ >] + + ungleich_service_models_MatrixServiceProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + MatrixServiceProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ domain + + CharField +
+ status + + CharField +
+ >] + + + + + opennebula_models_VM [label=< + + + + + + + + + + + + + + + +
+ + VM +
+ vmid + + IntegerField +
+ owner + + ForeignKey (id) +
+ data + + JSONField +
+ >] + + + + + // Relations + + django_contrib_admin_models_LogEntry -> uncloud_auth_models_User + [label=" user (logentry)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_admin_models_LogEntry -> django_contrib_contenttypes_models_ContentType + [label=" content_type (logentry)"] [arrowhead=none, arrowtail=dot, dir=both]; + + + django_contrib_auth_models_Permission -> django_contrib_contenttypes_models_ContentType + [label=" content_type (permission)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_auth_models_Group -> django_contrib_auth_models_Permission + [label=" permissions (group)"] [arrowhead=dot arrowtail=dot, dir=both]; + + + + django_contrib_sessions_models_Session -> django_contrib_sessions_base_session_AbstractBaseSession + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_StripeCustomer -> uncloud_auth_models_User + [label=" owner (stripecustomer)"] [arrowhead=none, arrowtail=none, dir=both]; + + uncloud_pay_models_Payment -> uncloud_auth_models_User + [label=" owner (payment)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_PaymentMethod -> uncloud_auth_models_User + [label=" owner (paymentmethod)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Bill -> uncloud_auth_models_User + [label=" owner (bill)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Order -> uncloud_auth_models_User + [label=" owner (order)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Order -> uncloud_pay_models_Bill + [label=" bill (order)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_pay_models_OrderRecord -> uncloud_pay_models_Order + [label=" order (orderrecord)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_auth_base_user_AbstractBaseUser [label=< + + +
+ AbstractBaseUser +
+ >] + django_contrib_auth_models_AbstractUser -> django_contrib_auth_base_user_AbstractBaseUser + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + django_contrib_auth_models_PermissionsMixin [label=< + + +
+ PermissionsMixin +
+ >] + django_contrib_auth_models_AbstractUser -> django_contrib_auth_models_PermissionsMixin + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_Group + [label=" groups (user)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_Permission + [label=" user_permissions (user)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_AbstractUser + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_Product -> uncloud_auth_models_User + [label=" owner (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Product -> uncloud_pay_models_Order + [label=" order (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMProduct -> uncloud_vm_models_VMHost + [label=" vmhost (vmproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_vm_models_VMWithOSProduct -> uncloud_vm_models_VMProduct + [label=" multi-table\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_vm_models_VMDiskImageProduct -> uncloud_auth_models_User + [label=" owner (vmdiskimageproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_auth_models_User + [label=" owner (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_vm_models_VMProduct + [label=" vm (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_vm_models_VMDiskImageProduct + [label=" image (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMNetworkCard -> uncloud_vm_models_VMProduct + [label=" vm (vmnetworkcard)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMSnapshotProduct -> uncloud_vm_models_VMProduct + [label=" vm (vmsnapshotproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMSnapshotProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_Product -> uncloud_auth_models_User + [label=" owner (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Product -> uncloud_pay_models_Order + [label=" order (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + ungleich_service_models_MatrixServiceProduct -> uncloud_vm_models_VMProduct + [label=" vm (matrixserviceproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + ungleich_service_models_MatrixServiceProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + opennebula_models_VM -> uncloud_auth_models_User + [label=" owner (vm)"] [arrowhead=none, arrowtail=dot, dir=both]; + + +} diff --git a/models.png b/models.png new file mode 100644 index 0000000..f9d0c2e Binary files /dev/null and b/models.png differ diff --git a/opennebula/__init__.py b/opennebula/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/opennebula/admin.py b/opennebula/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/opennebula/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/opennebula/apps.py b/opennebula/apps.py new file mode 100644 index 0000000..0750576 --- /dev/null +++ b/opennebula/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class OpennebulaConfig(AppConfig): + name = 'opennebula' diff --git a/opennebula/management/commands/opennebula-synchosts.py b/opennebula/management/commands/opennebula-synchosts.py new file mode 100644 index 0000000..29f9ac1 --- /dev/null +++ b/opennebula/management/commands/opennebula-synchosts.py @@ -0,0 +1,74 @@ +import json + +import uncloud.secrets as secrets + +from xmlrpc.client import ServerProxy as RPCClient + +from django.core.management.base import BaseCommand +from xmltodict import parse +from enum import IntEnum +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost +from django_auth_ldap.backend import LDAPBackend + + +class HostStates(IntEnum): + """ + The following flags are copied from + https://docs.opennebula.org/5.8/integration/system_interfaces/api.html#schemas-for-host + """ + INIT = 0 # Initial state for enabled hosts + MONITORING_MONITORED = 1 # Monitoring the host (from monitored) + MONITORED = 2 # The host has been successfully monitored + ERROR = 3 # An error ocurrer while monitoring the host + DISABLED = 4 # The host is disabled + MONITORING_ERROR = 5 # Monitoring the host (from error) + MONITORING_INIT = 6 # Monitoring the host (from init) + MONITORING_DISABLED = 7 # Monitoring the host (from disabled) + OFFLINE = 8 # The host is totally offline + + +class Command(BaseCommand): + help = 'Syncronize Host information from OpenNebula' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + with RPCClient(secrets.OPENNEBULA_URL) as rpc_client: + success, response, *_ = rpc_client.one.hostpool.info(secrets.OPENNEBULA_USER_PASS) + if success: + response = json.loads(json.dumps(parse(response))) + host_pool = response.get('HOST_POOL', {}).get('HOST', {}) + for host in host_pool: + host_share = host.get('HOST_SHARE', {}) + + host_name = host.get('NAME') + state = int(host.get('STATE', HostStates.OFFLINE.value)) + + if state == HostStates.MONITORED: + status = 'active' + elif state == HostStates.DISABLED: + status = 'disabled' + else: + status = 'unusable' + + usable_cores = host_share.get('TOTAL_CPU') + usable_ram_in_kb = int(host_share.get('TOTAL_MEM', 0)) + usable_ram_in_gb = int(usable_ram_in_kb / 2 ** 20) + + # vms cannot be created like this -- Nico, 2020-03-17 + # vms = host.get('VMS', {}) or {} + # vms = vms.get('ID', []) or [] + # vms = ','.join(vms) + + VMHost.objects.update_or_create( + hostname=host_name, + defaults={ + 'usable_cores': usable_cores, + 'usable_ram_in_gb': usable_ram_in_gb, + 'status': status + } + ) + else: + print(response) diff --git a/opennebula/management/commands/opennebula-syncvms.py b/opennebula/management/commands/opennebula-syncvms.py new file mode 100644 index 0000000..3c12fa9 --- /dev/null +++ b/opennebula/management/commands/opennebula-syncvms.py @@ -0,0 +1,44 @@ +import json + +from xmlrpc.client import ServerProxy as RPCClient +from django_auth_ldap.backend import LDAPBackend +from django.core.management.base import BaseCommand +from django.conf import settings +from xmltodict import parse + +from opennebula.models import VM as VMModel + + +class Command(BaseCommand): + help = 'Syncronize VM information from OpenNebula' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + with RPCClient(settings.OPENNEBULA_URL) as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + settings.OPENNEBULA_USER_PASS, -2, -1, -1, -1 + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + unknown_user = set() + + backend = LDAPBackend() + + for vm in vms: + vm_id = vm['ID'] + vm_owner = vm['UNAME'] + + user = backend.populate_user(username=vm_owner) + + if not user: + unknown_user.add(vm_owner) + else: + VMModel.objects.update_or_create( + vmid=vm_id, + defaults={'data': vm, 'owner': user} + ) + print('User not found in ldap:', unknown_user) + else: + print(response) diff --git a/opennebula/management/commands/opennebula-to-uncloud.py b/opennebula/management/commands/opennebula-to-uncloud.py new file mode 100644 index 0000000..230159a --- /dev/null +++ b/opennebula/management/commands/opennebula-to-uncloud.py @@ -0,0 +1,193 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand +from django.utils import timezone +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct + +from uncloud_pay.models import Order + +import logging + +log = logging.getLogger(__name__) + +def convert_mac_to_int(mac_address: str): + # Remove octet connecting characters + mac_address = mac_address.replace(':', '') + mac_address = mac_address.replace('.', '') + mac_address = mac_address.replace('-', '') + mac_address = mac_address.replace(' ', '') + + # Parse the resulting number as hexadecimal + mac_address = int(mac_address, base=16) + + return mac_address + + +def get_vm_price(core, ram, ssd_size, hdd_size, n_of_ipv4, n_of_ipv6): + total = 3 * core + 4 * ram + (3.5 * ssd_size/10.) + (1.5 * hdd_size/100.) + 8 * n_of_ipv4 + 0 * n_of_ipv6 + + # TODO: Find some reason about the following magical subtraction. + total -= 8 + + return total + + +def create_nics(one_vm, vm_product): + for nic in one_vm.nics: + mac_address = convert_mac_to_int(nic.get('MAC')) + ip_address = nic.get('IP', None) or nic.get('IP6_GLOBAL', None) + + VMNetworkCard.objects.update_or_create( + mac_address=mac_address, vm=vm_product, defaults={'ip_address': ip_address} + ) + + +def sync_disk_and_image(one_vm, vm_product, disk_owner): + """ + a) Check all opennebula disk if they are in the uncloud VM, if not add + b) Check all uncloud disks and remove them if they are not in the opennebula VM + """ + + vmdisknum = 0 + + one_disks_extra_data = [] + + for disk in one_vm.disks: + vmowner = one_vm.owner + name = disk.get('image') + vmdisknum += 1 + + log.info("Checking disk {} for VM {}".format(name, one_vm)) + + is_os_image, is_public, status = True, False, 'active' + + image_size_in_gb = disk.get('image_size_in_gb') + disk_size_in_gb = disk.get('size_in_gb') + storage_class = disk.get('storage_class') + image_source = disk.get('source') + image_source_type = disk.get('source_type') + + image, _ = VMDiskImageProduct.objects.update_or_create( + name=name, + defaults={ + 'owner': disk_owner, + 'is_os_image': is_os_image, + 'is_public': is_public, + 'size_in_gb': image_size_in_gb, + 'storage_class': storage_class, + 'image_source': image_source, + 'image_source_type': image_source_type, + 'status': status + } + ) + + # identify vmdisk from opennebula - primary mapping key + extra_data = { + 'opennebula_vm': one_vm.vmid, + 'opennebula_size_in_gb': disk_size_in_gb, + 'opennebula_source': disk.get('opennebula_source'), + 'opennebula_disk_num': vmdisknum + } + # Save for comparing later + one_disks_extra_data.append(extra_data) + + try: + vm_disk = VMDiskProduct.objects.get(extra_data=extra_data) + except VMDiskProduct.DoesNotExist: + vm_disk = VMDiskProduct.objects.create( + owner=vmowner, + vm=vm_product, + image=image, + size_in_gb=disk_size_in_gb, + extra_data=extra_data + ) + + # Now remove all disks that are not in above extra_data list + for disk in VMDiskProduct.objects.filter(vm=vm_product): + extra_data = disk.extra_data + if not extra_data in one_disks_extra_data: + log.info("Removing disk {} from VM {}".format(disk, vm_product)) + disk.delete() + + disks = [ disk.extra_data for disk in VMDiskProduct.objects.filter(vm=vm_product) ] + log.info("VM {} has disks: {}".format(vm_product, disks)) + +class Command(BaseCommand): + help = 'Migrate Opennebula VM to regular (uncloud) vm' + + def add_arguments(self, parser): + parser.add_argument('--disk-owner', required=True, help="The user who owns the the opennebula disks") + + def handle(self, *args, **options): + log.debug("{} {}".format(args, options)) + + disk_owner = get_user_model().objects.get(username=options['disk_owner']) + + for one_vm in VMModel.objects.all(): + + if not one_vm.last_host: + log.warning("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid)) + continue + + try: + vmhost = VMHost.objects.get(hostname=one_vm.last_host) + except VMHost.DoesNotExist: + log.error("VMHost {} does not exist, aborting".format(one_vm.last_host)) + raise + + cores = one_vm.cores + ram_in_gb = one_vm.ram_in_gb + owner = one_vm.owner + status = 'active' + + ssd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['ssd', 'one'] ]) + hdd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['hdd'] ]) + + # List of IPv4 addresses and Global IPv6 addresses + ipv4, ipv6 = one_vm.ips + + # TODO: Insert actual/real creation_date, starting_date, ending_date + # instead of pseudo one we are putting currently + creation_date = starting_date = datetime.now(tz=timezone.utc) + + # Price calculation based on datacenterlight.ch + one_time_price = 0 + recurring_period = 'per_month' + recurring_price = get_vm_price(cores, ram_in_gb, + ssd_size, hdd_size, + len(ipv4), len(ipv6)) + + try: + vm_product = VMProduct.objects.get(extra_data__opennebula_id=one_vm.vmid) + except VMProduct.DoesNotExist: + order = Order.objects.create( + owner=owner, + creation_date=creation_date, + starting_date=starting_date + ) + vm_product = VMProduct( + extra_data={ 'opennebula_id': one_vm.vmid }, + name=one_vm.uncloud_name, + order=order + ) + + # we don't use update_or_create, as filtering by json AND setting json + # at the same time does not work + + vm_product.vmhost = vmhost + vm_product.owner = owner + vm_product.cores = cores + vm_product.ram_in_gb = ram_in_gb + vm_product.status = status + + vm_product.save() + + # Create VMNetworkCards + create_nics(one_vm, vm_product) + + # Create VMDiskImageProduct and VMDiskProduct + sync_disk_and_image(one_vm, vm_product, disk_owner=disk_owner) diff --git a/opennebula/migrations/0001_initial.py b/opennebula/migrations/0001_initial.py new file mode 100644 index 0000000..9a135c6 --- /dev/null +++ b/opennebula/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='VM', + fields=[ + ('vmid', models.IntegerField(primary_key=True, serialize=False)), + ('data', models.JSONField()), + ], + ), + ] diff --git a/opennebula/migrations/__init__.py b/opennebula/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/opennebula/models.py b/opennebula/models.py new file mode 100644 index 0000000..f15b845 --- /dev/null +++ b/opennebula/models.py @@ -0,0 +1,90 @@ +import uuid +from django.db import models +from django.contrib.auth import get_user_model +from uncloud_pay.models import Product + +# ungleich specific +storage_class_mapping = { + 'one': 'ssd', + 'ssd': 'ssd', + 'hdd': 'hdd' +} + +class VM(models.Model): + vmid = models.IntegerField(primary_key=True) + data = models.JSONField() + + @property + def uncloud_name(self): + return "opennebula-{}".format(self.vmid) + + @property + def cores(self): + return int(self.data['TEMPLATE']['VCPU']) + + @property + def ram_in_gb(self): + return int(self.data['TEMPLATE']['MEMORY'])/1024 + + @property + def disks(self): + """ + If there is no disk then the key DISK does not exist. + + If there is only one disk, we have a dictionary in the database. + + If there are multiple disks, we have a list of dictionaries in the database. + """ + + disks = [] + + if 'DISK' in self.data['TEMPLATE']: + if type(self.data['TEMPLATE']['DISK']) is dict: + disks = [self.data['TEMPLATE']['DISK']] + else: + disks = self.data['TEMPLATE']['DISK'] + + disks = [ + { + 'size_in_gb': int(d['SIZE'])/1024, + 'opennebula_source': d['SOURCE'], + 'opennebula_name': d['IMAGE'], + 'image_size_in_gb': int(d['ORIGINAL_SIZE'])/1024, + 'pool_name': d['POOL_NAME'], + 'image': d['IMAGE'], + 'source': d['SOURCE'], + 'source_type': d['TM_MAD'], + 'storage_class': storage_class_mapping[d['POOL_NAME']] + + } + for d in disks + ] + + return disks + + @property + def last_host(self): + return ((self.data.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None) + + @property + def graphics(self): + return self.data.get('TEMPLATE', {}).get('GRAPHICS', {}) + + @property + def nics(self): + _nics = self.data.get('TEMPLATE', {}).get('NIC', {}) + if isinstance(_nics, dict): + _nics = [_nics] + return _nics + + @property + def ips(self): + ipv4, ipv6 = [], [] + for nic in self.nics: + ip = nic.get('IP') + ip6 = nic.get('IP6_GLOBAL') + if ip: + ipv4.append(ip) + if ip6: + ipv6.append(ip6) + return ipv4, ipv6 diff --git a/opennebula/serializers.py b/opennebula/serializers.py new file mode 100644 index 0000000..cd00622 --- /dev/null +++ b/opennebula/serializers.py @@ -0,0 +1,10 @@ +from rest_framework import serializers +from opennebula.models import VM + + +class OpenNebulaVMSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = VM + fields = [ 'vmid', 'owner', 'data', + 'uncloud_name', 'cores', 'ram_in_gb', + 'disks', 'nics', 'ips' ] diff --git a/opennebula/tests.py b/opennebula/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/opennebula/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/opennebula/views.py b/opennebula/views.py new file mode 100644 index 0000000..688f0b4 --- /dev/null +++ b/opennebula/views.py @@ -0,0 +1,16 @@ +from rest_framework import viewsets, permissions + +#from .models import VM +# from .serializers import OpenNebulaVMSerializer + +# class VMViewSet(viewsets.ModelViewSet): +# permission_classes = [permissions.IsAuthenticated] +# serializer_class = OpenNebulaVMSerializer + +# def get_queryset(self): +# if self.request.user.is_superuser: +# obj = VM.objects.all() +# else: +# obj = VM.objects.filter(owner=self.request.user) + +# return obj diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..adbda9c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,32 @@ +# Django basics +django +djangorestframework +django-auth-ldap + +psycopg2 +ldap3 + +xmltodict + +parsedatetime + +# Follow are for creating graph models +pyparsing +pydot +django-extensions + +# PDF creating +django-hardcopy + +# schema support +pyyaml +uritemplate + +# Payment & VAT +vat-validator +stripe + + +# Tasks +celery +redis diff --git a/resources/ci/.lock b/resources/ci/.lock new file mode 100644 index 0000000..e69de29 diff --git a/resources/ci/Dockerfile b/resources/ci/Dockerfile new file mode 100644 index 0000000..020b66e --- /dev/null +++ b/resources/ci/Dockerfile @@ -0,0 +1,3 @@ +FROM fedora:latest + +RUN dnf install -y python3-devel python3-pip python3-coverage libpq-devel openldap-devel gcc chromium diff --git a/resources/vat-rates.csv b/resources/vat-rates.csv new file mode 100644 index 0000000..17bdb99 --- /dev/null +++ b/resources/vat-rates.csv @@ -0,0 +1,325 @@ +start_date,stop_date,territory_codes,currency_code,rate,rate_type,description +2011-01-04,,AI,XCD,0,standard,Anguilla (British overseas territory) is exempted of VAT. +1984-01-01,,AT,EUR,0.2,standard,Austria (member state) standard VAT rate. +1976-01-01,1984-01-01,AT,EUR,0.18,standard, +1973-01-01,1976-01-01,AT,EUR,0.16,standard, +1984-01-01,,"AT-6691 +DE-87491",EUR,0.19,standard,Jungholz (Austrian town) special VAT rate. +1984-01-01,,"AT-6991 +AT-6992 +AT-6993 +DE-87567 +DE-87568 +DE-87569",EUR,0.19,standard,Mittelberg (Austrian town) special VAT rate. +1996-01-01,,BE,EUR,0.21,standard,Belgium (member state) standard VAT rate. +1994-01-01,1996-01-01,BE,EUR,0.205,standard, +1992-04-01,1994-01-01,BE,EUR,0.195,standard, +1983-01-01,1992-04-01,BE,EUR,0.19,standard, +1981-07-01,1983-01-01,BE,EUR,0.17,standard, +1978-07-01,1981-07-01,BE,EUR,0.16,standard, +1971-07-01,1978-07-01,BE,EUR,0.18,standard, +1999-01-01,,BG,BGN,0.2,standard,Bulgaria (member state) standard VAT rate. +1996-07-01,1999-01-01,BG,BGN,0.22,standard, +1994-04-01,1996-07-01,BG,BGN,0.18,standard, +2011-01-04,,BM,BMD,0,standard,Bermuda (British overseas territory) is exempted of VAT. +2014-01-13,,"CY +GB-BFPO 57 +GB-BFPO 58 +GB-BFPO 59 +UK-BFPO 57 +UK-BFPO 58 +UK-BFPO 59",EUR,0.19,standard,"Cyprus (member state) standard VAT rate. +Akrotiri and Dhekelia (British overseas territory) is subjected to Cyprus' standard VAT rate." +2013-01-14,2014-01-13,CY,EUR,0.18,standard, +2012-03-01,2013-01-14,CY,EUR,0.17,standard, +2003-01-01,2012-03-01,CY,EUR,0.15,standard, +2002-07-01,2003-01-01,CY,EUR,0.13,standard, +2000-07-01,2002-07-01,CY,EUR,0.1,standard, +1993-10-01,2000-07-01,CY,EUR,0.08,standard, +1992-07-01,1993-10-01,CY,EUR,0.05,standard, +2013-01-01,,CZ,CZK,0.21,standard,Czech Republic (member state) standard VAT rate. +2010-01-01,2013-01-01,CZ,CZK,0.2,standard, +2004-05-01,2010-01-01,CZ,CZK,0.19,standard, +1995-01-01,2004-05-01,CZ,CZK,0.22,standard, +1993-01-01,1995-01-01,CZ,CZK,0.23,standard, +2007-01-01,,DE,EUR,0.19,standard,Germany (member state) standard VAT rate. +1998-04-01,2007-01-01,DE,EUR,0.16,standard, +1993-01-01,1998-04-01,DE,EUR,0.15,standard, +1983-07-01,1993-01-01,DE,EUR,0.14,standard, +1979-07-01,1983-07-01,DE,EUR,0.13,standard, +1978-01-01,1979-07-01,DE,EUR,0.12,standard, +1968-07-01,1978-01-01,DE,EUR,0.11,standard, +1968-01-01,1968-07-01,DE,EUR,0.1,standard, +2007-01-01,,DE-27498,EUR,0,standard,Heligoland (German island) is exempted of VAT. +2007-01-01,,"DE-78266 +CH-8238",EUR,0,standard,Busingen am Hochrhein (German territory) is exempted of VAT. +1992-01-01,,DK,DKK,0.25,standard,Denmark (member state) standard VAT rate. +1980-06-30,1992-01-01,DK,DKK,0.22,standard, +1978-10-30,1980-06-30,DK,DKK,0.2025,standard, +1977-10-03,1978-10-30,DK,DKK,0.18,standard, +1970-06-29,1977-10-03,DK,DKK,0.15,standard, +1968-04-01,1970-06-29,DK,DKK,0.125,standard, +1967-07-03,1968-04-01,DK,DKK,0.1,standard, +2009-07-01,,EE,EUR,0.2,standard,Estonia (member state) standard VAT rate. +1993-01-01,2009-07-01,EE,EUR,0.18,standard, +1991-01-01,1993-01-01,EE,EUR,0.1,standard, +2016-06-01,,"GR +EL",EUR,0.24,standard,Greece (member state) standard VAT rate. +2010-07-01,2016-06-01,"GR +EL",EUR,0.23,standard, +2010-03-15,2010-07-01,"GR +EL",EUR,0.21,standard, +2005-04-01,2010-03-15,"GR +EL",EUR,0.19,standard, +1990-04-28,2005-04-01,"GR +EL",EUR,0.18,standard, +1988-01-01,1990-04-28,"GR +EL",EUR,0.16,standard, +1987-01-01,1988-01-01,"GR +EL",EUR,0.18,standard, +2012-09-01,,ES,EUR,0.21,standard,Spain (member state) standard VAT rate. +2010-07-01,2012-09-01,ES,EUR,0.18,standard, +1995-01-01,2010-07-01,ES,EUR,0.16,standard, +1992-08-01,1995-01-01,ES,EUR,0.15,standard, +1992-01-01,1992-08-01,ES,EUR,0.13,standard, +1986-01-01,1992-01-01,ES,EUR,0.12,standard, +2012-09-01,,"ES-CN +ES-GC +ES-TF +IC",EUR,0,standard,Canary Islands (Spanish autonomous community) is exempted of VAT. +2012-09-01,,"ES-ML +ES-CE +EA",EUR,0,standard,Ceuta and Melilla (Spanish autonomous cities) is exempted of VAT. +2013-01-01,,FI,EUR,0.24,standard,Finland (member state) standard VAT rate. +2010-07-01,2013-01-01,FI,EUR,0.23,standard, +1994-06-01,2010-07-01,FI,EUR,0.22,standard, +2013-01-01,,"FI-01 +AX",EUR,0,standard,Aland Islands (Finish autonomous region) is exempted of VAT. +2011-01-04,,FK,FKP,0,standard,Falkland Islands (British overseas territory) is exempted of VAT. +1992-01-01,,FO,DKK,0,standard,Faroe Islands (Danish autonomous country) is exempted of VAT. +2014-01-01,,"FR +MC",EUR,0.2,standard,"France (member state) standard VAT rate. +Monaco (sovereign city-state) is member of the EU VAT area and subjected to France's standard VAT rate." +2000-04-01,2014-01-01,"FR +MC",EUR,0.196,standard, +1995-08-01,2000-04-01,"FR +MC",EUR,0.206,standard, +1982-07-01,1995-08-01,"FR +MC",EUR,0.186,standard, +1977-01-01,1982-07-01,"FR +MC",EUR,0.176,standard, +1973-01-01,1977-01-01,"FR +MC",EUR,0.2,standard, +1970-01-01,1973-01-01,"FR +MC",EUR,0.23,standard, +1968-12-01,1970-01-01,"FR +MC",EUR,0.19,standard, +1968-01-01,1968-12-01,"FR +MC",EUR,0.1666,standard, +2014-01-01,,"FR-BL +BL",EUR,0,standard,Saint Barthelemy (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-GF +GF",EUR,0,standard,Guiana (French overseas department) is exempted of VAT. +2014-01-01,,"FR-GP +GP",EUR,0.085,standard,Guadeloupe (French overseas department) special VAT rate. +2014-01-01,,"FR-MF +MF",EUR,0,standard,Saint Martin (French overseas collectivity) is subjected to France's standard VAT rate. +2014-01-01,,"FR-MQ +MQ",EUR,0.085,standard,Martinique (French overseas department) special VAT rate. +2014-01-01,,"FR-NC +NC",XPF,0,standard,New Caledonia (French special collectivity) is exempted of VAT. +2014-01-01,,"FR-PF +PF",XPF,0,standard,French Polynesia (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-PM +PM",EUR,0,standard,Saint Pierre and Miquelon (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-RE +RE",EUR,0.085,standard,Reunion (French overseas department) special VAT rate. +2014-01-01,,"FR-TF +TF",EUR,0,standard,French Southern and Antarctic Lands (French overseas territory) is exempted of VAT. +2014-01-01,,"FR-WF +WF",XPF,0,standard,Wallis and Futuna (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-YT +YT",EUR,0,standard,Mayotte (French overseas department) is exempted of VAT. +2011-01-04,,GG,GBP,0,standard,Guernsey (British Crown dependency) is exempted of VAT. +2011-01-04,,GI,GIP,0,standard,Gibraltar (British overseas territory) is exempted of VAT. +1992-01-01,,GL,DKK,0,standard,Greenland (Danish autonomous country) is exempted of VAT. +2010-07-01,2016-06-01,"GR-34007 +EL-34007",EUR,0.16,standard,Skyros (Greek island) special VAT rate. +2010-07-01,2016-06-01,"GR-37002 +GR-37003 +GR-37005 +EL-37002 +EL-37003 +EL-37005",EUR,0.16,standard,Northern Sporades (Greek islands) special VAT rate. +2010-07-01,2016-06-01,"GR-64004 +EL-64004",EUR,0.16,standard,Thasos (Greek island) special VAT rate. +2010-07-01,2016-06-01,"GR-68002 +EL-68002",EUR,0.16,standard,Samothrace (Greek island) special VAT rate. +2010-07-01,,"GR-69 +EL-69",EUR,0,standard,Mount Athos (Greek self-governed part) is exempted of VAT. +2010-07-01,2016-06-01,"GR-81 +EL-81",EUR,0.16,standard,Dodecanese (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-82 +EL-82",EUR,0.16,standard,Cyclades (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-83 +EL-83",EUR,0.16,standard,Lesbos (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-84 +EL-84",EUR,0.16,standard,Samos (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-85 +EL-85",EUR,0.16,standard,Chios (Greek department) special VAT rate. +2011-01-04,,GS,GBP,0,standard,South Georgia and the South Sandwich Islands (British overseas territory) is exempted of VAT. +2012-03-01,,HR,HRK,0.25,standard,Croatia (member state) standard VAT rate. +2009-08-01,2012-03-01,HR,HRK,0.23,standard, +1998-08-01,2009-08-01,HR,HRK,0.22,standard, +2012-01-01,,HU,HUF,0.27,standard,Hungary (member state) standard VAT rate. +2009-07-01,2012-01-01,HU,HUF,0.25,standard, +2006-01-01,2009-07-01,HU,HUF,0.2,standard, +1988-01-01,2006-01-01,HU,HUF,0.25,standard, +2012-01-01,,IE,EUR,0.23,standard,Republic of Ireland (member state) standard VAT rate. +2010-01-01,2012-01-01,IE,EUR,0.21,standard, +2008-12-01,2010-01-01,IE,EUR,0.215,standard, +2002-03-01,2008-12-01,IE,EUR,0.21,standard, +2001-01-01,2002-03-01,IE,EUR,0.2,standard, +1991-03-01,2001-01-01,IE,EUR,0.21,standard, +1990-03-01,1991-03-01,IE,EUR,0.23,standard, +1986-03-01,1990-03-01,IE,EUR,0.25,standard, +1983-05-01,1986-03-01,IE,EUR,0.23,standard, +1983-03-01,1983-05-01,IE,EUR,0.35,standard, +1982-05-01,1983-03-01,IE,EUR,0.3,standard, +1980-05-01,1982-05-01,IE,EUR,0.25,standard, +1976-03-01,1980-05-01,IE,EUR,0.2,standard, +1973-09-03,1976-03-01,IE,EUR,0.195,standard, +1972-11-01,1973-09-03,IE,EUR,0.1637,standard, +2011-01-04,,IO,GBP,0,standard,British Indian Ocean Territory (British overseas territory) is exempted of VAT. +2013-10-01,,IT,EUR,0.22,standard,Italy (member state) standard VAT rate. +2011-09-17,2013-10-01,IT,EUR,0.21,standard, +1997-10-01,2011-09-17,IT,EUR,0.2,standard, +1988-08-01,1997-10-01,IT,EUR,0.19,standard, +1982-08-05,1988-08-01,IT,EUR,0.18,standard, +1981-01-01,1982-08-05,IT,EUR,0.15,standard, +1980-11-01,1981-01-01,IT,EUR,0.14,standard, +1980-07-03,1980-11-01,IT,EUR,0.15,standard, +1977-02-08,1980-07-03,IT,EUR,0.14,standard, +1973-01-01,1977-02-08,IT,EUR,0.12,standard, +2013-10-01,,"IT-22060 +CH-6911",CHF,0,standard,Campione (Italian town) is exempted of VAT. +2013-10-01,,IT-23030,EUR,0,standard,Livigno (Italian town) is exempted of VAT. +2011-01-04,,JE,GBP,0,standard,Jersey (British Crown dependency) is exempted of VAT. +2011-01-04,,KY,KYD,0,standard,Cayman Islands (British overseas territory) is exempted of VAT. +2009-09-01,,LT,EUR,0.21,standard,Lithuania (member state) standard VAT rate. +2009-01-01,2009-09-01,LT,EUR,0.19,standard, +1994-05-01,2009-01-01,LT,EUR,0.18,standard, +2015-01-01,,LU,EUR,0.17,standard,Luxembourg (member state) standard VAT rate. +1992-01-01,2015-01-01,LU,EUR,0.15,standard, +1983-07-01,1992-01-01,LU,EUR,0.12,standard, +1971-01-01,1983-07-01,LU,EUR,0.1,standard, +1970-01-01,1971-01-01,LU,EUR,0.8,standard, +2012-07-01,,LV,EUR,0.21,standard,Latvia (member state) standard VAT rate. +2011-01-01,2012-07-01,LV,EUR,0.22,standard, +2009-01-01,2011-01-01,LV,EUR,0.21,standard, +1995-05-01,2009-01-01,LV,EUR,0.18,standard, +2011-01-04,,MS,XCD,0,standard,Montserrat (British overseas territory) is exempted of VAT. +2004-01-01,,MT,EUR,0.18,standard,Malta (member state) standard VAT rate. +1995-01-01,2004-01-01,MT,EUR,0.15,standard, +2012-10-01,,NL,EUR,0.21,standard,Netherlands (member state) standard VAT rate. +2001-01-01,2012-10-01,NL,EUR,0.19,standard, +1992-10-01,2001-01-01,NL,EUR,0.175,standard, +1989-01-01,1992-10-01,NL,EUR,0.185,standard, +1986-10-01,1989-01-01,NL,EUR,0.2,standard, +1984-01-01,1986-10-01,NL,EUR,0.19,standard, +1976-01-01,1984-01-01,NL,EUR,0.18,standard, +1973-01-01,1976-01-01,NL,EUR,0.16,standard, +1971-01-01,1973-01-01,NL,EUR,0.14,standard, +1969-01-01,1971-01-01,NL,EUR,0.12,standard, +2012-10-01,,"NL-AW +AW",AWG,0,standard,Aruba (Dutch country) are exempted of VAT. +2012-10-01,,"NL-CW +NL-SX +CW +SX",ANG,0,standard,Curacao and Sint Maarten (Dutch countries) are exempted of VAT. +2012-10-01,,"NL-BQ1 +NL-BQ2 +NL-BQ3 +BQ +BQ-BO +BQ-SA +BQ-SE",USD,0,standard,"Bonaire, Saba and Sint Eustatius (Dutch special municipalities) are exempted of VAT." +2011-01-01,,PL,PLN,0.23,standard,Poland (member state) standard VAT rate. +1993-01-08,2011-01-01,PL,PLN,0.22,standard, +2011-01-04,,PN,NZD,0,standard,Pitcairn Islands (British overseas territory) is exempted of VAT. +2011-01-01,,PT,EUR,0.23,standard,Portugal (member state) standard VAT rate. +2010-07-01,2011-01-01,PT,EUR,0.21,standard, +2008-07-01,2010-07-01,PT,EUR,0.2,standard, +2005-07-01,2008-07-01,PT,EUR,0.21,standard, +2002-06-05,2005-07-01,PT,EUR,0.19,standard, +1995-01-01,2002-06-05,PT,EUR,0.17,standard, +1992-03-24,1995-01-01,PT,EUR,0.16,standard, +1988-02-01,1992-03-24,PT,EUR,0.17,standard, +1986-01-01,1988-02-01,PT,EUR,0.16,standard, +2011-01-01,,PT-20,EUR,0.18,standard,Azores (Portuguese autonomous region) special VAT rate. +2011-01-01,,PT-30,EUR,0.22,standard,Madeira (Portuguese autonomous region) special VAT rate. +2017-01-01,,RO,RON,0.19,standard,Romania (member state) standard VAT rate. +2016-01-01,2017-01-01,RO,RON,0.2,standard,Romania (member state) standard VAT rate. +2010-07-01,2016-01-01,RO,RON,0.24,standard, +2000-01-01,2010-07-01,RO,RON,0.19,standard, +1998-02-01,2000-01-01,RO,RON,0.22,standard, +1993-07-01,1998-02-01,RO,RON,0.18,standard, +1990-07-01,,SE,SEK,0.25,standard,Sweden (member state) standard VAT rate. +1983-01-01,1990-07-01,SE,SEK,0.2346,standard, +1981-11-16,1983-01-01,SE,SEK,0.2151,standard, +1980-09-08,1981-11-16,SE,SEK,0.2346,standard, +1977-06-01,1980-09-08,SE,SEK,0.2063,standard, +1971-01-01,1977-06-01,SE,SEK,0.1765,standard, +1969-01-01,1971-01-01,SE,SEK,0.1111,standard, +2011-01-04,,"AC +SH +SH-AC +SH-HL",SHP,0,standard,Ascension and Saint Helena (British overseas territory) is exempted of VAT. +2011-01-04,,"TA +SH-TA",GBP,0,standard,Tristan da Cunha (British oversea territory) is exempted of VAT. +2013-07-01,,SI,EUR,0.22,standard,Slovenia (member state) standard VAT rate. +2002-01-01,2013-07-01,SI,EUR,0.2,standard, +1999-07-01,2002-01-01,SI,EUR,0.19,standard, +2011-01-01,,SK,EUR,0.2,standard,Slovakia (member state) standard VAT rate. +2004-01-01,2011-01-01,SK,EUR,0.19,standard, +2003-01-01,2004-01-01,SK,EUR,0.2,standard, +1996-01-01,2003-01-01,SK,EUR,0.23,standard, +1993-08-01,1996-01-01,SK,EUR,0.25,standard, +1993-01-01,1993-08-01,SK,EUR,0.23,standard, +2011-01-04,,TC,USD,0,standard,Turks and Caicos Islands (British overseas territory) is exempted of VAT. +2011-01-04,,"GB +UK +IM",GBP,0.2,standard,"United Kingdom (member state) standard VAT rate. +Isle of Man (British self-governing dependency) is member of the EU VAT area and subjected to UK's standard VAT rate." +2010-01-01,2011-01-04,"GB +UK +IM",GBP,0.175,standard, +2008-12-01,2010-01-01,"GB +UK +IM",GBP,0.15,standard, +1991-04-01,2008-12-01,"GB +UK +IM",GBP,0.175,standard, +1979-06-18,1991-04-01,"GB +UK +IM",GBP,0.15,standard, +1974-07-29,1979-06-18,"GB +UK +IM",GBP,0.08,standard, +1973-04-01,1974-07-29,"GB +UK +IM",GBP,0.1,standard, +2011-01-04,,VG,USD,0,standard,British Virgin Islands (British overseas territory) is exempted of VAT. +2014-01-01,,CP,EUR,0,standard,Clipperton Island (French overseas possession) is exempted of VAT. +2019-11-15,,CH,CHF,0.077,standard,Switzerland standard VAT (added manually) +2019-11-15,,MC,EUR,0.196,standard,Monaco standard VAT (added manually) +2019-11-15,,FR,EUR,0.2,standard,France standard VAT (added manually) +2019-11-15,,GR,EUR,0.24,standard,Greece standard VAT (added manually) +2019-11-15,,GB,EUR,0.2,standard,UK standard VAT (added manually) +2019-12-17,,AD,EUR,0.045,standard,Andorra standard VAT (added manually) +2019-12-17,,TK,EUR,0.18,standard,Turkey standard VAT (added manually) +2019-12-17,,IS,EUR,0.24,standard,Iceland standard VAT (added manually) +2019-12-17,,FX,EUR,0.20,standard,France metropolitan standard VAT (added manually) +2020-01-04,,CY,EUR,0.19,standard,Cyprus standard VAT (added manually) +2019-01-04,,IL,EUR,0.23,standard,Ireland standard VAT (added manually) +2019-01-04,,LI,EUR,0.077,standard,Liechtenstein standard VAT (added manually) diff --git a/scripts/ucloud b/scripts/ucloud deleted file mode 100755 index 7be6b24..0000000 --- a/scripts/ucloud +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 - -import argparse -import multiprocessing as mp -import logging - -from os.path import join as join_path -from ucloud.sanity_checks import check - -if __name__ == "__main__": - arg_parser = argparse.ArgumentParser(prog='ucloud', - description='Open Source Cloud Management Software') - arg_parser.add_argument('component', - choices=['api', 'scheduler', 'host', - 'filescanner', 'imagescanner', - 'metadata']) - arg_parser.add_argument('component_args', nargs='*') - args = arg_parser.parse_args() - - logging.basicConfig( - level=logging.DEBUG, - filename=join_path("/", "etc", "ucloud", "log.txt"), - filemode="a", - format="%(name)s %(asctime)s: %(levelname)s - %(message)s", - datefmt="%d-%b-%y %H:%M:%S", - ) - try: - check() - - if args.component == 'api': - from ucloud.api.main import main - - main() - elif args.component == 'host': - from ucloud.host.main import main - - hostname = args.component_args - mp.set_start_method('spawn') - main(*hostname) - elif args.component == 'scheduler': - from ucloud.scheduler.main import main - - main() - elif args.component == 'filescanner': - from ucloud.filescanner.main import main - - main() - elif args.component == 'imagescanner': - from ucloud.imagescanner.main import main - - main() - elif args.component == 'metadata': - from ucloud.metadata.main import main - - main() - - except Exception as e: - logging.exception(e) - print(e) \ No newline at end of file diff --git a/setup.py b/setup.py deleted file mode 100644 index 9a35f27..0000000 --- a/setup.py +++ /dev/null @@ -1,35 +0,0 @@ -from setuptools import setup, find_packages - -with open("README.md", "r") as fh: - long_description = fh.read() - -setup(name='ucloud', - version='0.0.1', - description='All ucloud server components.', - url='https://code.ungleich.ch/ucloud/ucloud', - long_description=long_description, - long_description_content_type='text/markdown', - classifiers=[ - 'Development Status :: 3 - Alpha', - 'License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)', - 'Programming Language :: Python :: 3' - ], - author='ungleich', - author_email='technik@ungleich.ch', - packages=find_packages(), - install_requires=[ - 'requests', - 'python-decouple', - 'flask', - 'flask-restful', - 'bitmath', - 'pyotp', - 'sshtunnel', - 'sphinx', - 'pynetbox', - 'sphinx-rtd-theme', - 'etcd3_wrapper @ https://code.ungleich.ch/ungleich-public/etcd3_wrapper/repository/master/archive.tar.gz#egg=etcd3_wrapper', - 'etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3', - ], - scripts=['scripts/ucloud'], - zip_safe=False) diff --git a/ucloud/api/create_image_store.py b/ucloud/api/create_image_store.py deleted file mode 100755 index 17fa63c..0000000 --- a/ucloud/api/create_image_store.py +++ /dev/null @@ -1,16 +0,0 @@ -import json -import os - -from uuid import uuid4 - -from ucloud.config import etcd_client, env_vars - -data = { - "is_public": True, - "type": "ceph", - "name": "images", - "description": "first ever public image-store", - "attributes": {"list": [], "key": [], "pool": "images"}, -} - -etcd_client.put(os.path.join(env_vars.get('IMAGE_STORE_PREFIX'), uuid4().hex), json.dumps(data)) diff --git a/ucloud/api/main.py b/ucloud/api/main.py deleted file mode 100644 index 1475fb0..0000000 --- a/ucloud/api/main.py +++ /dev/null @@ -1,517 +0,0 @@ -import json -import pynetbox - -from uuid import uuid4 -from os.path import join as join_path - -from flask import Flask, request -from flask_restful import Resource, Api - -from ucloud.common import counters -from ucloud.common.vm import VMStatus -from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import (etcd_client, request_pool, vm_pool, host_pool, env_vars, image_storage_handler) -from . import schemas -from .helper import generate_mac, mac2ipv6 -from . import logger - -app = Flask(__name__) -api = Api(app) - - -class CreateVM(Resource): - """API Request to Handle Creation of VM""" - - @staticmethod - def post(): - data = request.json - validator = schemas.CreateVMSchema(data) - if validator.is_valid(): - vm_uuid = uuid4().hex - vm_key = join_path(env_vars.get("VM_PREFIX"), vm_uuid) - specs = { - "cpu": validator.specs["cpu"], - "ram": validator.specs["ram"], - "os-ssd": validator.specs["os-ssd"], - "hdd": validator.specs["hdd"], - } - macs = [generate_mac() for _ in range(len(data["network"]))] - tap_ids = [counters.increment_etcd_counter(etcd_client, "/v1/counter/tap") - for _ in range(len(data["network"]))] - vm_entry = { - "name": data["vm_name"], - "owner": data["name"], - "owner_realm": data["realm"], - "specs": specs, - "hostname": "", - "status": VMStatus.stopped, - "image_uuid": validator.image_uuid, - "log": [], - "vnc_socket": "", - "network": list(zip(data["network"], macs, tap_ids)), - "metadata": {"ssh-keys": []}, - } - etcd_client.put(vm_key, vm_entry, value_in_json=True) - - # Create ScheduleVM Request - r = RequestEntry.from_scratch( - type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=env_vars.get("REQUEST_PREFIX") - ) - request_pool.put(r) - - return {"message": "VM Creation Queued"}, 200 - return validator.get_errors(), 400 - - -class VmStatus(Resource): - @staticmethod - def get(): - data = request.json - validator = schemas.VMStatusSchema(data) - if validator.is_valid(): - vm = vm_pool.get( - join_path(env_vars.get("VM_PREFIX"), data["uuid"]) - ) - vm_value = vm.value.copy() - vm_value["ip"] = [] - for network_mac_and_tap in vm.network: - network_name, mac, tap = network_mac_and_tap - network = etcd_client.get( - join_path( - env_vars.get("NETWORK_PREFIX"), - data["name"], - network_name, - ), - value_in_json=True, - ) - ipv6_addr = network.value.get("ipv6").split("::")[0] + "::" - vm_value["ip"].append(mac2ipv6(mac, ipv6_addr)) - vm.value = vm_value - return vm.value - else: - return validator.get_errors(), 400 - - -class CreateImage(Resource): - @staticmethod - def post(): - data = request.json - validator = schemas.CreateImageSchema(data) - if validator.is_valid(): - file_entry = etcd_client.get( - join_path(env_vars.get("FILE_PREFIX"), data["uuid"]) - ) - file_entry_value = json.loads(file_entry.value) - - image_entry_json = { - "status": "TO_BE_CREATED", - "owner": file_entry_value["owner"], - "filename": file_entry_value["filename"], - "name": data["name"], - "store_name": data["image_store"], - "visibility": "public", - } - etcd_client.put( - join_path(env_vars.get("IMAGE_PREFIX"), data["uuid"]), - json.dumps(image_entry_json), - ) - - return {"message": "Image queued for creation."} - return validator.get_errors(), 400 - - -class ListPublicImages(Resource): - @staticmethod - def get(): - images = etcd_client.get_prefix( - env_vars.get("IMAGE_PREFIX"), value_in_json=True - ) - r = { - "images": [] - } - for image in images: - image_key = "{}:{}".format( - image.value["store_name"], image.value["name"] - ) - r["images"].append( - {"name": image_key, "status": image.value["status"]} - ) - return r, 200 - - -class VMAction(Resource): - @staticmethod - def post(): - data = request.json - validator = schemas.VmActionSchema(data) - - if validator.is_valid(): - vm_entry = vm_pool.get( - join_path(env_vars.get("VM_PREFIX"), data["uuid"]) - ) - action = data["action"] - - if action == "start": - action = "schedule" - - if action == "delete" and vm_entry.hostname == "": - if image_storage_handler.is_vm_image_exists(vm_entry.uuid): - r_status = image_storage_handler.delete_vm_image(vm_entry.uuid) - if r_status: - etcd_client.client.delete(vm_entry.key) - return {"message": "VM successfully deleted"} - else: - logger.error("Some Error Occurred while deleting VM") - return {"message": "VM deletion unsuccessfull"} - else: - etcd_client.client.delete(vm_entry.key) - return {"message": "VM successfully deleted"} - - r = RequestEntry.from_scratch( - type="{}VM".format(action.title()), - uuid=data["uuid"], - hostname=vm_entry.hostname, - request_prefix=env_vars.get("REQUEST_PREFIX") - ) - request_pool.put(r) - return {"message": "VM {} Queued".format(action.title())}, 200 - else: - return validator.get_errors(), 400 - - -class VMMigration(Resource): - @staticmethod - def post(): - data = request.json - validator = schemas.VmMigrationSchema(data) - - if validator.is_valid(): - vm = vm_pool.get(data["uuid"]) - - r = RequestEntry.from_scratch( - type=RequestType.ScheduleVM, - uuid=vm.uuid, - destination=join_path( - env_vars.get("HOST_PREFIX"), validator.destination.value - ), - migration=True, - request_prefix=env_vars.get("REQUEST_PREFIX") - ) - request_pool.put(r) - return {"message": "VM Migration Initialization Queued"}, 200 - else: - return validator.get_errors(), 400 - - -class ListUserVM(Resource): - @staticmethod - def get(): - data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - vms = etcd_client.get_prefix( - env_vars.get("VM_PREFIX"), value_in_json=True - ) - return_vms = [] - user_vms = filter(lambda v: v.value["owner"] == data["name"], vms) - for vm in user_vms: - return_vms.append( - { - "name": vm.value["name"], - "vm_uuid": vm.key.split("/")[-1], - "specs": vm.value["specs"], - "status": vm.value["status"], - "hostname": vm.value["hostname"], - # "mac": vm.value["mac"], - "vnc_socket": None - if vm.value.get("vnc_socket", None) is None - else vm.value["vnc_socket"], - } - ) - if return_vms: - return {"message": return_vms}, 200 - return {"message": "No VM found"}, 404 - - else: - return validator.get_errors(), 400 - - -class ListUserFiles(Resource): - @staticmethod - def get(): - data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - files = etcd_client.get_prefix( - env_vars.get("FILE_PREFIX"), value_in_json=True - ) - return_files = [] - user_files = list( - filter(lambda f: f.value["owner"] == data["name"], files) - ) - for file in user_files: - return_files.append( - { - "filename": file.value["filename"], - "uuid": file.key.split("/")[-1], - } - ) - return {"message": return_files}, 200 - else: - return validator.get_errors(), 400 - - -class CreateHost(Resource): - @staticmethod - def post(): - data = request.json - validator = schemas.CreateHostSchema(data) - if validator.is_valid(): - host_key = join_path(env_vars.get("HOST_PREFIX"), uuid4().hex) - host_entry = { - "specs": data["specs"], - "hostname": data["hostname"], - "status": "DEAD", - "last_heartbeat": "", - } - etcd_client.put(host_key, host_entry, value_in_json=True) - - return {"message": "Host Created"}, 200 - - return validator.get_errors(), 400 - - -class ListHost(Resource): - @staticmethod - def get(): - hosts = host_pool.hosts - r = { - host.key: { - "status": host.status, - "specs": host.specs, - "hostname": host.hostname, - } - for host in hosts - } - return r, 200 - - -class GetSSHKeys(Resource): - @staticmethod - def get(): - data = request.json - validator = schemas.GetSSHSchema(data) - if validator.is_valid(): - if not validator.key_name.value: - - # {user_prefix}/{realm}/{name}/key/ - etcd_key = join_path( - env_vars.get('USER_PREFIX'), - data["realm"], - data["name"], - "key", - ) - etcd_entry = etcd_client.get_prefix( - etcd_key, value_in_json=True - ) - - keys = { - key.key.split("/")[-1]: key.value for key in etcd_entry - } - return {"keys": keys} - else: - - # {user_prefix}/{realm}/{name}/key/{key_name} - etcd_key = join_path( - env_vars.get('USER_PREFIX'), - data["realm"], - data["name"], - "key", - data["key_name"], - ) - etcd_entry = etcd_client.get(etcd_key, value_in_json=True) - - if etcd_entry: - return { - "keys": { - etcd_entry.key.split("/")[-1]: etcd_entry.value - } - } - else: - return {"keys": {}} - else: - return validator.get_errors(), 400 - - -class AddSSHKey(Resource): - @staticmethod - def post(): - data = request.json - validator = schemas.AddSSHSchema(data) - if validator.is_valid(): - - # {user_prefix}/{realm}/{name}/key/{key_name} - etcd_key = join_path( - env_vars.get("USER_PREFIX"), - data["realm"], - data["name"], - "key", - data["key_name"], - ) - etcd_entry = etcd_client.get(etcd_key, value_in_json=True) - if etcd_entry: - return { - "message": "Key with name '{}' already exists".format( - data["key_name"] - ) - } - else: - # Key Not Found. It implies user' haven't added any key yet. - etcd_client.put(etcd_key, data["key"], value_in_json=True) - return {"message": "Key added successfully"} - else: - return validator.get_errors(), 400 - - -class RemoveSSHKey(Resource): - @staticmethod - def get(): - data = request.json - validator = schemas.RemoveSSHSchema(data) - if validator.is_valid(): - - # {user_prefix}/{realm}/{name}/key/{key_name} - etcd_key = join_path( - env_vars.get("USER_PREFIX"), - data["realm"], - data["name"], - "key", - data["key_name"], - ) - etcd_entry = etcd_client.get(etcd_key, value_in_json=True) - if etcd_entry: - etcd_client.client.delete(etcd_key) - return {"message": "Key successfully removed."} - else: - return { - "message": "No Key with name '{}' Exists at all.".format( - data["key_name"] - ) - } - else: - return validator.get_errors(), 400 - - -class CreateNetwork(Resource): - @staticmethod - def post(): - data = request.json - validator = schemas.CreateNetwork(data) - - if validator.is_valid(): - - network_entry = { - "id": counters.increment_etcd_counter( - etcd_client, "/v1/counter/vxlan" - ), - "type": data["type"], - } - if validator.user.value: - nb = pynetbox.api( - url=env_vars.get("NETBOX_URL"), - token=env_vars.get("NETBOX_TOKEN"), - ) - nb_prefix = nb.ipam.prefixes.get( - prefix=env_vars.get("PREFIX") - ) - - prefix = nb_prefix.available_prefixes.create( - data={ - "prefix_length": env_vars.get( - "PREFIX_LENGTH", cast=int - ), - "description": '{}\'s network "{}"'.format( - data["name"], data["network_name"] - ), - "is_pool": True, - } - ) - network_entry["ipv6"] = prefix["prefix"] - else: - network_entry["ipv6"] = "fd00::/64" - - network_key = join_path( - env_vars.get("NETWORK_PREFIX"), - data["name"], - data["network_name"], - ) - etcd_client.put(network_key, network_entry, value_in_json=True) - return {"message": "Network successfully added."} - else: - return validator.get_errors(), 400 - - -class ListUserNetwork(Resource): - @staticmethod - def get(): - data = request.json - validator = schemas.OTPSchema(data) - - if validator.is_valid(): - prefix = join_path( - env_vars.get("NETWORK_PREFIX"), data["name"] - ) - networks = etcd_client.get_prefix(prefix, value_in_json=True) - user_networks = [] - for net in networks: - net.value["name"] = net.key.split("/")[-1] - user_networks.append(net.value) - return {"networks": user_networks}, 200 - else: - return validator.get_errors(), 400 - - -api.add_resource(CreateVM, "/vm/create") -api.add_resource(VmStatus, "/vm/status") - -api.add_resource(VMAction, "/vm/action") -api.add_resource(VMMigration, "/vm/migrate") - -api.add_resource(CreateImage, "/image/create") -api.add_resource(ListPublicImages, "/image/list-public") - -api.add_resource(ListUserVM, "/user/vms") -api.add_resource(ListUserFiles, "/user/files") -api.add_resource(ListUserNetwork, "/user/networks") - -api.add_resource(AddSSHKey, "/user/add-ssh") -api.add_resource(RemoveSSHKey, "/user/remove-ssh") -api.add_resource(GetSSHKeys, "/user/get-ssh") - -api.add_resource(CreateHost, "/host/create") -api.add_resource(ListHost, "/host/list") - -api.add_resource(CreateNetwork, "/network/create") - - -def main(): - image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'), value_in_json=True)) - if len(image_stores) == 0: - data = { - "is_public": True, - "type": "ceph", - "name": "images", - "description": "first ever public image-store", - "attributes": {"list": [], "key": [], "pool": "images"}, - } - - etcd_client.put(join_path(env_vars.get('IMAGE_STORE_PREFIX'), uuid4().hex), json.dumps(data)) - - app.run(host="::", debug=True) - - -if __name__ == "__main__": - main() diff --git a/ucloud/common/helpers.py b/ucloud/common/helpers.py deleted file mode 100644 index 1bdf0b4..0000000 --- a/ucloud/common/helpers.py +++ /dev/null @@ -1,54 +0,0 @@ -import logging -import socket -import requests -import json - -from ipaddress import ip_address - -from os.path import join as join_path - - -def create_package_loggers(packages, base_path, mode="a"): - loggers = {} - for pkg in packages: - logger = logging.getLogger(pkg) - logger_handler = logging.FileHandler( - join_path(base_path, "{}.txt".format(pkg)), - mode=mode - ) - logger.setLevel(logging.DEBUG) - logger_handler.setFormatter(logging.Formatter(fmt="%(asctime)s: %(levelname)s - %(message)s", - datefmt="%d-%b-%y %H:%M:%S")) - logger.addHandler(logger_handler) - loggers[pkg] = logger - - -# TODO: Should be removed as soon as migration -# mechanism is finalized inside ucloud -def get_ipv4_address(): - # If host is connected to internet - # Return IPv4 address of machine - # Otherwise, return 127.0.0.1 - with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: - try: - s.connect(("8.8.8.8", 80)) - except socket.timeout: - address = "127.0.0.1" - except Exception as e: - logging.getLogger().exception(e) - address = "127.0.0.1" - else: - address = s.getsockname()[0] - - return address - - -def get_ipv6_address(): - try: - r = requests.get("https://api6.ipify.org?format=json") - content = json.loads(r.content.decode("utf-8")) - ip = ip_address(content["ip"]).exploded - except Exception as e: - logging.exception(e) - else: - return ip diff --git a/ucloud/config.py b/ucloud/config.py deleted file mode 100644 index 7c141a3..0000000 --- a/ucloud/config.py +++ /dev/null @@ -1,41 +0,0 @@ -from etcd3_wrapper import Etcd3Wrapper - -from ucloud.common.host import HostPool -from ucloud.common.request import RequestPool -from ucloud.common.vm import VmPool -from ucloud.common.storage_handlers import FileSystemBasedImageStorageHandler, CEPHBasedImageStorageHandler -from decouple import Config, RepositoryEnv, RepositoryEmpty - - -# Try importing config, but don't fail if it does not exist -try: - env_vars = Config(RepositoryEnv('/etc/ucloud/ucloud.conf')) -except FileNotFoundError: - env_vars = Config(RepositoryEmpty()) - - -etcd_wrapper_args = () -etcd_wrapper_kwargs = { - 'host': env_vars.get('ETCD_URL', 'localhost'), - 'port': env_vars.get('ETCD_PORT', 2379), - 'ca_cert': env_vars.get('CA_CERT', None), - 'cert_cert': env_vars.get('CERT_CERT', None), - 'cert_key': env_vars.get('CERT_KEY', None) -} - -etcd_client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) - -host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX')) -vm_pool = VmPool(etcd_client, env_vars.get('VM_PREFIX')) -request_pool = RequestPool(etcd_client, env_vars.get('REQUEST_PREFIX')) - -running_vms = [] - -__storage_backend = env_vars.get("STORAGE_BACKEND") -if __storage_backend == "filesystem": - image_storage_handler = FileSystemBasedImageStorageHandler(vm_base=env_vars.get("VM_DIR"), - image_base=env_vars.get("IMAGE_DIR")) -elif __storage_backend == "ceph": - image_storage_handler = CEPHBasedImageStorageHandler(vm_base="ssd", image_base="ssd") -else: - raise Exception("Unknown Image Storage Handler") diff --git a/ucloud/filescanner/main.py b/ucloud/filescanner/main.py deleted file mode 100755 index b70cb5b..0000000 --- a/ucloud/filescanner/main.py +++ /dev/null @@ -1,126 +0,0 @@ -import glob -import os -import pathlib -import subprocess as sp -import time -from uuid import uuid4 - -from . import logger -from ucloud.config import env_vars, etcd_client - - -def getxattr(file, attr): - """Get specified user extended attribute (arg:attr) of a file (arg:file)""" - try: - attr = "user." + attr - value = sp.check_output(['getfattr', file, - '--name', attr, - '--only-values', - '--absolute-names'], stderr=sp.DEVNULL) - value = value.decode("utf-8") - except sp.CalledProcessError as e: - logger.exception(e) - value = None - - return value - - -def setxattr(file, attr, value): - """Set specified user extended attribute (arg:attr) equal to (arg:value) - of a file (arg:file)""" - - attr = "user." + attr - sp.check_output(['setfattr', file, - '--name', attr, - '--value', str(value)]) - - -def sha512sum(file: str): - """Use sha512sum utility to compute sha512 sum of arg:file - - IF arg:file does not exists: - raise FileNotFoundError exception - ELSE IF sum successfully computer: - return computed sha512 sum - ELSE: - return None - """ - if not isinstance(file, str): raise TypeError - try: - output = sp.check_output(["sha512sum", file], stderr=sp.PIPE) - except sp.CalledProcessError as e: - error = e.stderr.decode("utf-8") - if "No such file or directory" in error: - raise FileNotFoundError from None - else: - output = output.decode("utf-8").strip() - output = output.split(" ") - return output[0] - return None - - -try: - sp.check_output(['which', 'getfattr']) - sp.check_output(['which', 'setfattr']) -except Exception as e: - logger.exception(e) - print('Make sure you have getfattr and setfattr available') - exit(1) - - -def main(): - BASE_DIR = env_vars.get("BASE_DIR") - - FILE_PREFIX = env_vars.get("FILE_PREFIX") - - # Recursively Get All Files and Folder below BASE_DIR - files = glob.glob("{}/**".format(BASE_DIR), recursive=True) - - # Retain only Files - files = list(filter(os.path.isfile, files)) - - untracked_files = list( - filter(lambda f: not bool(getxattr(f, "user.utracked")), files) - ) - - tracked_files = list( - filter(lambda f: f not in untracked_files, files) - ) - for file in untracked_files: - file_id = uuid4() - - # Get Username - owner = pathlib.Path(file).parts[3] - # Get Creation Date of File - # Here, we are assuming that ctime is creation time - # which is mostly not true. - creation_date = time.ctime(os.stat(file).st_ctime) - - # Get File Size - size = os.path.getsize(file) - - # Compute sha512 sum - sha_sum = sha512sum(file) - - # File Path excluding base and username - file_path = pathlib.Path(file).parts[4:] - file_path = os.path.join(*file_path) - - # Create Entry - entry_key = os.path.join(FILE_PREFIX, str(file_id)) - entry_value = { - "filename": file_path, - "owner": owner, - "sha512sum": sha_sum, - "creation_date": creation_date, - "size": size - } - - print("Tracking {}".format(file)) - # Insert Entry - etcd_client.put(entry_key, entry_value, value_in_json=True) - setxattr(file, "user.utracked", True) - - -if __name__ == "__main__": - main() diff --git a/ucloud/host/helper.py b/ucloud/host/helper.py deleted file mode 100644 index edcb82d..0000000 --- a/ucloud/host/helper.py +++ /dev/null @@ -1,13 +0,0 @@ -import socket -from contextlib import closing - - -def find_free_port(): - with closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as s: - try: - s.bind(('', 0)) - s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - except Exception: - return None - else: - return s.getsockname()[1] diff --git a/ucloud/host/main.py b/ucloud/host/main.py deleted file mode 100755 index ccf0a8d..0000000 --- a/ucloud/host/main.py +++ /dev/null @@ -1,143 +0,0 @@ -import argparse -import multiprocessing as mp -import time - -from etcd3_wrapper import Etcd3Wrapper - -from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import (vm_pool, request_pool, - etcd_client, running_vms, - etcd_wrapper_args, etcd_wrapper_kwargs, - HostPool, env_vars) - -from .helper import find_free_port -from . import virtualmachine -from ucloud.host import logger - - -def update_heartbeat(hostname): - """Update Last HeartBeat Time for :param hostname: in etcd""" - client = Etcd3Wrapper(*etcd_wrapper_args, **etcd_wrapper_kwargs) - host_pool = HostPool(client, env_vars.get('HOST_PREFIX')) - this_host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) - - while True: - this_host.update_heartbeat() - host_pool.put(this_host) - time.sleep(10) - - -def maintenance(host): - # To capture vm running according to running_vms list - - # This is to capture successful migration of a VM. - # Suppose, this host is running "vm1" and user initiated - # request to migrate this "vm1" to some other host. On, - # successful migration the destination host would set - # the vm hostname to itself. Thus, we are checking - # whether this host vm is successfully migrated. If yes - # then we shutdown "vm1" on this host. - - to_be_removed = [] - for running_vm in running_vms: - with vm_pool.get_put(running_vm.key) as vm_entry: - if vm_entry.hostname != host.key and not vm_entry.in_migration: - running_vm.handle.shutdown() - logger.info("VM migration not completed successfully.") - to_be_removed.append(running_vm) - - for r in to_be_removed: - running_vms.remove(r) - - # To check vm running according to etcd entries - alleged_running_vms = vm_pool.by_status("RUNNING", vm_pool.by_host(host.key)) - - for vm_entry in alleged_running_vms: - _vm = virtualmachine.get_vm(running_vms, vm_entry.key) - # Whether, the allegedly running vm is in our - # running_vms list or not if it is said to be - # running on this host but it is not then we - # need to shut it down - - # This is to capture poweroff/shutdown of a VM - # initiated by user inside VM. OR crash of VM by some - # user running process - if (_vm and not _vm.handle.is_running()) or not _vm: - logger.debug("_vm = %s, is_running() = %s" % (_vm, _vm.handle.is_running())) - vm_entry.add_log("""{} is not running but is said to be running. - So, shutting it down and declare it killed""".format(vm_entry.key)) - vm_entry.declare_killed() - vm_pool.put(vm_entry) - if _vm: - running_vms.remove(_vm) - - -def main(hostname): - heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) - - host_pool = HostPool(etcd_client, env_vars.get('HOST_PREFIX')) - host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) - assert host is not None, "No such host with name = {}".format(hostname) - - try: - heartbeat_updating_process.start() - except Exception as e: - logger.info("No Need To Go Further. Our heartbeat updating mechanism is not working") - logger.exception(e) - exit(-1) - - logger.info("%s Session Started %s", '*' * 5, '*' * 5) - - # It is seen that under heavy load, timeout event doesn't come - # in a predictive manner (which is intentional because we give - # higher priority to customer's requests) which delays heart - # beat update which in turn misunderstood by scheduler that the - # host is dead when it is actually alive. So, to ensure that we - # update the heart beat in a predictive manner we start Heart - # beat updating mechanism in separated thread - - for events_iterator in [ - etcd_client.get_prefix(env_vars.get('REQUEST_PREFIX'), value_in_json=True), - etcd_client.watch_prefix(env_vars.get('REQUEST_PREFIX'), timeout=10, value_in_json=True), - ]: - for request_event in events_iterator: - request_event = RequestEntry(request_event) - - if request_event.type == "TIMEOUT": - maintenance(host) - continue - - # If the event is directed toward me OR I am destination of a InitVMMigration - if request_event.hostname == host.key or request_event.destination == host.key: - logger.debug("VM Request: %s", request_event) - - request_pool.client.client.delete(request_event.key) - vm_entry = vm_pool.get(request_event.uuid) - - if vm_entry: - if request_event.type == RequestType.StartVM: - virtualmachine.start(vm_entry) - - elif request_event.type == RequestType.StopVM: - virtualmachine.stop(vm_entry) - - elif request_event.type == RequestType.DeleteVM: - virtualmachine.delete(vm_entry) - - elif request_event.type == RequestType.InitVMMigration: - virtualmachine.start(vm_entry, host.key, find_free_port()) - - elif request_event.type == RequestType.TransferVM: - virtualmachine.transfer(request_event) - else: - logger.info("VM Entry missing") - - logger.info("Running VMs %s", running_vms) - - -if __name__ == "__main__": - argparser = argparse.ArgumentParser() - argparser.add_argument("hostname", help="Name of this host. e.g /v1/host/1") - args = argparser.parse_args() - mp.set_start_method('spawn') - main(args.hostname) diff --git a/ucloud/host/qmp/__init__.py b/ucloud/host/qmp/__init__.py deleted file mode 100755 index 775b397..0000000 --- a/ucloud/host/qmp/__init__.py +++ /dev/null @@ -1,537 +0,0 @@ -# QEMU library -# -# Copyright (C) 2015-2016 Red Hat Inc. -# Copyright (C) 2012 IBM Corp. -# -# Authors: -# Fam Zheng -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. -# -# Based on qmp.py. -# - -import errno -import logging -import os -import shutil -import socket -import subprocess -import tempfile - -from . import qmp - -LOG = logging.getLogger(__name__) - -# Mapping host architecture to any additional architectures it can -# support which often includes its 32 bit cousin. -ADDITIONAL_ARCHES = { - "x86_64": "i386", - "aarch64": "armhf" -} - - -def kvm_available(target_arch=None): - host_arch = os.uname()[4] - if target_arch and target_arch != host_arch: - if target_arch != ADDITIONAL_ARCHES.get(host_arch): - return False - return os.access("/dev/kvm", os.R_OK | os.W_OK) - - -class QEMUMachineError(Exception): - """ - Exception called when an error in QEMUMachine happens. - """ - - -class QEMUMachineAddDeviceError(QEMUMachineError): - """ - Exception raised when a request to add a device can not be fulfilled - - The failures are caused by limitations, lack of information or conflicting - requests on the QEMUMachine methods. This exception does not represent - failures reported by the QEMU binary itself. - """ - - -class MonitorResponseError(qmp.QMPError): - """ - Represents erroneous QMP monitor reply - """ - - def __init__(self, reply): - try: - desc = reply["error"]["desc"] - except KeyError: - desc = reply - super(MonitorResponseError, self).__init__(desc) - self.reply = reply - - -class QEMUMachine(object): - """ - A QEMU VM - - Use this object as a context manager to ensure the QEMU process terminates:: - - with VM(binary) as vm: - ... - # vm is guaranteed to be shut down here - """ - - def __init__(self, binary, args=None, wrapper=None, name=None, - test_dir="/var/tmp", monitor_address=None, - socket_scm_helper=None): - ''' - Initialize a QEMUMachine - - @param binary: path to the qemu binary - @param args: list of extra arguments - @param wrapper: list of arguments used as prefix to qemu binary - @param name: prefix for socket and log file names (default: qemu-PID) - @param test_dir: where to create socket and log file - @param monitor_address: address for QMP monitor - @param socket_scm_helper: helper program, required for send_fd_scm() - @note: Qemu process is not started until launch() is used. - ''' - if args is None: - args = [] - if wrapper is None: - wrapper = [] - if name is None: - name = "qemu-%d" % os.getpid() - self._name = name - self._monitor_address = monitor_address - self._vm_monitor = None - self._qemu_log_path = None - self._qemu_log_file = None - self._popen = None - self._binary = binary - self._args = list(args) # Force copy args in case we modify them - self._wrapper = wrapper - self._events = [] - self._iolog = None - self._socket_scm_helper = socket_scm_helper - self._qmp = None - self._qemu_full_args = None - self._test_dir = test_dir - self._temp_dir = None - self._launched = False - self._machine = None - self._console_set = False - self._console_device_type = None - self._console_address = None - self._console_socket = None - - # just in case logging wasn't configured by the main script: - logging.basicConfig(level=logging.DEBUG) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.shutdown() - return False - - # This can be used to add an unused monitor instance. - def add_monitor_null(self): - self._args.append('-monitor') - self._args.append('null') - - def add_fd(self, fd, fdset, opaque, opts=''): - """ - Pass a file descriptor to the VM - """ - options = ['fd=%d' % fd, - 'set=%d' % fdset, - 'opaque=%s' % opaque] - if opts: - options.append(opts) - - # This did not exist before 3.4, but since then it is - # mandatory for our purpose - if hasattr(os, 'set_inheritable'): - os.set_inheritable(fd, True) - - self._args.append('-add-fd') - self._args.append(','.join(options)) - return self - - # Exactly one of fd and file_path must be given. - # (If it is file_path, the helper will open that file and pass its - # own fd) - def send_fd_scm(self, fd=None, file_path=None): - # In iotest.py, the qmp should always use unix socket. - assert self._qmp.is_scm_available() - if self._socket_scm_helper is None: - raise QEMUMachineError("No path to socket_scm_helper set") - if not os.path.exists(self._socket_scm_helper): - raise QEMUMachineError("%s does not exist" % - self._socket_scm_helper) - - # This did not exist before 3.4, but since then it is - # mandatory for our purpose - if hasattr(os, 'set_inheritable'): - os.set_inheritable(self._qmp.get_sock_fd(), True) - if fd is not None: - os.set_inheritable(fd, True) - - fd_param = ["%s" % self._socket_scm_helper, - "%d" % self._qmp.get_sock_fd()] - - if file_path is not None: - assert fd is None - fd_param.append(file_path) - else: - assert fd is not None - fd_param.append(str(fd)) - - devnull = open(os.path.devnull, 'rb') - proc = subprocess.Popen(fd_param, stdin=devnull, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, close_fds=False) - output = proc.communicate()[0] - if output: - LOG.debug(output) - - return proc.returncode - - @staticmethod - def _remove_if_exists(path): - """ - Remove file object at path if it exists - """ - try: - os.remove(path) - except OSError as exception: - if exception.errno == errno.ENOENT: - return - raise - - def is_running(self): - return self._popen is not None and self._popen.poll() is None - - def exitcode(self): - if self._popen is None: - return None - return self._popen.poll() - - def get_pid(self): - if not self.is_running(): - return None - return self._popen.pid - - def _load_io_log(self): - if self._qemu_log_path is not None: - with open(self._qemu_log_path, "r") as iolog: - self._iolog = iolog.read() - - def _base_args(self): - if isinstance(self._monitor_address, tuple): - moncdev = "socket,id=mon,host=%s,port=%s" % ( - self._monitor_address[0], - self._monitor_address[1]) - else: - moncdev = 'socket,id=mon,path=%s' % self._vm_monitor - args = ['-chardev', moncdev, - '-mon', 'chardev=mon,mode=control'] - if self._machine is not None: - args.extend(['-machine', self._machine]) - if self._console_set: - self._console_address = os.path.join(self._temp_dir, - self._name + "-console.sock") - chardev = ('socket,id=console,path=%s,server,nowait' % - self._console_address) - args.extend(['-chardev', chardev]) - if self._console_device_type is None: - args.extend(['-serial', 'chardev:console']) - else: - device = '%s,chardev=console' % self._console_device_type - args.extend(['-device', device]) - return args - - def _pre_launch(self): - self._temp_dir = tempfile.mkdtemp(dir=self._test_dir) - if self._monitor_address is not None: - self._vm_monitor = self._monitor_address - else: - self._vm_monitor = os.path.join(self._temp_dir, - self._name + "-monitor.sock") - self._qemu_log_path = os.path.join(self._temp_dir, self._name + ".log") - self._qemu_log_file = open(self._qemu_log_path, 'wb') - - self._qmp = qmp.QEMUMonitorProtocol(self._vm_monitor, - server=True) - - def _post_launch(self): - self._qmp.accept() - - def _post_shutdown(self): - if self._qemu_log_file is not None: - self._qemu_log_file.close() - self._qemu_log_file = None - - self._qemu_log_path = None - - if self._console_socket is not None: - self._console_socket.close() - self._console_socket = None - - if self._temp_dir is not None: - shutil.rmtree(self._temp_dir) - self._temp_dir = None - - def launch(self): - """ - Launch the VM and make sure we cleanup and expose the - command line/output in case of exception - """ - - if self._launched: - raise QEMUMachineError('VM already launched') - - self._iolog = None - self._qemu_full_args = None - try: - self._launch() - self._launched = True - except: - self.shutdown() - - LOG.debug('Error launching VM') - if self._qemu_full_args: - LOG.debug('Command: %r', ' '.join(self._qemu_full_args)) - if self._iolog: - LOG.debug('Output: %r', self._iolog) - raise Exception(self._iolog) - raise - - def _launch(self): - """ - Launch the VM and establish a QMP connection - """ - devnull = open(os.path.devnull, 'rb') - self._pre_launch() - self._qemu_full_args = (self._wrapper + [self._binary] + - self._base_args() + self._args) - LOG.debug('VM launch command: %r', ' '.join(self._qemu_full_args)) - self._popen = subprocess.Popen(self._qemu_full_args, - stdin=devnull, - stdout=self._qemu_log_file, - stderr=subprocess.STDOUT, - shell=False, - close_fds=False) - self._post_launch() - - def wait(self): - """ - Wait for the VM to power off - """ - self._popen.wait() - self._qmp.close() - self._load_io_log() - self._post_shutdown() - - def shutdown(self): - """ - Terminate the VM and clean up - """ - if self.is_running(): - try: - self._qmp.cmd('quit') - self._qmp.close() - except: - self._popen.kill() - self._popen.wait() - - self._load_io_log() - self._post_shutdown() - - exitcode = self.exitcode() - if exitcode is not None and exitcode < 0: - msg = 'qemu received signal %i: %s' - if self._qemu_full_args: - command = ' '.join(self._qemu_full_args) - else: - command = '' - LOG.warn(msg, -exitcode, command) - - self._launched = False - - def qmp(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command and return the response dict - """ - qmp_args = dict() - for key, value in args.items(): - if conv_keys: - qmp_args[key.replace('_', '-')] = value - else: - qmp_args[key] = value - - return self._qmp.cmd(cmd, args=qmp_args) - - def command(self, cmd, conv_keys=True, **args): - """ - Invoke a QMP command. - On success return the response dict. - On failure raise an exception. - """ - reply = self.qmp(cmd, conv_keys, **args) - if reply is None: - raise qmp.QMPError("Monitor is closed") - if "error" in reply: - raise MonitorResponseError(reply) - return reply["return"] - - def get_qmp_event(self, wait=False): - """ - Poll for one queued QMP events and return it - """ - if len(self._events) > 0: - return self._events.pop(0) - return self._qmp.pull_event(wait=wait) - - def get_qmp_events(self, wait=False): - """ - Poll for queued QMP events and return a list of dicts - """ - events = self._qmp.get_events(wait=wait) - events.extend(self._events) - del self._events[:] - self._qmp.clear_events() - return events - - @staticmethod - def event_match(event, match=None): - """ - Check if an event matches optional match criteria. - - The match criteria takes the form of a matching subdict. The event is - checked to be a superset of the subdict, recursively, with matching - values whenever the subdict values are not None. - - This has a limitation that you cannot explicitly check for None values. - - Examples, with the subdict queries on the left: - - None matches any object. - - {"foo": None} matches {"foo": {"bar": 1}} - - {"foo": None} matches {"foo": 5} - - {"foo": {"abc": None}} does not match {"foo": {"bar": 1}} - - {"foo": {"rab": 2}} matches {"foo": {"bar": 1, "rab": 2}} - """ - if match is None: - return True - - try: - for key in match: - if key in event: - if not QEMUMachine.event_match(event[key], match[key]): - return False - else: - return False - return True - except TypeError: - # either match or event wasn't iterable (not a dict) - return match == event - - def event_wait(self, name, timeout=60.0, match=None): - """ - event_wait waits for and returns a named event from QMP with a timeout. - - name: The event to wait for. - timeout: QEMUMonitorProtocol.pull_event timeout parameter. - match: Optional match criteria. See event_match for details. - """ - return self.events_wait([(name, match)], timeout) - - def events_wait(self, events, timeout=60.0): - """ - events_wait waits for and returns a named event from QMP with a timeout. - - events: a sequence of (name, match_criteria) tuples. - The match criteria are optional and may be None. - See event_match for details. - timeout: QEMUMonitorProtocol.pull_event timeout parameter. - """ - - def _match(event): - for name, match in events: - if (event['event'] == name and - self.event_match(event, match)): - return True - return False - - # Search cached events - for event in self._events: - if _match(event): - self._events.remove(event) - return event - - # Poll for new events - while True: - event = self._qmp.pull_event(wait=timeout) - if _match(event): - return event - self._events.append(event) - - return None - - def get_log(self): - """ - After self.shutdown or failed qemu execution, this returns the output - of the qemu process. - """ - return self._iolog - - def add_args(self, *args): - """ - Adds to the list of extra arguments to be given to the QEMU binary - """ - self._args.extend(args) - - def set_machine(self, machine_type): - """ - Sets the machine type - - If set, the machine type will be added to the base arguments - of the resulting QEMU command line. - """ - self._machine = machine_type - - def set_console(self, device_type=None): - """ - Sets the device type for a console device - - If set, the console device and a backing character device will - be added to the base arguments of the resulting QEMU command - line. - - This is a convenience method that will either use the provided - device type, or default to a "-serial chardev:console" command - line argument. - - The actual setting of command line arguments will be be done at - machine launch time, as it depends on the temporary directory - to be created. - - @param device_type: the device type, such as "isa-serial". If - None is given (the default value) a "-serial - chardev:console" command line argument will - be used instead, resorting to the machine's - default device type. - """ - self._console_set = True - self._console_device_type = device_type - - @property - def console_socket(self): - """ - Returns a socket connected to the console - """ - if self._console_socket is None: - self._console_socket = socket.socket(socket.AF_UNIX, - socket.SOCK_STREAM) - self._console_socket.connect(self._console_address) - return self._console_socket diff --git a/ucloud/host/qmp/qmp.py b/ucloud/host/qmp/qmp.py deleted file mode 100755 index bf35d71..0000000 --- a/ucloud/host/qmp/qmp.py +++ /dev/null @@ -1,255 +0,0 @@ -# QEMU Monitor Protocol Python class -# -# Copyright (C) 2009, 2010 Red Hat Inc. -# -# Authors: -# Luiz Capitulino -# -# This work is licensed under the terms of the GNU GPL, version 2. See -# the COPYING file in the top-level directory. - -import errno -import json -import logging -import socket - - -class QMPError(Exception): - pass - - -class QMPConnectError(QMPError): - pass - - -class QMPCapabilitiesError(QMPError): - pass - - -class QMPTimeoutError(QMPError): - pass - - -class QEMUMonitorProtocol(object): - #: Logger object for debugging messages - logger = logging.getLogger('QMP') - #: Socket's error class - error = socket.error - #: Socket's timeout - timeout = socket.timeout - - def __init__(self, address, server=False): - """ - Create a QEMUMonitorProtocol class. - - @param address: QEMU address, can be either a unix socket path (string) - or a tuple in the form ( address, port ) for a TCP - connection - @param server: server mode listens on the socket (bool) - @raise socket.error on socket connection errors - @note No connection is established, this is done by the connect() or - accept() methods - """ - self.__events = [] - self.__address = address - self.__sock = self.__get_sock() - self.__sockfile = None - if server: - self.__sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - self.__sock.bind(self.__address) - self.__sock.listen(1) - - def __get_sock(self): - if isinstance(self.__address, tuple): - family = socket.AF_INET - else: - family = socket.AF_UNIX - return socket.socket(family, socket.SOCK_STREAM) - - def __negotiate_capabilities(self): - greeting = self.__json_read() - if greeting is None or "QMP" not in greeting: - raise QMPConnectError - # Greeting seems ok, negotiate capabilities - resp = self.cmd('qmp_capabilities') - if "return" in resp: - return greeting - raise QMPCapabilitiesError - - def __json_read(self, only_event=False): - while True: - data = self.__sockfile.readline() - if not data: - return - resp = json.loads(data) - if 'event' in resp: - self.logger.debug("<<< %s", resp) - self.__events.append(resp) - if not only_event: - continue - return resp - - def __get_events(self, wait=False): - """ - Check for new events in the stream and cache them in __events. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - """ - - # Check for new events regardless and pull them into the cache: - self.__sock.setblocking(0) - try: - self.__json_read() - except socket.error as err: - if err[0] == errno.EAGAIN: - # No data available - pass - self.__sock.setblocking(1) - - # Wait for new events, if needed. - # if wait is 0.0, this means "no wait" and is also implicitly false. - if not self.__events and wait: - if isinstance(wait, float): - self.__sock.settimeout(wait) - try: - ret = self.__json_read(only_event=True) - except socket.timeout: - raise QMPTimeoutError("Timeout waiting for event") - except: - raise QMPConnectError("Error while reading from socket") - if ret is None: - raise QMPConnectError("Error while reading from socket") - self.__sock.settimeout(None) - - def connect(self, negotiate=True): - """ - Connect to the QMP Monitor and perform capabilities negotiation. - - @return QMP greeting dict - @raise socket.error on socket connection errors - @raise QMPConnectError if the greeting is not received - @raise QMPCapabilitiesError if fails to negotiate capabilities - """ - self.__sock.connect(self.__address) - self.__sockfile = self.__sock.makefile() - if negotiate: - return self.__negotiate_capabilities() - - def accept(self): - """ - Await connection from QMP Monitor and perform capabilities negotiation. - - @return QMP greeting dict - @raise socket.error on socket connection errors - @raise QMPConnectError if the greeting is not received - @raise QMPCapabilitiesError if fails to negotiate capabilities - """ - self.__sock.settimeout(15) - self.__sock, _ = self.__sock.accept() - self.__sockfile = self.__sock.makefile() - return self.__negotiate_capabilities() - - def cmd_obj(self, qmp_cmd): - """ - Send a QMP command to the QMP Monitor. - - @param qmp_cmd: QMP command to be sent as a Python dict - @return QMP response as a Python dict or None if the connection has - been closed - """ - self.logger.debug(">>> %s", qmp_cmd) - try: - self.__sock.sendall(json.dumps(qmp_cmd).encode('utf-8')) - except socket.error as err: - if err[0] == errno.EPIPE: - return - raise socket.error(err) - resp = self.__json_read() - self.logger.debug("<<< %s", resp) - return resp - - def cmd(self, name, args=None, cmd_id=None): - """ - Build a QMP command and send it to the QMP Monitor. - - @param name: command name (string) - @param args: command arguments (dict) - @param cmd_id: command id (dict, list, string or int) - """ - qmp_cmd = {'execute': name} - if args: - qmp_cmd['arguments'] = args - if cmd_id: - qmp_cmd['id'] = cmd_id - return self.cmd_obj(qmp_cmd) - - def command(self, cmd, **kwds): - """ - Build and send a QMP command to the monitor, report errors if any - """ - ret = self.cmd(cmd, kwds) - if "error" in ret: - raise Exception(ret['error']['desc']) - return ret['return'] - - def pull_event(self, wait=False): - """ - Pulls a single event. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - - @return The first available QMP event, or None. - """ - self.__get_events(wait) - - if self.__events: - return self.__events.pop(0) - return None - - def get_events(self, wait=False): - """ - Get a list of available QMP events. - - @param wait (bool): block until an event is available. - @param wait (float): If wait is a float, treat it as a timeout value. - - @raise QMPTimeoutError: If a timeout float is provided and the timeout - period elapses. - @raise QMPConnectError: If wait is True but no events could be - retrieved or if some other error occurred. - - @return The list of available QMP events. - """ - self.__get_events(wait) - return self.__events - - def clear_events(self): - """ - Clear current list of pending events. - """ - self.__events = [] - - def close(self): - self.__sock.close() - self.__sockfile.close() - - def settimeout(self, timeout): - self.__sock.settimeout(timeout) - - def get_sock_fd(self): - return self.__sock.fileno() - - def is_scm_available(self): - return self.__sock.family == socket.AF_UNIX diff --git a/ucloud/host/virtualmachine.py b/ucloud/host/virtualmachine.py deleted file mode 100755 index 7524083..0000000 --- a/ucloud/host/virtualmachine.py +++ /dev/null @@ -1,384 +0,0 @@ -# QEMU Manual -# https://qemu.weilnetz.de/doc/qemu-doc.html - -# For QEMU Monitor Protocol Commands Information, See -# https://qemu.weilnetz.de/doc/qemu-doc.html#pcsys_005fmonitor - -import os -import random -import subprocess as sp -import tempfile -import time - -from functools import wraps -from string import Template -from typing import Union -from os.path import join as join_path - -import bitmath -import sshtunnel - -from ucloud.common.helpers import get_ipv6_address -from ucloud.common.request import RequestEntry, RequestType -from ucloud.common.vm import VMEntry, VMStatus -from ucloud.config import (etcd_client, request_pool, - running_vms, vm_pool, env_vars, - image_storage_handler) -from . import qmp -from ucloud.host import logger - - -class VM: - def __init__(self, key, handle, vnc_socket_file): - self.key = key # type: str - self.handle = handle # type: qmp.QEMUMachine - self.vnc_socket_file = vnc_socket_file # type: tempfile.NamedTemporaryFile - - def __repr__(self): - return "VM({})".format(self.key) - - -def delete_network_interface(iface): - try: - sp.check_output(['ip', 'link', 'del', iface]) - except Exception: - pass - - -def resolve_network(network_name, network_owner): - network = etcd_client.get(join_path(env_vars.get("NETWORK_PREFIX"), - network_owner, - network_name), - value_in_json=True) - return network - - -def delete_vm_network(vm_entry): - try: - for network in vm_entry.network: - network_name = network[0] - tap_mac = network[1] - tap_id = network[2] - - delete_network_interface('tap{}'.format(tap_id)) - - owners_vms = vm_pool.by_owner(vm_entry.owner) - owners_running_vms = vm_pool.by_status(VMStatus.running, - _vms=owners_vms) - - networks = map(lambda n: n[0], - map(lambda vm: vm.network, owners_running_vms) - ) - networks_in_use_by_user_vms = [vm[0] for vm in networks] - if network_name not in networks_in_use_by_user_vms: - network_entry = resolve_network(network[0], vm_entry.owner) - if network_entry: - network_type = network_entry.value["type"] - network_id = network_entry.value["id"] - if network_type == "vxlan": - delete_network_interface('br{}'.format(network_id)) - delete_network_interface('vxlan{}'.format(network_id)) - except Exception: - logger.exception("Exception in network interface deletion") - - -def create_dev(script, _id, dev, ip=None): - command = [script, _id, dev] - if ip: - command.append(ip) - try: - output = sp.check_output(command, stderr=sp.PIPE) - except Exception as e: - print(e.stderr) - return None - else: - return output.decode("utf-8").strip() - - -def create_vxlan_br_tap(_id, _dev, tap_id, ip=None): - network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') - vxlan = create_dev(script=os.path.join(network_script_base, 'create-vxlan.sh'), - _id=_id, dev=_dev) - if vxlan: - bridge = create_dev(script=os.path.join(network_script_base, 'create-bridge.sh'), - _id=_id, dev=vxlan, ip=ip) - if bridge: - tap = create_dev(script=os.path.join(network_script_base, 'create-tap.sh'), - _id=str(tap_id), dev=bridge) - if tap: - return tap - - -def random_bytes(num=6): - return [random.randrange(256) for _ in range(num)] - - -def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'): - mac = random_bytes() - if oui: - if type(oui) == str: - oui = [int(chunk) for chunk in oui.split(separator)] - mac = oui + random_bytes(num=6 - len(oui)) - else: - if multicast: - mac[0] |= 1 # set bit 0 - else: - mac[0] &= ~1 # clear bit 0 - if uaa: - mac[0] &= ~(1 << 1) # clear bit 1 - else: - mac[0] |= 1 << 1 # set bit 1 - return separator.join(byte_fmt % b for b in mac) - - -def update_radvd_conf(etcd_client): - network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network') - - networks = { - net.value['ipv6']: net.value['id'] - for net in etcd_client.get_prefix('/v1/network/', value_in_json=True) - if net.value.get('ipv6') - } - radvd_template = open(os.path.join(network_script_base, - 'radvd-template.conf'), 'r').read() - radvd_template = Template(radvd_template) - - content = [radvd_template.safe_substitute(bridge='br{}'.format(networks[net]), - prefix=net) - for net in networks if networks.get(net)] - - with open('/etc/radvd.conf', 'w') as radvd_conf: - radvd_conf.writelines(content) - try: - sp.check_output(['systemctl', 'restart', 'radvd']) - except Exception: - sp.check_output(['service', 'radvd', 'restart']) - - -def get_start_command_args(vm_entry, vnc_sock_filename: str, migration=False, migration_port=None): - threads_per_core = 1 - vm_memory = int(bitmath.parse_string_unsafe(vm_entry.specs["ram"]).to_MB()) - vm_cpus = int(vm_entry.specs["cpu"]) - vm_uuid = vm_entry.uuid - vm_networks = vm_entry.network - - command = "-name {}_{}".format(vm_entry.owner, vm_entry.name) - - command += " -drive file={},format=raw,if=virtio,cache=none".format( - image_storage_handler.qemu_path_string(vm_uuid) - ) - command += " -device virtio-rng-pci -vnc unix:{}".format(vnc_sock_filename) - command += " -m {} -smp cores={},threads={}".format( - vm_memory, vm_cpus, threads_per_core - ) - - if migration: - command += " -incoming tcp:[::]:{}".format(migration_port) - - tap = None - for network_mac_and_tap in vm_networks: - network_name, mac, tap = network_mac_and_tap - - _key = os.path.join(env_vars.get('NETWORK_PREFIX'), vm_entry.owner, network_name) - network = etcd_client.get(_key, value_in_json=True) - network_type = network.value["type"] - network_id = str(network.value["id"]) - network_ipv6 = network.value["ipv6"] - - if network_type == "vxlan": - tap = create_vxlan_br_tap(_id=network_id, - _dev=env_vars.get("VXLAN_PHY_DEV"), - tap_id=tap, - ip=network_ipv6) - update_radvd_conf(etcd_client) - - command += " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" \ - " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" \ - .format(tap=tap, net_id=network_id, mac=mac) - - return command.split(" ") - - -def create_vm_object(vm_entry, migration=False, migration_port=None): - # NOTE: If migration suddenly stop working, having different - # VNC unix filename on source and destination host can - # be a possible cause of it. - - # REQUIREMENT: Use Unix Socket instead of TCP Port for VNC - vnc_sock_file = tempfile.NamedTemporaryFile() - - qemu_args = get_start_command_args( - vm_entry=vm_entry, - vnc_sock_filename=vnc_sock_file.name, - migration=migration, - migration_port=migration_port, - ) - qemu_machine = qmp.QEMUMachine("/usr/bin/qemu-system-x86_64", args=qemu_args) - return VM(vm_entry.key, qemu_machine, vnc_sock_file) - - -def get_vm(vm_list: list, vm_key) -> Union[VM, None]: - return next((vm for vm in vm_list if vm.key == vm_key), None) - - -def need_running_vm(func): - @wraps(func) - def wrapper(e): - vm = get_vm(running_vms, e.key) - if vm: - try: - status = vm.handle.command("query-status") - logger.debug("VM Status Check - %s", status) - except Exception as exception: - logger.info("%s failed - VM %s %s", func.__name__, e, exception) - else: - return func(e) - - return None - else: - logger.info("%s failed because VM %s is not running", func.__name__, e.key) - return None - - return wrapper - - -def create(vm_entry: VMEntry): - if image_storage_handler.is_vm_image_exists(vm_entry.uuid): - # File Already exists. No Problem Continue - logger.debug("Image for vm %s exists", vm_entry.uuid) - else: - vm_hdd = int(bitmath.parse_string_unsafe(vm_entry.specs["os-ssd"]).to_MB()) - if image_storage_handler.make_vm_image(src=vm_entry.image_uuid, dest=vm_entry.uuid): - if not image_storage_handler.resize_vm_image(path=vm_entry.uuid, size=vm_hdd): - vm_entry.status = VMStatus.error - else: - logger.info("New VM Created") - - -def start(vm_entry: VMEntry, destination_host_key=None, migration_port=None): - _vm = get_vm(running_vms, vm_entry.key) - - # VM already running. No need to proceed further. - if _vm: - logger.info("VM %s already running" % vm_entry.uuid) - return - else: - logger.info("Trying to start %s" % vm_entry.uuid) - if destination_host_key: - launch_vm(vm_entry, migration=True, migration_port=migration_port, - destination_host_key=destination_host_key) - else: - create(vm_entry) - launch_vm(vm_entry) - - -@need_running_vm -def stop(vm_entry): - vm = get_vm(running_vms, vm_entry.key) - vm.handle.shutdown() - if not vm.handle.is_running(): - vm_entry.add_log("Shutdown successfully") - vm_entry.declare_stopped() - vm_pool.put(vm_entry) - running_vms.remove(vm) - delete_vm_network(vm_entry) - - -def delete(vm_entry): - logger.info("Deleting VM | %s", vm_entry) - stop(vm_entry) - - if image_storage_handler.is_vm_image_exists(vm_entry.uuid): - r_status = image_storage_handler.delete_vm_image(vm_entry.uuid) - if r_status: - etcd_client.client.delete(vm_entry.key) - else: - etcd_client.client.delete(vm_entry.key) - -def transfer(request_event): - # This function would run on source host i.e host on which the vm - # is running initially. This host would be responsible for transferring - # vm state to destination host. - - _host, _port = request_event.parameters["host"], request_event.parameters["port"] - _uuid = request_event.uuid - _destination = request_event.destination_host_key - vm = get_vm(running_vms, join_path(env_vars.get('VM_PREFIX'), _uuid)) - - if vm: - tunnel = sshtunnel.SSHTunnelForwarder( - _host, - ssh_username=env_vars.get("ssh_username"), - ssh_pkey=env_vars.get("ssh_pkey"), - remote_bind_address=("127.0.0.1", _port), - ssh_proxy_enabled=True, - ssh_proxy=(_host, 22) - ) - try: - tunnel.start() - except sshtunnel.BaseSSHTunnelForwarderError: - logger.exception("Couldn't establish connection to (%s, 22)", _host) - else: - vm.handle.command( - "migrate", uri="tcp:0.0.0.0:{}".format(tunnel.local_bind_port) - ) - - status = vm.handle.command("query-migrate")["status"] - while status not in ["failed", "completed"]: - time.sleep(2) - status = vm.handle.command("query-migrate")["status"] - - with vm_pool.get_put(request_event.uuid) as source_vm: - if status == "failed": - source_vm.add_log("Migration Failed") - elif status == "completed": - # If VM is successfully migrated then shutdown the VM - # on this host and update hostname to destination host key - source_vm.add_log("Successfully migrated") - source_vm.hostname = _destination - running_vms.remove(vm) - vm.handle.shutdown() - source_vm.in_migration = False # VM transfer finished - finally: - tunnel.close() - - -def launch_vm(vm_entry, migration=False, migration_port=None, destination_host_key=None): - logger.info("Starting %s" % vm_entry.key) - - vm = create_vm_object(vm_entry, migration=migration, migration_port=migration_port) - try: - vm.handle.launch() - except Exception: - logger.exception("Error Occured while starting VM") - vm.handle.shutdown() - - if migration: - # We don't care whether MachineError or any other error occurred - pass - else: - # Error during typical launch of a vm - vm.handle.shutdown() - vm_entry.declare_killed() - vm_pool.put(vm_entry) - else: - vm_entry.vnc_socket = vm.vnc_socket_file.name - running_vms.append(vm) - - if migration: - vm_entry.in_migration = True - r = RequestEntry.from_scratch( - type=RequestType.TransferVM, - hostname=vm_entry.hostname, - parameters={"host": get_ipv6_address(), "port": migration_port}, - uuid=vm_entry.uuid, - destination_host_key=destination_host_key, - request_prefix=env_vars.get("REQUEST_PREFIX") - ) - request_pool.put(r) - else: - # Typical launching of a vm - vm_entry.status = VMStatus.running - vm_entry.add_log("Started successfully") - - vm_pool.put(vm_entry) diff --git a/ucloud/imagescanner/main.py b/ucloud/imagescanner/main.py deleted file mode 100755 index 20ce9d5..0000000 --- a/ucloud/imagescanner/main.py +++ /dev/null @@ -1,78 +0,0 @@ -import json -import os -import subprocess - -from os.path import join as join_path -from ucloud.config import etcd_client, env_vars, image_storage_handler -from ucloud.imagescanner import logger - - -def qemu_img_type(path): - qemu_img_info_command = ["qemu-img", "info", "--output", "json", path] - try: - qemu_img_info = subprocess.check_output(qemu_img_info_command) - except Exception as e: - logger.exception(e) - return None - else: - qemu_img_info = json.loads(qemu_img_info.decode("utf-8")) - return qemu_img_info["format"] - - -def main(): - # We want to get images entries that requests images to be created - images = etcd_client.get_prefix(env_vars.get('IMAGE_PREFIX'), value_in_json=True) - images_to_be_created = list(filter(lambda im: im.value['status'] == 'TO_BE_CREATED', images)) - - for image in images_to_be_created: - try: - image_uuid = image.key.split('/')[-1] - image_owner = image.value['owner'] - image_filename = image.value['filename'] - image_store_name = image.value['store_name'] - image_full_path = join_path(env_vars.get('BASE_DIR'), image_owner, image_filename) - - image_stores = etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX'), value_in_json=True) - user_image_store = next(filter( - lambda s, store_name=image_store_name: s.value["name"] == store_name, - image_stores - )) - - image_store_pool = user_image_store.value['attributes']['pool'] - - except Exception as e: - logger.exception(e) - else: - # At least our basic data is available - qemu_img_convert_command = ["qemu-img", "convert", "-f", "qcow2", - "-O", "raw", image_full_path, "image.raw"] - - if qemu_img_type(image_full_path) == "qcow2": - try: - # Convert .qcow2 to .raw - subprocess.check_output(qemu_img_convert_command) - except Exception as e: - logger.exception(e) - else: - # Import and Protect - r_status = image_storage_handler.import_image(src="image.raw", - dest=image_uuid, - protect=True) - if r_status: - # Everything is successfully done - image.value["status"] = "CREATED" - etcd_client.put(image.key, json.dumps(image.value)) - - else: - # The user provided image is either not found or of invalid format - image.value["status"] = "INVALID_IMAGE" - etcd_client.put(image.key, json.dumps(image.value)) - - try: - os.remove("image.raw") - except Exception: - pass - - -if __name__ == "__main__": - main() diff --git a/ucloud/metadata/main.py b/ucloud/metadata/main.py deleted file mode 100644 index e7cb33b..0000000 --- a/ucloud/metadata/main.py +++ /dev/null @@ -1,91 +0,0 @@ -import os - -from flask import Flask, request -from flask_restful import Resource, Api - -from ucloud.config import etcd_client, env_vars, vm_pool - -app = Flask(__name__) -api = Api(app) - - -def get_vm_entry(mac_addr): - return next(filter(lambda vm: mac_addr in list(zip(*vm.network))[1], vm_pool.vms), None) - - -# https://stackoverflow.com/questions/37140846/how-to-convert-ipv6-link-local-address-to-mac-address-in-python -def ipv62mac(ipv6): - # remove subnet info if given - subnet_index = ipv6.find('/') - if subnet_index != -1: - ipv6 = ipv6[:subnet_index] - - ipv6_parts = ipv6.split(':') - mac_parts = list() - for ipv6_part in ipv6_parts[-4:]: - while len(ipv6_part) < 4: - ipv6_part = '0' + ipv6_part - mac_parts.append(ipv6_part[:2]) - mac_parts.append(ipv6_part[-2:]) - - # modify parts to match MAC value - mac_parts[0] = '%02x' % (int(mac_parts[0], 16) ^ 2) - del mac_parts[4] - del mac_parts[3] - return ':'.join(mac_parts) - - -class Root(Resource): - @staticmethod - def get(): - data = get_vm_entry(ipv62mac(request.remote_addr)) - - if not data: - return {'message': 'Metadata for such VM does not exists.'}, 404 - else: - - # {env_vars.get('USER_PREFIX')}/{realm}/{name}/key - etcd_key = os.path.join(env_vars.get('USER_PREFIX'), data.value['owner_realm'], - data.value['owner'], 'key') - etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True) - user_personal_ssh_keys = [key.value for key in etcd_entry] - data.value['metadata']['ssh-keys'] += user_personal_ssh_keys - return data.value['metadata'], 200 - - @staticmethod - def post(): - return {'message': 'Previous Implementation is deprecated.'} - # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True) - # print(data) - # if data: - # for k in request.json: - # if k not in data.value: - # data.value[k] = request.json[k] - # if k.endswith("-list"): - # data.value[k] = [request.json[k]] - # else: - # if k.endswith("-list"): - # data.value[k].append(request.json[k]) - # else: - # data.value[k] = request.json[k] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data.value, value_in_json=True) - # else: - # data = {} - # for k in request.json: - # data[k] = request.json[k] - # if k.endswith("-list"): - # data[k] = [request.json[k]] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data, value_in_json=True) - - -api.add_resource(Root, '/') - - -def main(): - app.run(debug=True, host="::", port="80") - - -if __name__ == '__main__': - main() diff --git a/ucloud/sanity_checks.py b/ucloud/sanity_checks.py deleted file mode 100644 index 143f767..0000000 --- a/ucloud/sanity_checks.py +++ /dev/null @@ -1,33 +0,0 @@ -import sys -import subprocess as sp - -from os.path import isdir -from ucloud.config import env_vars - - -def check(): - ######################### - # ucloud-image-scanner # - ######################### - if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('IMAGE_DIR')): - print("You have set STORAGE_BACKEND to filesystem. So," - "the {} must exists. But, it don't".format(env_vars.get('IMAGE_DIR'))) - sys.exit(1) - - try: - sp.check_output(['which', 'qemu-img']) - except Exception: - print("qemu-img missing") - sys.exit(1) - - ############### - # ucloud-host # - ############### - - if env_vars.get('STORAGE_BACKEND') == 'filesystem' and not isdir(env_vars.get('VM_DIR')): - print("You have set STORAGE_BACKEND to filesystem. So, the vm directory mentioned" - " in .env file must exists. But, it don't.") - sys.exit(1) - -if __name__ == "__main__": - check() \ No newline at end of file diff --git a/ucloud/scheduler/__init__.py b/ucloud/scheduler/__init__.py deleted file mode 100644 index 95e1be0..0000000 --- a/ucloud/scheduler/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import logging - -logger = logging.getLogger(__name__) \ No newline at end of file diff --git a/ucloud/scheduler/main.py b/ucloud/scheduler/main.py deleted file mode 100755 index e2c975a..0000000 --- a/ucloud/scheduler/main.py +++ /dev/null @@ -1,93 +0,0 @@ -# TODO -# 1. send an email to an email address defined by env['admin-email'] -# if resources are finished -# 2. Introduce a status endpoint of the scheduler - -# maybe expose a prometheus compatible output - -from ucloud.common.request import RequestEntry, RequestType -from ucloud.config import etcd_client -from ucloud.config import host_pool, request_pool, vm_pool, env_vars -from .helper import (get_suitable_host, dead_host_mitigation, dead_host_detection, - assign_host, NoSuitableHostFound) -from . import logger - - -def main(): - logger.info("%s SESSION STARTED %s", '*' * 5, '*' * 5) - - pending_vms = [] - - for request_iterator in [ - etcd_client.get_prefix(env_vars.get('REQUEST_PREFIX'), value_in_json=True), - etcd_client.watch_prefix(env_vars.get('REQUEST_PREFIX'), timeout=5, value_in_json=True), - ]: - for request_event in request_iterator: - request_entry = RequestEntry(request_event) - # Never Run time critical mechanism inside timeout - # mechanism because timeout mechanism only comes - # when no other event is happening. It means under - # heavy load there would not be a timeout event. - if request_entry.type == "TIMEOUT": - - # Detect hosts that are dead and set their status - # to "DEAD", and their VMs' status to "KILLED" - dead_hosts = dead_host_detection() - if dead_hosts: - logger.debug("Dead hosts: %s", dead_hosts) - dead_host_mitigation(dead_hosts) - - # If there are VMs that weren't assigned a host - # because there wasn't a host available which - # meets requirement of that VM then we would - # create a new ScheduleVM request for that VM - # on our behalf. - while pending_vms: - pending_vm_entry = pending_vms.pop() - r = RequestEntry.from_scratch(type="ScheduleVM", - uuid=pending_vm_entry.uuid, - hostname=pending_vm_entry.hostname, - request_prefix=env_vars.get("REQUEST_PREFIX")) - request_pool.put(r) - - elif request_entry.type == RequestType.ScheduleVM: - logger.debug("%s, %s", request_entry.key, request_entry.value) - - vm_entry = vm_pool.get(request_entry.uuid) - if vm_entry is None: - logger.info("Trying to act on {} but it is deleted".format(request_entry.uuid)) - continue - etcd_client.client.delete(request_entry.key) # consume Request - - # If the Request is about a VM which is labelled as "migration" - # and has a destination - if hasattr(request_entry, "migration") and request_entry.migration \ - and hasattr(request_entry, "destination") and request_entry.destination: - try: - get_suitable_host(vm_specs=vm_entry.specs, - hosts=[host_pool.get(request_entry.destination)]) - except NoSuitableHostFound: - logger.info("Requested destination host doesn't have enough capacity" - "to hold %s" % vm_entry.uuid) - else: - r = RequestEntry.from_scratch(type=RequestType.InitVMMigration, - uuid=request_entry.uuid, - destination=request_entry.destination, - request_prefix=env_vars.get("REQUEST_PREFIX")) - request_pool.put(r) - - # If the Request is about a VM that just want to get started/created - else: - # assign_host only returns None when we couldn't be able to assign - # a host to a VM because of resource constraints - try: - assign_host(vm_entry) - except NoSuitableHostFound: - vm_entry.add_log("Can't schedule VM. No Resource Left.") - vm_pool.put(vm_entry) - - pending_vms.append(vm_entry) - logger.info("No Resource Left. Emailing admin....") - - -if __name__ == "__main__": - main() diff --git a/uncloud/.gitignore b/uncloud/.gitignore new file mode 100644 index 0000000..b03e0a5 --- /dev/null +++ b/uncloud/.gitignore @@ -0,0 +1,2 @@ +local_settings.py +ldap_max_uid_file \ No newline at end of file diff --git a/uncloud/__init__.py b/uncloud/__init__.py new file mode 100644 index 0000000..e073dd5 --- /dev/null +++ b/uncloud/__init__.py @@ -0,0 +1,254 @@ +from django.utils.translation import gettext_lazy as _ +import decimal +from .celery import app as celery_app + +# Define DecimalField properties, used to represent amounts of money. +AMOUNT_MAX_DIGITS=10 +AMOUNT_DECIMALS=2 + +decimal.getcontext().prec = AMOUNT_DECIMALS + +# http://xml.coverpages.org/country3166.html +COUNTRIES = ( + ('AD', _('Andorra')), + ('AE', _('United Arab Emirates')), + ('AF', _('Afghanistan')), + ('AG', _('Antigua & Barbuda')), + ('AI', _('Anguilla')), + ('AL', _('Albania')), + ('AM', _('Armenia')), + ('AN', _('Netherlands Antilles')), + ('AO', _('Angola')), + ('AQ', _('Antarctica')), + ('AR', _('Argentina')), + ('AS', _('American Samoa')), + ('AT', _('Austria')), + ('AU', _('Australia')), + ('AW', _('Aruba')), + ('AZ', _('Azerbaijan')), + ('BA', _('Bosnia and Herzegovina')), + ('BB', _('Barbados')), + ('BD', _('Bangladesh')), + ('BE', _('Belgium')), + ('BF', _('Burkina Faso')), + ('BG', _('Bulgaria')), + ('BH', _('Bahrain')), + ('BI', _('Burundi')), + ('BJ', _('Benin')), + ('BM', _('Bermuda')), + ('BN', _('Brunei Darussalam')), + ('BO', _('Bolivia')), + ('BR', _('Brazil')), + ('BS', _('Bahama')), + ('BT', _('Bhutan')), + ('BV', _('Bouvet Island')), + ('BW', _('Botswana')), + ('BY', _('Belarus')), + ('BZ', _('Belize')), + ('CA', _('Canada')), + ('CC', _('Cocos (Keeling) Islands')), + ('CF', _('Central African Republic')), + ('CG', _('Congo')), + ('CH', _('Switzerland')), + ('CI', _('Ivory Coast')), + ('CK', _('Cook Iislands')), + ('CL', _('Chile')), + ('CM', _('Cameroon')), + ('CN', _('China')), + ('CO', _('Colombia')), + ('CR', _('Costa Rica')), + ('CU', _('Cuba')), + ('CV', _('Cape Verde')), + ('CX', _('Christmas Island')), + ('CY', _('Cyprus')), + ('CZ', _('Czech Republic')), + ('DE', _('Germany')), + ('DJ', _('Djibouti')), + ('DK', _('Denmark')), + ('DM', _('Dominica')), + ('DO', _('Dominican Republic')), + ('DZ', _('Algeria')), + ('EC', _('Ecuador')), + ('EE', _('Estonia')), + ('EG', _('Egypt')), + ('EH', _('Western Sahara')), + ('ER', _('Eritrea')), + ('ES', _('Spain')), + ('ET', _('Ethiopia')), + ('FI', _('Finland')), + ('FJ', _('Fiji')), + ('FK', _('Falkland Islands (Malvinas)')), + ('FM', _('Micronesia')), + ('FO', _('Faroe Islands')), + ('FR', _('France')), + ('FX', _('France, Metropolitan')), + ('GA', _('Gabon')), + ('GB', _('United Kingdom (Great Britain)')), + ('GD', _('Grenada')), + ('GE', _('Georgia')), + ('GF', _('French Guiana')), + ('GH', _('Ghana')), + ('GI', _('Gibraltar')), + ('GL', _('Greenland')), + ('GM', _('Gambia')), + ('GN', _('Guinea')), + ('GP', _('Guadeloupe')), + ('GQ', _('Equatorial Guinea')), + ('GR', _('Greece')), + ('GS', _('South Georgia and the South Sandwich Islands')), + ('GT', _('Guatemala')), + ('GU', _('Guam')), + ('GW', _('Guinea-Bissau')), + ('GY', _('Guyana')), + ('HK', _('Hong Kong')), + ('HM', _('Heard & McDonald Islands')), + ('HN', _('Honduras')), + ('HR', _('Croatia')), + ('HT', _('Haiti')), + ('HU', _('Hungary')), + ('ID', _('Indonesia')), + ('IE', _('Ireland')), + ('IL', _('Israel')), + ('IN', _('India')), + ('IO', _('British Indian Ocean Territory')), + ('IQ', _('Iraq')), + ('IR', _('Islamic Republic of Iran')), + ('IS', _('Iceland')), + ('IT', _('Italy')), + ('JM', _('Jamaica')), + ('JO', _('Jordan')), + ('JP', _('Japan')), + ('KE', _('Kenya')), + ('KG', _('Kyrgyzstan')), + ('KH', _('Cambodia')), + ('KI', _('Kiribati')), + ('KM', _('Comoros')), + ('KN', _('St. Kitts and Nevis')), + ('KP', _('Korea, Democratic People\'s Republic of')), + ('KR', _('Korea, Republic of')), + ('KW', _('Kuwait')), + ('KY', _('Cayman Islands')), + ('KZ', _('Kazakhstan')), + ('LA', _('Lao People\'s Democratic Republic')), + ('LB', _('Lebanon')), + ('LC', _('Saint Lucia')), + ('LI', _('Liechtenstein')), + ('LK', _('Sri Lanka')), + ('LR', _('Liberia')), + ('LS', _('Lesotho')), + ('LT', _('Lithuania')), + ('LU', _('Luxembourg')), + ('LV', _('Latvia')), + ('LY', _('Libyan Arab Jamahiriya')), + ('MA', _('Morocco')), + ('MC', _('Monaco')), + ('MD', _('Moldova, Republic of')), + ('MG', _('Madagascar')), + ('MH', _('Marshall Islands')), + ('ML', _('Mali')), + ('MN', _('Mongolia')), + ('MM', _('Myanmar')), + ('MO', _('Macau')), + ('MP', _('Northern Mariana Islands')), + ('MQ', _('Martinique')), + ('MR', _('Mauritania')), + ('MS', _('Monserrat')), + ('MT', _('Malta')), + ('MU', _('Mauritius')), + ('MV', _('Maldives')), + ('MW', _('Malawi')), + ('MX', _('Mexico')), + ('MY', _('Malaysia')), + ('MZ', _('Mozambique')), + ('NA', _('Namibia')), + ('NC', _('New Caledonia')), + ('NE', _('Niger')), + ('NF', _('Norfolk Island')), + ('NG', _('Nigeria')), + ('NI', _('Nicaragua')), + ('NL', _('Netherlands')), + ('NO', _('Norway')), + ('NP', _('Nepal')), + ('NR', _('Nauru')), + ('NU', _('Niue')), + ('NZ', _('New Zealand')), + ('OM', _('Oman')), + ('PA', _('Panama')), + ('PE', _('Peru')), + ('PF', _('French Polynesia')), + ('PG', _('Papua New Guinea')), + ('PH', _('Philippines')), + ('PK', _('Pakistan')), + ('PL', _('Poland')), + ('PM', _('St. Pierre & Miquelon')), + ('PN', _('Pitcairn')), + ('PR', _('Puerto Rico')), + ('PT', _('Portugal')), + ('PW', _('Palau')), + ('PY', _('Paraguay')), + ('QA', _('Qatar')), + ('RE', _('Reunion')), + ('RO', _('Romania')), + ('RU', _('Russian Federation')), + ('RW', _('Rwanda')), + ('SA', _('Saudi Arabia')), + ('SB', _('Solomon Islands')), + ('SC', _('Seychelles')), + ('SD', _('Sudan')), + ('SE', _('Sweden')), + ('SG', _('Singapore')), + ('SH', _('St. Helena')), + ('SI', _('Slovenia')), + ('SJ', _('Svalbard & Jan Mayen Islands')), + ('SK', _('Slovakia')), + ('SL', _('Sierra Leone')), + ('SM', _('San Marino')), + ('SN', _('Senegal')), + ('SO', _('Somalia')), + ('SR', _('Suriname')), + ('ST', _('Sao Tome & Principe')), + ('SV', _('El Salvador')), + ('SY', _('Syrian Arab Republic')), + ('SZ', _('Swaziland')), + ('TC', _('Turks & Caicos Islands')), + ('TD', _('Chad')), + ('TF', _('French Southern Territories')), + ('TG', _('Togo')), + ('TH', _('Thailand')), + ('TJ', _('Tajikistan')), + ('TK', _('Tokelau')), + ('TM', _('Turkmenistan')), + ('TN', _('Tunisia')), + ('TO', _('Tonga')), + ('TP', _('East Timor')), + ('TR', _('Turkey')), + ('TT', _('Trinidad & Tobago')), + ('TV', _('Tuvalu')), + ('TW', _('Taiwan, Province of China')), + ('TZ', _('Tanzania, United Republic of')), + ('UA', _('Ukraine')), + ('UG', _('Uganda')), + ('UM', _('United States Minor Outlying Islands')), + ('US', _('United States of America')), + ('UY', _('Uruguay')), + ('UZ', _('Uzbekistan')), + ('VA', _('Vatican City State (Holy See)')), + ('VC', _('St. Vincent & the Grenadines')), + ('VE', _('Venezuela')), + ('VG', _('British Virgin Islands')), + ('VI', _('United States Virgin Islands')), + ('VN', _('Viet Nam')), + ('VU', _('Vanuatu')), + ('WF', _('Wallis & Futuna Islands')), + ('WS', _('Samoa')), + ('YE', _('Yemen')), + ('YT', _('Mayotte')), + ('YU', _('Yugoslavia')), + ('ZA', _('South Africa')), + ('ZM', _('Zambia')), + ('ZR', _('Zaire')), + ('ZW', _('Zimbabwe')), +) + + +__all__ = ('celery_app',) diff --git a/uncloud/admin.py b/uncloud/admin.py new file mode 100644 index 0000000..a89a574 --- /dev/null +++ b/uncloud/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin + +from .models import * + +for m in [ UncloudProvider, UncloudNetwork, UncloudTask ]: + admin.site.register(m) diff --git a/uncloud/asgi.py b/uncloud/asgi.py new file mode 100644 index 0000000..2b5a7a3 --- /dev/null +++ b/uncloud/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for uncloud project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + +application = get_asgi_application() diff --git a/uncloud/celery.py b/uncloud/celery.py new file mode 100644 index 0000000..3408634 --- /dev/null +++ b/uncloud/celery.py @@ -0,0 +1,17 @@ +import os + +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + +app = Celery('uncloud') + +# Using a string here means the worker doesn't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() diff --git a/uncloud/management/commands/db-add-defaults.py b/uncloud/management/commands/db-add-defaults.py new file mode 100644 index 0000000..605c8f5 --- /dev/null +++ b/uncloud/management/commands/db-add-defaults.py @@ -0,0 +1,43 @@ +import random +import string + +from django.core.management.base import BaseCommand +from django.core.exceptions import ObjectDoesNotExist +from django.contrib.auth import get_user_model +from django.conf import settings + +from uncloud_pay.models import BillingAddress, RecurringPeriod, Product +from uncloud.models import UncloudProvider, UncloudNetwork + + +class Command(BaseCommand): + help = 'Add standard uncloud values' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + # Order matters, objects can be dependent on each other + + admin_username="uncloud-admin" + pw_length = 32 + + # Only set password if the user did not exist before + try: + admin_user = get_user_model().objects.get(username=settings.UNCLOUD_ADMIN_NAME) + except ObjectDoesNotExist: + random_password = ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(pw_length)) + + admin_user = get_user_model().objects.create_user(username=settings.UNCLOUD_ADMIN_NAME, password=random_password) + admin_user.is_superuser=True + admin_user.is_staff=True + admin_user.save() + + print(f"Created admin user '{admin_username}' with password '{random_password}'") + + BillingAddress.populate_db_defaults() + RecurringPeriod.populate_db_defaults() + Product.populate_db_defaults() + + UncloudNetwork.populate_db_defaults() + UncloudProvider.populate_db_defaults() diff --git a/uncloud/management/commands/uncloud.py b/uncloud/management/commands/uncloud.py new file mode 100644 index 0000000..bd47c6b --- /dev/null +++ b/uncloud/management/commands/uncloud.py @@ -0,0 +1,28 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand + +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster + +import logging +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'General uncloud commands' + + def add_arguments(self, parser): + parser.add_argument('--bootstrap', action='store_true', help='Bootstrap a typical uncloud installation') + + def handle(self, *args, **options): + + if options['bootstrap']: + self.bootstrap() + + def bootstrap(self): + default_cluster = VMCluster.objects.get_or_create(name="default") +# local_host = diff --git a/uncloud/migrations/0001_initial.py b/uncloud/migrations/0001_initial.py new file mode 100644 index 0000000..10d1144 --- /dev/null +++ b/uncloud/migrations/0001_initial.py @@ -0,0 +1,46 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import uncloud.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='UncloudNetwork', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('network_address', models.GenericIPAddressField(unique=True)), + ('network_mask', models.IntegerField(validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(128)])), + ('description', models.CharField(max_length=256)), + ], + ), + migrations.CreateModel( + name='UncloudProvider', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('full_name', models.CharField(max_length=256)), + ('organization', models.CharField(blank=True, max_length=256, null=True)), + ('street', models.CharField(max_length=256)), + ('city', models.CharField(max_length=256)), + ('postal_code', models.CharField(max_length=64)), + ('country', uncloud.models.CountryField(blank=True, choices=[('AD', 'Andorra'), ('AE', 'United Arab Emirates'), ('AF', 'Afghanistan'), ('AG', 'Antigua & Barbuda'), ('AI', 'Anguilla'), ('AL', 'Albania'), ('AM', 'Armenia'), ('AN', 'Netherlands Antilles'), ('AO', 'Angola'), ('AQ', 'Antarctica'), ('AR', 'Argentina'), ('AS', 'American Samoa'), ('AT', 'Austria'), ('AU', 'Australia'), ('AW', 'Aruba'), ('AZ', 'Azerbaijan'), ('BA', 'Bosnia and Herzegovina'), ('BB', 'Barbados'), ('BD', 'Bangladesh'), ('BE', 'Belgium'), ('BF', 'Burkina Faso'), ('BG', 'Bulgaria'), ('BH', 'Bahrain'), ('BI', 'Burundi'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BN', 'Brunei Darussalam'), ('BO', 'Bolivia'), ('BR', 'Brazil'), ('BS', 'Bahama'), ('BT', 'Bhutan'), ('BV', 'Bouvet Island'), ('BW', 'Botswana'), ('BY', 'Belarus'), ('BZ', 'Belize'), ('CA', 'Canada'), ('CC', 'Cocos (Keeling) Islands'), ('CF', 'Central African Republic'), ('CG', 'Congo'), ('CH', 'Switzerland'), ('CI', 'Ivory Coast'), ('CK', 'Cook Iislands'), ('CL', 'Chile'), ('CM', 'Cameroon'), ('CN', 'China'), ('CO', 'Colombia'), ('CR', 'Costa Rica'), ('CU', 'Cuba'), ('CV', 'Cape Verde'), ('CX', 'Christmas Island'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DE', 'Germany'), ('DJ', 'Djibouti'), ('DK', 'Denmark'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('DZ', 'Algeria'), ('EC', 'Ecuador'), ('EE', 'Estonia'), ('EG', 'Egypt'), ('EH', 'Western Sahara'), ('ER', 'Eritrea'), ('ES', 'Spain'), ('ET', 'Ethiopia'), ('FI', 'Finland'), ('FJ', 'Fiji'), ('FK', 'Falkland Islands (Malvinas)'), ('FM', 'Micronesia'), ('FO', 'Faroe Islands'), ('FR', 'France'), ('FX', 'France, Metropolitan'), ('GA', 'Gabon'), ('GB', 'United Kingdom (Great Britain)'), ('GD', 'Grenada'), ('GE', 'Georgia'), ('GF', 'French Guiana'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GL', 'Greenland'), ('GM', 'Gambia'), ('GN', 'Guinea'), ('GP', 'Guadeloupe'), ('GQ', 'Equatorial Guinea'), ('GR', 'Greece'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GT', 'Guatemala'), ('GU', 'Guam'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HK', 'Hong Kong'), ('HM', 'Heard & McDonald Islands'), ('HN', 'Honduras'), ('HR', 'Croatia'), ('HT', 'Haiti'), ('HU', 'Hungary'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('IL', 'Israel'), ('IN', 'India'), ('IO', 'British Indian Ocean Territory'), ('IQ', 'Iraq'), ('IR', 'Islamic Republic of Iran'), ('IS', 'Iceland'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JO', 'Jordan'), ('JP', 'Japan'), ('KE', 'Kenya'), ('KG', 'Kyrgyzstan'), ('KH', 'Cambodia'), ('KI', 'Kiribati'), ('KM', 'Comoros'), ('KN', 'St. Kitts and Nevis'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('KW', 'Kuwait'), ('KY', 'Cayman Islands'), ('KZ', 'Kazakhstan'), ('LA', "Lao People's Democratic Republic"), ('LB', 'Lebanon'), ('LC', 'Saint Lucia'), ('LI', 'Liechtenstein'), ('LK', 'Sri Lanka'), ('LR', 'Liberia'), ('LS', 'Lesotho'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('LV', 'Latvia'), ('LY', 'Libyan Arab Jamahiriya'), ('MA', 'Morocco'), ('MC', 'Monaco'), ('MD', 'Moldova, Republic of'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('ML', 'Mali'), ('MN', 'Mongolia'), ('MM', 'Myanmar'), ('MO', 'Macau'), ('MP', 'Northern Mariana Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MS', 'Monserrat'), ('MT', 'Malta'), ('MU', 'Mauritius'), ('MV', 'Maldives'), ('MW', 'Malawi'), ('MX', 'Mexico'), ('MY', 'Malaysia'), ('MZ', 'Mozambique'), ('NA', 'Namibia'), ('NC', 'New Caledonia'), ('NE', 'Niger'), ('NF', 'Norfolk Island'), ('NG', 'Nigeria'), ('NI', 'Nicaragua'), ('NL', 'Netherlands'), ('NO', 'Norway'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('NU', 'Niue'), ('NZ', 'New Zealand'), ('OM', 'Oman'), ('PA', 'Panama'), ('PE', 'Peru'), ('PF', 'French Polynesia'), ('PG', 'Papua New Guinea'), ('PH', 'Philippines'), ('PK', 'Pakistan'), ('PL', 'Poland'), ('PM', 'St. Pierre & Miquelon'), ('PN', 'Pitcairn'), ('PR', 'Puerto Rico'), ('PT', 'Portugal'), ('PW', 'Palau'), ('PY', 'Paraguay'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SA', 'Saudi Arabia'), ('SB', 'Solomon Islands'), ('SC', 'Seychelles'), ('SD', 'Sudan'), ('SE', 'Sweden'), ('SG', 'Singapore'), ('SH', 'St. Helena'), ('SI', 'Slovenia'), ('SJ', 'Svalbard & Jan Mayen Islands'), ('SK', 'Slovakia'), ('SL', 'Sierra Leone'), ('SM', 'San Marino'), ('SN', 'Senegal'), ('SO', 'Somalia'), ('SR', 'Suriname'), ('ST', 'Sao Tome & Principe'), ('SV', 'El Salvador'), ('SY', 'Syrian Arab Republic'), ('SZ', 'Swaziland'), ('TC', 'Turks & Caicos Islands'), ('TD', 'Chad'), ('TF', 'French Southern Territories'), ('TG', 'Togo'), ('TH', 'Thailand'), ('TJ', 'Tajikistan'), ('TK', 'Tokelau'), ('TM', 'Turkmenistan'), ('TN', 'Tunisia'), ('TO', 'Tonga'), ('TP', 'East Timor'), ('TR', 'Turkey'), ('TT', 'Trinidad & Tobago'), ('TV', 'Tuvalu'), ('TW', 'Taiwan, Province of China'), ('TZ', 'Tanzania, United Republic of'), ('UA', 'Ukraine'), ('UG', 'Uganda'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VA', 'Vatican City State (Holy See)'), ('VC', 'St. Vincent & the Grenadines'), ('VE', 'Venezuela'), ('VG', 'British Virgin Islands'), ('VI', 'United States Virgin Islands'), ('VN', 'Viet Nam'), ('VU', 'Vanuatu'), ('WF', 'Wallis & Futuna Islands'), ('WS', 'Samoa'), ('YE', 'Yemen'), ('YT', 'Mayotte'), ('YU', 'Yugoslavia'), ('ZA', 'South Africa'), ('ZM', 'Zambia'), ('ZR', 'Zaire'), ('ZW', 'Zimbabwe')], default='CH', max_length=2)), + ('starting_date', models.DateField()), + ('ending_date', models.DateField(blank=True, null=True)), + ('billing_network', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='uncloudproviderbill', to='uncloud.uncloudnetwork')), + ('coupon_network', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='uncloudprovidercoupon', to='uncloud.uncloudnetwork')), + ('referral_network', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='uncloudproviderreferral', to='uncloud.uncloudnetwork')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/uncloud/migrations/0002_uncloudtasks.py b/uncloud/migrations/0002_uncloudtasks.py new file mode 100644 index 0000000..9c69606 --- /dev/null +++ b/uncloud/migrations/0002_uncloudtasks.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2020-12-20 17:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='UncloudTasks', + fields=[ + ('task_id', models.UUIDField(primary_key=True, serialize=False)), + ], + ), + ] diff --git a/uncloud/migrations/0003_auto_20201220_1728.py b/uncloud/migrations/0003_auto_20201220_1728.py new file mode 100644 index 0000000..2ec0eec --- /dev/null +++ b/uncloud/migrations/0003_auto_20201220_1728.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1 on 2020-12-20 17:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud', '0002_uncloudtasks'), + ] + + operations = [ + migrations.RenameModel( + old_name='UncloudTasks', + new_name='UncloudTask', + ), + ] diff --git a/uncloud/migrations/__init__.py b/uncloud/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/models.py b/uncloud/models.py new file mode 100644 index 0000000..5545303 --- /dev/null +++ b/uncloud/models.py @@ -0,0 +1,172 @@ +from django.db import models +from django.db.models import JSONField, Q +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from django.core.validators import MinValueValidator, MaxValueValidator +from django.core.exceptions import FieldError + +from uncloud import COUNTRIES + +class UncloudModel(models.Model): + """ + This class extends the standard model with an + extra_data field that can be used to include public, + but internal information. + + For instance if you migrate from an existing virtualisation + framework to uncloud. + + The extra_data attribute should be considered a hack and whenever + data is necessary for running uncloud, it should **not** be stored + in there. + + """ + + extra_data = JSONField(editable=False, blank=True, null=True) + + class Meta: + abstract = True + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class UncloudStatus(models.TextChoices): + PENDING = 'PENDING', _('Pending') + AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment') + BEING_CREATED = 'BEING_CREATED', _('Being created') + SCHEDULED = 'SCHEDULED', _('Scheduled') # resource selected, waiting for dispatching + ACTIVE = 'ACTIVE', _('Active') + MODIFYING = 'MODIFYING', _('Modifying') # Resource is being changed + DELETED = 'DELETED', _('Deleted') # Resource has been deleted + DISABLED = 'DISABLED', _('Disabled') # Is usable, but cannot be used for new things + UNUSABLE = 'UNUSABLE', _('Unusable'), # Has some kind of error + + + +### +# General address handling +class CountryField(models.CharField): + def __init__(self, *args, **kwargs): + kwargs.setdefault('choices', COUNTRIES) + kwargs.setdefault('default', 'CH') + kwargs.setdefault('max_length', 2) + + super().__init__(*args, **kwargs) + + def get_internal_type(self): + return "CharField" + + +class UncloudAddress(models.Model): + full_name = models.CharField(max_length=256) + organization = models.CharField(max_length=256, blank=True, null=True) + street = models.CharField(max_length=256) + city = models.CharField(max_length=256) + postal_code = models.CharField(max_length=64) + country = CountryField(blank=True) + + class Meta: + abstract = True + + +### +# UncloudNetworks are used as identifiers - such they are a base of uncloud + +class UncloudNetwork(models.Model): + """ + Storing IP networks + """ + + network_address = models.GenericIPAddressField(null=False, unique=True) + network_mask = models.IntegerField(null=False, + validators=[MinValueValidator(0), + MaxValueValidator(128)] + ) + + description = models.CharField(max_length=256) + + @classmethod + def populate_db_defaults(cls): + for net, desc in [ + ( "2a0a:e5c0:11::", "uncloud Billing" ), + ( "2a0a:e5c0:11:1::", "uncloud Referral" ), + ( "2a0a:e5c0:11:2::", "uncloud Coupon" ) + ]: + obj, created = cls.objects.get_or_create(network_address=net, + defaults= { + 'network_mask': 64, + 'description': desc + } + ) + + + def save(self, *args, **kwargs): + if not ':' in self.network_address and self.network_mask > 32: + raise FieldError("Mask cannot exceed 32 for IPv4") + + super().save(*args, **kwargs) + + + def __str__(self): + return f"{self.network_address}/{self.network_mask} {self.description}" + +### +# Who is running / providing this instance of uncloud? + +class UncloudProvider(UncloudAddress): + """ + A class resembling who is running this uncloud instance. + This might change over time so we allow starting/ending dates + + This also defines the taxation rules. + + starting/ending date define from when to when this is valid. This way + we can model address changes and have it correct in the bills. + """ + + # Meta: + # FIXMe: only allow non overlapping time frames -- how to define this as a constraint? + starting_date = models.DateField() + ending_date = models.DateField(blank=True, null=True) + + billing_network = models.ForeignKey(UncloudNetwork, related_name="uncloudproviderbill", on_delete=models.CASCADE) + referral_network = models.ForeignKey(UncloudNetwork, related_name="uncloudproviderreferral", on_delete=models.CASCADE) + coupon_network = models.ForeignKey(UncloudNetwork, related_name="uncloudprovidercoupon", on_delete=models.CASCADE) + + + @classmethod + def get_provider(cls, when=None): + """ + Find active provide at a certain time - if there was any + """ + + if not when: + when = timezone.now() + + + return cls.objects.get(Q(starting_date__gte=when, ending_date__lte=when) | + Q(starting_date__gte=when, ending_date__isnull=True)) + + + @classmethod + def populate_db_defaults(cls): + obj, created = cls.objects.get_or_create(full_name="ungleich glarus ag", + street="Bahnhofstrasse 1", + postal_code="8783", + city="Linthal", + country="CH", + starting_date=timezone.now(), + billing_network=UncloudNetwork.objects.get(description="uncloud Billing"), + referral_network=UncloudNetwork.objects.get(description="uncloud Referral"), + coupon_network=UncloudNetwork.objects.get(description="uncloud Coupon") + ) + + + def __str__(self): + return f"{self.full_name} {self.country}" + + +class UncloudTask(models.Model): + """ + Class to store dispatched tasks to be handled + """ + + task_id = models.UUIDField(primary_key=True) diff --git a/uncloud/settings.py b/uncloud/settings.py new file mode 100644 index 0000000..ae734dc --- /dev/null +++ b/uncloud/settings.py @@ -0,0 +1,240 @@ +""" +Django settings for uncloud project. + +Generated by 'django-admin startproject' using Django 3.0.3. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os +import re +import ldap + +from django.core.management.utils import get_random_secret_key +from django_auth_ldap.config import LDAPSearch, LDAPSearchUnion + + +LOGGING = {} + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django_extensions', + 'rest_framework', + 'uncloud', + 'uncloud_pay', + 'uncloud_auth', + 'uncloud_net', + 'uncloud_storage', + 'uncloud_vm', + 'uncloud_service', + 'opennebula' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'uncloud.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'uncloud.wsgi.application' + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +################################################################################ +# AUTH/LDAP + +AUTH_LDAP_SERVER_URI = "" +AUTH_LDAP_BIND_DN = "" +AUTH_LDAP_BIND_PASSWORD = "" +AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=example,dc=com", + ldap.SCOPE_SUBTREE, + "(uid=%(user)s)") + +AUTH_LDAP_USER_ATTR_MAP = { + "first_name": "givenName", + "last_name": "sn", + "email": "mail" +} + +################################################################################ +# AUTH/Django +AUTHENTICATION_BACKENDS = [ + "django_auth_ldap.backend.LDAPBackend", + "django.contrib.auth.backends.ModelBackend" +] + +AUTH_USER_MODEL = 'uncloud_auth.User' + + +################################################################################ +# AUTH/REST +REST_FRAMEWORK = { + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.BasicAuthentication', + 'rest_framework.authentication.SessionAuthentication', + ] +} + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ +STATIC_URL = '/static/' +STATICFILES_DIRS = [ os.path.join(BASE_DIR, "static") ] + +# XML-RPC interface of opennebula +OPENNEBULA_URL = 'https://opennebula.example.com:2634/RPC2' + +# user:pass for accessing opennebula +OPENNEBULA_USER_PASS = 'user:password' + +# Stripe (Credit Card payments) +STRIPE_KEY="" +STRIPE_PUBLIC_KEY="" + +# The django secret key +SECRET_KEY=get_random_secret_key() + +ALLOWED_HOSTS = [] + +# required for hardcopy / pdf rendering: https://github.com/loftylabs/django-hardcopy +CHROME_PATH = '/usr/bin/chromium-browser' + +# Username that is created by default and owns the configuration objects +UNCLOUD_ADMIN_NAME = "uncloud-admin" + +LOGIN_REDIRECT_URL = '/' +LOGOUT_REDIRECT_URL = '/' + +# replace these in local_settings.py +AUTH_LDAP_SERVER_URI = "ldaps://ldap1.example.com,ldaps://ldap2.example.com" +AUTH_LDAP_BIND_DN="uid=django,ou=system,dc=example,dc=com" +AUTH_LDAP_BIND_PASSWORD="a very secure ldap password" +AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=example,dc=com", + ldap.SCOPE_SUBTREE, + "(uid=%(user)s)") + +# where to create customers +LDAP_CUSTOMER_DN="ou=customer,dc=example,dc=com" + +# def route_task(name, args, kwargs, options, task=None, **kw): +# print(f"{name} - {args} - {kwargs}") +# # if name == 'myapp.tasks.compress_video': +# return {'queue': 'vpn1' } +# # 'exchange_type': 'topic', +# # 'routing_key': 'video.compress'} + + +# CELERY_TASK_ROUTES = (route_task,) + +# CELERY_TASK_ROUTES = { +# '*': { +# 'queue': 'vpn1' +# } +# } + + +CELERY_BROKER_URL = 'redis://:uncloud.example.com:6379/0' +CELERY_RESULT_BACKEND = 'redis://:uncloud.example.com:6379/0' + +CELERY_TASK_ROUTES = { + re.compile(r'.*.tasks.cdist.*'): { 'queue': 'cdist' } # cdist tasks go into cdist queue +} + +CELERY_BEAT_SCHEDULE = { + 'cleanup_tasks': { + 'task': 'uncloud.tasks.cleanup_tasks', + 'schedule': 10 + } +} + +# CELERY_TASK_CREATE_MISSING_QUEUES = False + +# Overwrite settings with local settings, if existing +try: + from uncloud.local_settings import * +except (ModuleNotFoundError, ImportError): + pass diff --git a/uncloud/tasks.py b/uncloud/tasks.py new file mode 100644 index 0000000..5a13ec5 --- /dev/null +++ b/uncloud/tasks.py @@ -0,0 +1,19 @@ +from celery import shared_task +from celery.result import AsyncResult + +from .models import UncloudTask + +@shared_task(bind=True) +def cleanup_tasks(self): + print(f"Cleanup time from {self}: {self.request.id}") + for task in UncloudTask.objects.all(): + print(f"Pruning {task}...") + + if str(task.task_id) == str(self.request.id): + print("Skipping myself") + continue + + res = AsyncResult(id=str(task.task_id)) + if res.ready(): + print(res.get()) + task.delete() diff --git a/uncloud/templates/uncloud/base.html b/uncloud/templates/uncloud/base.html new file mode 100644 index 0000000..034fa7c --- /dev/null +++ b/uncloud/templates/uncloud/base.html @@ -0,0 +1,14 @@ + + + + + + + + {% block title %}Welcome to uncloud{% endblock %} + {% block header %}{% endblock %} + + + {% block body %}{% endblock %} + + diff --git a/uncloud/templates/uncloud/index.html b/uncloud/templates/uncloud/index.html new file mode 100644 index 0000000..b40c3b4 --- /dev/null +++ b/uncloud/templates/uncloud/index.html @@ -0,0 +1,15 @@ +{% extends 'uncloud/base.html' %} +{% block title %}{% endblock %} + +{% block body %} +
+

Welcome to uncloud

+ Welcome to uncloud, checkout the following locations: + + + +
+{% endblock %} diff --git a/uncloud/urls.py b/uncloud/urls.py new file mode 100644 index 0000000..169be7f --- /dev/null +++ b/uncloud/urls.py @@ -0,0 +1,94 @@ +"""uncloud URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +""" + +from django.contrib import admin +from django.urls import path, include +from django.conf import settings +from django.conf.urls.static import static + +from rest_framework import routers +from rest_framework.schemas import get_schema_view + +#from opennebula import views as oneviews +from uncloud import views as uncloudviews +from uncloud_auth import views as authviews +from uncloud_net import views as netviews +from uncloud_pay import views as payviews +from uncloud_vm import views as vmviews +from uncloud_service import views as serviceviews + +router = routers.DefaultRouter() + +# Beta endpoints +router.register(r'beta/vm', vmviews.NicoVMProductViewSet, basename='nicovmproduct') + +# VM +router.register(r'v1/vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct') +router.register(r'v1/vm/diskimage', vmviews.VMDiskImageProductViewSet, basename='vmdiskimageproduct') +router.register(r'v1/vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct') +router.register(r'v1/vm/vm', vmviews.VMProductViewSet, basename='vmproduct') + + +# creates VM from os image +#router.register(r'vm/ipv6onlyvm', vmviews.VMProductViewSet, basename='vmproduct') +# ... AND adds IPv4 mapping +#router.register(r'vm/dualstackvm', vmviews.VMProductViewSet, basename='vmproduct') + +# Services +router.register(r'v1/service/matrix', serviceviews.MatrixServiceProductViewSet, basename='matrixserviceproduct') +router.register(r'v1/service/generic', serviceviews.GenericServiceProductViewSet, basename='genericserviceproduct') + + + +# Pay +router.register(r'v1/my/address', payviews.BillingAddressViewSet, basename='billingaddress') +router.register(r'v1/my/bill', payviews.BillViewSet, basename='bill') +router.register(r'v1/my/order', payviews.OrderViewSet, basename='order') +router.register(r'v1/my/payment', payviews.PaymentViewSet, basename='payment') +router.register(r'v1/my/payment-method', payviews.PaymentMethodViewSet, basename='payment-method') + +# admin/staff urls +router.register(r'v1/admin/bill', payviews.AdminBillViewSet, basename='admin/bill') +router.register(r'v1/admin/payment', payviews.AdminPaymentViewSet, basename='admin/payment') +router.register(r'v1/admin/order', payviews.AdminOrderViewSet, basename='admin/order') +router.register(r'v1/admin/vmhost', vmviews.VMHostViewSet) +router.register(r'v1/admin/vmcluster', vmviews.VMClusterViewSet) +#router.register(r'v1/admin/vpnpool', netviews.VPNPoolViewSet) +#router.register(r'v1/admin/opennebula', oneviews.VMViewSet, basename='opennebula') + +# User/Account +router.register(r'v1/my/user', authviews.UserViewSet, basename='user') +router.register(r'v1/admin/user', authviews.AdminUserViewSet, basename='useradmin') +router.register(r'v1/user/register', authviews.AccountManagementViewSet, basename='user/register') + + +################################################################################ +# v2 + +# Net +router.register(r'v2/net/wireguardvpn', netviews.WireGuardVPNViewSet, basename='wireguardvpnnetwork') +router.register(r'v2/net/wireguardvpnsizes', netviews.WireGuardVPNSizes, basename='wireguardvpnnetworksizes') + + + +urlpatterns = [ + path(r'api/', include(router.urls)), + + path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), # for login to REST API + path('openapi', get_schema_view( + title="uncloud", + description="uncloud API", + version="1.0.0" + ), name='openapi-schema'), + + # web/ = stuff to view in the browser +# path('web/vpn/create/', netviews.WireGuardVPNCreateView.as_view(), name="vpncreate"), + path('login/', authviews.LoginView.as_view(), name="login"), + path('logout/', authviews.LogoutView.as_view(), name="logout"), + path('admin/', admin.site.urls), + path('cc/reg/', payviews.RegisterCard.as_view(), name="cc_register"), + path('', uncloudviews.UncloudIndex.as_view(), name="uncloudindex"), +] diff --git a/uncloud/views.py b/uncloud/views.py new file mode 100644 index 0000000..198abd0 --- /dev/null +++ b/uncloud/views.py @@ -0,0 +1,4 @@ +from django.views.generic.base import TemplateView + +class UncloudIndex(TemplateView): + template_name = "uncloud/index.html" diff --git a/uncloud/wsgi.py b/uncloud/wsgi.py new file mode 100644 index 0000000..c4a07b8 --- /dev/null +++ b/uncloud/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for uncloud project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + +application = get_wsgi_application() diff --git a/uncloud_auth/__init__.py b/uncloud_auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_auth/admin.py b/uncloud_auth/admin.py new file mode 100644 index 0000000..f91be8f --- /dev/null +++ b/uncloud_auth/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin +from .models import User + +admin.site.register(User, UserAdmin) diff --git a/uncloud_auth/apps.py b/uncloud_auth/apps.py new file mode 100644 index 0000000..c16bd7a --- /dev/null +++ b/uncloud_auth/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class AuthConfig(AppConfig): + name = 'uncloud_auth' diff --git a/uncloud_auth/management/commands/make-admin.py b/uncloud_auth/management/commands/make-admin.py new file mode 100644 index 0000000..9157439 --- /dev/null +++ b/uncloud_auth/management/commands/make-admin.py @@ -0,0 +1,21 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model +import sys + +class Command(BaseCommand): + help = 'Give Admin rights to existing user' + + def add_arguments(self, parser): + parser.add_argument('username', type=str) + parser.add_argument('--superuser', action='store_true') + + def handle(self, *args, **options): + user = get_user_model().objects.get(username=options['username']) + user.is_staff = True + + if options['superuser']: + user.is_superuser = True + + user.save() + + print(f"{user.username} is now admin (superuser={user.is_superuser})") diff --git a/uncloud_auth/migrations/0001_initial.py b/uncloud_auth/migrations/0001_initial.py new file mode 100644 index 0000000..b263dc6 --- /dev/null +++ b/uncloud_auth/migrations/0001_initial.py @@ -0,0 +1,46 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +import django.contrib.auth.models +import django.contrib.auth.validators +import django.core.validators +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('maximum_credit', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/uncloud_auth/migrations/__init__.py b/uncloud_auth/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_auth/models.py b/uncloud_auth/models.py new file mode 100644 index 0000000..90463e1 --- /dev/null +++ b/uncloud_auth/models.py @@ -0,0 +1,17 @@ +from django.contrib.auth.models import AbstractUser +from django.db import models +from django.core.validators import MinValueValidator + +from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS + +class User(AbstractUser): + """ + We use the standard user and add a maximum credit that is allowed + to be accumulated. After that we need to have warnings, cancellation, etc. + """ + + maximum_credit = models.DecimalField( + default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) diff --git a/uncloud_auth/serializers.py b/uncloud_auth/serializers.py new file mode 100644 index 0000000..c3f6694 --- /dev/null +++ b/uncloud_auth/serializers.py @@ -0,0 +1,72 @@ +from django.contrib.auth import get_user_model +from django.db import transaction +from ldap3.core.exceptions import LDAPEntryAlreadyExistsResult +from rest_framework import serializers + +from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS +from uncloud_pay.models import BillingAddress + +from .ungleich_ldap import LdapManager + + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = get_user_model() + read_only_fields = [ 'username', 'balance', 'maximum_credit' ] + fields = read_only_fields + [ 'email' ] # , 'primary_billing_address' ] + + def validate(self, data): + """ + Ensure that the primary billing address belongs to the user + """ + # The following is raising exceptions probably, it is WIP somewhere + # if 'primary_billing_address' in data: + # if not data['primary_billing_address'].owner == self.instance: + # raise serializers.ValidationError('Invalid data') + + return data + + def update(self, instance, validated_data): + ldap_manager = LdapManager() + return_val, _ = ldap_manager.change_user_details( + instance.username, {'mail': validated_data.get('email')} + ) + if not return_val: + raise serializers.ValidationError('Couldn\'t update email') + instance.email = validated_data.get('email') + instance.save() + return instance + + +class UserRegistrationSerializer(serializers.ModelSerializer): + class Meta: + model = get_user_model() + fields = ['username', 'first_name', 'last_name', 'email', 'password'] + extra_kwargs = { + 'password': {'style': {'input_type': 'password'}}, + 'first_name': {'allow_blank': False, 'required': True}, + 'last_name': {'allow_blank': False, 'required': True}, + 'email': {'allow_blank': False, 'required': True}, + } + + def create(self, validated_data): + ldap_manager = LdapManager() + try: + data = { + 'user': validated_data['username'], + 'password': validated_data['password'], + 'email': validated_data['email'], + 'firstname': validated_data['first_name'], + 'lastname': validated_data['last_name'], + } + ldap_manager.create_user(**data) + except LDAPEntryAlreadyExistsResult: + raise serializers.ValidationError( + {'username': ['A user with that username already exists.']} + ) + else: + return get_user_model().objects.create_user(**validated_data) + + +class ImportUserSerializer(serializers.Serializer): + username = serializers.CharField() diff --git a/uncloud_auth/templates/uncloud_auth/login.html b/uncloud_auth/templates/uncloud_auth/login.html new file mode 100644 index 0000000..04f9a15 --- /dev/null +++ b/uncloud_auth/templates/uncloud_auth/login.html @@ -0,0 +1,13 @@ +{% extends 'uncloud/base.html' %} + +{% block body %} +
+ +
+ {% csrf_token %} + {{ form }} + +
+
+ +{% endblock %} diff --git a/uncloud_auth/uldap.py b/uncloud_auth/uldap.py new file mode 100644 index 0000000..aa90c77 --- /dev/null +++ b/uncloud_auth/uldap.py @@ -0,0 +1,42 @@ +import ldap +# from django.conf import settings + +AUTH_LDAP_SERVER_URI = "ldaps://ldap1.ungleich.ch,ldaps://ldap2.ungleich.ch" +AUTH_LDAP_BIND_DN="uid=django-create,ou=system,dc=ungleich,dc=ch" +AUTH_LDAP_BIND_PASSWORD="kS#e+v\zjKn]L!,RIu2}V+DUS" +# AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=ungleich,dc=ch", +# ldap.SCOPE_SUBTREE, +# "(uid=%(user)s)") + + + +ldap_object = ldap.initialize(AUTH_LDAP_SERVER_URI) +cancelid = ldap_object.bind(AUTH_LDAP_BIND_DN, AUTH_LDAP_BIND_PASSWORD) + +res = ldap_object.search_s("dc=ungleich,dc=ch", ldap.SCOPE_SUBTREE, "(uid=nico)") +print(res) + +# class LDAP(object): +# """ +# Managing users in LDAP + +# Requires the following settings? + +# LDAP_USER_DN: where to create users in the tree + +# LDAP_ADMIN_DN: which DN to use for managing users +# LDAP_ADMIN_PASSWORD: which password to used + +# This module will reuse information from djagno_auth_ldap, including: + +# AUTH_LDAP_SERVER_URI + +# """ +# def __init__(self): +# pass + +# def create_user(self): +# pass + +# def change_password(self): +# pass diff --git a/uncloud_auth/ungleich_ldap.py b/uncloud_auth/ungleich_ldap.py new file mode 100644 index 0000000..f22b423 --- /dev/null +++ b/uncloud_auth/ungleich_ldap.py @@ -0,0 +1,284 @@ +import base64 +import hashlib +import logging +import random + +import ldap3 +from django.conf import settings + +logger = logging.getLogger(__name__) + + +class LdapManager: + __instance = None + def __new__(cls): + if LdapManager.__instance is None: + LdapManager.__instance = object.__new__(cls) + return LdapManager.__instance + + def __init__(self): + """ + Initialize the LDAP subsystem. + """ + self.rng = random.SystemRandom() + self.server = ldap3.Server(settings.AUTH_LDAP_SERVER) + + + def get_admin_conn(self): + """ + Return a bound :class:`ldap3.Connection` instance which has write + permissions on the dn in which the user accounts reside. + """ + conn = self.get_conn(user=settings.LDAP_ADMIN_DN, + password=settings.LDAP_ADMIN_PASSWORD, + raise_exceptions=True) + conn.bind() + return conn + + + def get_conn(self, **kwargs): + """ + Return an unbound :class:`ldap3.Connection` which talks to the configured + LDAP server. + + The *kwargs* are passed to the constructor of :class:`ldap3.Connection` and + can be used to set *user*, *password* and other useful arguments. + """ + return ldap3.Connection(self.server, **kwargs) + + + def _ssha_password(self, password): + """ + Apply the SSHA password hashing scheme to the given *password*. + *password* must be a :class:`bytes` object, containing the utf-8 + encoded password. + + Return a :class:`bytes` object containing ``ascii``-compatible data + which can be used as LDAP value, e.g. after armoring it once more using + base64 or decoding it to unicode from ``ascii``. + """ + SALT_BYTES = 15 + + sha1 = hashlib.sha1() + salt = self.rng.getrandbits(SALT_BYTES * 8).to_bytes(SALT_BYTES, + "little") + sha1.update(password) + sha1.update(salt) + + digest = sha1.digest() + passwd = b"{SSHA}" + base64.b64encode(digest + salt) + return passwd + + + def create_user(self, user, password, firstname, lastname, email): + conn = self.get_admin_conn() + uidNumber = self._get_max_uid() + 1 + logger.debug("uidNumber={uidNumber}".format(uidNumber=uidNumber)) + user_exists = True + while user_exists: + user_exists, _ = self.check_user_exists( + "", + '(&(objectClass=inetOrgPerson)(objectClass=posixAccount)' + '(objectClass=top)(uidNumber={uidNumber}))'.format( + uidNumber=uidNumber + ) + ) + if user_exists: + logger.debug( + "{uid} exists. Trying next.".format(uid=uidNumber) + ) + uidNumber += 1 + logger.debug("{uid} does not exist. Using it".format(uid=uidNumber)) + self._set_max_uid(uidNumber) + try: + uid = user # user.encode("utf-8") + conn.add("uid={uid},{customer_dn}".format( + uid=uid, customer_dn=settings.LDAP_CUSTOMER_DN + ), + ["inetOrgPerson", "posixAccount", "ldapPublickey"], + { + "uid": [uid], + "sn": [lastname.encode("utf-8")], + "givenName": [firstname.encode("utf-8")], + "cn": [uid], + "displayName": ["{} {}".format(firstname, lastname).encode("utf-8")], + "uidNumber": [str(uidNumber)], + "gidNumber": [str(settings.LDAP_CUSTOMER_GROUP_ID)], + "loginShell": ["/bin/bash"], + "homeDirectory": ["/home/{}".format(user).encode("utf-8")], + "mail": email.encode("utf-8"), + "userPassword": [self._ssha_password( + password.encode("utf-8") + )] + } + ) + logger.debug('Created user %s %s' % (user.encode('utf-8'), + uidNumber)) + except Exception as ex: + logger.debug('Could not create user %s' % user.encode('utf-8')) + logger.error("Exception: " + str(ex)) + raise + finally: + conn.unbind() + + + def change_password(self, uid, new_password): + """ + Changes the password of the user identified by user_dn + + :param uid: str The uid that identifies the user + :param new_password: str The new password string + :return: True if password was changed successfully False otherwise + """ + conn = self.get_admin_conn() + + # Make sure the user exists first to change his/her details + user_exists, entries = self.check_user_exists( + uid=uid, + search_base=settings.ENTIRE_SEARCH_BASE + ) + return_val = False + if user_exists: + try: + return_val = conn.modify( + entries[0].entry_dn, + { + "userpassword": ( + ldap3.MODIFY_REPLACE, + [self._ssha_password(new_password.encode("utf-8"))] + ) + } + ) + except Exception as ex: + logger.error("Exception: " + str(ex)) + else: + logger.error("User {} not found".format(uid)) + + conn.unbind() + return return_val + + def change_user_details(self, uid, details): + """ + Updates the user details as per given values in kwargs of the user + identified by user_dn. + + Assumes that all attributes passed in kwargs are valid. + + :param uid: str The uid that identifies the user + :param details: dict A dictionary containing the new values + :return: True if user details were updated successfully False otherwise + """ + conn = self.get_admin_conn() + + # Make sure the user exists first to change his/her details + user_exists, entries = self.check_user_exists( + uid=uid, + search_base=settings.ENTIRE_SEARCH_BASE + ) + + return_val = False + if user_exists: + details_dict = {k: (ldap3.MODIFY_REPLACE, [v.encode("utf-8")]) for + k, v in details.items()} + try: + return_val = conn.modify(entries[0].entry_dn, details_dict) + msg = "success" + except Exception as ex: + msg = str(ex) + logger.error("Exception: " + msg) + finally: + conn.unbind() + else: + msg = "User {} not found".format(uid) + logger.error(msg) + conn.unbind() + return return_val, msg + + def check_user_exists(self, uid, search_filter="", attributes=None, + search_base=settings.LDAP_CUSTOMER_DN): + """ + Check if the user with the given uid exists in the customer group. + + :param uid: str representing the user + :param search_filter: str representing the filter condition to find + users. If its empty, the search finds the user with + the given uid. + :param attributes: list A list of str representing all the attributes + to be obtained in the result entries + :param search_base: str + :return: tuple (bool, [ldap3.abstract.entry.Entry ..]) + A bool indicating if the user exists + A list of all entries obtained in the search + """ + conn = self.get_admin_conn() + entries = [] + try: + result = conn.search( + search_base=search_base, + search_filter=search_filter if len(search_filter)> 0 else + '(uid={uid})'.format(uid=uid), + attributes=attributes + ) + entries = conn.entries + finally: + conn.unbind() + return result, entries + + def delete_user(self, uid): + """ + Deletes the user with the given uid from ldap + + :param uid: str representing the user + :return: True if the delete was successful False otherwise + """ + conn = self.get_admin_conn() + try: + return_val = conn.delete( + ("uid={uid}," + settings.LDAP_CUSTOMER_DN).format(uid=uid), + ) + msg = "success" + except Exception as ex: + msg = str(ex) + logger.error("Exception: " + msg) + return_val = False + finally: + conn.unbind() + return return_val, msg + + def _set_max_uid(self, max_uid): + """ + a utility function to save max_uid value to a file + + :param max_uid: an integer representing the max uid + :return: + """ + with open(settings.LDAP_MAX_UID_FILE_PATH, 'w+') as handler: + handler.write(str(max_uid)) + + def _get_max_uid(self): + """ + A utility function to read the max uid value that was previously set + + :return: An integer representing the max uid value that was previously + set + """ + try: + with open(settings.LDAP_MAX_UID_FILE_PATH, 'r+') as handler: + try: + return_value = int(handler.read()) + except ValueError as ve: + logger.error( + "Error reading int value from {}. {}" + "Returning default value {} instead".format( + settings.LDAP_MAX_UID_PATH, + str(ve), + settings.LDAP_DEFAULT_START_UID + ) + ) + return_value = settings.LDAP_DEFAULT_START_UID + return return_value + except FileNotFoundError as fnfe: + logger.error("File not found : " + str(fnfe)) + return_value = settings.LDAP_DEFAULT_START_UID + logger.error("So, returning UID={}".format(return_value)) + return return_value diff --git a/uncloud_auth/views.py b/uncloud_auth/views.py new file mode 100644 index 0000000..9310a4c --- /dev/null +++ b/uncloud_auth/views.py @@ -0,0 +1,77 @@ +from django.contrib.auth import views as auth_views +from django.contrib.auth import logout + +from django_auth_ldap.backend import LDAPBackend +from rest_framework import mixins, permissions, status, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response + +from .serializers import * + + +class LoginView(auth_views.LoginView): + template_name = 'uncloud_auth/login.html' + +class LogoutView(auth_views.LogoutView): + pass +# template_name = 'uncloud_auth/logo.html' + + +class UserViewSet(viewsets.GenericViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = UserSerializer + + def get_queryset(self): + return self.request.user + + def list(self, request, format=None): + # This is a bit stupid: we have a user, we create a queryset by + # matching on the username. But I don't know a "nicer" way. + # Nico, 2020-03-18 + user = request.user + serializer = self.get_serializer(user, context = {'request': request}) + return Response(serializer.data) + + @action(detail=False, methods=['post']) + def change_email(self, request): + serializer = self.get_serializer( + request.user, data=request.data, context={'request': request} + ) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data) + + +class AccountManagementViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet): + serializer_class = UserRegistrationSerializer + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + headers = self.get_success_headers(serializer.data) + return Response( + serializer.data, status=status.HTTP_201_CREATED, headers=headers + ) + + +class AdminUserViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAdminUser] + + def get_serializer_class(self): + if self.action == 'import_from_ldap': + return ImportUserSerializer + else: + return UserSerializer + + def get_queryset(self): + return get_user_model().objects.all() + + @action(detail=False, methods=['post'], url_path='import_from_ldap') + def import_from_ldap(self, request, pk=None): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + ldap_username = serializer.validated_data.pop("username") + user = LDAPBackend().populate_user(ldap_username) + + return Response(UserSerializer(user, context = {'request': request}).data) diff --git a/uncloud_net/__init__.py b/uncloud_net/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_net/admin.py b/uncloud_net/admin.py new file mode 100644 index 0000000..ca6aaa1 --- /dev/null +++ b/uncloud_net/admin.py @@ -0,0 +1,7 @@ +from django.contrib import admin + +from .models import * + + +for m in [ ReverseDNSEntry, WireGuardVPNPool, WireGuardVPN ]: + admin.site.register(m) diff --git a/uncloud_net/apps.py b/uncloud_net/apps.py new file mode 100644 index 0000000..489beb1 --- /dev/null +++ b/uncloud_net/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudNetConfig(AppConfig): + name = 'uncloud_net' diff --git a/uncloud_net/forms.py b/uncloud_net/forms.py new file mode 100644 index 0000000..ad4e013 --- /dev/null +++ b/uncloud_net/forms.py @@ -0,0 +1,11 @@ +from django import forms + +from .models import * +from .selectors import * + +class WireGuardVPNForm(forms.ModelForm): + network_size = forms.ChoiceField(choices=allowed_vpn_network_reservation_size) + + class Meta: + model = WireGuardVPN + fields = [ "wireguard_public_key" ] diff --git a/uncloud_net/management/commands/vpn.py b/uncloud_net/management/commands/vpn.py new file mode 100644 index 0000000..9fdc80d --- /dev/null +++ b/uncloud_net/management/commands/vpn.py @@ -0,0 +1,44 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand + +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster + +import logging +log = logging.getLogger(__name__) + + + +peer_template=""" +# {username} +[Peer] +PublicKey = {public_key} +AllowedIPs = {vpnnetwork} +""" + +class Command(BaseCommand): + help = 'General uncloud commands' + + def add_arguments(self, parser): + parser.add_argument('--hostname', + action='store_true', + help='Name of this VPN Host', + required=True) + + def handle(self, *args, **options): + if options['bootstrap']: + self.bootstrap() + + self.create_vpn_config(options['hostname']) + + def create_vpn_config(self, hostname): + configs = [] + + for pool in VPNPool.objects.filter(vpn_hostname=hostname): + configs.append(pool_config) + + print(configs) diff --git a/uncloud_net/migrations/0001_initial.py b/uncloud_net/migrations/0001_initial.py new file mode 100644 index 0000000..6794156 --- /dev/null +++ b/uncloud_net/migrations/0001_initial.py @@ -0,0 +1,62 @@ +# Generated by Django 3.1 on 2020-12-13 13:42 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='MACAdress', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + ), + migrations.CreateModel( + name='WireGuardVPNPool', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('network', models.GenericIPAddressField(unique=True)), + ('network_mask', models.IntegerField(validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(128)])), + ('subnetwork_mask', models.IntegerField(validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(128)])), + ('vpn_server_hostname', models.CharField(max_length=256)), + ('wireguard_private_key', models.CharField(max_length=48)), + ], + ), + migrations.CreateModel( + name='WireGuardVPNFreeLeases', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pool_index', models.IntegerField(unique=True)), + ('vpnpool', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_net.wireguardvpnpool')), + ], + ), + migrations.CreateModel( + name='WireGuardVPN', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pool_index', models.IntegerField(unique=True)), + ('wireguard_public_key', models.CharField(max_length=48)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('vpnpool', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_net.wireguardvpnpool')), + ], + ), + migrations.CreateModel( + name='ReverseDNSEntry', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ip_address', models.GenericIPAddressField(unique=True)), + ('name', models.CharField(max_length=253)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/uncloud_net/migrations/0002_wireguardvpnpool_wireguard_public_key.py b/uncloud_net/migrations/0002_wireguardvpnpool_wireguard_public_key.py new file mode 100644 index 0000000..479aba1 --- /dev/null +++ b/uncloud_net/migrations/0002_wireguardvpnpool_wireguard_public_key.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2020-12-13 17:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='wireguardvpnpool', + name='wireguard_public_key', + field=models.CharField(default='', max_length=48), + preserve_default=False, + ), + ] diff --git a/uncloud_net/migrations/0003_wireguardvpnpool_wg_name.py b/uncloud_net/migrations/0003_wireguardvpnpool_wg_name.py new file mode 100644 index 0000000..9ecf52c --- /dev/null +++ b/uncloud_net/migrations/0003_wireguardvpnpool_wg_name.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2020-12-13 17:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0002_wireguardvpnpool_wireguard_public_key'), + ] + + operations = [ + migrations.AddField( + model_name='wireguardvpnpool', + name='wg_name', + field=models.CharField(default='wg0', max_length=15), + preserve_default=False, + ), + ] diff --git a/uncloud_net/migrations/0004_auto_20201213_1734.py b/uncloud_net/migrations/0004_auto_20201213_1734.py new file mode 100644 index 0000000..24e46e7 --- /dev/null +++ b/uncloud_net/migrations/0004_auto_20201213_1734.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1 on 2020-12-13 17:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0003_wireguardvpnpool_wg_name'), + ] + + operations = [ + migrations.AddConstraint( + model_name='wireguardvpnpool', + constraint=models.UniqueConstraint(fields=('wg_name', 'vpn_server_hostname'), name='unique_interface_name_per_host'), + ), + ] diff --git a/uncloud_net/migrations/0005_auto_20201220_1837.py b/uncloud_net/migrations/0005_auto_20201220_1837.py new file mode 100644 index 0000000..1dbabe6 --- /dev/null +++ b/uncloud_net/migrations/0005_auto_20201220_1837.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1 on 2020-12-20 18:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0004_auto_20201213_1734'), + ] + + operations = [ + migrations.AlterField( + model_name='wireguardvpn', + name='wireguard_public_key', + field=models.CharField(max_length=48, unique=True), + ), + ] diff --git a/uncloud_net/migrations/0006_auto_20201224_1626.py b/uncloud_net/migrations/0006_auto_20201224_1626.py new file mode 100644 index 0000000..c0dd2ef --- /dev/null +++ b/uncloud_net/migrations/0006_auto_20201224_1626.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1 on 2020-12-24 16:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0005_auto_20201220_1837'), + ] + + operations = [ + migrations.AddConstraint( + model_name='wireguardvpn', + constraint=models.UniqueConstraint(fields=('vpnpool', 'wireguard_public_key'), name='wg_key_unique_per_pool'), + ), + ] diff --git a/uncloud_net/migrations/__init__.py b/uncloud_net/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_net/models.py b/uncloud_net/models.py new file mode 100644 index 0000000..0c8b02a --- /dev/null +++ b/uncloud_net/models.py @@ -0,0 +1,192 @@ +import uuid +import ipaddress + +from django.db import models +from django.contrib.auth import get_user_model +from django.core.validators import MinValueValidator, MaxValueValidator +from django.core.exceptions import FieldError, ValidationError + +from uncloud_pay.models import Order + +class WireGuardVPNPool(models.Model): + """ + Network address pools from which VPNs can be created + """ + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['wg_name', 'vpn_server_hostname' ], + name='unique_interface_name_per_host') + ] + + + # Linux interface naming is restricing to max 15 characters + wg_name = models.CharField(max_length=15) + + network = models.GenericIPAddressField(unique=True) + network_mask = models.IntegerField(validators=[MinValueValidator(0), + MaxValueValidator(128)]) + + subnetwork_mask = models.IntegerField(validators=[ + MinValueValidator(0), + MaxValueValidator(128) + ]) + + vpn_server_hostname = models.CharField(max_length=256) + wireguard_private_key = models.CharField(max_length=48) + wireguard_public_key = models.CharField(max_length=48) + + @property + def max_pool_index(self): + """ + Return the highest possible network / last network id + """ + + bits = self.subnetwork_mask - self.network_mask + + return (2**bits)-1 + + @property + def ip_network(self): + return ipaddress.ip_network(f"{self.network}/{self.network_mask}") + + def __str__(self): + return f"{self.ip_network} (subnets: /{self.subnetwork_mask})" + + @property + def wireguard_config(self): + wireguard_config = [ f"[Interface]\nListenPort = 51820\nPrivateKey = {self.wireguard_private_key}\n" ] + + peers = [] + + for vpn in self.wireguardvpn_set.all(): + public_key = vpn.wireguard_public_key + peer_network = f"{vpn.address}/{self.subnetwork_mask}" + owner = vpn.owner + + peers.append(f"# Owner: {owner}\n[Peer]\nPublicKey = {public_key}\nAllowedIPs = {peer_network}\n\n") + + wireguard_config.extend(peers) + + return "\n".join(wireguard_config) + + +class WireGuardVPN(models.Model): + """ + Created VPNNetworks + """ + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + vpnpool = models.ForeignKey(WireGuardVPNPool, + on_delete=models.CASCADE) + + pool_index = models.IntegerField(unique=True) + + wireguard_public_key = models.CharField(max_length=48, unique=True) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['vpnpool', 'wireguard_public_key'], + name='wg_key_unique_per_pool') + ] + + + @property + def network_mask(self): + return self.vpnpool.subnetwork_mask + + @property + def vpn_server(self): + return self.vpnpool.vpn_server_hostname + + @property + def vpn_server_public_key(self): + return self.vpnpool.wireguard_public_key + + @property + def address(self): + """ + Locate the correct subnet in the supernet + + First get the network itself + + """ + + net = self.vpnpool.ip_network + subnet = net[(2**(128-self.vpnpool.subnetwork_mask)) * self.pool_index] + + return str(subnet) + + def __str__(self): + return f"{self.address} ({self.pool_index})" + + +class WireGuardVPNFreeLeases(models.Model): + """ + Previously used VPNNetworks + """ + vpnpool = models.ForeignKey(WireGuardVPNPool, + on_delete=models.CASCADE) + + pool_index = models.IntegerField(unique=True) + +################################################################################ + +class MACAdress(models.Model): + default_prefix = 0x420000000000 + + +class ReverseDNSEntry(models.Model): + """ + A reverse DNS entry + """ + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + ip_address = models.GenericIPAddressField(null=False, unique=True) + + name = models.CharField(max_length=253, null=False) + + @property + def reverse_pointer(self): + return ipaddress.ip_address(self.ip_address).reverse_pointer + + def implement(self): + """ + The implement function implements the change + """ + + # Get all DNS entries (?) / update this DNS entry + # convert to DNS name + # + pass + + + def save(self, *args, **kwargs): + # Product.objects.filter(config__parameters__contains='reverse_dns_network') + # FIXME: check if order is still active / not replaced + + allowed = False + product = None + + for order in Order.objects.filter(config__parameters__reverse_dns_network__isnull=False, + owner=self.owner): + network = order.config['parameters']['reverse_dns_network'] + + net = ipaddress.ip_network(network) + addr = ipaddress.ip_address(self.ip_address) + + if addr in net: + allowed = True + product = order.product + break + + + if not allowed: + raise ValidationError(f"User {self.owner} does not have the right to create reverse DNS entry for {self.ip_address}") + + super().save(*args, **kwargs) + + + def __str__(self): + return f"{self.ip_address} - {self.name}" diff --git a/uncloud_net/selectors.py b/uncloud_net/selectors.py new file mode 100644 index 0000000..6e12e8b --- /dev/null +++ b/uncloud_net/selectors.py @@ -0,0 +1,43 @@ +from django.db import transaction +from django.db.models import Count, F +from .models import * + +def get_suitable_pools(subnetwork_mask): + """ + Find suitable pools for a certain network size. + + First, filter for all pools that offer the requested subnetwork_size. + + Then find those pools that are not fully exhausted: + + The number of available networks in a pool is 2^(subnetwork_size-network_size. + + The number of available networks in a pool is given by the number of VPNNetworkreservations. + + """ + + return WireGuardVPNPool.objects.annotate( + num_reservations=Count('wireguardvpn'), + max_reservations=2**(F('subnetwork_mask')-F('network_mask'))).filter( + num_reservations__lt=F('max_reservations'), + subnetwork_mask=subnetwork_mask) + + +def allowed_vpn_network_reservation_size(): + """ + Find all possible sizes of subnetworks that are available. + + Select all pools with free networks. + + Get their subnetwork sizes, reduce to a set + + """ + + pools = WireGuardVPNPool.objects.annotate(num_reservations=Count('wireguardvpn'), + max_reservations=2**(F('subnetwork_mask')-F('network_mask'))).filter( + num_reservations__lt=F('max_reservations')) + + # Need to return set of tuples, see + # https://docs.djangoproject.com/en/3.1/ref/models/fields/#field-choices +# return set([ (pool.subnetwork_mask, pool.subnetwork_mask) for pool in pools ]) + return set([pool.subnetwork_mask for pool in pools ]) diff --git a/uncloud_net/serializers.py b/uncloud_net/serializers.py new file mode 100644 index 0000000..09baa59 --- /dev/null +++ b/uncloud_net/serializers.py @@ -0,0 +1,57 @@ +import base64 + +from django.contrib.auth import get_user_model +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from .models import * +from .services import * +from .selectors import * + + +class WireGuardVPNSerializer(serializers.ModelSerializer): + address = serializers.CharField(read_only=True) + vpn_server = serializers.CharField(read_only=True) + vpn_server_public_key = serializers.CharField(read_only=True) + network_mask = serializers.IntegerField() + + class Meta: + model = WireGuardVPN + fields = [ 'wireguard_public_key', 'address', 'network_mask', 'vpn_server', + 'vpn_server_public_key' ] + + extra_kwargs = { + 'network_mask': {'write_only': True } + } + + + def validate_network_mask(self, value): + msg = _(f"No pool for network size {value}") + sizes = allowed_vpn_network_reservation_size() + + if not value in sizes: + raise serializers.ValidationError(msg) + + return value + + def validate_wireguard_public_key(self, value): + msg = _("Supplied key is not a valid wireguard public key") + + """ + Verify wireguard key. + See https://lists.zx2c4.com/pipermail/wireguard/2020-December/006221.html + """ + + try: + decoded_key = base64.standard_b64decode(value) + except Exception as e: + raise serializers.ValidationError(msg) + + if not len(decoded_key) == 32: + raise serializers.ValidationError(msg) + + return value + + +class WireGuardVPNSizesSerializer(serializers.Serializer): + size = serializers.IntegerField(min_value=0, max_value=128) diff --git a/uncloud_net/services.py b/uncloud_net/services.py new file mode 100644 index 0000000..4f80c44 --- /dev/null +++ b/uncloud_net/services.py @@ -0,0 +1,47 @@ +from django.db import transaction + +from .models import * +from .selectors import * +from .tasks import * + +@transaction.atomic +def create_wireguard_vpn(owner, public_key, network_mask): + + pool = get_suitable_pools(network_mask)[0] + count = pool.wireguardvpn_set.count() + + # Try re-using previously used networks first + try: + free_lease = WireGuardVPNFreeLeases.objects.get(vpnpool=pool) + + vpn = WireGuardVPN.objects.create(owner=owner, + vpnpool=pool, + pool_index=free_lease.pool_index, + wireguard_public_key=public_key) + + free_lease.delete() + + except WireGuardVPNFreeLeases.DoesNotExist: + + # First object + if count == 0: + vpn = WireGuardVPN.objects.create(owner=owner, + vpnpool=pool, + pool_index=0, + wireguard_public_key=public_key) + + else: # Select last network and try +1 it + last_net = WireGuardVPN.objects.filter(vpnpool=pool).order_by('pool_index').last() + + next_index = last_net.pool_index + 1 + + if next_index <= pool.max_pool_index: + vpn = WireGuardVPN.objects.create(owner=owner, + vpnpool=pool, + pool_index=next_index, + wireguard_public_key=public_key) + + + + configure_wireguard_server(pool) + return vpn diff --git a/uncloud_net/tasks.py b/uncloud_net/tasks.py new file mode 100644 index 0000000..78ae80c --- /dev/null +++ b/uncloud_net/tasks.py @@ -0,0 +1,60 @@ +from celery import shared_task +from .models import * + +from uncloud.models import UncloudTask + +import os +import subprocess +import logging +import uuid + +log = logging.getLogger(__name__) + +@shared_task +def whereami(): + print(os.uname()) + return os.uname() + +def configure_wireguard_server(wireguardvpnpool): + """ + - Create wireguard config (DB query -> string) + - Submit config to cdist worker + - Change config locally on worker / commit / shared + + """ + + config = wireguardvpnpool.wireguard_config + server = wireguardvpnpool.vpn_server_hostname + + log.info(f"Configuring VPN server {server} (async)") + + task_id = uuid.UUID(cdist_configure_wireguard_server.apply_async((config, server)).id) + UncloudTask.objects.create(task_id=task_id) + + +@shared_task +def cdist_configure_wireguard_server(config, server): + """ + Create config and configure server. + + To be executed on the cdist workers. + """ + + dirname= "/home/app/.cdist/type/__ungleich_wireguard/files/" + fname = os.path.join(dirname,server) + + log.info(f"Configuring VPN server {server} (on cdist host)") + with open(fname, "w") as fd: + fd.write(config) + + log.debug("git committing wireguard changes") + subprocess.run(f"cd {dirname} && git pull && git add {server} && git commit -m 'Updating config for {server}' && git push", + shell=True, check=True) + + log.debug(f"Configuring VPN server {server} with cdist") + subprocess.run(f"cdist config {server}", shell=True, check=True) + + # FIXME: + # ensure logs are on the server + # ensure exit codes are known + return True diff --git a/uncloud_net/templates/uncloud_net/wireguardvpn_form.html b/uncloud_net/templates/uncloud_net/wireguardvpn_form.html new file mode 100644 index 0000000..1463f41 --- /dev/null +++ b/uncloud_net/templates/uncloud_net/wireguardvpn_form.html @@ -0,0 +1,25 @@ +{% extends 'uncloud/base.html' %} + +{% block body %} +
+
+
+

+

Create a VPN Network

+

+ Create a new wireguard based VPN network. +

+ +
+ +
+
+ {% csrf_token %} + {{ form }} + +
+
+
+
+ +{% endblock %} diff --git a/uncloud_net/tests.py b/uncloud_net/tests.py new file mode 100644 index 0000000..4491551 --- /dev/null +++ b/uncloud_net/tests.py @@ -0,0 +1,102 @@ +from django.test import TestCase +from rest_framework.test import APIRequestFactory, force_authenticate + +from rest_framework.reverse import reverse +from django.contrib.auth import get_user_model +from django.core.exceptions import ValidationError, FieldError + +from .views import * +from .models import * + +from uncloud_pay.models import BillingAddress, Order +from uncloud.models import UncloudNetwork + +class UncloudNetworkTests(TestCase): + def test_invalid_IPv4_network(self): + with self.assertRaises(FieldError): + UncloudNetwork.objects.create(network_address="192.168.1.0", + network_mask=33) + +class VPNTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_user('django-test-user', 'noreply@ungleich.ch') + self.admin_user = get_user_model().objects.create_user('django-test-adminuser', + 'noreply-admin@ungleich.ch') + + + + self.admin_user.is_staff = True + self.admin_user.save() + + self.pool_network = '2001:db8::' + self.pool_network2 = '2001:db8:1::' + self.pool_network_size = '48' + self.pool_subnetwork_size = '64' + self.pool_vpn_hostname = 'vpn.example.org' + self.pool_wireguard_private_key = 'MOz8kk0m4jhNtAXlge0qzexZh1MipIhu4HJwtdvZ2EY=' + + self.vpn_wireguard_public_key = 'B2b78eWBIXPMM1x4DDjkCDZepS0qDgcLN3T3PjcgXkY=' + + self.vpnpool = VPNPool.objects.get_or_create(network=self.pool_network, + network_size=self.pool_network_size, + subnetwork_size=self.pool_subnetwork_size, + vpn_hostname=self.pool_vpn_hostname, + wireguard_private_key=self.pool_wireguard_private_key + ) + + self.factory = APIRequestFactory() + + + def test_create_vpnpool(self): + url = reverse("vpnpool-list") + view = VPNPoolViewSet.as_view({'post': 'create'}) + request = self.factory.post(url, { 'network': self.pool_network2, + 'network_size': self.pool_network_size, + 'subnetwork_size': self.pool_subnetwork_size, + 'vpn_hostname': self.pool_vpn_hostname, + 'wireguard_private_key': self.pool_wireguard_private_key + + }) + force_authenticate(request, user=self.admin_user) + response = view(request) + + # This raises an exception if the request was not successful + # No assert needed + pool = VPNPool.objects.get(network=self.pool_network2) + + # def test_create_vpn(self): + # url = reverse("vpnnetwork-list") + # view = VPNNetworkViewSet.as_view({'post': 'create'}) + # request = self.factory.post(url, { 'network_size': self.pool_subnetwork_size, + # 'wireguard_public_key': self.vpn_wireguard_public_key + + # }) + # force_authenticate(request, user=self.user) + + + # # we don't have a billing address -> should raise an error + # # with self.assertRaises(ValidationError): + # # response = view(request) + + # addr = BillingAddress.objects.get_or_create( + # owner=self.user, + # active=True, + # defaults={'organization': 'ungleich', + # 'name': 'Nico Schottelius', + # 'street': 'Hauptstrasse 14', + # 'city': 'Luchsingen', + # 'postal_code': '8775', + # 'country': 'CH' } + # ) + + # # This should work now + # response = view(request) + + # # Verify that an order was created successfully - there should only be one order at + # # this point in time + # order = Order.objects.get(owner=self.user) + + + def tearDown(self): + self.user.delete() + self.admin_user.delete() diff --git a/uncloud_net/views.py b/uncloud_net/views.py new file mode 100644 index 0000000..77ba952 --- /dev/null +++ b/uncloud_net/views.py @@ -0,0 +1,70 @@ +from django.views.generic.edit import CreateView +from django.contrib.auth.mixins import LoginRequiredMixin +from django.contrib.messages.views import SuccessMessageMixin +from rest_framework.response import Response + +from django.shortcuts import render + +from rest_framework import viewsets, permissions + +from .models import * +from .serializers import * +from .selectors import * +from .services import * +from .forms import * +from .tasks import * + +class WireGuardVPNViewSet(viewsets.ModelViewSet): + serializer_class = WireGuardVPNSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + if self.request.user.is_superuser: + obj = WireGuardVPN.objects.all() + else: + obj = WireGuardVPN.objects.filter(owner=self.request.user) + + return obj + + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + vpn = create_wireguard_vpn( + owner=self.request.user, + public_key=serializer.validated_data['wireguard_public_key'], + network_mask=serializer.validated_data['network_mask'] + ) + configure_wireguard_server(vpn.vpnpool) + return Response(WireGuardVPNSerializer(vpn).data) + + +class WireGuardVPNCreateView(LoginRequiredMixin, SuccessMessageMixin, CreateView): + model = WireGuardVPN + + login_url = '/login/' + success_url = '/' + success_message = "%(network) was created successfully" + + form_class = WireGuardVPNForm + + def get_success_message(self, cleaned_data): + return self.success_message % dict(cleaned_data, + the_prefix = self.object.prefix) + +class WireGuardVPNSizes(viewsets.ViewSet): + def list(self, request): + sizes = allowed_vpn_network_reservation_size() + print(sizes) + + sizes = [ { 'size': size } for size in sizes ] + print(sizes) + + return Response(WireGuardVPNSizesSerializer(sizes, many=True).data) + + + +# class VPNPoolViewSet(viewsets.ModelViewSet): +# serializer_class = VPNPoolSerializer +# permission_classes = [permissions.IsAdminUser] +# queryset = VPNPool.objects.all() diff --git a/uncloud_pay/__init__.py b/uncloud_pay/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/uncloud_pay/__init__.py @@ -0,0 +1 @@ + diff --git a/uncloud_pay/admin.py b/uncloud_pay/admin.py new file mode 100644 index 0000000..2c72274 --- /dev/null +++ b/uncloud_pay/admin.py @@ -0,0 +1,92 @@ +from django.contrib import admin +from django.template.response import TemplateResponse +from django.urls import path +from django.shortcuts import render +from django.conf.urls import url + +from uncloud_pay.views import BillViewSet +from hardcopy import bytestring_to_pdf +from django.core.files.temp import NamedTemporaryFile +from django.http import FileResponse +from django.template.loader import render_to_string + + +from uncloud_pay.models import * + + +class BillRecordInline(admin.TabularInline): + model = BillRecord + +class RecurringPeriodInline(admin.TabularInline): + model = ProductToRecurringPeriod + +class ProductAdmin(admin.ModelAdmin): + inlines = [ RecurringPeriodInline ] + +class BillAdmin(admin.ModelAdmin): + inlines = [ BillRecordInline ] + + def get_urls(self): + """ + Create URLs for PDF view + """ + + info = "%s_%s" % (self.model._meta.app_label, self.model._meta.model_name) + pat = lambda regex, fn: url(regex, self.admin_site.admin_view(fn), name='%s_%s' % (info, fn.__name__)) + + url_patterns = [ + pat(r'^([0-9]+)/as_pdf/$', self.as_pdf), + pat(r'^([0-9]+)/as_html/$', self.as_html), + ] + super().get_urls() + + return url_patterns + + def as_pdf(self, request, object_id): + bill = self.get_object(request, object_id=object_id) + print(bill) + + if bill is None: + raise self._get_404_exception(object_id) + + output_file = NamedTemporaryFile() + bill_html = render_to_string("bill.html.j2", {'bill': bill, + 'bill_records': bill.billrecord_set.all() + }) + + bytestring_to_pdf(bill_html.encode('utf-8'), output_file) + response = FileResponse(output_file, content_type="application/pdf") + response['Content-Disposition'] = f'filename="bill_{bill}.pdf"' + + return response + + def as_html(self, request, object_id): + bill = self.get_object(request, object_id=object_id) + + if bill is None: + raise self._get_404_exception(object_id) + + return render(request, 'bill.html.j2', + {'bill': bill, + 'bill_records': bill.billrecord_set.all() + }) + + + bill_html = render_to_string("bill.html.j2", {'bill': bill, + 'bill_records': bill.billrecord_set.all() + }) + + bytestring_to_pdf(bill_html.encode('utf-8'), output_file) + response = FileResponse(output_file, content_type="application/pdf") + + response['Content-Disposition'] = f'filename="bill_{bill}.pdf"' + + return HttpResponse(template.render(context, request)) + return response + + +admin.site.register(Bill, BillAdmin) +admin.site.register(ProductToRecurringPeriod) +admin.site.register(Product, ProductAdmin) + +for m in [ Order, BillRecord, BillingAddress, RecurringPeriod, VATRate, StripeCustomer ]: + admin.site.register(m) diff --git a/uncloud_pay/apps.py b/uncloud_pay/apps.py new file mode 100644 index 0000000..051ffb4 --- /dev/null +++ b/uncloud_pay/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudPayConfig(AppConfig): + name = 'uncloud_pay' diff --git a/uncloud_pay/helpers.py b/uncloud_pay/helpers.py new file mode 100644 index 0000000..f791564 --- /dev/null +++ b/uncloud_pay/helpers.py @@ -0,0 +1,26 @@ +from functools import reduce +from datetime import datetime +from rest_framework import mixins +from rest_framework.viewsets import GenericViewSet +from django.utils import timezone +from calendar import monthrange + +def beginning_of_month(year, month): + tz = timezone.get_current_timezone() + return datetime(year=year, month=month, day=1, tzinfo=tz) + +def end_of_month(year, month): + (_, days) = monthrange(year, month) + tz = timezone.get_current_timezone() + return datetime(year=year, month=month, day=days, + hour=23, minute=59, second=59, tzinfo=tz) + +class ProductViewSet(mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.ListModelMixin, + GenericViewSet): + """ + A customer-facing viewset that provides default `create()`, `retrieve()` + and `list()`. + """ + pass diff --git a/uncloud_pay/management/commands/.gitignore b/uncloud_pay/management/commands/.gitignore new file mode 100644 index 0000000..cf5c7fa --- /dev/null +++ b/uncloud_pay/management/commands/.gitignore @@ -0,0 +1,2 @@ +# Customer tests +customer-*.py diff --git a/uncloud_pay/management/commands/add-opennebula-vm-orders.py b/uncloud_pay/management/commands/add-opennebula-vm-orders.py new file mode 100644 index 0000000..e0b6758 --- /dev/null +++ b/uncloud_pay/management/commands/add-opennebula-vm-orders.py @@ -0,0 +1,152 @@ +import datetime +import sys + +from django.contrib.auth import get_user_model +from django.core.management.base import BaseCommand +from django.utils import timezone + +from uncloud_pay.models import ( + BillingAddress +) +from uncloud_vm.models import ( + VMDiskType, VMProduct +) + + +def vm_price_2020(cpu=1, ram=2, v6only=False): + if v6only: + discount = 9 + else: + discount = 0 + + return cpu * 3 + ram * 4 - discount + + +def disk_price_2020(size_in_gb, disk_type): + if disk_type == VMDiskType.CEPH_SSD: + price = 3.5 / 10 + elif disk_type == VMDiskType.CEPH_HDD: + price = 1.5 / 100 + else: + raise Exception("not yet defined price") + + return size_in_gb * price + + +class Command(BaseCommand): + help = 'Adding VMs / creating orders for user' + + def add_arguments(self, parser): + parser.add_argument('--username', type=str, required=True) + + def handle(self, *args, **options): + user = get_user_model().objects.get(username=options['username']) + + addr, created = BillingAddress.objects.get_or_create( + owner=user, + active=True, + defaults={'organization': 'Undefined organisation', + 'full_name': 'Undefined name', + 'street': 'Undefined Street', + 'city': 'Undefined city', + 'postal_code': '8750', + 'country': 'CH', + 'active': True + } + ) + + # 25206 + SSD + vm25206 = VMProduct.objects.create(name="one-25206", cores=1, + ram_in_gb=4, owner=user) + vm25206.create_order_at( + timezone.make_aware(datetime.datetime(2020, 3, 3))) + + # vm25206_ssd = VMDiskProduct.objects.create(vm=vm25206, owner=user, size_in_gb=30) + # vm25206_ssd.create_order_at(timezone.make_aware(datetime.datetime(2020,3,3))) + + # change 1 + vm25206.cores = 2 + vm25206.ram_in_gb = 8 + vm25206.save() + vm25206.create_or_update_order( + when_to_start=timezone.make_aware(datetime.datetime(2020, 4, 17))) + + sys.exit(0) + + # change 2 + # vm25206_ssd.size_in_gb = 50 + # vm25206_ssd.save() + # vm25206_ssd.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,8,5))) + + # 25206 done. + + # 25615 + vm25615 = VMProduct.objects.create(name="one-25615", cores=1, + ram_in_gb=4, owner=user) + vm25615.create_order_at( + timezone.make_aware(datetime.datetime(2020, 3, 3))) + + # Change 2020-04-17 + vm25615.cores = 2 + vm25615.ram_in_gb = 8 + vm25615.save() + vm25615.create_or_update_order( + when_to_start=timezone.make_aware(datetime.datetime(2020, 4, 17))) + + # vm25615_ssd = VMDiskProduct(vm=vm25615, owner=user, size_in_gb=30) + # vm25615_ssd.create_order_at(timezone.make_aware(datetime.datetime(2020,3,3))) + # vm25615_ssd.save() + + vm25208 = VMProduct.objects.create(name="one-25208", cores=1, + ram_in_gb=4, owner=user) + vm25208.create_order_at( + timezone.make_aware(datetime.datetime(2020, 3, 5))) + + vm25208.cores = 2 + vm25208.ram_in_gb = 8 + vm25208.save() + vm25208.create_or_update_order( + when_to_start=timezone.make_aware(datetime.datetime(2020, 4, 17))) + + Bill.create_next_bills_for_user(user, ending_date=end_of_month( + timezone.make_aware(datetime.datetime(2020, 7, 31)))) + + sys.exit(0) + + vm25615_ssd.size_in_gb = 50 + vm25615_ssd.save() + vm25615_ssd.create_or_update_order( + when_to_start=timezone.make_aware(datetime.datetime(2020, 8, 5))) + + vm25208_ssd = VMDiskProduct.objects.create(vm=vm25208, + owner=user, + size_in_gb=30) + + vm25208_ssd.size_in_gb = 50 + vm25208_ssd.save() + vm25208_ssd.create_or_update_order( + when_to_start=timezone.make_aware(datetime.datetime(2020, 8, 5))) + + # 25207 + vm25207 = VMProduct.objects.create(name="OpenNebula 25207", + cores=1, + ram_in_gb=4, + owner=user) + + vm25207_ssd = VMDiskProduct.objects.create(vm=vm25207, + owner=user, + size_in_gb=30) + + vm25207_ssd.size_in_gb = 50 + vm25207_ssd.save() + vm25207_ssd.create_or_update_order( + when_to_start=timezone.make_aware(datetime.datetime(2020, 8, 5))) + + vm25207.cores = 2 + vm25207.ram_in_gb = 8 + vm25207.save() + vm25207.create_or_update_order( + when_to_start=timezone.make_aware(datetime.datetime(2020, 6, 19))) + + # FIXES: check starting times (they are slightly different) + # add vm 25236 diff --git a/uncloud_pay/management/commands/bootstrap-user.py b/uncloud_pay/management/commands/bootstrap-user.py new file mode 100644 index 0000000..b78e80c --- /dev/null +++ b/uncloud_pay/management/commands/bootstrap-user.py @@ -0,0 +1,40 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model +import datetime + +from uncloud_pay.models import * + +class Command(BaseCommand): + help = 'Bootstrap user (for testing)' + + def add_arguments(self, parser): + parser.add_argument('--username', type=str, required=True) + + def handle(self, *args, **options): + user = get_user_model().objects.get(username=options['username']) + + addr = BillingAddress.objects.get_or_create( + owner=user, + active=True, + defaults={'organization': 'ungleich', + 'name': 'Nico Schottelius', + 'street': 'Hauptstrasse 14', + 'city': 'Luchsingen', + 'postal_code': '8775', + 'country': 'CH' } + ) + + + bills = Bill.objects.filter(owner=user) + + # not even one bill? create! + if bills: + bill = bills[0] + else: + bill = Bill.objects.create(owner=user) + + # find any order that is associated to this bill + orders = Order.objects.filter(owner=user) + + print(f"Addr: {addr}") + print(f"Bill: {bill}") diff --git a/uncloud_pay/management/commands/charge-negative-balance.py b/uncloud_pay/management/commands/charge-negative-balance.py new file mode 100644 index 0000000..8ee8736 --- /dev/null +++ b/uncloud_pay/management/commands/charge-negative-balance.py @@ -0,0 +1,31 @@ +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Order, Bill, PaymentMethod, get_balance_for_user + +from datetime import timedelta +from django.utils import timezone + +class Command(BaseCommand): + help = 'Generate bills and charge customers if necessary.' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + users = User.objects.all() + print("Processing {} users.".format(users.count())) + for user in users: + balance = get_balance_for_user(user) + if balance < 0: + print("User {} has negative balance ({}), charging.".format(user.username, balance)) + payment_method = PaymentMethod.get_primary_for(user) + if payment_method != None: + amount_to_be_charged = abs(balance) + charge_ok = payment_method.charge(amount_to_be_charged) + if not charge_ok: + print("ERR: charging {} with method {} failed" + .format(user.username, payment_method.uuid) + ) + else: + print("ERR: no payment method registered for {}".format(user.username)) + print("=> Done.") diff --git a/uncloud_pay/management/commands/generate-bills.py b/uncloud_pay/management/commands/generate-bills.py new file mode 100644 index 0000000..5bd4519 --- /dev/null +++ b/uncloud_pay/management/commands/generate-bills.py @@ -0,0 +1,35 @@ +import logging + +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Order, Bill +from django.core.exceptions import ObjectDoesNotExist + +from datetime import timedelta, date +from django.utils import timezone +from uncloud_pay.models import Bill + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = 'Generate bills and charge customers if necessary.' + + def add_arguments(self, parser): + pass + + # TODO: use logger.* + def handle(self, *args, **options): + # Iterate over all 'active' users. + # TODO: filter out inactive users. + users = User.objects.all() + print("Processing {} users.".format(users.count())) + + for user in users: + now = timezone.now() + Bill.generate_for( + year=now.year, + month=now.month, + user=user) + + # We're done for this round :-) + print("=> Done.") diff --git a/uncloud_pay/management/commands/handle-overdue-bills.py b/uncloud_pay/management/commands/handle-overdue-bills.py new file mode 100644 index 0000000..595fbc2 --- /dev/null +++ b/uncloud_pay/management/commands/handle-overdue-bills.py @@ -0,0 +1,23 @@ +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Bill + +from datetime import timedelta +from django.utils import timezone + +class Command(BaseCommand): + help = 'Take action on overdue bills.' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + users = User.objects.all() + print("Processing {} users.".format(users.count())) + for user in users: + for bill in Bill.get_overdue_for(user): + print("/!\ Overdue bill for {}, {} with amount {}" + .format(user.username, bill.uuid, bill.amount)) + # TODO: take action? + + print("=> Done.") diff --git a/uncloud_pay/management/commands/import-vat-rates.py b/uncloud_pay/management/commands/import-vat-rates.py new file mode 100644 index 0000000..46848cd --- /dev/null +++ b/uncloud_pay/management/commands/import-vat-rates.py @@ -0,0 +1,35 @@ +from django.core.management.base import BaseCommand +from uncloud_pay.models import VATRate + +import urllib +import csv +import sys +import io + +class Command(BaseCommand): + help = '''Imports VAT Rates. Assume vat rates of format https://github.com/kdeldycke/vat-rates/blob/master/vat_rates.csv''' + vat_url = "https://raw.githubusercontent.com/ungleich/vat-rates/main/vat_rates.csv" + + + def add_arguments(self, parser): + parser.add_argument('--vat-url', default=self.vat_url) + + def handle(self, *args, **options): + vat_url = options['vat_url'] + url_open = urllib.request.urlopen(vat_url) + + # map to fileio using stringIO + csv_file = io.StringIO(url_open.read().decode('utf-8')) + reader = csv.DictReader(csv_file) + + for row in reader: +# print(row) + obj, created = VATRate.objects.get_or_create( + starting_date=row["start_date"], + ending_date=row["stop_date"] if row["stop_date"] != "" else None, + territory_codes=row["territory_codes"], + currency_code=row["currency_code"], + rate=row["rate"], + rate_type=row["rate_type"], + description=row["description"] + ) diff --git a/uncloud_pay/migrations/0001_initial.py b/uncloud_pay/migrations/0001_initial.py new file mode 100644 index 0000000..b1b68c5 --- /dev/null +++ b/uncloud_pay/migrations/0001_initial.py @@ -0,0 +1,181 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import uncloud.models +import uncloud_pay.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_auth', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Bill', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField(default=uncloud_pay.models.start_of_this_month)), + ('ending_date', models.DateTimeField()), + ('due_date', models.DateField(default=uncloud_pay.models.default_payment_delay)), + ('is_final', models.BooleanField(default=False)), + ], + ), + migrations.CreateModel( + name='BillingAddress', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('full_name', models.CharField(max_length=256)), + ('organization', models.CharField(blank=True, max_length=256, null=True)), + ('street', models.CharField(max_length=256)), + ('city', models.CharField(max_length=256)), + ('postal_code', models.CharField(max_length=64)), + ('country', uncloud.models.CountryField(blank=True, choices=[('AD', 'Andorra'), ('AE', 'United Arab Emirates'), ('AF', 'Afghanistan'), ('AG', 'Antigua & Barbuda'), ('AI', 'Anguilla'), ('AL', 'Albania'), ('AM', 'Armenia'), ('AN', 'Netherlands Antilles'), ('AO', 'Angola'), ('AQ', 'Antarctica'), ('AR', 'Argentina'), ('AS', 'American Samoa'), ('AT', 'Austria'), ('AU', 'Australia'), ('AW', 'Aruba'), ('AZ', 'Azerbaijan'), ('BA', 'Bosnia and Herzegovina'), ('BB', 'Barbados'), ('BD', 'Bangladesh'), ('BE', 'Belgium'), ('BF', 'Burkina Faso'), ('BG', 'Bulgaria'), ('BH', 'Bahrain'), ('BI', 'Burundi'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BN', 'Brunei Darussalam'), ('BO', 'Bolivia'), ('BR', 'Brazil'), ('BS', 'Bahama'), ('BT', 'Bhutan'), ('BV', 'Bouvet Island'), ('BW', 'Botswana'), ('BY', 'Belarus'), ('BZ', 'Belize'), ('CA', 'Canada'), ('CC', 'Cocos (Keeling) Islands'), ('CF', 'Central African Republic'), ('CG', 'Congo'), ('CH', 'Switzerland'), ('CI', 'Ivory Coast'), ('CK', 'Cook Iislands'), ('CL', 'Chile'), ('CM', 'Cameroon'), ('CN', 'China'), ('CO', 'Colombia'), ('CR', 'Costa Rica'), ('CU', 'Cuba'), ('CV', 'Cape Verde'), ('CX', 'Christmas Island'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DE', 'Germany'), ('DJ', 'Djibouti'), ('DK', 'Denmark'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('DZ', 'Algeria'), ('EC', 'Ecuador'), ('EE', 'Estonia'), ('EG', 'Egypt'), ('EH', 'Western Sahara'), ('ER', 'Eritrea'), ('ES', 'Spain'), ('ET', 'Ethiopia'), ('FI', 'Finland'), ('FJ', 'Fiji'), ('FK', 'Falkland Islands (Malvinas)'), ('FM', 'Micronesia'), ('FO', 'Faroe Islands'), ('FR', 'France'), ('FX', 'France, Metropolitan'), ('GA', 'Gabon'), ('GB', 'United Kingdom (Great Britain)'), ('GD', 'Grenada'), ('GE', 'Georgia'), ('GF', 'French Guiana'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GL', 'Greenland'), ('GM', 'Gambia'), ('GN', 'Guinea'), ('GP', 'Guadeloupe'), ('GQ', 'Equatorial Guinea'), ('GR', 'Greece'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GT', 'Guatemala'), ('GU', 'Guam'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HK', 'Hong Kong'), ('HM', 'Heard & McDonald Islands'), ('HN', 'Honduras'), ('HR', 'Croatia'), ('HT', 'Haiti'), ('HU', 'Hungary'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('IL', 'Israel'), ('IN', 'India'), ('IO', 'British Indian Ocean Territory'), ('IQ', 'Iraq'), ('IR', 'Islamic Republic of Iran'), ('IS', 'Iceland'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JO', 'Jordan'), ('JP', 'Japan'), ('KE', 'Kenya'), ('KG', 'Kyrgyzstan'), ('KH', 'Cambodia'), ('KI', 'Kiribati'), ('KM', 'Comoros'), ('KN', 'St. Kitts and Nevis'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('KW', 'Kuwait'), ('KY', 'Cayman Islands'), ('KZ', 'Kazakhstan'), ('LA', "Lao People's Democratic Republic"), ('LB', 'Lebanon'), ('LC', 'Saint Lucia'), ('LI', 'Liechtenstein'), ('LK', 'Sri Lanka'), ('LR', 'Liberia'), ('LS', 'Lesotho'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('LV', 'Latvia'), ('LY', 'Libyan Arab Jamahiriya'), ('MA', 'Morocco'), ('MC', 'Monaco'), ('MD', 'Moldova, Republic of'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('ML', 'Mali'), ('MN', 'Mongolia'), ('MM', 'Myanmar'), ('MO', 'Macau'), ('MP', 'Northern Mariana Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MS', 'Monserrat'), ('MT', 'Malta'), ('MU', 'Mauritius'), ('MV', 'Maldives'), ('MW', 'Malawi'), ('MX', 'Mexico'), ('MY', 'Malaysia'), ('MZ', 'Mozambique'), ('NA', 'Namibia'), ('NC', 'New Caledonia'), ('NE', 'Niger'), ('NF', 'Norfolk Island'), ('NG', 'Nigeria'), ('NI', 'Nicaragua'), ('NL', 'Netherlands'), ('NO', 'Norway'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('NU', 'Niue'), ('NZ', 'New Zealand'), ('OM', 'Oman'), ('PA', 'Panama'), ('PE', 'Peru'), ('PF', 'French Polynesia'), ('PG', 'Papua New Guinea'), ('PH', 'Philippines'), ('PK', 'Pakistan'), ('PL', 'Poland'), ('PM', 'St. Pierre & Miquelon'), ('PN', 'Pitcairn'), ('PR', 'Puerto Rico'), ('PT', 'Portugal'), ('PW', 'Palau'), ('PY', 'Paraguay'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SA', 'Saudi Arabia'), ('SB', 'Solomon Islands'), ('SC', 'Seychelles'), ('SD', 'Sudan'), ('SE', 'Sweden'), ('SG', 'Singapore'), ('SH', 'St. Helena'), ('SI', 'Slovenia'), ('SJ', 'Svalbard & Jan Mayen Islands'), ('SK', 'Slovakia'), ('SL', 'Sierra Leone'), ('SM', 'San Marino'), ('SN', 'Senegal'), ('SO', 'Somalia'), ('SR', 'Suriname'), ('ST', 'Sao Tome & Principe'), ('SV', 'El Salvador'), ('SY', 'Syrian Arab Republic'), ('SZ', 'Swaziland'), ('TC', 'Turks & Caicos Islands'), ('TD', 'Chad'), ('TF', 'French Southern Territories'), ('TG', 'Togo'), ('TH', 'Thailand'), ('TJ', 'Tajikistan'), ('TK', 'Tokelau'), ('TM', 'Turkmenistan'), ('TN', 'Tunisia'), ('TO', 'Tonga'), ('TP', 'East Timor'), ('TR', 'Turkey'), ('TT', 'Trinidad & Tobago'), ('TV', 'Tuvalu'), ('TW', 'Taiwan, Province of China'), ('TZ', 'Tanzania, United Republic of'), ('UA', 'Ukraine'), ('UG', 'Uganda'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VA', 'Vatican City State (Holy See)'), ('VC', 'St. Vincent & the Grenadines'), ('VE', 'Venezuela'), ('VG', 'British Virgin Islands'), ('VI', 'United States Virgin Islands'), ('VN', 'Viet Nam'), ('VU', 'Vanuatu'), ('WF', 'Wallis & Futuna Islands'), ('WS', 'Samoa'), ('YE', 'Yemen'), ('YT', 'Mayotte'), ('YU', 'Yugoslavia'), ('ZA', 'South Africa'), ('ZM', 'Zambia'), ('ZR', 'Zaire'), ('ZW', 'Zimbabwe')], default='CH', max_length=2)), + ('vat_number', models.CharField(blank=True, default='', max_length=100)), + ('active', models.BooleanField(default=False)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Product', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=256, unique=True)), + ('description', models.CharField(max_length=1024)), + ('config', models.JSONField()), + ('currency', models.CharField(choices=[('CHF', 'Swiss Franc'), ('EUR', 'Euro'), ('USD', 'US Dollar')], default='CHF', max_length=32)), + ], + ), + migrations.CreateModel( + name='RecurringPeriod', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True)), + ('duration_seconds', models.IntegerField(unique=True)), + ], + ), + migrations.CreateModel( + name='StripeCustomer', + fields=[ + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='uncloud_auth.user')), + ('stripe_id', models.CharField(max_length=32)), + ], + ), + migrations.CreateModel( + name='VATRate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('starting_date', models.DateField(blank=True, null=True)), + ('ending_date', models.DateField(blank=True, null=True)), + ('territory_codes', models.TextField(blank=True, default='')), + ('currency_code', models.CharField(max_length=10)), + ('rate', models.FloatField()), + ('rate_type', models.TextField(blank=True, default='')), + ('description', models.TextField(blank=True, default='')), + ], + ), + migrations.CreateModel( + name='ProductToRecurringPeriod', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('is_default', models.BooleanField(default=False)), + ('product', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.product')), + ('recurring_period', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.recurringperiod')), + ], + ), + migrations.AddField( + model_name='product', + name='recurring_periods', + field=models.ManyToManyField(through='uncloud_pay.ProductToRecurringPeriod', to='uncloud_pay.RecurringPeriod'), + ), + migrations.CreateModel( + name='PaymentMethod', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source', models.CharField(choices=[('stripe', 'Stripe'), ('unknown', 'Unknown')], default='stripe', max_length=256)), + ('description', models.TextField()), + ('primary', models.BooleanField(default=False, editable=False)), + ('stripe_payment_method_id', models.CharField(blank=True, max_length=32, null=True)), + ('stripe_setup_intent_id', models.CharField(blank=True, max_length=32, null=True)), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Payment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('amount', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('source', models.CharField(choices=[('wire', 'Wire Transfer'), ('stripe', 'Stripe'), ('voucher', 'Voucher'), ('referral', 'Referral'), ('unknown', 'Unknown')], default='unknown', max_length=256)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Order', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.TextField()), + ('config', models.JSONField()), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField(default=django.utils.timezone.now)), + ('ending_date', models.DateTimeField(blank=True, null=True)), + ('one_time_price', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('recurring_price', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('currency', models.CharField(choices=[('CHF', 'Swiss Franc'), ('EUR', 'Euro'), ('USD', 'US Dollar')], default='CHF', max_length=32)), + ('should_be_billed', models.BooleanField(default=True)), + ('billing_address', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.billingaddress')), + ('depends_on', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='parent_of', to='uncloud_pay.order')), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('product', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.product')), + ('recurring_period', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.recurringperiod')), + ('replaces', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='replaced_by', to='uncloud_pay.order')), + ], + ), + migrations.CreateModel( + name='BillRecord', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField()), + ('ending_date', models.DateTimeField()), + ('is_recurring_record', models.BooleanField()), + ('bill', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.bill')), + ('order', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.order')), + ], + ), + migrations.AddField( + model_name='bill', + name='billing_address', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.billingaddress'), + ), + migrations.AddField( + model_name='bill', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.AddConstraint( + model_name='producttorecurringperiod', + constraint=models.UniqueConstraint(condition=models.Q(is_default=True), fields=('product',), name='one_default_recurring_period_per_product'), + ), + migrations.AddConstraint( + model_name='producttorecurringperiod', + constraint=models.UniqueConstraint(fields=('product', 'recurring_period'), name='recurring_period_once_per_product'), + ), + migrations.AddConstraint( + model_name='billingaddress', + constraint=models.UniqueConstraint(condition=models.Q(active=True), fields=('owner',), name='one_active_billing_address_per_user'), + ), + migrations.AddConstraint( + model_name='bill', + constraint=models.UniqueConstraint(fields=('owner', 'starting_date', 'ending_date'), name='one_bill_per_month_per_user'), + ), + ] diff --git a/uncloud_pay/migrations/__init__.py b/uncloud_pay/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_pay/models.py b/uncloud_pay/models.py new file mode 100644 index 0000000..18e6f85 --- /dev/null +++ b/uncloud_pay/models.py @@ -0,0 +1,1263 @@ +import logging +import itertools +import datetime +from math import ceil +from calendar import monthrange +from decimal import Decimal +from functools import reduce + +from django.db import models +from django.db.models import Q +from django.contrib.auth import get_user_model +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType +from django.utils.translation import gettext_lazy as _ +from django.core.validators import MinValueValidator +from django.utils import timezone +from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.conf import settings + +import uncloud_pay.stripe +from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS +from uncloud.models import UncloudAddress + +# Used to generate bill due dates. +BILL_PAYMENT_DELAY=datetime.timedelta(days=10) + +# Initialize logger. +logger = logging.getLogger(__name__) + +def start_of_month(a_day): + """ Returns first of the month of a given datetime object""" + return a_day.replace(day=1,hour=0,minute=0,second=0, microsecond=0) + +def end_of_month(a_day): + """ Returns first of the month of a given datetime object""" + + _, last_day = monthrange(a_day.year, a_day.month) + return a_day.replace(day=last_day,hour=23,minute=59,second=59, microsecond=0) + +def start_of_this_month(): + """ Returns first of this month""" + a_day = timezone.now() + return a_day.replace(day=1,hour=0,minute=0,second=0, microsecond=0) + +def end_of_this_month(): + """ Returns first of this month""" + a_day = timezone.now() + + _, last_day = monthrange(a_day.year, a_day.month) + return a_day.replace(day=last_day,hour=23,minute=59,second=59, microsecond=0) + +def end_before(a_date): + """ Return suitable datetimefield for ending just before a_date """ + return a_date - datetime.timedelta(seconds=1) + +def start_after(a_date): + """ Return suitable datetimefield for starting just after a_date """ + return a_date + datetime.timedelta(seconds=1) + +def default_payment_delay(): + return timezone.now() + BILL_PAYMENT_DELAY + +class Currency(models.TextChoices): + """ + Possible currencies to be billed + """ + CHF = 'CHF', _('Swiss Franc') + EUR = 'EUR', _('Euro') + USD = 'USD', _('US Dollar') + + +def get_balance_for_user(user): + bills = reduce( + lambda acc, entry: acc + entry.total, + Bill.objects.filter(owner=user), + 0) + payments = reduce( + lambda acc, entry: acc + entry.amount, + Payment.objects.filter(owner=user), + 0) + return payments - bills + +### +# Stripe + +class StripeCustomer(models.Model): + owner = models.OneToOneField( get_user_model(), + primary_key=True, + on_delete=models.CASCADE) + stripe_id = models.CharField(max_length=32) + + def __str__(self): + return self.owner.username + +### +# Payments and Payment Methods. + +class Payment(models.Model): + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + amount = models.DecimalField( + default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + source = models.CharField(max_length=256, + choices = ( + ('wire', 'Wire Transfer'), + ('stripe', 'Stripe'), + ('voucher', 'Voucher'), + ('referral', 'Referral'), + ('unknown', 'Unknown') + ), + default='unknown') + timestamp = models.DateTimeField(editable=False, auto_now_add=True) + + # We override save() in order to active products awaiting payment. + def save(self, *args, **kwargs): + # _state.adding is switched to false after super(...) call. + being_created = self._state.adding + + unpaid_bills_before_payment = Bill.get_unpaid_for(self.owner) + super(Payment, self).save(*args, **kwargs) # Save payment in DB. + unpaid_bills_after_payment = Bill.get_unpaid_for(self.owner) + + newly_paid_bills = list( + set(unpaid_bills_before_payment) - set(unpaid_bills_after_payment)) + for bill in newly_paid_bills: + bill.activate_products() + + +class PaymentMethod(models.Model): + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE, + editable=False) + source = models.CharField(max_length=256, + choices = ( + ('stripe', 'Stripe'), + ('unknown', 'Unknown'), + ), + default='stripe') + description = models.TextField() + primary = models.BooleanField(default=False, editable=False) + + # Only used for "Stripe" source + stripe_payment_method_id = models.CharField(max_length=32, blank=True, null=True) + stripe_setup_intent_id = models.CharField(max_length=32, blank=True, null=True) + + @property + def stripe_card_last4(self): + if self.source == 'stripe' and self.active: + payment_method = uncloud_pay.stripe.get_payment_method( + self.stripe_payment_method_id) + return payment_method.card.last4 + else: + return None + + @property + def active(self): + if self.source == 'stripe' and self.stripe_payment_method_id != None: + return True + else: + return False + + def charge(self, amount): + if not self.active: + raise Exception('This payment method is inactive.') + + if amount < 0: # Make sure we don't charge negative amount by errors... + raise Exception('Cannot charge negative amount.') + + if self.source == 'stripe': + stripe_customer = StripeCustomer.objects.get(owner=self.owner).stripe_id + stripe_payment = uncloud_pay.stripe.charge_customer( + amount, stripe_customer, self.stripe_payment_method_id) + if 'paid' in stripe_payment and stripe_payment['paid'] == False: + raise Exception(stripe_payment['error']) + else: + payment = Payment.objects.create( + owner=self.owner, source=self.source, amount=amount) + + return payment + else: + raise Exception('This payment method is unsupported/cannot be charged.') + + def set_as_primary_for(self, user): + methods = PaymentMethod.objects.filter(owner=user, primary=True) + for method in methods: + print(method) + method.primary = False + method.save() + + self.primary = True + self.save() + + def get_primary_for(user): + methods = PaymentMethod.objects.filter(owner=user) + for method in methods: + # Do we want to do something with non-primary method? + if method.active and method.primary: + return method + + return None + + class Meta: + # TODO: limit to one primary method per user. + # unique_together is no good since it won't allow more than one + # non-primary method. + pass + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class RecurringPeriodDefaultChoices(models.IntegerChoices): + """ + This is an old class and being superseeded by the database model below + """ + PER_365D = 365*24*3600, _('Per 365 days') + PER_30D = 30*24*3600, _('Per 30 days') + PER_WEEK = 7*24*3600, _('Per Week') + PER_DAY = 24*3600, _('Per Day') + PER_HOUR = 3600, _('Per Hour') + PER_MINUTE = 60, _('Per Minute') + PER_SECOND = 1, _('Per Second') + ONE_TIME = 0, _('Onetime') + +# RecurringPeriods +class RecurringPeriod(models.Model): + """ + Available recurring periods. + By default seeded from RecurringPeriodChoices + """ + + name = models.CharField(max_length=100, unique=True) + duration_seconds = models.IntegerField(unique=True) + + @classmethod + def populate_db_defaults(cls): + for (seconds, name) in RecurringPeriodDefaultChoices.choices: + obj, created = cls.objects.get_or_create(name=name, + defaults={ 'duration_seconds': seconds }) + + @staticmethod + def secs_to_name(secs): + name = "" + days = 0 + hours = 0 + + if secs > 24*3600: + days = secs // (24*3600) + secs -= (days*24*3600) + + if secs > 3600: + hours = secs // 3600 + secs -= hours*3600 + + return f"{days} days {hours} hours {secs} seconds" + + def __str__(self): + duration = self.secs_to_name(self.duration_seconds) + + return f"{self.name} ({duration})" + + +### +# Bills. + +class BillingAddress(UncloudAddress): + owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + vat_number = models.CharField(max_length=100, default="", blank=True) + active = models.BooleanField(default=False) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['owner'], + condition=Q(active=True), + name='one_active_billing_address_per_user') + ] + + @classmethod + def populate_db_defaults(cls): + """ + Ensure we have at least one billing address that is associated with the uncloud-admin. + + This way we are sure that an UncloudProvider can be created. + + Cannot use get_or_create as that looks for exactly one. + + """ + + owner = get_user_model().objects.get(username=settings.UNCLOUD_ADMIN_NAME) + billing_address = cls.objects.filter(owner=owner).first() + + if not billing_address: + billing_address = cls.objects.create(owner=owner, + organization="uncloud admins", + name="Uncloud Admin", + street="Uncloudstreet. 42", + city="Luchsingen", + postal_code="8775", + country="CH", + active=True) + + + @staticmethod + def get_address_for(user): + return BillingAddress.objects.get(owner=user, active=True) + + def __str__(self): + return "{} - {}, {}, {} {}, {}".format( + self.owner, + self.full_name, self.street, self.postal_code, self.city, + self.country) + +### +# VAT + +class VATRate(models.Model): + starting_date = models.DateField(blank=True, null=True) + ending_date = models.DateField(blank=True, null=True) + territory_codes = models.TextField(blank=True, default='') + currency_code = models.CharField(max_length=10) + rate = models.FloatField() + rate_type = models.TextField(blank=True, default='') + description = models.TextField(blank=True, default='') + + @staticmethod + def get_for_country(country_code): + vat_rate = None + try: + vat_rate = VATRate.objects.get( + territory_codes=country_code, start_date__isnull=False, stop_date=None + ) + return vat_rate.rate + except VATRate.DoesNotExist as dne: + logger.debug(str(dne)) + logger.debug("Did not find VAT rate for %s, returning 0" % country_code) + return 0 + + + def __str__(self): + return f"{self.territory_codes}: {self.starting_date} - {self.ending_date}: {self.rate_type}" + +### +# Products + +class Product(models.Model): + """ + A product is something a user can order. To record the pricing, we + create order that define a state in time. + + A product can have *one* one_time_order and/or *one* + recurring_order. + + If either of them needs to be updated, a new order of the same + type will be created and links to the previous order. + + """ + + name = models.CharField(max_length=256, unique=True) + description = models.CharField(max_length=1024) + config = models.JSONField() + recurring_periods = models.ManyToManyField(RecurringPeriod, through='ProductToRecurringPeriod') + currency = models.CharField(max_length=32, choices=Currency.choices, default=Currency.CHF) + + @property + def default_recurring_period(self): + """ + Return the default recurring Period + """ + return self.recurring_periods.get(producttorecurringperiod__is_default=True) + + @classmethod + def populate_db_defaults(cls): + recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + obj, created = cls.objects.get_or_create(name="Dual Stack Virtual Machine v1", + description="A standard virtual machine", + currency=Currency.CHF, + config={ + 'features': { + 'cores': + { 'min': 1, + 'max': 48, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 3 + }, + 'ram_gb': + { 'min': 1, + 'max': 256, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 4 + }, + 'ssd_gb': + { 'min': 10, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 0.35 + }, + 'hdd_gb': + { 'min': 0, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 15/1000 + }, + 'additional_ipv4_address': + { 'min': 0, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 8 + }, + } + } + ) + + obj.recurring_periods.add(recurring_period, through_defaults= { 'is_default': True }) + + obj, created = cls.objects.get_or_create(name="Dual Stack Virtual Machine v2", + description="A standard virtual machine", + currency=Currency.CHF, + config={ + 'features': { + 'base': + { 'min': 1, + 'max': 1, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 1 + }, + 'cores': + { 'min': 1, + 'max': 48, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 3 + }, + 'ram_gb': + { 'min': 1, + 'max': 256, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 4 + }, + 'ssd_gb': + { 'min': 10, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 0.35 + }, + 'hdd_gb': + { 'min': 0, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 15/1000 + }, + 'additional_ipv4_address': + { 'min': 0, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 9 + }, + } + } + ) + + obj.recurring_periods.add(recurring_period, through_defaults= { 'is_default': True }) + + obj, created = cls.objects.get_or_create(name="reverse DNS", + description="Reverse DNS network", + currency=Currency.CHF, + config={ + 'parameters': [ + 'network' + ] + }) + obj.recurring_periods.add(recurring_period, through_defaults= { 'is_default': True }) + + + def __str__(self): + return f"{self.name} - {self.description}" + + @property + def recurring_orders(self): + return self.orders.order_by('id').exclude(recurring_period=RecurringPeriod.objects.get(name="ONE_TIME")) + + @property + def last_recurring_order(self): + return self.recurring_orders.last() + + @property + def one_time_orders(self): + return self.orders.order_by('id').filter(recurring_period=RecurringPeriod.objects.get(name="ONE_TIME")) + + @property + def last_one_time_order(self): + return self.one_time_orders.last() + + def create_order(self, when_to_start=None, recurring_period=None): + billing_address = BillingAddress.get_address_for(self.owner) + + if not billing_address: + raise ValidationError("Cannot order without a billing address") + + if not when_to_start: + when_to_start = timezone.now() + + if not recurring_period: + recurring_period = self.default_recurring_period + + + # Create one time order if we did not create one already + if self.one_time_price > 0 and not self.last_one_time_order: + one_time_order = Order.objects.create(owner=self.owner, + billing_address=billing_address, + starting_date=when_to_start, + price=self.one_time_price, + recurring_period=RecurringPeriod.objects.get(name="ONE_TIME"), + description=str(self)) + self.orders.add(one_time_order) + else: + one_time_order = None + + if recurring_period != RecurringPeriod.objects.get(name="ONE_TIME"): + if one_time_order: + recurring_order = Order.objects.create(owner=self.owner, + billing_address=billing_address, + starting_date=when_to_start, + price=self.recurring_price, + recurring_period=recurring_period, + depends_on=one_time_order, + description=str(self)) + else: + recurring_order = Order.objects.create(owner=self.owner, + billing_address=billing_address, + starting_date=when_to_start, + price=self.recurring_price, + recurring_period=recurring_period, + description=str(self)) + self.orders.add(recurring_order) + + + # FIXME: this could/should be part of Order (?) + def create_or_update_recurring_order(self, when_to_start=None, recurring_period=None): + if not self.recurring_price: + return + + if not recurring_period: + recurring_period = self.default_recurring_period + + if not when_to_start: + when_to_start = timezone.now() + + if self.last_recurring_order: + if self.recurring_price < self.last_recurring_order.price: + + if when_to_start < self.last_recurring_order.next_cancel_or_downgrade_date: + when_to_start = start_after(self.last_recurring_order.next_cancel_or_downgrade_date) + + when_to_end = end_before(when_to_start) + + new_order = Order.objects.create(owner=self.owner, + billing_address=self.last_recurring_order.billing_address, + starting_date=when_to_start, + price=self.recurring_price, + recurring_period=recurring_period, + description=str(self), + replaces=self.last_recurring_order) + + self.last_recurring_order.replace_with(new_order) + self.orders.add(new_order) + else: + self.create_order(when_to_start, recurring_period) + + @property + def is_recurring(self): + return self.recurring_price > 0 + + @property + def billing_address(self): + return self.order.billing_address + + def discounted_price_by_period(self, requested_period): + """ + Each product has a standard recurring period for which + we define a pricing. I.e. VPN is usually year, VM is usually monthly. + + The user can opt-in to use a different period, which influences the price: + The longer a user commits, the higher the discount. + + Products can also be limited in the available periods. For instance + a VPN only makes sense to be bought for at least one day. + + Rules are as follows: + + given a standard recurring period of ..., changing to ... modifies price ... + + + # One month for free if buying / year, compared to a month: about 8.33% discount + per_year -> per_month -> /11 + per_month -> per_year -> *11 + + # Month has 30.42 days on average. About 7.9% discount to go monthly + per_month -> per_day -> /28 + per_day -> per_month -> *28 + + # Day has 24h, give one for free + per_day -> per_hour -> /23 + per_hour -> per_day -> /23 + + + Examples + + VPN @ 120CHF/y becomes + - 10.91 CHF/month (130.91 CHF/year) + - 0.39 CHF/day (142.21 CHF/year) + + VM @ 15 CHF/month becomes + - 165 CHF/month (13.75 CHF/month) + - 0.54 CHF/day (16.30 CHF/month) + + """ + + # FIXME: This logic needs to be phased out / replaced by product specific (?) + # proportions. Maybe using the RecurringPeriod table to link the possible discounts/add ups + + if self.default_recurring_period == RecurringPeriod.PER_365D: + if requested_period == RecurringPeriod.PER_365D: + return self.recurring_price + if requested_period == RecurringPeriod.PER_30D: + return self.recurring_price/11. + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price/11./28. + + elif self.default_recurring_period == RecurringPeriod.PER_30D: + if requested_period == RecurringPeriod.PER_365D: + return self.recurring_price*11 + if requested_period == RecurringPeriod.PER_30D: + return self.recurring_price + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price/28. + + elif self.default_recurring_period == RecurringPeriod.PER_DAY: + if requested_period == RecurringPeriod.PER_365D: + return self.recurring_price*11*28 + if requested_period == RecurringPeriod.PER_30D: + return self.recurring_price*28 + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price + else: + # FIXME: use the right type of exception here! + raise Exception("Did not implement the discounter for this case") + + + def save(self, *args, **kwargs): + # try: + # ba = BillingAddress.get_address_for(self.owner) + # except BillingAddress.DoesNotExist: + # raise ValidationError("User does not have a billing address") + + # if not ba.active: + # raise ValidationError("User does not have an active billing address") + + + # Verify the required JSON fields + + super().save(*args, **kwargs) + + + +### +# Orders. + +class Order(models.Model): + """ + Order are assumed IMMUTABLE and used as SOURCE OF TRUST for generating + bills. Do **NOT** mutate then! + + An one time order is "closed" (does not need to be billed anymore) + if it has one bill record. Having more than one is a programming + error. + + A recurring order is closed if it has been replaced + (replaces__isnull=False) AND the ending_date is set AND it was + billed the last time it needed to be billed (how to check the last + item?) + + BOTH are closed, if they are ended/closed AND have been fully + charged. + + Fully charged == fully billed: sum_of_order_usage == sum_of_bill_records + + """ + + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE, + editable=True) + + billing_address = models.ForeignKey(BillingAddress, + on_delete=models.CASCADE) + + description = models.TextField() + + product = models.ForeignKey(Product, blank=False, null=False, on_delete=models.CASCADE) + config = models.JSONField() + + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField(default=timezone.now) + ending_date = models.DateTimeField(blank=True, null=True) + + recurring_period = models.ForeignKey(RecurringPeriod, + on_delete=models.CASCADE, + editable=True) + + one_time_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + recurring_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + currency = models.CharField(max_length=32, choices=Currency.choices, default=Currency.CHF) + + replaces = models.ForeignKey('self', + related_name='replaced_by', + on_delete=models.CASCADE, + blank=True, + null=True) + + depends_on = models.ForeignKey('self', + related_name='parent_of', + on_delete=models.CASCADE, + blank=True, + null=True) + + should_be_billed = models.BooleanField(default=True) + + @property + def earliest_ending_date(self): + """ + Recurring orders cannot end before finishing at least one recurring period. + + One time orders have a recurring period of 0, so this work universally + """ + + return self.starting_date + datetime.timedelta(seconds=self.recurring_period.duration_seconds) + + + def next_cancel_or_downgrade_date(self, until_when=None): + """ + Return the next proper ending date after n times the + recurring_period, where n is an integer that applies for downgrading + or cancelling. + """ + + if not until_when: + until_when = timezone.now() + + if until_when < self.starting_date: + raise ValidationError("Cannot end before start of start of order") + + if self.recurring_period.duration_seconds > 0: + delta = until_when - self.starting_date + + num_times = ceil(delta.total_seconds() / self.recurring_period.duration_seconds) + + next_date = self.starting_date + datetime.timedelta(seconds=num_times * self.recurring_period.duration_seconds) + else: + next_date = self.starting_date + + return next_date + + def get_ending_date_for_bill(self, bill): + """ + Determine the ending date given a specific bill + """ + + # If the order is quit, charge the final amount / finish (????) + # Probably not a good idea -- FIXME :continue until usual + if self.ending_date: + this_ending_date = self.ending_date + else: + if self.next_cancel_or_downgrade_date(bill.ending_date) > bill.ending_date: + this_ending_date = self.next_cancel_or_downgrade_date(bill.ending_date) + else: + this_ending_date = bill.ending_date + + return this_ending_date + + + @property + def count_billed(self): + """ + How many times this order was billed so far. + This logic is mainly thought to be for recurring bills, but also works for one time bills + """ + + return sum([ br.quantity for br in self.bill_records.all() ]) + + def count_used(self, when=None): + """ + How many times this order was billed so far. + This logic is mainly thought to be for recurring bills, but also works for one time bills + """ + + if self.is_one_time: + return 1 + + if not when: + when = timezone.now() + + # Cannot be used after it ended + if self.ending_date and when > self.ending_date: + when = self.ending_date + + return (when - self.starting_date) / self.default_recurring_period + + @property + def all_usage_billed(self, when=None): + """ + Returns true if this order does not need any further billing + ever. In other words: is this order "closed"? + """ + + if self.count_billed == self.count_used(when): + return True + else: + return False + + @property + def is_closed(self): + if self.all_usage_billed and self.ending_date: + return True + else: + return False + + @property + def is_recurring(self): + return not self.recurring_period == RecurringPeriod.objects.get(name="ONE_TIME") + + @property + def is_one_time(self): + return not self.is_recurring + + def replace_with(self, new_order): + new_order.replaces = self + self.ending_date = end_before(new_order.starting_date) + self.save() + + def update_order(self, config, starting_date=None): + """ + Updating an order means creating a new order and reference the previous order + """ + + if not starting_date: + starting_date = timezone.now() + + new_order = self.__class__(owner=self.owner, + billing_address=self.billing_address, + description=self.description, + product=self.product, + config=config, + starting_date=starting_date, + currency=self.currency + ) + + (new_order.one_time_price, new_order.recurring_price, new_order.config) = new_order.calculate_prices_and_config() + + + + new_order.replaces = self + new_order.save() + + self.ending_date = end_before(new_order.starting_date) + self.save() + + return new_order + + + def create_bill_record(self, bill): + br = None + + # Note: check for != 0 not > 0, as we allow discounts to be expressed with < 0 + if self.one_time_price != 0 and self.billrecord_set.count() == 0: + br = BillRecord.objects.create(bill=bill, + order=self, + starting_date=self.starting_date, + ending_date=self.starting_date, + is_recurring_record=False) + + if self.recurring_price != 0: + br = BillRecord.objects.filter(bill=bill, order=self, is_recurring_record=True).first() + + if br: + self.update_bill_record_for_recurring_order(br, bill) + else: + br = self.create_new_bill_record_for_recurring_order(bill) + + return br + + def update_bill_record_for_recurring_order(self, + bill_record, + bill): + """ + Possibly update a bill record according to the information in the bill + """ + + # If the order has an ending date set, we might need to adjust the bill_record + if self.ending_date: + if bill_record_for_this_bill.ending_date != self.ending_date: + bill_record_for_this_bill.ending_date = self.ending_date + + else: + # recurring, not terminated, should go until at least end of bill + if bill_record_for_this_bill.ending_date < bill.ending_date: + bill_record_for_this_bill.ending_date = bill.ending_date + + bill_record_for_this_bill.save() + + def create_new_bill_record_for_recurring_order(self, bill): + """ + Create a new bill record + """ + + last_bill_record = BillRecord.objects.filter(order=self, is_recurring_record=True).order_by('id').last() + + starting_date=self.starting_date + + if last_bill_record: + # We already charged beyond the end of this bill's period + if last_bill_record.ending_date >= bill.ending_date: + return + + # This order is terminated or replaced + if self.ending_date: + # And the last bill record already covered us -> nothing to be done anymore + if last_bill_record.ending_date == self.ending_date: + return + + starting_date = start_after(last_bill_record.ending_date) + + ending_date = self.get_ending_date_for_bill(bill) + + return BillRecord.objects.create(bill=bill, + order=self, + starting_date=starting_date, + ending_date=ending_date, + is_recurring_record=True) + + def calculate_prices_and_config(self): + one_time_price = 0 + recurring_price = 0 + + if self.config: + config = self.config + + if 'features' not in self.config: + self.config['features'] = {} + + else: + config = { + 'features': {} + } + + # FIXME: adjust prices to the selected recurring_period to the + + if 'features' in self.product.config: + for feature in self.product.config['features']: + + # Set min to 0 if not specified + min_val = self.product.config['features'][feature].get('min', 0) + + # We might not even have 'features' cannot use .get() on it + try: + value = self.config['features'][feature] + except (KeyError, TypeError): + value = self.product.config['features'][feature]['min'] + + # Set max to current value if not specified + max_val = self.product.config['features'][feature].get('max', value) + + + if value < min_val or value > max_val: + raise ValidationError(f"Feature '{feature}' must be at least {min_val} and at maximum {max_val}. Value is: {value}") + + one_time_price += self.product.config['features'][feature]['one_time_price_per_unit'] * value + recurring_price += self.product.config['features'][feature]['recurring_price_per_unit'] * value + config['features'][feature] = value + + return (one_time_price, recurring_price, config) + + def check_parameters(self): + if 'parameters' in self.product.config: + for parameter in self.product.config['parameters']: + if not parameter in self.config['parameters']: + raise ValidationError(f"Required parameter '{parameter}' is missing.") + + + def save(self, *args, **kwargs): + # Calculate the price of the order when we create it + # IMMUTABLE fields -- need to create new order to modify them + # However this is not enforced here... + if self._state.adding: + (self.one_time_price, self.recurring_price, self.config) = self.calculate_prices_and_config() + + if self.recurring_period_id is None: + self.recurring_period = self.product.default_recurring_period + + try: + prod_period = self.product.recurring_periods.get(producttorecurringperiod__recurring_period=self.recurring_period) + except ObjectDoesNotExist: + raise ValidationError(f"Recurring Period {self.recurring_period} not allowed for product {self.product}") + + self.check_parameters() + + if self.ending_date and self.ending_date < self.starting_date: + raise ValidationError("End date cannot be before starting date") + + + super().save(*args, **kwargs) + + + def __str__(self): + try: + conf = " ".join([ f"{key}:{val}" for key,val in self.config['features'].items() if val != 0 ]) + except KeyError: + conf = "" + + return f"Order {self.id}: {self.description} {conf}" + +class Bill(models.Model): + """ + A bill is a representation of usage at a specific time + """ + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField(default=start_of_this_month) + ending_date = models.DateTimeField() + due_date = models.DateField(default=default_payment_delay) + + + billing_address = models.ForeignKey(BillingAddress, + on_delete=models.CASCADE, + editable=True, + null=False) + + # FIXME: editable=True -> is in the admin, but also editable in DRF + # Maybe filter fields in the serializer? + + is_final = models.BooleanField(default=False) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['owner', + 'starting_date', + 'ending_date' ], + name='one_bill_per_month_per_user') + ] + + def close(self): + """ + Close/finish a bill + """ + + self.is_final = True + self.save() + + @property + def sum(self): + bill_records = BillRecord.objects.filter(bill=self) + return sum([ br.sum for br in bill_records ]) + + @property + def vat_rate(self): + """ + Handling VAT is a tricky business - thus we only implement the cases + that we clearly now and leave it open to fellow developers to implement + correct handling for other cases. + + Case CH: + + - If the customer is in .ch -> apply standard rate + - If the customer is in EU AND private -> apply country specific rate + - If the customer is in EU AND business -> do not apply VAT + - If the customer is outside EU and outside CH -> do not apply VAT + """ + + provider = UncloudProvider.objects.get() + + # Assume always VAT inside the country + if provider.country == self.billing_address.country: + vat_rate = VATRate.objects.get(country=provider.country, + when=self.ending_date) + elif self.billing_address.country in EU: + # FIXME: need to check for validated vat number + if self.billing_address.vat_number: + return 0 + else: + return VATRate.objects.get(country=self.biling_address.country, + when=self.ending_date) + else: # non-EU, non-national + return 0 + + + @classmethod + def create_bills_for_all_users(cls): + """ + Create next bill for each user + """ + + for owner in get_user_model().objects.all(): + cls.create_next_bills_for_user(owner) + + @classmethod + def create_next_bills_for_user(cls, owner, ending_date=None): + """ + Create one bill per billing address, as the VAT rates might be different + for each address + """ + + bills = [] + + for billing_address in BillingAddress.objects.filter(owner=owner): + bills.append(cls.create_next_bill_for_user_address(billing_address, ending_date)) + + return bills + + @classmethod + def create_next_bill_for_user_address(cls, billing_address, ending_date=None): + """ + Create the next bill for a specific billing address of a user + """ + + owner = billing_address.owner + + all_orders = Order.objects.filter(owner=owner, + billing_address=billing_address).order_by('id') + + bill = cls.get_or_create_bill(billing_address, ending_date=ending_date) + + for order in all_orders: + order.create_bill_record(bill) + + return bill + + + @classmethod + def get_or_create_bill(cls, billing_address, ending_date=None): + """ + Get / reuse last bill if it is not yet closed + + Create bill, if there is no bill or if bill is closed. + """ + + last_bill = cls.objects.filter(billing_address=billing_address).order_by('id').last() + + all_orders = Order.objects.filter(billing_address=billing_address).order_by('id') + first_order = all_orders.first() + + bill = None + + # Get date & bill from previous bill, if it exists + if last_bill: + if not last_bill.is_final: + bill = last_bill + starting_date = last_bill.starting_date + ending_date = bill.ending_date + else: + starting_date = last_bill.ending_date + datetime.timedelta(seconds=1) + else: + # Might be an idea to make this the start of the month, too + if first_order: + starting_date = first_order.starting_date + else: + starting_date = timezone.now() + + if not ending_date: + ending_date = end_of_month(starting_date) + + if not bill: + bill = cls.objects.create( + owner=billing_address.owner, + starting_date=starting_date, + ending_date=ending_date, + billing_address=billing_address) + + + return bill + + def __str__(self): + return f"Bill {self.owner}-{self.id}" + + +class BillRecord(models.Model): + """ + Entry of a bill, dynamically generated from an order. + """ + + bill = models.ForeignKey(Bill, on_delete=models.CASCADE) + order = models.ForeignKey(Order, on_delete=models.CASCADE) + + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField() + ending_date = models.DateTimeField() + + is_recurring_record = models.BooleanField(blank=False, null=False) + + @property + def quantity(self): + """ Determine the quantity by the duration""" + if not self.is_recurring_record: + return 1 + + record_delta = self.ending_date - self.starting_date + + return record_delta.total_seconds()/self.order.recurring_period.duration_seconds + + @property + def sum(self): + if self.is_recurring_record: + return self.order.recurring_price * Decimal(self.quantity) + else: + return self.order.one_time_price + + @property + def price(self): + if self.is_recurring_record: + return self.order.recurring_price + else: + return self.order.one_time_price + + def __str__(self): + if self.is_recurring_record: + bill_line = f"{self.starting_date} - {self.ending_date}: {self.quantity} x {self.order}" + else: + bill_line = f"{self.starting_date}: {self.order}" + + return bill_line + + def save(self, *args, **kwargs): + if self.ending_date < self.starting_date: + raise ValidationError("End date cannot be before starting date") + + super().save(*args, **kwargs) + + +class ProductToRecurringPeriod(models.Model): + """ + Intermediate manytomany mapping class that allows storing the default recurring period + for a product + """ + + recurring_period = models.ForeignKey(RecurringPeriod, on_delete=models.CASCADE) + product = models.ForeignKey(Product, on_delete=models.CASCADE) + + is_default = models.BooleanField(default=False) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['product'], + condition=Q(is_default=True), + name='one_default_recurring_period_per_product'), + models.UniqueConstraint(fields=['product', 'recurring_period'], + name='recurring_period_once_per_product') + ] + + def __str__(self): + return f"{self.product} - {self.recurring_period} (default: {self.is_default})" diff --git a/uncloud_pay/serializers.py b/uncloud_pay/serializers.py new file mode 100644 index 0000000..9214105 --- /dev/null +++ b/uncloud_pay/serializers.py @@ -0,0 +1,112 @@ +from django.contrib.auth import get_user_model +from rest_framework import serializers +from uncloud_auth.serializers import UserSerializer +from django.utils.translation import gettext_lazy as _ + +from .models import * + +### +# Payments and Payment Methods. + +class PaymentSerializer(serializers.ModelSerializer): + class Meta: + model = Payment + fields = '__all__' + +class PaymentMethodSerializer(serializers.ModelSerializer): + stripe_card_last4 = serializers.IntegerField() + + class Meta: + model = PaymentMethod + fields = ['uuid', 'source', 'description', 'primary', 'stripe_card_last4', 'active'] + +class UpdatePaymentMethodSerializer(serializers.ModelSerializer): + class Meta: + model = PaymentMethod + fields = ['description', 'primary'] + +class ChargePaymentMethodSerializer(serializers.Serializer): + amount = serializers.DecimalField(max_digits=10, decimal_places=2) + +class CreatePaymentMethodSerializer(serializers.ModelSerializer): + please_visit = serializers.CharField(read_only=True) + class Meta: + model = PaymentMethod + fields = ['source', 'description', 'primary', 'please_visit'] + +### +# Orders & Products. + +class OrderSerializer(serializers.ModelSerializer): + owner = serializers.PrimaryKeyRelatedField(queryset=get_user_model().objects.all()) + + def __init__(self, *args, **kwargs): + # Don't pass the 'fields' arg up to the superclass + admin = kwargs.pop('admin', None) + + # Instantiate the superclass normally + super(OrderSerializer, self).__init__(*args, **kwargs) + + # Only allows owner in admin mode. + if not admin: + self.fields.pop('owner') + + def create(self, validated_data): + billing_address = BillingAddress.get_preferred_address_for(validated_data["owner"]) + instance = Order(billing_address=billing_address, **validated_data) + instance.save() + + return instance + + def validate_owner(self, value): + if BillingAddress.get_preferred_address_for(value) == None: + raise serializers.ValidationError("Owner does not have a valid billing address.") + + return value + + class Meta: + model = Order + read_only_fields = ['replaced_by', 'depends_on'] + fields = ['uuid', 'owner', 'description', 'creation_date', 'starting_date', 'ending_date', + 'bill', 'recurring_period', 'recurring_price', 'one_time_price'] + read_only_fields + + +### +# Bills + +# TODO: remove magic numbers for decimal fields +class BillRecordSerializer(serializers.Serializer): + order = serializers.HyperlinkedRelatedField( + view_name='order-detail', + read_only=True) + description = serializers.CharField() + one_time_price = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + recurring_price = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) +# recurring_period = serializers.ChoiceField() + recurring_count = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + vat_rate = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + vat_amount = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + amount = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + total = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + +class BillingAddressSerializer(serializers.ModelSerializer): + class Meta: + model = BillingAddress + fields = ['uuid', 'organization', 'name', 'street', 'city', 'postal_code', 'country', 'vat_number'] + +class BillSerializer(serializers.ModelSerializer): + billing_address = BillingAddressSerializer(read_only=True) + records = BillRecordSerializer(many=True, read_only=True) + + class Meta: + model = Bill + fields = ['uuid', 'reference', 'owner', 'amount', 'vat_amount', 'total', + 'due_date', 'creation_date', 'starting_date', 'ending_date', + 'records', 'final', 'billing_address'] + +# We do not want users to mutate the country / VAT number of an address, as it +# will change VAT on existing bills. +class UpdateBillingAddressSerializer(serializers.ModelSerializer): + class Meta: + model = BillingAddress + fields = ['uuid', 'street', 'city', 'postal_code'] diff --git a/uncloud_pay/stripe.py b/uncloud_pay/stripe.py new file mode 100644 index 0000000..2ed4ef2 --- /dev/null +++ b/uncloud_pay/stripe.py @@ -0,0 +1,114 @@ +import stripe +import stripe.error +import logging + +from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings + +import uncloud_pay.models + +# Static stripe configuration used below. +CURRENCY = 'chf' + +# README: We use the Payment Intent API as described on +# https://stripe.com/docs/payments/save-and-reuse + +# For internal use only. +stripe.api_key = settings.STRIPE_KEY + +# Helper (decorator) used to catch errors raised by stripe logic. +# Catch errors that should not be displayed to the end user, raise again. +def handle_stripe_error(f): + def handle_problems(*args, **kwargs): + response = { + 'paid': False, + 'response_object': None, + 'error': None + } + + common_message = "Currently it is not possible to make payments. Please try agin later." + try: + response_object = f(*args, **kwargs) + return response_object + except stripe.error.CardError as e: + # Since it's a decline, stripe.error.CardError will be caught + body = e.json_body + logging.error(str(e)) + + raise e # For error handling. + except stripe.error.RateLimitError: + logging.error("Too many requests made to the API too quickly.") + raise Exception(common_message) + except stripe.error.InvalidRequestError as e: + logging.error(str(e)) + raise Exception('Invalid parameters.') + except stripe.error.AuthenticationError as e: + # Authentication with Stripe's API failed + # (maybe you changed API keys recently) + logging.error(str(e)) + raise Exception(common_message) + except stripe.error.APIConnectionError as e: + logging.error(str(e)) + raise Exception(common_message) + except stripe.error.StripeError as e: + # XXX: maybe send email + logging.error(str(e)) + raise Exception(common_message) + except Exception as e: + # maybe send email + logging.error(str(e)) + raise Exception(common_message) + + return handle_problems + +# Actual Stripe logic. + +def public_api_key(): + return settings.STRIPE_PUBLIC_KEY + +def get_customer_id_for(user): + try: + # .get() raise if there is no matching entry. + return uncloud_pay.models.StripeCustomer.objects.get(owner=user).stripe_id + except ObjectDoesNotExist: + # No entry yet - making a new one. + try: + customer = create_customer(user.username, user.email) + uncloud_stripe_mapping = uncloud_pay.models.StripeCustomer.objects.create( + owner=user, stripe_id=customer.id) + return uncloud_stripe_mapping.stripe_id + except Exception as e: + return None + +@handle_stripe_error +def create_setup_intent(customer_id): + return stripe.SetupIntent.create(customer=customer_id) + +@handle_stripe_error +def get_setup_intent(setup_intent_id): + return stripe.SetupIntent.retrieve(setup_intent_id) + +def get_payment_method(payment_method_id): + return stripe.PaymentMethod.retrieve(payment_method_id) + +@handle_stripe_error +def charge_customer(amount, customer_id, card_id): + # Amount is in CHF but stripes requires smallest possible unit. + # https://stripe.com/docs/api/payment_intents/create#create_payment_intent-amount + adjusted_amount = int(amount * 100) + return stripe.PaymentIntent.create( + amount=adjusted_amount, + currency=CURRENCY, + customer=customer_id, + payment_method=card_id, + off_session=True, + confirm=True, + ) + +@handle_stripe_error +def create_customer(name, email): + return stripe.Customer.create(name=name, email=email) + +@handle_stripe_error +def get_customer(customer_id): + return stripe.Customer.retrieve(customer_id) diff --git a/uncloud_pay/templates/bill.html.j2 b/uncloud_pay/templates/bill.html.j2 new file mode 100644 index 0000000..c227f43 --- /dev/null +++ b/uncloud_pay/templates/bill.html.j2 @@ -0,0 +1,1061 @@ +{% load static %} + + + + + + + + + {{ bill }} + + + + + + +
+ +
+
+ ungleich glarus ag +
Bahnhofstrasse 1 +
8783 Linthal +
Switzerland +
+
+
+ {{ bill.billing_address.organization }}
+ {{ bill.billing_address.name }}
+ {{ bill.owner.email }}
+ {{ bill.billing_address.street }}
+ {{ bill.billing_address.country }} {{ bill.billing_address.postal_code }} {{ bill.billing_address.city }}
+ +
+
+
+ {{ bill.starting_date|date:"c" }} - + {{ bill.ending_date|date:"c" }} +
Bill id: {{ bill }} +
Due: {{ bill.due_date }} + +
+
+
+
+

Invoice

+
+ + + + + + + + + + + {% for record in bill_records %} + + + + + + + {% endfor %} + +
DetailPrice/UnitUnitsTotal price
{{ record.starting_date|date:"c" }} + - {{ record.ending_date|date:"c" }} + {{ record.order }} + {{ record.price|floatformat:2 }}{{ record.quantity|floatformat:2 }}{{ record.sum|floatformat:2 }}
+
+

+ Total (excl. VAT) + {{ bill.amount }} +

+

+ VAT 7.7% + {{ bill.vat_amount|floatformat:2 }} +

+
+
+

+ Total amount to be paid + {{ bill.sum|floatformat:2 }} +

+
+
+
+

+ +41 (0) 55 505-62-66

+ +

+ buchhaltung-ag@ungleich.ch

+ +

+ https://ungleich.ch

+

+ @ungleich

+
+
+

Glarner Kantonalbank

+

+ IBAN: CH 4300 7730 0055 5931 177 +

+

+ BIC: GLKBCH22 +

+

Mwst-Nummer: CHE-156.970.649 MWST

+
+
+ + + diff --git a/uncloud_pay/templates/error.html.j2 b/uncloud_pay/templates/error.html.j2 new file mode 100644 index 0000000..ba9209c --- /dev/null +++ b/uncloud_pay/templates/error.html.j2 @@ -0,0 +1,18 @@ + + + + Error + + + +
+

Error

+

{{ error }}

+
+ + diff --git a/uncloud_pay/templates/stripe-payment.html.j2 b/uncloud_pay/templates/stripe-payment.html.j2 new file mode 100644 index 0000000..6c59740 --- /dev/null +++ b/uncloud_pay/templates/stripe-payment.html.j2 @@ -0,0 +1,76 @@ + + + + Stripe Card Registration + + + + + + + + +
+

Registering Stripe Credit Card

+ + + +
+
+ +
+ + +
+
+ + + + + + + + diff --git a/uncloud_pay/templates/uncloud_pay/stripe.html b/uncloud_pay/templates/uncloud_pay/stripe.html new file mode 100644 index 0000000..3051bf0 --- /dev/null +++ b/uncloud_pay/templates/uncloud_pay/stripe.html @@ -0,0 +1,72 @@ +{% extends 'uncloud/base.html' %} + +{% block header %} + + +{% endblock %} + +{% block body %} +
+

Registering Stripe Credit Card

+ + + +
+
+ +
+ + +
+
+ + + + + + + +{% endblock %} diff --git a/uncloud_pay/tests.py b/uncloud_pay/tests.py new file mode 100644 index 0000000..ca91cc9 --- /dev/null +++ b/uncloud_pay/tests.py @@ -0,0 +1,465 @@ +from django.test import TestCase +from django.contrib.auth import get_user_model +from datetime import datetime, date, timedelta +from django.utils import timezone + +from .models import * +from uncloud_service.models import GenericServiceProduct + +import json + +chocolate_product_config = { + 'features': { + 'gramm': + { 'min': 100, + 'max': 5000, + 'one_time_price_per_unit': 0.2, + 'recurring_price_per_unit': 0 + }, + }, +} + +chocolate_order_config = { + 'features': { + 'gramm': 500, + } +} + +chocolate_one_time_price = chocolate_order_config['features']['gramm'] * chocolate_product_config['features']['gramm']['one_time_price_per_unit'] + +vm_product_config = { + 'features': { + 'cores': + { 'min': 1, + 'max': 48, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 4 + }, + 'ram_gb': + { 'min': 1, + 'max': 256, + 'one_time_price_per_unit': 0, + 'recurring_price_per_unit': 4 + }, + }, +} + +vm_order_config = { + 'features': { + 'cores': 2, + 'ram_gb': 2 + } +} + +vm_order_downgrade_config = { + 'features': { + 'cores': 1, + 'ram_gb': 1 + } +} + +vm_order_upgrade_config = { + 'features': { + 'cores': 4, + 'ram_gb': 4 + } +} + + +class ProductTestCase(TestCase): + """ + Test products and products <-> order interaction + """ + + def setUp(self): + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + + self.ba = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="somewhere else", + active=True) + + RecurringPeriod.populate_db_defaults() + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + def test_create_product(self): + """ + Create a sample product + """ + + p = Product.objects.create(name="Testproduct", + description="Only for testing", + config=vm_product_config) + + p.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + +class OrderTestCase(TestCase): + """ + The heart of ordering products + """ + + def setUp(self): + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + + self.ba = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="somewhere else", + active=True) + + self.product = Product.objects.create(name="Testproduct", + description="Only for testing", + config=vm_product_config) + + RecurringPeriod.populate_db_defaults() + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + self.product.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + + def test_order_invalid_recurring_period(self): + """ + Order a products with a recurringperiod that is not added to the product + """ + + o = Order.objects.create(owner=self.user, + billing_address=self.ba, + product=self.product, + config=vm_order_config) + + + def test_order_product(self): + """ + Order a product, ensure the order has correct price setup + """ + + o = Order.objects.create(owner=self.user, + billing_address=self.ba, + product=self.product) + + self.assertEqual(o.one_time_price, 0) + self.assertEqual(o.recurring_price, 16) + + def test_change_order(self): + """ + Change an order and ensure that + - a new order is created + - the price is correct in the new order + """ + order1 = Order.objects.create(owner=self.user, + billing_address=self.ba, + product=self.product, + config=vm_order_config) + + + self.assertEqual(order1.one_time_price, 0) + self.assertEqual(order1.recurring_price, 16) + + +class ModifyOrderTestCase(TestCase): + """ + Test typical order flows like + - cancelling + - downgrading + - upgrading + """ + + def setUp(self): + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + + self.ba = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="somewhere else", + active=True) + + self.product = Product.objects.create(name="Testproduct", + description="Only for testing", + config=vm_product_config) + + RecurringPeriod.populate_db_defaults() + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + self.product.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + + def test_change_order(self): + """ + Test changing an order + + Expected result: + + - Old order should be closed before new order starts + - New order should start at starting data + """ + + user = self.user + + starting_price = 16 + downgrade_price = 8 + + starting_date = timezone.make_aware(datetime.datetime(2019,3,3)) + ending1_date = starting_date + datetime.timedelta(days=15) + change1_date = start_after(ending1_date) + + bill_ending_date = change1_date + datetime.timedelta(days=1) + + + order1 = Order.objects.create(owner=self.user, + billing_address=BillingAddress.get_address_for(self.user), + product=self.product, + config=vm_order_config, + starting_date=starting_date) + + order1.update_order(vm_order_downgrade_config, starting_date=change1_date) + + bills = Bill.create_next_bills_for_user(user, ending_date=bill_ending_date) + + bill = bills[0] + bill_records = BillRecord.objects.filter(bill=bill) + + self.assertEqual(len(bill_records), 2) + + self.assertEqual(bill_records[0].starting_date, starting_date) + self.assertEqual(bill_records[0].ending_date, ending1_date) + + self.assertEqual(bill_records[1].starting_date, change1_date) + + + + def test_downgrade_product(self): + """ + Test downgrading behaviour: + + We create a recurring product (recurring time: 30 days) and downgrade after 15 days. + + We create the bill right AFTER the end of the first order. + + Expected result: + + - First bill record for 30 days + - Second bill record starting after 30 days + - Bill contains two bill records + + """ + + user = self.user + + starting_price = 16 + downgrade_price = 8 + + starting_date = timezone.make_aware(datetime.datetime(2019,3,3)) + first_order_should_end_at = starting_date + datetime.timedelta(days=30) + change1_date = start_after(starting_date + datetime.timedelta(days=15)) + bill_ending_date = change1_date + datetime.timedelta(days=1) + + order1 = Order.objects.create(owner=self.user, + billing_address=BillingAddress.get_address_for(self.user), + product=self.product, + config=vm_order_config, + starting_date=starting_date) + + order1.update_order(vm_order_downgrade_config, starting_date=change1_date) + + bills = Bill.create_next_bills_for_user(user, ending_date=bill_ending_date) + + bill = bills[0] + bill_records = BillRecord.objects.filter(bill=bill) + + self.assertEqual(len(bill_records), 2) + + self.assertEqual(bill_records[0].starting_date, starting_date) + self.assertEqual(bill_records[0].order.ending_date, first_order_should_end_at) + + +class BillTestCase(TestCase): + """ + Test aspects of billing / creating a bill + """ + + def setUp(self): + RecurringPeriod.populate_db_defaults() + + self.user_without_address = get_user_model().objects.create( + username='no_home_person', + email='far.away@domain.tld') + + self.user = get_user_model().objects.create( + username='jdoe', + email='john.doe@domain.tld') + + self.recurring_user = get_user_model().objects.create( + username='recurrent_product_user', + email='jane.doe@domain.tld') + + self.user_addr = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="unknown", + active=True) + + self.recurring_user_addr = BillingAddress.objects.create( + owner=self.recurring_user, + organization = 'Test org', + street="Somewhere", + city="Else", + postal_code="unknown", + active=True) + + self.order_meta = {} + self.order_meta[1] = { + 'starting_date': timezone.make_aware(datetime.datetime(2020,3,3)), + 'ending_date': timezone.make_aware(datetime.datetime(2020,4,17)), + 'price': 15, + 'description': 'One chocolate bar' + } + + self.chocolate = Product.objects.create(name="Swiss Chocolate", + description="Not only for testing, but for joy", + config=chocolate_product_config) + + + self.vm = Product.objects.create(name="Super Fast VM", + description="Zooooom", + config=vm_product_config) + + + RecurringPeriod.populate_db_defaults() + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + self.onetime_recurring_period = RecurringPeriod.objects.get(name="Onetime") + + self.chocolate.recurring_periods.add(self.onetime_recurring_period, + through_defaults= { 'is_default': True }) + + self.vm.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + + # used for generating multiple bills + self.bill_dates = [ + timezone.make_aware(datetime.datetime(2020,3,31)), + timezone.make_aware(datetime.datetime(2020,4,30)), + timezone.make_aware(datetime.datetime(2020,5,31)), + ] + + + def order_chocolate(self): + return Order.objects.create( + owner=self.user, + recurring_period=RecurringPeriod.objects.get(name="Onetime"), + product=self.chocolate, + billing_address=BillingAddress.get_address_for(self.user), + starting_date=self.order_meta[1]['starting_date'], + ending_date=self.order_meta[1]['ending_date'], + config=chocolate_order_config) + + def order_vm(self, owner=None): + + if not owner: + owner = self.recurring_user + + return Order.objects.create( + owner=owner, + product=self.vm, + config=vm_order_config, + billing_address=BillingAddress.get_address_for(self.recurring_user), + starting_date=timezone.make_aware(datetime.datetime(2020,3,3)), + ) + + return Order.objects.create( + owner=self.user, + recurring_period=RecurringPeriod.objects.get(name="Onetime"), + product=self.chocolate, + billing_address=BillingAddress.get_address_for(self.user), + starting_date=self.order_meta[1]['starting_date'], + ending_date=self.order_meta[1]['ending_date'], + config=chocolate_order_config) + + + + def test_bill_one_time_one_bill_record(self): + """ + Ensure there is only 1 bill record per order + """ + + order = self.order_chocolate() + + bill = Bill.create_next_bill_for_user_address(self.user_addr) + + self.assertEqual(order.billrecord_set.count(), 1) + + def test_bill_sum_onetime(self): + """ + Check the bill sum for a single one time order + """ + + order = self.order_chocolate() + bill = Bill.create_next_bill_for_user_address(self.user_addr) + self.assertEqual(bill.sum, chocolate_one_time_price) + + + def test_bill_creates_record_for_recurring_order(self): + """ + Ensure there is only 1 bill record per order + """ + + order = self.order_vm() + bill = Bill.create_next_bill_for_user_address(self.recurring_user_addr) + + self.assertEqual(order.billrecord_set.count(), 1) + self.assertEqual(bill.billrecord_set.count(), 1) + + + def test_new_bill_after_closing(self): + """ + After closing a bill and the user has a recurring product, + the next bill run should create e new bill + """ + + order = self.order_vm() + + for ending_date in self.bill_dates: + b = Bill.create_next_bill_for_user_address(self.recurring_user_addr, ending_date) + b.close() + + bill_count = Bill.objects.filter(owner=self.recurring_user).count() + + self.assertEqual(len(self.bill_dates), bill_count) + + + +class BillingAddressTestCase(TestCase): + def setUp(self): + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + + + def test_user_no_address(self): + """ + Raise an error, when there is no address + """ + + self.assertRaises(uncloud_pay.models.BillingAddress.DoesNotExist, + BillingAddress.get_address_for, + self.user) diff --git a/uncloud_pay/views.py b/uncloud_pay/views.py new file mode 100644 index 0000000..53d6ef4 --- /dev/null +++ b/uncloud_pay/views.py @@ -0,0 +1,398 @@ +from django.contrib.auth.mixins import LoginRequiredMixin +from django.views.generic.base import TemplateView + + +from django.shortcuts import render +from django.db import transaction +from django.contrib.auth import get_user_model +from rest_framework import viewsets, mixins, permissions, status, views +from rest_framework.renderers import TemplateHTMLRenderer +from rest_framework.response import Response +from rest_framework.decorators import action +from rest_framework.reverse import reverse +from rest_framework.decorators import renderer_classes +from vat_validator import validate_vat, vies +from vat_validator.countries import EU_COUNTRY_CODES +from hardcopy import bytestring_to_pdf +from django.core.files.temp import NamedTemporaryFile +from django.http import FileResponse +from django.template.loader import render_to_string +from copy import deepcopy + +import json +import logging + +from .models import * +from .serializers import * +from datetime import datetime +from vat_validator import sanitize_vat +import uncloud_pay.stripe as uncloud_stripe + +logger = logging.getLogger(__name__) + +### +# Payments and Payment Methods. + +class PaymentViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = PaymentSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Payment.objects.filter(owner=self.request.user) + +class OrderViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = OrderSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Order.objects.filter(owner=self.request.user) + + +class RegisterCard(LoginRequiredMixin, TemplateView): + login_url = '/login/' + + # This is not supposed to be "static" -- + # the idea is to be able to switch the provider when needed + template_name = "uncloud_pay/stripe.html" + + def get_context_data(self, **kwargs): + customer_id = uncloud_stripe.get_customer_id_for(self.request.user) + setup_intent = uncloud_stripe.create_setup_intent(customer_id) + + context = super().get_context_data(**kwargs) + context['client_secret'] = setup_intent.client_secret + context['username'] = self.request.user + context['stripe_pk'] = uncloud_stripe.public_api_key + return context + + +class PaymentMethodViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_serializer_class(self): + if self.action == 'create': + return CreatePaymentMethodSerializer + elif self.action == 'update': + return UpdatePaymentMethodSerializer + elif self.action == 'charge': + return ChargePaymentMethodSerializer + else: + return PaymentMethodSerializer + + def get_queryset(self): + return PaymentMethod.objects.filter(owner=self.request.user) + + # XXX: Handling of errors is far from great down there. + @transaction.atomic + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Set newly created method as primary if no other method is. + if PaymentMethod.get_primary_for(request.user) == None: + serializer.validated_data['primary'] = True + + if serializer.validated_data['source'] == "stripe": + # Retrieve Stripe customer ID for user. + customer_id = uncloud_stripe.get_customer_id_for(request.user) + if customer_id == None: + return Response( + {'error': 'Could not resolve customer stripe ID.'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + try: + setup_intent = uncloud_stripe.create_setup_intent(customer_id) + except Exception as e: + return Response({'error': str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + payment_method = PaymentMethod.objects.create( + owner=request.user, + stripe_setup_intent_id=setup_intent.id, + **serializer.validated_data) + + # TODO: find a way to use reverse properly: + # https://www.django-rest-framework.org/api-guide/reverse/ + path = "payment-method/{}/register-stripe-cc".format( + payment_method.uuid) + stripe_registration_url = reverse('api-root', request=request) + path + return Response({'please_visit': stripe_registration_url}) + else: + serializer.save(owner=request.user, **serializer.validated_data) + return Response(serializer.data) + + @action(detail=True, methods=['post']) + def charge(self, request, pk=None): + payment_method = self.get_object() + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + amount = serializer.validated_data['amount'] + try: + payment = payment_method.charge(amount) + output_serializer = PaymentSerializer(payment) + return Response(output_serializer.data) + except Exception as e: + return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=True, methods=['get'], url_path='register-stripe-cc', renderer_classes=[TemplateHTMLRenderer]) + def register_stripe_cc(self, request, pk=None): + payment_method = self.get_object() + + if payment_method.source != 'stripe': + return Response( + {'error': 'This is not a Stripe-based payment method.'}, + template_name='error.html.j2') + + if payment_method.active: + return Response( + {'error': 'This payment method is already active'}, + template_name='error.html.j2') + + try: + setup_intent = uncloud_stripe.get_setup_intent( + payment_method.stripe_setup_intent_id) + except Exception as e: + return Response( + {'error': str(e)}, + template_name='error.html.j2') + + # TODO: find a way to use reverse properly: + # https://www.django-rest-framework.org/api-guide/reverse/ + callback_path= "payment-method/{}/activate-stripe-cc/".format( + payment_method.uuid) + callback = reverse('api-root', request=request) + callback_path + + # Render stripe card registration form. + template_args = { + 'client_secret': setup_intent.client_secret, + 'stripe_pk': uncloud_stripe.public_api_key, + 'callback': callback + } + return Response(template_args, template_name='stripe-payment.html.j2') + + @action(detail=True, methods=['post'], url_path='activate-stripe-cc') + def activate_stripe_cc(self, request, pk=None): + payment_method = self.get_object() + try: + setup_intent = uncloud_stripe.get_setup_intent( + payment_method.stripe_setup_intent_id) + except Exception as e: + return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + # Card had been registered, fetching payment method. + print(setup_intent) + if setup_intent.payment_method: + payment_method.stripe_payment_method_id = setup_intent.payment_method + payment_method.save() + + return Response({ + 'uuid': payment_method.uuid, + 'activated': payment_method.active}) + else: + error = 'Could not fetch payment method from stripe. Please try again.' + return Response({'error': error}) + + @action(detail=True, methods=['post'], url_path='set-as-primary') + def set_as_primary(self, request, pk=None): + payment_method = self.get_object() + payment_method.set_as_primary_for(request.user) + + serializer = self.get_serializer(payment_method) + return Response(serializer.data) + +### +# Bills and Orders. + +class BillViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = BillSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Bill.objects.filter(owner=self.request.user) + + + @action(detail=False, methods=['get']) + def unpaid(self, request): + serializer = self.get_serializer( + Bill.get_unpaid_for(self.request.user), + many=True) + return Response(serializer.data) + + @action(detail=True, methods=['get']) + def download(self, *args, **kwargs): + """ + Allow to download + """ + bill = self.get_object() + provider = UncloudProvider.get_provider() + output_file = NamedTemporaryFile() + bill_html = render_to_string("bill.html.j2", {'bill': bill}) + + bytestring_to_pdf(bill_html.encode('utf-8'), output_file) + response = FileResponse(output_file, content_type="application/pdf") + response['Content-Disposition'] = 'filename="{}_{}.pdf"'.format( + bill.reference, bill.uuid + ) + + return response + + +class OrderViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = OrderSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Order.objects.filter(owner=self.request.user) + +class BillingAddressViewSet(mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_serializer_class(self): + if self.action == 'update': + return UpdateBillingAddressSerializer + else: + return BillingAddressSerializer + + def get_queryset(self): + return self.request.user.billingaddress_set.all() + + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Validate VAT numbers. + country = serializer.validated_data["country"] + + # We ignore empty VAT numbers. + if 'vat_number' in serializer.validated_data and serializer.validated_data["vat_number"] != "": + vat_number = serializer.validated_data["vat_number"] + + if not validate_vat(country, vat_number): + return Response( + {'error': 'Malformed VAT number.'}, + status=status.HTTP_400_BAD_REQUEST) + elif country in EU_COUNTRY_CODES: + # XXX: make a synchroneous call to a third patry API here might not be a good idea.. + try: + vies_state = vies.check_vat(country, vat_number) + if not vies_state.valid: + return Response( + {'error': 'European VAT number does not exist in VIES.'}, + status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + logger.warning(e) + return Response( + {'error': 'Could not validate EU VAT number against VIES. Try again later..'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + + serializer.save(owner=request.user) + return Response(serializer.data) + +### +# Admin stuff. + +class AdminPaymentViewSet(viewsets.ModelViewSet): + serializer_class = PaymentSerializer + permission_classes = [permissions.IsAdminUser] + + def get_queryset(self): + return Payment.objects.all() + + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save(timestamp=datetime.now()) + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + +# Bills are generated from orders and should not be created or updated by hand. +class AdminBillViewSet(BillViewSet): + serializer_class = BillSerializer + permission_classes = [permissions.IsAdminUser] + + def get_queryset(self): + return Bill.objects.all() + + @action(detail=False, methods=['get']) + def unpaid(self, request): + unpaid_bills = [] + # XXX: works but we can do better than number of users + 1 SQL requests... + for user in get_user_model().objects.all(): + unpaid_bills = unpaid_bills + Bill.get_unpaid_for(self.request.user) + + serializer = self.get_serializer(unpaid_bills, many=True) + return Response(serializer.data) + + @action(detail=False, methods=['post']) + def generate(self, request): + users = get_user_model().objects.all() + + generated_bills = [] + for user in users: + now = timezone.now() + generated_bills = generated_bills + Bill.generate_for( + year=now.year, + month=now.month, + user=user) + + return Response( + map(lambda b: b.reference, generated_bills), + status=status.HTTP_200_OK) + +class AdminOrderViewSet(mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.UpdateModelMixin, + viewsets.GenericViewSet): + serializer_class = OrderSerializer + permission_classes = [permissions.IsAdminUser] + + def get_serializer(self, *args, **kwargs): + return self.serializer_class(*args, **kwargs, admin=True) + + def get_queryset(self): + return Order.objects.all() + + # Updates create a new order and terminate the 'old' one. + @transaction.atomic + def update(self, request, *args, **kwargs): + order = self.get_object() + partial = kwargs.pop('partial', False) + serializer = self.get_serializer(order, data=request.data, partial=partial) + serializer.is_valid(raise_exception=True) + + # Clone existing order for replacement. + replacing_order = deepcopy(order) + + # Yes, that's how you make a new entry in DB: + # https://docs.djangoproject.com/en/3.0/topics/db/queries/#copying-model-instances + replacing_order.pk = None + + for attr, value in serializer.validated_data.items(): + setattr(replacing_order, attr, value) + + # Save replacing order and terminate 'previous' one. + replacing_order.save() + order.replaced_by = replacing_order + order.save() + order.terminate() + + return Response(replacing_order) + + @action(detail=True, methods=['post']) + def terminate(self, request, pk): + order = self.get_object() + if order.is_terminated: + return Response( + {'error': 'Order is already terminated.'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + else: + order.terminate() + return Response({}, status=status.HTTP_200_OK) diff --git a/uncloud_service/__init__.py b/uncloud_service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_service/admin.py b/uncloud_service/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_service/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_service/apps.py b/uncloud_service/apps.py new file mode 100644 index 0000000..190bd35 --- /dev/null +++ b/uncloud_service/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UngleichServiceConfig(AppConfig): + name = 'uncloud_service' diff --git a/uncloud_service/models.py b/uncloud_service/models.py new file mode 100644 index 0000000..a37e42b --- /dev/null +++ b/uncloud_service/models.py @@ -0,0 +1,63 @@ +from django.db import models +from uncloud_pay.models import Product, RecurringPeriod, AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS +from uncloud_vm.models import VMProduct, VMDiskImageProduct +from django.core.validators import MinValueValidator + +class MatrixServiceProduct(models.Model): + monthly_managment_fee = 20 + + description = "Managed Matrix HomeServer" + + # Specific to Matrix-as-a-Service + vm = models.ForeignKey( + VMProduct, on_delete=models.CASCADE + ) + domain = models.CharField(max_length=255, default='domain.tld') + + # Default recurring price is PER_MONT, see Product class. + # def recurring_price(self, recurring_period=RecurringPeriod.PER_30D): + # return self.monthly_managment_fee + + @staticmethod + def base_image(): + # TODO: find a way to safely reference debian 10 image. +#e return VMDiskImageProduct.objects.get(uuid="93e564c5-adb3-4741-941f-718f76075f02") + return False + + # @staticmethod + # def allowed_recurring_periods(): + # return list(filter( + # lambda pair: pair[0] in [RecurringPeriod.PER_30D], + # RecurringPeriod.choices)) + + @property + def one_time_price(self): + return 30 + +class GenericServiceProduct(models.Model): + custom_description = models.TextField() + custom_recurring_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + custom_one_time_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + @property + def recurring_price(self): + # FIXME: handle recurring_period somehow. + return self.custom_recurring_price + + @property + def description(self): + return self.custom_description + + @property + def one_time_price(self): + return self.custom_one_time_price + + @staticmethod + def allowed_recurring_periods(): + return RecurringPeriod.choices diff --git a/uncloud_service/serializers.py b/uncloud_service/serializers.py new file mode 100644 index 0000000..bc6d753 --- /dev/null +++ b/uncloud_service/serializers.py @@ -0,0 +1,60 @@ +from rest_framework import serializers +from .models import * +from uncloud_vm.serializers import ManagedVMProductSerializer +from uncloud_vm.models import VMProduct +from uncloud_pay.models import RecurringPeriod, BillingAddress + +# XXX: the OrderSomethingSomthingProductSerializer classes add a lot of +# boilerplate: can we reduce it somehow? + +class MatrixServiceProductSerializer(serializers.ModelSerializer): + vm = ManagedVMProductSerializer() + + class Meta: + model = MatrixServiceProduct + fields = ['order', 'owner', 'status', 'vm', 'domain', + 'recurring_period'] + read_only_fields = ['order', 'owner', 'status'] + +class OrderMatrixServiceProductSerializer(MatrixServiceProductSerializer): + # recurring_period = serializers.ChoiceField( + # choices=MatrixServiceProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(OrderMatrixServiceProductSerializer, self).__init__(*args, **kwargs) + self.fields['billing_address'] = serializers.ChoiceField( + choices=BillingAddress.get_addresses_for( + self.context['request'].user) + ) + + class Meta: + model = MatrixServiceProductSerializer.Meta.model + fields = MatrixServiceProductSerializer.Meta.fields + [ + 'recurring_period', 'billing_address' + ] + read_only_fields = MatrixServiceProductSerializer.Meta.read_only_fields + +class GenericServiceProductSerializer(serializers.ModelSerializer): + class Meta: + model = GenericServiceProduct + fields = ['order', 'owner', 'status', 'custom_recurring_price', + 'custom_description', 'custom_one_time_price'] + read_only_fields = [ 'owner', 'status'] + +class OrderGenericServiceProductSerializer(GenericServiceProductSerializer): + # recurring_period = serializers.ChoiceField( + # choices=GenericServiceProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(OrderGenericServiceProductSerializer, self).__init__(*args, **kwargs) + self.fields['billing_address'] = serializers.ChoiceField( + choices=BillingAddress.get_addresses_for( + self.context['request'].user) + ) + + class Meta: + model = GenericServiceProductSerializer.Meta.model + fields = GenericServiceProductSerializer.Meta.fields + [ + 'recurring_period', 'billing_address' + ] + read_only_fields = GenericServiceProductSerializer.Meta.read_only_fields diff --git a/uncloud_service/tests.py b/uncloud_service/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/uncloud_service/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/uncloud_service/views.py b/uncloud_service/views.py new file mode 100644 index 0000000..abd4a05 --- /dev/null +++ b/uncloud_service/views.py @@ -0,0 +1,128 @@ +from rest_framework import viewsets, permissions +from rest_framework.response import Response +from django.db import transaction +from django.utils import timezone + +from .models import * +from .serializers import * + +from uncloud_pay.helpers import ProductViewSet +from uncloud_pay.models import Order +from uncloud_vm.models import VMProduct, VMDiskProduct + +def create_managed_vm(cores, ram, disk_size, image, order): + # Create VM + disk = VMDiskProduct( + owner=order.owner, + order=order, + size_in_gb=disk_size, + image=image) + vm = VMProduct( + name="Managed Service Host", + owner=order.owner, + cores=cores, + ram_in_gb=ram, + primary_disk=disk) + disk.vm = vm + + vm.save() + disk.save() + + return vm + + +class MatrixServiceProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = MatrixServiceProductSerializer + + def get_queryset(self): + return MatrixServiceProduct.objects.filter(owner=self.request.user) + + def get_serializer_class(self): + if self.action == 'create': + return OrderMatrixServiceProductSerializer + else: + return MatrixServiceProductSerializer + + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + order_billing_address = serializer.validated_data.pop("billing_address") + + # Create base order.) + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user, + billing_address=order_billing_address, + starting_date=timezone.now() + ) + order.save() + + # Create unerderlying VM. + data = serializer.validated_data.pop('vm') + vm = create_managed_vm( + order=order, + cores=data['cores'], + ram=data['ram_in_gb'], + disk_size=data['primary_disk']['size_in_gb'], + image=MatrixServiceProduct.base_image()) + + # Create service. + service = serializer.save( + order=order, + owner=request.user, + vm=vm) + + return Response(serializer.data) + +class GenericServiceProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return GenericServiceProduct.objects.filter(owner=self.request.user) + + def get_serializer_class(self): + if self.action == 'create': + return OrderGenericServiceProductSerializer + else: + return GenericServiceProductSerializer + + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + order_billing_address = serializer.validated_data.pop("billing_address") + + # Create base order. + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user, + billing_address=order_billing_address, + starting_date=timezone.now() + ) + order.save() + + # Create service. + print(serializer.validated_data) + service = serializer.save(order=order, owner=request.user) + + # XXX: Move this to some kind of on_create hook in parent + # Product class? + order.add_record( + service.one_time_price, + service.recurring_price, + service.description) + + # XXX: Move this to some kind of on_create hook in parent + # Product class? + order.add_record( + service.one_time_price, + service.recurring_price, + service.description) + + return Response(serializer.data) diff --git a/uncloud_storage/__init__.py b/uncloud_storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_storage/admin.py b/uncloud_storage/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_storage/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_storage/apps.py b/uncloud_storage/apps.py new file mode 100644 index 0000000..38b2301 --- /dev/null +++ b/uncloud_storage/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudStorageConfig(AppConfig): + name = 'uncloud_storage' diff --git a/uncloud_storage/models.py b/uncloud_storage/models.py new file mode 100644 index 0000000..0dac5c2 --- /dev/null +++ b/uncloud_storage/models.py @@ -0,0 +1,7 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class StorageClass(models.TextChoices): + HDD = 'HDD', _('HDD') + SSD = 'SSD', _('SSD') diff --git a/uncloud_storage/tests.py b/uncloud_storage/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/uncloud_storage/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/uncloud_storage/views.py b/uncloud_storage/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/uncloud_storage/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/uncloud_vm/__init__.py b/uncloud_vm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_vm/admin.py b/uncloud_vm/admin.py new file mode 100644 index 0000000..6f3bc50 --- /dev/null +++ b/uncloud_vm/admin.py @@ -0,0 +1,19 @@ +from django.contrib import admin + +# Register your models here. +from uncloud_vm.models import * +from uncloud_pay.models import Order + +class VMDiskInline(admin.TabularInline): + model = VMDiskProduct + +class OrderInline(admin.TabularInline): + model = Order + +class VMProductAdmin(admin.ModelAdmin): + inlines = [ + VMDiskInline + ] + +admin.site.register(VMProduct, VMProductAdmin) +admin.site.register(VMDiskProduct) diff --git a/uncloud_vm/apps.py b/uncloud_vm/apps.py new file mode 100644 index 0000000..c5e94a5 --- /dev/null +++ b/uncloud_vm/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudVmConfig(AppConfig): + name = 'uncloud_vm' diff --git a/uncloud_vm/management/commands/vm.py b/uncloud_vm/management/commands/vm.py new file mode 100644 index 0000000..667c5ad --- /dev/null +++ b/uncloud_vm/management/commands/vm.py @@ -0,0 +1,119 @@ +import json + +import uncloud.secrets as secrets + +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model + +from uncloud_vm.models import VMSnapshotProduct, VMProduct, VMHost +from datetime import datetime + +class Command(BaseCommand): + help = 'Select VM Host for VMs' + + def add_arguments(self, parser): + parser.add_argument('--this-hostname', required=True) + parser.add_argument('--this-cluster', required=True) + + parser.add_argument('--create-vm-snapshots', action='store_true') + parser.add_argument('--schedule-vms', action='store_true') + parser.add_argument('--start-vms', action='store_true') + + + def handle(self, *args, **options): + for cmd in [ 'create_vm_snapshots', 'schedule_vms', 'start_vms' ]: + if options[cmd]: + f = getattr(self, cmd) + f(args, options) + + def schedule_vms(self, *args, **options): + for pending_vm in VMProduct.objects.filter(status='PENDING'): + cores_needed = pending_vm.cores + ram_needed = pending_vm.ram_in_gb + + # Database filtering + possible_vmhosts = VMHost.objects.filter(physical_cores__gte=cores_needed) + + # Logical filtering + possible_vmhosts = [ vmhost for vmhost in possible_vmhosts + if vmhost.available_cores >=cores_needed + and vmhost.available_ram_in_gb >= ram_needed ] + + if not possible_vmhosts: + log.error("No suitable Host found - cannot schedule VM {}".format(pending_vm)) + continue + + vmhost = possible_vmhosts[0] + pending_vm.vmhost = vmhost + pending_vm.status = 'SCHEDULED' + pending_vm.save() + + print("Scheduled VM {} on VMHOST {}".format(pending_vm, pending_vm.vmhost)) + + print(self) + + def start_vms(self, *args, **options): + vmhost = VMHost.objects.get(hostname=options['this_hostname']) + + if not vmhost: + raise Exception("No vmhost {} exists".format(options['vmhostname'])) + + # not active? done here + if not vmhost.status = 'ACTIVE': + return + + vms_to_start = VMProduct.objects.filter(vmhost=vmhost, + status='SCHEDULED') + for vm in vms_to_start: + """ run qemu: + check if VM is not already active / qemu running + prepare / create the Qemu arguments + """ + print("Starting VM {}".format(VM)) + + def check_vms(self, *args, **options): + """ + Check if all VMs that are supposed to run are running + """ + + def modify_vms(self, *args, **options): + """ + Check all VMs that are requested to be modified and restart them + """ + + def create_vm_snapshots(self, *args, **options): + this_cluster = VMCluster(option['this_cluster']) + + for snapshot in VMSnapshotProduct.objects.filter(status='PENDING', + cluster=this_cluster): + if not snapshot.extra_data: + snapshot.extra_data = {} + + # TODO: implement locking here + if 'creating_hostname' in snapshot.extra_data: + pass + + snapshot.extra_data['creating_hostname'] = options['this_hostname'] + snapshot.extra_data['creating_start'] = str(datetime.now()) + snapshot.save() + + # something on the line of: + # for disk im vm.disks: + # rbd snap create pool/image-name@snapshot name + # snapshot.extra_data['snapshots'] + # register the snapshot names in extra_data (?) + + print(snapshot) + + def check_health(self, *args, **options): + pending_vms = VMProduct.objects.filter(status='PENDING') + vmhosts = VMHost.objects.filter(status='active') + + # 1. Check that all active hosts reported back N seconds ago + # 2. Check that no VM is running on a dead host + # 3. Migrate VMs if necessary + # 4. Check that no VMs have been pending for longer than Y seconds + + # If VM snapshots exist without a VM -> notify user (?) + + print("Nothing is good, you should implement me") diff --git a/uncloud_vm/migrations/0001_initial.py b/uncloud_vm/migrations/0001_initial.py new file mode 100644 index 0000000..4ec089a --- /dev/null +++ b/uncloud_vm/migrations/0001_initial.py @@ -0,0 +1,111 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='VMCluster', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('extra_data', models.JSONField(blank=True, editable=False, null=True)), + ('name', models.CharField(max_length=128, unique=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VMDiskImageProduct', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('extra_data', models.JSONField(blank=True, editable=False, null=True)), + ('name', models.CharField(max_length=256)), + ('is_os_image', models.BooleanField(default=False)), + ('is_public', models.BooleanField(default=False, editable=False)), + ('size_in_gb', models.FloatField(blank=True, null=True)), + ('import_url', models.URLField(blank=True, null=True)), + ('image_source', models.CharField(max_length=128, null=True)), + ('image_source_type', models.CharField(max_length=128, null=True)), + ('storage_class', models.CharField(choices=[('HDD', 'HDD'), ('SSD', 'SSD')], default='SSD', max_length=32)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32)), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VMHost', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('extra_data', models.JSONField(blank=True, editable=False, null=True)), + ('hostname', models.CharField(max_length=253, unique=True)), + ('physical_cores', models.IntegerField(default=0)), + ('usable_cores', models.IntegerField(default=0)), + ('usable_ram_in_gb', models.FloatField(default=0)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32)), + ('vmcluster', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmcluster')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VMProduct', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, max_length=32, null=True)), + ('cores', models.IntegerField()), + ('ram_in_gb', models.FloatField()), + ('vmcluster', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmcluster')), + ('vmhost', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmhost')), + ], + ), + migrations.CreateModel( + name='VMSnapshotProduct', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('gb_ssd', models.FloatField(editable=False)), + ('gb_hdd', models.FloatField(editable=False)), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='snapshots', to='uncloud_vm.vmproduct')), + ], + ), + migrations.CreateModel( + name='VMNetworkCard', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('mac_address', models.BigIntegerField()), + ('ip_address', models.GenericIPAddressField(blank=True, null=True)), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmproduct')), + ], + ), + migrations.CreateModel( + name='VMDiskProduct', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('size_in_gb', models.FloatField(blank=True)), + ('disk_type', models.CharField(choices=[('ceph/ssd', 'Ceph Ssd'), ('ceph/hdd', 'Ceph Hdd'), ('local/ssd', 'Local Ssd'), ('local/hdd', 'Local Hdd')], default='ceph/ssd', max_length=20)), + ('image', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmdiskimageproduct')), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmproduct')), + ], + ), + migrations.CreateModel( + name='VMWithOSProduct', + fields=[ + ('vmproduct_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='uncloud_vm.vmproduct')), + ('primary_disk', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmdiskproduct')), + ], + bases=('uncloud_vm.vmproduct',), + ), + ] diff --git a/uncloud_vm/migrations/0002_vmproduct_owner.py b/uncloud_vm/migrations/0002_vmproduct_owner.py new file mode 100644 index 0000000..3b96a87 --- /dev/null +++ b/uncloud_vm/migrations/0002_vmproduct_owner.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1.4 on 2021-04-14 10:40 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_vm', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='vmproduct', + name='owner', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/uncloud_vm/migrations/0003_vmproduct_created_order_at.py b/uncloud_vm/migrations/0003_vmproduct_created_order_at.py new file mode 100644 index 0000000..8f5d0c4 --- /dev/null +++ b/uncloud_vm/migrations/0003_vmproduct_created_order_at.py @@ -0,0 +1,20 @@ +# Generated by Django 3.1.4 on 2021-04-14 10:46 + +import datetime +from django.db import migrations, models +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0002_vmproduct_owner'), + ] + + operations = [ + migrations.AddField( + model_name='vmproduct', + name='created_order_at', + field=models.DateTimeField(default=datetime.datetime(2021, 4, 14, 10, 46, 14, 96330, tzinfo=utc)), + ), + ] diff --git a/uncloud_vm/migrations/0004_auto_20210414_1048.py b/uncloud_vm/migrations/0004_auto_20210414_1048.py new file mode 100644 index 0000000..20214bc --- /dev/null +++ b/uncloud_vm/migrations/0004_auto_20210414_1048.py @@ -0,0 +1,24 @@ +# Generated by Django 3.1.4 on 2021-04-14 10:48 + +import datetime +from django.db import migrations, models +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0003_vmproduct_created_order_at'), + ] + + operations = [ + migrations.RemoveField( + model_name='vmproduct', + name='created_order_at', + ), + migrations.AddField( + model_name='vmproduct', + name='create_order_at', + field=models.DateTimeField(default=datetime.datetime(2021, 4, 14, 10, 48, 6, 641056, tzinfo=utc)), + ), + ] diff --git a/uncloud_vm/migrations/0005_auto_20210414_1119.py b/uncloud_vm/migrations/0005_auto_20210414_1119.py new file mode 100644 index 0000000..ef9df79 --- /dev/null +++ b/uncloud_vm/migrations/0005_auto_20210414_1119.py @@ -0,0 +1,24 @@ +# Generated by Django 3.1.4 on 2021-04-14 11:19 + +import datetime +from django.db import migrations, models +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0004_auto_20210414_1048'), + ] + + operations = [ + migrations.RemoveField( + model_name='vmproduct', + name='create_order_at', + ), + migrations.AddField( + model_name='vmproduct', + name='created_order_at', + field=models.DateTimeField(default=datetime.datetime(2021, 4, 14, 11, 19, 39, 447274, tzinfo=utc)), + ), + ] diff --git a/uncloud_vm/migrations/0006_auto_20210414_1122.py b/uncloud_vm/migrations/0006_auto_20210414_1122.py new file mode 100644 index 0000000..2c302fb --- /dev/null +++ b/uncloud_vm/migrations/0006_auto_20210414_1122.py @@ -0,0 +1,20 @@ +# Generated by Django 3.1.4 on 2021-04-14 11:22 + +import datetime +from django.db import migrations, models +from django.utils.timezone import utc + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0005_auto_20210414_1119'), + ] + + operations = [ + migrations.AlterField( + model_name='vmproduct', + name='created_order_at', + field=models.DateTimeField(default=datetime.datetime(2021, 4, 14, 11, 22, 11, 352536, tzinfo=utc)), + ), + ] diff --git a/uncloud_vm/migrations/__init__.py b/uncloud_vm/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_vm/models.py b/uncloud_vm/models.py new file mode 100644 index 0000000..952cde9 --- /dev/null +++ b/uncloud_vm/models.py @@ -0,0 +1,200 @@ +import datetime +from django.utils import timezone + +from django.db import models +from django.contrib.auth import get_user_model + +from uncloud_pay.models import Product, RecurringPeriod +from uncloud.models import UncloudModel, UncloudStatus + +import uncloud_pay.models as pay_models +import uncloud_storage.models + +class VMCluster(UncloudModel): + name = models.CharField(max_length=128, unique=True) + +class VMHost(UncloudModel): + # 253 is the maximum DNS name length + hostname = models.CharField(max_length=253, unique=True) + + vmcluster = models.ForeignKey( + VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + # indirectly gives a maximum number of cores / VM - f.i. 32 + physical_cores = models.IntegerField(default=0) + + # determines the maximum usable cores - f.i. 320 if you overbook by a factor of 10 + usable_cores = models.IntegerField(default=0) + + # ram that can be used of the server + usable_ram_in_gb = models.FloatField(default=0) + + status = models.CharField( + max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING + ) + + @property + def vms(self): + return VMProduct.objects.filter(vmhost=self) + + @property + def used_ram_in_gb(self): + return sum([vm.ram_in_gb for vm in VMProduct.objects.filter(vmhost=self)]) + + @property + def available_ram_in_gb(self): + return self.usable_ram_in_gb - self.used_ram_in_gb + + @property + def available_cores(self): + return self.usable_cores - sum([vm.cores for vm in self.vms ]) + + + +class VMProduct(models.Model): + owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE, + blank=True, null=True) + vmhost = models.ForeignKey( + VMHost, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + vmcluster = models.ForeignKey( + VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + name = models.CharField(max_length=32, blank=True, null=True) + cores = models.IntegerField() + ram_in_gb = models.FloatField() + created_order_at = models.DateTimeField(default=timezone.make_aware(datetime.datetime.now())) + + @property + def recurring_price(self): + return self.cores * 3 + self.ram_in_gb * 4 + + @property + def description(self): + return "Virtual machine '{}': {} core(s), {}GB memory".format( + self.name, self.cores, self.ram_in_gb) + + # @staticmethod + # def allowed_recurring_periods(): + # return list(filter( + # lambda pair: pair[0] in [RecurringPeriod.PER_365D, + # RecurringPeriod.PER_30D, RecurringPeriod.PER_HOUR], + # RecurringPeriod.choices)) + + def create_order_at(self, dt): + self.created_order_at = dt + + def create_or_update_order(self, when_to_start): + self.created_order_at = when_to_start + + def __str__(self): + return f"VM id={self.id},name={self.name},cores={self.cores},ram_in_gb={self.ram_in_gb}" + + +class VMWithOSProduct(VMProduct): + primary_disk = models.ForeignKey('VMDiskProduct', on_delete=models.CASCADE, null=True) + + +class VMDiskImageProduct(UncloudModel): + """ + Images are used for cloning/linking. + + They are the base for images. + + """ + + owner = models.ForeignKey( + get_user_model(), on_delete=models.CASCADE, editable=False + ) + + name = models.CharField(max_length=256) + is_os_image = models.BooleanField(default=False) + is_public = models.BooleanField(default=False, editable=False) # only allow admins to set this + + size_in_gb = models.FloatField(null=True, blank=True) + import_url = models.URLField(null=True, blank=True) + image_source = models.CharField(max_length=128, null=True) + image_source_type = models.CharField(max_length=128, null=True) + + storage_class = models.CharField(max_length=32, + choices = uncloud_storage.models.StorageClass.choices, + default = uncloud_storage.models.StorageClass.SSD) + + status = models.CharField( + max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING + ) + + def __str__(self): + return "VMDiskImage {} ({}): {} gb".format(self.id, + self.name, + self.size_in_gb) + + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class VMDiskType(models.TextChoices): + """ + Types of disks that can be attached to VMs + """ + CEPH_SSD = 'ceph/ssd' + CEPH_HDD = 'ceph/hdd' + LOCAL_SSD = 'local/ssd' + LOCAL_HDD = 'local/hdd' + + +class VMDiskProduct(models.Model): + """ + The VMDiskProduct is attached to a VM. + + It is based on a VMDiskImageProduct that will be used as a basis. + + It can be enlarged, but not shrinked compared to the VMDiskImageProduct. + """ + + vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE) + + image = models.ForeignKey(VMDiskImageProduct, on_delete=models.CASCADE, + blank=True, null=True) + + size_in_gb = models.FloatField(blank=True) + + disk_type = models.CharField( + max_length=20, + choices=VMDiskType.choices, + default=VMDiskType.CEPH_SSD) + + def __str__(self): + return f"Disk {self.size_in_gb}GB ({self.disk_type}) for {self.vm}" + + @property + def recurring_price(self): + if self.disk_type == VMDiskType.CEPH_SSD: + price_per_gb = 3.5/10 + elif self.disk_type == VMDiskType.CEPH_HDD: + price_per_gb = 1.5/100 + elif self.disk_type == VMDiskType.LOCAL_SSD: + price_per_gb = 3.5/10 + elif self.disk_type == VMDiskType.CEPH_HDD: + price_per_gb = 1.5/100 + + return self.size_in_gb * price_per_gb + + +class VMNetworkCard(models.Model): + vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE) + + mac_address = models.BigIntegerField() + + ip_address = models.GenericIPAddressField(blank=True, + null=True) + + +class VMSnapshotProduct(models.Model): + gb_ssd = models.FloatField(editable=False) + gb_hdd = models.FloatField(editable=False) + + vm = models.ForeignKey(VMProduct, + related_name='snapshots', + on_delete=models.CASCADE) diff --git a/uncloud_vm/serializers.py b/uncloud_vm/serializers.py new file mode 100644 index 0000000..a60d10b --- /dev/null +++ b/uncloud_vm/serializers.py @@ -0,0 +1,143 @@ +from django.contrib.auth import get_user_model + +from rest_framework import serializers + +from .models import * +from uncloud_pay.models import RecurringPeriod, BillingAddress + +# XXX: does not seem to be used? + +GB_SSD_PER_DAY=0.012 +GB_HDD_PER_DAY=0.0006 + +GB_SSD_PER_DAY=0.012 +GB_HDD_PER_DAY=0.0006 + +### +# Admin views. + +class VMHostSerializer(serializers.HyperlinkedModelSerializer): + vms = serializers.PrimaryKeyRelatedField(many=True, read_only=True) + + class Meta: + model = VMHost + fields = '__all__' + read_only_fields = [ 'vms' ] + +class VMClusterSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = VMCluster + fields = '__all__' + + +### +# Disks. + +class VMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = '__all__' + +class CreateVMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = ['size_in_gb', 'image'] + +class CreateManagedVMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = ['size_in_gb'] + +class VMDiskImageProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskImageProduct + fields = '__all__' + +class VMSnapshotProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMSnapshotProduct + fields = '__all__' + + + # verify that vm.owner == user.request + def validate_vm(self, value): + if not value.owner == self.context['request'].user: + raise serializers.ValidationError("VM {} not found for owner {}.".format(value, + self.context['request'].user)) + disks = VMDiskProduct.objects.filter(vm=value) + + if len(disks) == 0: + raise serializers.ValidationError("VM {} does not have any disks, cannot snapshot".format(value.id)) + + return value + + pricing = {} + pricing['per_gb_ssd'] = 0.012 + pricing['per_gb_hdd'] = 0.0006 + pricing['recurring_period'] = 'per_day' + +### +# VMs + +# Helper used in uncloud_service for services allocating VM. +class ManagedVMProductSerializer(serializers.ModelSerializer): + """ + Managed VM serializer used in ungleich_service app. + """ + primary_disk = CreateManagedVMDiskProductSerializer() + class Meta: + model = VMWithOSProduct + fields = [ 'cores', 'ram_in_gb', 'primary_disk'] + +class VMProductSerializer(serializers.ModelSerializer): + primary_disk = CreateVMDiskProductSerializer() + snapshots = VMSnapshotProductSerializer(many=True, read_only=True) + disks = VMDiskProductSerializer(many=True, read_only=True) + + class Meta: + model = VMWithOSProduct + fields = ['order', 'owner', 'status', 'name', 'cores', + 'ram_in_gb', 'primary_disk', 'snapshots', 'disks', 'extra_data'] + read_only_fields = ['order', 'owner', 'status'] + +class OrderVMProductSerializer(VMProductSerializer): + # recurring_period = serializers.ChoiceField( + # choices=VMWithOSProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(VMProductSerializer, self).__init__(*args, **kwargs) + + class Meta: + model = VMProductSerializer.Meta.model + fields = VMProductSerializer.Meta.fields + [ 'recurring_period' ] + read_only_fields = VMProductSerializer.Meta.read_only_fields + +# Nico's playground. +class NicoVMProductSerializer(serializers.ModelSerializer): + snapshots = VMSnapshotProductSerializer(many=True, read_only=True) + order = serializers.StringRelatedField() + + class Meta: + model = VMProduct + read_only_fields = ['order', 'owner', 'status', + 'vmhost', 'vmcluster', 'snapshots', + 'extra_data' ] + fields = read_only_fields + [ 'name', + 'cores', + 'ram_in_gb' + ] + +class DCLVMProductSerializer(serializers.HyperlinkedModelSerializer): + """ + Create an interface similar to standard DCL + """ + + # Custom field used at creation (= ordering) only. + # recurring_period = serializers.ChoiceField( + # choices=VMProduct.allowed_recurring_periods()) + + os_disk_uuid = serializers.UUIDField() + # os_disk_size = + + class Meta: + model = VMProduct diff --git a/uncloud_vm/tests.py b/uncloud_vm/tests.py new file mode 100644 index 0000000..e5d403f --- /dev/null +++ b/uncloud_vm/tests.py @@ -0,0 +1,98 @@ +import datetime + +import parsedatetime + +from django.test import TestCase +from django.contrib.auth import get_user_model +from django.utils import timezone +from django.core.exceptions import ValidationError + +from uncloud_vm.models import VMDiskImageProduct, VMDiskProduct, VMProduct, VMHost +from uncloud_pay.models import Order, RecurringPeriod + +User = get_user_model() +cal = parsedatetime.Calendar() + + +# If you want to check the test database using some GUI/cli tool +# then use the following connecting parameters + +# host: localhost +# database: test_uncloud +# user: root +# password: +# port: 5432 + +class VMTestCase(TestCase): + @classmethod + def setUpClass(cls): + # Setup vm host + cls.vm_host, created = VMHost.objects.get_or_create( + hostname='serverx.placey.ungleich.ch', physical_cores=32, usable_cores=320, + usable_ram_in_gb=512.0, status='active' + ) + super().setUpClass() + + def setUp(self) -> None: + # Setup two users as it is common to test with different user + self.user = User.objects.create_user( + username='testuser', email='test@test.com', first_name='Test', last_name='User' + ) + self.user2 = User.objects.create_user( + username='Meow', email='meow123@test.com', first_name='Meow', last_name='Cat' + ) + super().setUp() + + def create_sample_vm(self, owner): + one_month_later, parse_status = cal.parse("1 month later") + return VMProduct.objects.create( + vmhost=self.vm_host, cores=2, ram_in_gb=4, owner=owner, + order=Order.objects.create( + owner=owner, + creation_date=datetime.datetime.now(tz=timezone.utc), + starting_date=datetime.datetime.now(tz=timezone.utc), + ending_date=datetime.datetime(*one_month_later[:6], tzinfo=timezone.utc), + recurring_period=RecurringPeriod.PER_MONTH + ) + ) + +# TODO: the logic tested by this test is not implemented yet. +# def test_disk_product(self): +# """Ensures that a VMDiskProduct can only be created from a VMDiskImageProduct +# that is in status 'active'""" +# +# vm = self.create_sample_vm(owner=self.user) +# +# pending_disk_image = VMDiskImageProduct.objects.create( +# owner=self.user, name='pending_disk_image', is_os_image=True, is_public=True, size_in_gb=10, +# status='pending' +# ) +# try: +# vm_disk_product = VMDiskProduct.objects.create( +# owner=self.user, vm=vm, image=pending_disk_image, size_in_gb=10 +# ) +# except ValidationError: +# vm_disk_product = None +# +# self.assertIsNone( +# vm_disk_product, +# msg='VMDiskProduct created with disk image whose status is not active.' +# ) + +# TODO: the logic tested by this test is not implemented yet. +# def test_vm_disk_product_creation_for_someone_else(self): +# """Ensure that a user can only create a VMDiskProduct for his/her own VM""" +# +# # Create a VM which is ownership of self.user2 +# someone_else_vm = self.create_sample_vm(owner=self.user2) +# +# # 'self.user' would try to create a VMDiskProduct for 'user2's VM +# with self.assertRaises(ValidationError, msg='User created a VMDiskProduct for someone else VM.'): +# vm_disk_product = VMDiskProduct.objects.create( +# owner=self.user, vm=someone_else_vm, +# size_in_gb=10, +# image=VMDiskImageProduct.objects.create( +# owner=self.user, name='disk_image', is_os_image=True, is_public=True, size_in_gb=10, +# status='active' +# ) +# ) diff --git a/uncloud_vm/views.py b/uncloud_vm/views.py new file mode 100644 index 0000000..67f8656 --- /dev/null +++ b/uncloud_vm/views.py @@ -0,0 +1,261 @@ +from django.db import transaction +from django.shortcuts import render +from django.utils import timezone + +from django.contrib.auth.models import User +from django.shortcuts import get_object_or_404 + +from rest_framework import viewsets, permissions +from rest_framework.response import Response +from rest_framework.exceptions import ValidationError + +from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster +from uncloud_pay.models import Order, BillingAddress + +from .serializers import * +from uncloud_pay.helpers import ProductViewSet + +import datetime + +### +# Generic disk image views. Do not require orders / billing. + +class VMDiskImageProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskImageProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMDiskImageProduct.objects.all() + else: + obj = VMDiskImageProduct.objects.filter(owner=self.request.user) | VMDiskImageProduct.objects.filter(is_public=True) + + return obj + + + def create(self, request): + serializer = VMDiskImageProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + + # did not specify size NOR import url? + if not serializer.validated_data['size_in_gb']: + if not serializer.validated_data['import_url']: + raise ValidationError(detail={ 'error_mesage': 'Specify either import_url or size_in_gb' }) + + serializer.save(owner=request.user) + return Response(serializer.data) + +class VMDiskImageProductPublicViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskImageProductSerializer + + def get_queryset(self): + return VMDiskImageProduct.objects.filter(is_public=True) + +### +# User VM disk and snapshots. + +class VMDiskProductViewSet(viewsets.ModelViewSet): + """ + Let a user modify their own VMDisks + """ + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMDiskProduct.objects.all() + else: + obj = VMDiskProduct.objects.filter(owner=self.request.user) + + return obj + + def create(self, request): + serializer = VMDiskProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + + # get disk size from image, if not specified + if not 'size_in_gb' in serializer.validated_data: + size_in_gb = serializer.validated_data['image'].size_in_gb + else: + size_in_gb = serializer.validated_data['size_in_gb'] + + if size_in_gb < serializer.validated_data['image'].size_in_gb: + raise ValidationError(detail={ 'error_mesage': 'Size is smaller than original image' }) + + serializer.save(owner=request.user, size_in_gb=size_in_gb) + return Response(serializer.data) + +class VMSnapshotProductViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMSnapshotProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMSnapshotProduct.objects.all() + else: + obj = VMSnapshotProduct.objects.filter(owner=self.request.user) + + return obj + + def create(self, request): + serializer = VMSnapshotProductSerializer(data=request.data, context={'request': request}) + + # This verifies that the VM belongs to the request user + serializer.is_valid(raise_exception=True) + + vm = vm=serializer.validated_data['vm'] + disks = VMDiskProduct.objects.filter(vm=vm) + ssds_size = sum([d.size_in_gb for d in disks if d.image.storage_class == 'ssd']) + hdds_size = sum([d.size_in_gb for d in disks if d.image.storage_class == 'hdd']) + + recurring_price = serializer.pricing['per_gb_ssd'] * ssds_size + serializer.pricing['per_gb_hdd'] * hdds_size + recurring_period = serializer.pricing['recurring_period'] + + # Create order + now = datetime.datetime.now() + order = Order(owner=request.user, + recurring_period=recurring_period) + order.save() + order.add_record(one_time_price=0, + recurring_price=recurring_price, + description="Snapshot of VM {} from {}".format(vm, now)) + + serializer.save(owner=request.user, + order=order, + gb_ssd=ssds_size, + gb_hdd=hdds_size) + + return Response(serializer.data) + +### +# User VMs. + +class VMProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMWithOSProduct.objects.all() + else: + obj = VMWithOSProduct.objects.filter(owner=self.request.user) + + return obj + + def get_serializer_class(self): + if self.action == 'create': + return OrderVMProductSerializer + else: + return VMProductSerializer + + # Use a database transaction so that we do not get half-created structure + # if something goes wrong. + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + + # Create disk image. + disk = VMDiskProduct(owner=request.user, + **serializer.validated_data.pop("primary_disk")) + vm = VMWithOSProduct(owner=request.user, primary_disk=disk, + **serializer.validated_data) + disk.vm = vm # XXX: Is this really needed? + + # Create VM and Disk orders. + vm_order = Order.from_product( + vm, + recurring_period=order_recurring_period, + starting_date=timezone.now() + ) + + disk_order = Order.from_product( + disk, + recurring_period=order_recurring_period, + starting_date=timezone.now(), + depends_on=vm_order + ) + + + # Commit to DB. + vm.order = vm_order + vm.save() + vm_order.save() + + disk.order = disk_order + disk_order.save() + disk.save() + + return Response(VMProductSerializer(vm, context={'request': request}).data) + +class NicoVMProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = NicoVMProductSerializer + + def get_queryset(self): + obj = VMProduct.objects.filter(owner=self.request.user) + return obj + + def create(self, request): + serializer = self.serializer_class(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + vm = serializer.save(owner=request.user) + + return Response(serializer.data) + + +### +# Admin stuff. + +class VMHostViewSet(viewsets.ModelViewSet): + serializer_class = VMHostSerializer + queryset = VMHost.objects.all() + permission_classes = [permissions.IsAdminUser] + +class VMClusterViewSet(viewsets.ModelViewSet): + serializer_class = VMClusterSerializer + queryset = VMCluster.objects.all() + permission_classes = [permissions.IsAdminUser] + +## +# Nico's playground. + +# Also create: +# - /dcl/available_os +# Basically a view of public and my disk images +# - +class DCLCreateVMProductViewSet(ProductViewSet): + """ + This view resembles the way how DCL VMs are created by default. + + The user chooses an OS, os disk size, ram, cpu and whether or not to have a mapped IPv4 address + """ + + permission_classes = [permissions.IsAuthenticated] + serializer_class = DCLVMProductSerializer + + def get_queryset(self): + return VMProduct.objects.filter(owner=self.request.user) + + # Use a database transaction so that we do not get half-created structure + # if something goes wrong. + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = VMProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + + # Create base order. + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user + ) + order.save() + + # Create VM. + vm = serializer.save(owner=request.user, order=order) + + return Response(serializer.data)