diff --git a/.gitignore b/.gitignore index 5c55899..426f11b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ -.idea -.vscode +.idea/ +.vscode/ +__pycache__/ +pay.conf +log.txt +test.py +STRIPE +venv/ uncloud/docs/build logs.txt @@ -16,3 +22,6 @@ uncloud/version.py build/ venv/ dist/ +.history/ +*.iso +*.sqlite3 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..e468591 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,18 @@ +stages: + - lint + - test + +run-tests: + stage: test + image: code.ungleich.ch:5050/uncloud/uncloud/uncloud-ci:latest + services: + - postgres:latest + variables: + DATABASE_HOST: postgres + DATABASE_USER: postgres + POSTGRES_HOST_AUTH_METHOD: trust + coverage: /^TOTAL.+?(\d+\%)$/ + script: + - pip install -r requirements.txt + - coverage run --source='.' ./manage.py test + - coverage report diff --git a/README.md b/README.md index 0e32f57..07f5c91 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,70 @@ -# ucloud +# Uncloud -Checkout https://ungleich.ch/ucloud/ for the documentation of ucloud. +Cloud management platform, the ungleich way. + + +[![pipeline status](https://code.ungleich.ch/uncloud/uncloud/badges/master/pipeline.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master) +[![coverage report](https://code.ungleich.ch/uncloud/uncloud/badges/master/coverage.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master) + +## Useful commands + +* `./manage.py import-vat-rates path/to/csv` +* `./manage.py createsuperuser` + +## Development setup + +Install system dependencies: + +* On Fedora, you will need the following packages: `python3-virtualenv python3-devel openldap-devel gcc chromium` +* sudo apt-get install libpq-dev python-dev libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libffi-dev + + +NOTE: you will need to configure a LDAP server and credentials for authentication. See `uncloud/settings.py`. + +``` +# Initialize virtualenv. +» virtualenv .venv +Using base prefix '/usr' +New python executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python3 +Also creating executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python +Installing setuptools, pip, wheel... +done. + +# Enter virtualenv. +» source .venv/bin/activate + +# Install dependencies. +» pip install -r requirements.txt +[...] + +# Run migrations. +» ./manage.py migrate +Operations to perform: + Apply all migrations: admin, auth, contenttypes, opennebula, sessions, uncloud_auth, uncloud_net, uncloud_pay, uncloud_service, uncloud_vm +Running migrations: + [...] + +# Run webserver. +» ./manage.py runserver +Watching for file changes with StatReloader +Performing system checks... + +System check identified no issues (0 silenced). +May 07, 2020 - 10:17:08 +Django version 3.0.6, using settings 'uncloud.settings' +Starting development server at http://127.0.0.1:8000/ +Quit the server with CONTROL-C. +``` +### Run Background Job Queue +We use Django Q to handle the asynchronous code and Background Cron jobs +To start the workers make sure first that Redis or the Django Q broker is working and you can edit it's settings in the settings file. +``` +./manage.py qcluster +``` + +### Note on PGSQL + +If you want to use Postgres: + +* Install on configure PGSQL on your base system. +* OR use a container! `podman run --rm -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -it postgres:latest` diff --git a/archive/issues.org b/archive/issues.org new file mode 100644 index 0000000..840ec3c --- /dev/null +++ b/archive/issues.org @@ -0,0 +1,6 @@ +* Intro + This file lists issues that should be handled, are small and likely + not yet high prio. +* Issues +** TODO Register prefered address in User model +** TODO Allow to specify different recurring periods diff --git a/archive/uncloud_django_based/hacks/abk-hacks.py b/archive/uncloud_django_based/hacks/abk-hacks.py new file mode 100644 index 0000000..abc63d3 --- /dev/null +++ b/archive/uncloud_django_based/hacks/abk-hacks.py @@ -0,0 +1,55 @@ +""" +investigate into a simple python function that maps an ldap user to a vat percentage. Basically you need to +lookup the customer address, check if she is a business/registered tax number and if not apply the local +vat +""" + +import iso3166 +import datetime + +from csv import DictReader + + +def get_vat(street_address, city, postal_code, country, vat_number=None): + vat = { + 'Austria': [ + {'period': '1984-01-01/', 'rate': 0.2}, + {'period': '1976-01-01/1984-01-01', 'rate': 0.18}, + {'period': '1973-01-01/1976-01-01', 'rate': 0.16}, + ] + } + return iso3166.countries.get(country) + + # return iso3166.countries_by_name[country] + + +def main(): + # vat = get_vat( + # street_address='82 Nasheman-e-Iqbal near Wapda Town', + # city='Lahore', + # postal_code=53700, + # country='Pakistan', + # ) + # print(vat) + vat_rates = {} + with open('vat_rates.csv', newline='') as csvfile: + reader = DictReader(csvfile) + for row in reader: + territory_codes = row['territory_codes'].split('\n') + for code in territory_codes: + if code not in vat_rates: + vat_rates[code] = {} + + start_date = row['start_date'] + stop_data = row['stop_date'] + time_period = f'{start_date}|{stop_data}' + r = row.copy() + del r['start_date'] + del r['stop_date'] + del r['territory_codes'] + vat_rates[code][time_period] = r + print(vat_rates) + + +if __name__ == '__main__': + main() diff --git a/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py b/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py new file mode 100644 index 0000000..c0bbaf8 --- /dev/null +++ b/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py @@ -0,0 +1,46 @@ +import importlib +import sys +import os + +from os.path import join as join_path +from xmlrpc.client import ServerProxy as RPCClient + +root = os.path.dirname(os.getcwd()) +sys.path.append(join_path(root, 'uncloud')) +secrets = importlib.import_module('uncloud.secrets') + + +class OpenNebula: + def __init__(self, url, session_string): + self.session_string = session_string + self.client = RPCClient(secrets.OPENNEBULA_URL) + + def create_user(self, username, password, authentication_driver='', group_id=None): + # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-user-allocate + + if group_id is None: + group_id = [] + + return self.client.one.user.allocate( + self.session_string, + username, + password, + authentication_driver, + group_id + ) + + def chmod(self, vm_id, user_id=-1, group_id=-1): + # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-vm-chown + + return self.client.one.vm.chown(self.session_string, vm_id, user_id, group_id) + + +one = OpenNebula(secrets.OPENNEBULA_URL, secrets.OPENNEBULA_USER_PASS) + +# Create User in OpenNebula +# success, response, *_ = one.create_user(username='meow12345', password='hello_world') +# print(success, response) + +# Change owner of a VM +# success, response, *_ = one.chmod(vm_id=25589, user_id=706) +# print(success, response) diff --git a/archive/uncloud_django_based/hacks/command-wrapper.sh b/archive/uncloud_django_based/hacks/command-wrapper.sh new file mode 100644 index 0000000..d6ddd13 --- /dev/null +++ b/archive/uncloud_django_based/hacks/command-wrapper.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +dbhost=$1; shift + +ssh -L5432:localhost:5432 "$dbhost" & + +python manage.py "$@" + + + +# command only needs to be active while manage command is running + +# -T no pseudo terminal + + +# alternatively: commands output shell code + +# ssh uncloud@dbhost "python manage.py --hostname xxx ..." diff --git a/archive/uncloud_django_based/meow-payv1/README.md b/archive/uncloud_django_based/meow-payv1/README.md new file mode 100644 index 0000000..fe6a2a3 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/README.md @@ -0,0 +1,51 @@ +# uncloud-pay + +The generic product/payment system. + +## Installation + +```shell script +pip3 install -r requirements.txt +``` + +## Getting Started + +```shell script +python ucloud_pay.py +``` + +## Usage + +#### 1. Adding of products +```shell script +http --json http://[::]:5000/product/add username=your_username_here password=your_password_here specs:=@ipv6-only-vm.json +``` + +#### 2. Listing of products +```shell script +http --json http://[::]:5000/product/list +``` + +#### 3. Registering user's payment method (credit card for now using Stripe) + +```shell script +http --json http://[::]:5000/user/register_payment card_number=4111111111111111 cvc=123 expiry_year=2020 expiry_month=8 card_holder_name="The test user" username=your_username_here password=your_password_here line1="your_billing_address" city="your_city" country="your_country" +``` + +#### 4. Ordering products + +First of all, user have to buy the membership first. + +```shell script +http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=membership pay=True +``` + +```shell script +http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=ipv6-only-vm cpu=1 ram=1 os-disk-space=10 os=alpine pay=True +``` + +#### 5. Listing users orders + +```shell script +http --json POST http://[::]:5000/order/list username=your_username_here password=your_password_here +``` diff --git a/archive/uncloud_django_based/meow-payv1/config.py b/archive/uncloud_django_based/meow-payv1/config.py new file mode 100644 index 0000000..16804af --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/config.py @@ -0,0 +1,21 @@ +import os + +from ungleich_common.ldap.ldap_manager import LdapManager +from ungleich_common.std.configparser import StrictConfigParser +from ungleich_common.etcd.etcd_wrapper import EtcdWrapper + +config_file = os.environ.get('meow-pay-config-file', default='pay.conf') + +config = StrictConfigParser(allow_no_value=True) +config.read(config_file) + +etcd_client = EtcdWrapper( + host=config.get('etcd', 'host'), port=config.get('etcd', 'port'), + ca_cert=config.get('etcd', 'ca_cert'), cert_key=config.get('etcd', 'cert_key'), + cert_cert=config.get('etcd', 'cert_cert') +) + +ldap_manager = LdapManager( + server=config.get('ldap', 'server'), admin_dn=config.get('ldap', 'admin_dn'), + admin_password=config.get('ldap', 'admin_password') +) diff --git a/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py b/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py new file mode 100644 index 0000000..e6bfb43 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py @@ -0,0 +1,213 @@ +from flask import Flask, request +from flask_restful import Resource, Api +import etcd3 +import json +import logging +from functools import wraps + +from ldaptest import is_valid_ldap_user + +def authenticate(func): + @wraps(func) + def wrapper(*args, **kwargs): + if not getattr(func, 'authenticated', True): + return func(*args, **kwargs) + + # pass in username/password ! + acct = basic_authentication() # custom account lookup function + + if acct: + return func(*args, **kwargs) + + flask_restful.abort(401) + return wrapper + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + + +class DB(object): + def __init__(self, config, prefix="/"): + self.config = config + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + + self.connect() + + @readable_errors + def connect(self): + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) + + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) + + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: + value = json.dumps(value) + + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + +class Membership(Resource): + def __init__(self, config): + self.config = config + + def get(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + return {'message': 'Order successful' }, 200 + + def post(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + return {'message': 'Order 2x successful' }, 200 + + +class Order(Resource): + def __init__(self, config): + self.config = config + + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + +class Product(Resource): + def __init__(self, config): + self.config = config + + self.products = [] + self.products.append( + { "name": "membership-free", + "description": """ +This membership gives you access to the API and includes a VPN +with 1 IPv6 address. +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "a3883466-0012-4d01-80ff-cbf7469957af", + "recurring": True, + "recurring_time_frame": "per_year", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 0 + } + ] + } + ) + self.products.append( + { "name": "membership-standard", + "description": """ +This membership gives you access to the API and includes an IPv6-VPN with +one IPv6 address ("Road warrior") +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "1d85296b-0863-4dd6-a543-a6d5a4fbe4a6", + "recurring": True, + "recurring_time_frame": "per_month", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 5 + } + + ] + } + ) + self.products.append( + { "name": "membership-premium", + "description": """ +This membership gives you access to the API and includes an +IPv6-VPN with a /48 IPv6 network. +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "bfd63fd2-d227-436f-a8b8-600de74dd6ce", + "recurring": True, + "recurring_time_frame": "per_month", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 5 + } + + ] + } + ) + self.products.append( + { "name": "ipv6-vpn-with-/48", + "description": """ +An IPv6 VPN with a /48 network included. +""", + "uuid": "fe5753f8-6fe1-4dc4-9b73-7b803de4c597", + "recurring": True, + "recurring_time_frame": "per_year", + "features": [ + { "name": "vpn", + "price_one_time": 0, + "price_recurring": 120 + } + ] + } + ) + + + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + def get(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + return self.products + + + + + +if __name__ == '__main__': + app = Flask(__name__) + + config = {} + + config['etcd_url']="https://etcd1.ungleich.ch" + config['ldap_url']="ldaps://ldap1.ungleich.ch" + + api = Api(app) + api.add_resource(Order, '/orders', resource_class_args=( config, )) + api.add_resource(Product, '/products', resource_class_args=( config, )) + api.add_resource(Membership, '/membership', resource_class_args=( config, )) + + app.run(host='::', port=5000, debug=True) diff --git a/archive/uncloud_django_based/meow-payv1/helper.py b/archive/uncloud_django_based/meow-payv1/helper.py new file mode 100644 index 0000000..65a5155 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/helper.py @@ -0,0 +1,87 @@ +import logging + +import parsedatetime + +from datetime import datetime +from stripe_utils import StripeUtils + + +def get_plan_id_from_product(product): + plan_id = 'ucloud-v1-' + plan_id += product['name'].strip().replace(' ', '-') + return plan_id + + +def get_pricing(price_in_chf_cents, product_type, recurring_period): + if product_type == 'recurring': + return 'CHF {}/{}'.format(price_in_chf_cents/100, recurring_period) + elif product_type == 'one-time': + return 'CHF {} (One time charge)'.format(price_in_chf_cents/100) + + +def get_user_friendly_product(product_dict): + uf_product = { + 'name': product_dict['name'], + 'description': product_dict['description'], + 'product_id': product_dict['usable-id'], + 'pricing': get_pricing( + product_dict['price'], product_dict['type'], product_dict['recurring_period'] + ) + } + if product_dict['type'] == 'recurring': + uf_product['minimum_subscription_period'] = product_dict['minimum_subscription_period'] + return uf_product + + +def get_token(card_number, cvc, exp_month, exp_year): + stripe_utils = StripeUtils() + token_response = stripe_utils.get_token_from_card( + card_number, cvc, exp_month, exp_year + ) + if token_response['response_object']: + return token_response['response_object'].id + else: + return None + + +def resolve_product(usable_id, etcd_client): + products = etcd_client.get_prefix('/v1/products/', value_in_json=True) + for p in products: + if p.value['usable-id'] == usable_id: + return p.value + return None + + +def calculate_charges(specification, data): + logging.debug('Calculating charges for specs:{} and data:{}'.format(specification, data)) + one_time_charge = 0 + recurring_charge = 0 + for feature_name, feature_detail in specification['features'].items(): + if feature_detail['constant']: + data[feature_name] = 1 + + if feature_detail['unit']['type'] != 'str': + one_time_charge += feature_detail['one_time_fee'] + recurring_charge += ( + feature_detail['price_per_unit_per_period'] * data[feature_name] / + feature_detail['unit']['value'] + ) + return one_time_charge, recurring_charge + + +def is_order_valid(order_timestamp, renewal_period): + """ + Sample Code Usage + + >> current_datetime, status = cal.parse('Now') + >> current_datetime = datetime(*current_datetime[:6]) + + >> print('Is order valid: ', is_order_valid(current_datetime, '1 month')) + >> True + """ + cal = parsedatetime.Calendar() + + renewal_datetime, status = cal.parse(renewal_period) + renewal_datetime = datetime(*renewal_datetime[:6]) + + return order_timestamp <= renewal_datetime diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json new file mode 100644 index 0000000..110027a --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json @@ -0,0 +1,28 @@ +{ + "usable-id": "ipv6-only-django-hosting", + "active": true, + "name": "IPv6 Only Django Hosting", + "description": "Host your Django application on our shiny IPv6 Only VM", + "recurring_period": "month", + "quantity": "inf", + "features": { + "cpu": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 3, + "one_time_fee": 0, + "constant": false + }, + "ram": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 4, + "one_time_fee": 0, + "constant": false + }, + "os-disk-space": { + "unit": {"value": 10, "type":"int"}, + "one_time_fee": 0, + "price_per_unit_per_period": 3.5, + "constant": false + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json new file mode 100644 index 0000000..d07ad6c --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json @@ -0,0 +1,34 @@ +{ + "usable-id": "ipv6-only-vm", + "active": true, + "name": "IPv6 Only VM", + "description": "IPv6 Only VM are accessible to only those having IPv6 for themselves", + "recurring_period": "month", + "quantity": "inf", + "features": { + "cpu": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 3, + "one_time_fee": 0, + "constant": false + }, + "ram": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 4, + "one_time_fee": 0, + "constant": false + }, + "os-disk-space": { + "unit": {"value": 10, "type":"int"}, + "one_time_fee": 0, + "price_per_unit_per_period": 4, + "constant": false + }, + "os": { + "unit": {"value": 1, "type":"str"}, + "one_time_fee": 0, + "price_per_unit_per_period": 0, + "constant": false + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json new file mode 100644 index 0000000..38c6201 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json @@ -0,0 +1,16 @@ +{ + "usable-id": "ipv6-only-vpn", + "active": true, + "name": "IPv6 Only VPN", + "description": "IPv6 VPN enable you to access IPv6 only websites and more", + "recurring_period": "month", + "quantity": "inf", + "features": { + "vpn": { + "unit": {"value": 1, "type": "int"}, + "price_per_unit_per_period": 10, + "one_time_fee": 0, + "constant": true + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6box.json b/archive/uncloud_django_based/meow-payv1/products/ipv6box.json new file mode 100644 index 0000000..eca11f0 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6box.json @@ -0,0 +1,16 @@ +{ + "usable-id": "ipv6-box", + "active": true, + "name": "IPv6 Box", + "description": "A ready-to-go IPv6 Box: it creates a VPN to ungleich and distributes IPv6 addresses to all your computers.", + "recurring_period": "eternity", + "quantity": 4, + "features": { + "ipv6-box": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 0, + "one_time_fee": 250, + "constant": true + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/membership.json b/archive/uncloud_django_based/meow-payv1/products/membership.json new file mode 100644 index 0000000..4003330 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/membership.json @@ -0,0 +1,17 @@ +{ + "usable-id": "membership", + "active": true, + "name": "Membership", + "description": "Membership to use uncloud-pay", + "recurring_period": "month", + "quantity": "inf", + "features": { + "membership": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 5, + "one_time_fee": 0, + "constant": true + } + }, + "max_per_user": "1" +} diff --git a/archive/uncloud_django_based/meow-payv1/requirements.txt b/archive/uncloud_django_based/meow-payv1/requirements.txt new file mode 100644 index 0000000..0b758ca --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/requirements.txt @@ -0,0 +1,7 @@ +stripe +flask +Flask-RESTful +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-etcd&subdirectory=etcd +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-std&subdirectory=std +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-schemas&subdirectory=schemas diff --git a/archive/uncloud_django_based/meow-payv1/sample-pay.conf b/archive/uncloud_django_based/meow-payv1/sample-pay.conf new file mode 100644 index 0000000..5d1fe61 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/sample-pay.conf @@ -0,0 +1,17 @@ +[etcd] +host = 127.0.0.1 +port = 2379 +ca_cert +cert_cert +cert_key + +[stripe] +private_key=stripe_private_key + +[app] +port = 5000 + +[ldap] +server = ldap_server_url +admin_dn = ldap_admin_dn +admin_password = ldap_admin_password diff --git a/archive/uncloud_django_based/meow-payv1/schemas.py b/archive/uncloud_django_based/meow-payv1/schemas.py new file mode 100644 index 0000000..2e3aef7 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/schemas.py @@ -0,0 +1,136 @@ +import logging +import config +import json +import math + +from config import ldap_manager, etcd_client +from helper import resolve_product +from ungleich_common.schemas.schemas import BaseSchema, Field, ValidationException + + +class AddProductSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.add_schema(UserCredentialSchema, data) + self.specs = Field('specs', dict, **self.get(data, 'specs')) + self.update = Field('update', bool, **self.get(data, 'update', return_default=True, default=False)) + + def validation(self): + user = self.objects['user'] + user = json.loads(user.entry_to_json()) + uid, ou, *dc = user['dn'].replace('ou=', '').replace('dc=', '').replace('uid=', '').split(',') + if ou != config.config.get('ldap', 'internal_user_ou', fallback='users'): + raise ValidationException('You do not have access to create product.') + + product = resolve_product(self.specs.value['usable-id'], etcd_client) + if product: + self.objects['product'] = product + + +class AddressSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.line1 = Field('line1', str, **self.get(data, 'line1')) + self.line2 = Field('line2', str, **self.get(data, 'line2', return_default=True)) + self.city = Field('city', str, **self.get(data, 'city')) + self.country = Field('country', str, **self.get(data, 'country')) + self.state = Field('state', str, **self.get(data, 'state', return_default=True)) + self.postal_code = Field('postal_code', str, **self.get(data, 'postal_code', return_default=True)) + + +class UserRegisterPaymentSchema(BaseSchema): + def __init__(self, data): + super().__init__() + + self.add_schema(UserCredentialSchema, data) + self.add_schema(AddressSchema, data, under_field_name='address') + + self.card_number = Field('card_number', str, **self.get(data, 'card_number')) + self.cvc = Field('cvc', str, **self.get(data, 'cvc')) + self.expiry_year = Field('expiry_year', int, **self.get(data, 'expiry_year')) + self.expiry_month = Field('expiry_month', int, **self.get(data, 'expiry_month')) + self.card_holder_name = Field('card_holder_name', str, **self.get(data, 'card_holder_name')) + + +class UserCredentialSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.username = Field('username', str, **self.get(data, 'username')) + self.password = Field('password', str, **self.get(data, 'password')) + + def validation(self): + try: + entry = ldap_manager.is_password_valid(self.username.value, self.password.value, query_key='uid') + except ValueError: + raise ValidationException('No user with \'{}\' username found. You can create account at ' + 'https://account.ungleich.ch'.format(self.username.value)) + except Exception: + raise ValidationException('Invalid username/password.') + else: + self.objects['user'] = entry + + +class ProductOrderSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.product_id = Field( + 'product_id', str, **self.get(data, 'product_id'), validators=[self.product_id_validation] + ) + self.pay_consent = Field('pay', bool, **self.get(data, 'pay', return_default=True, default=False)) + self.add_schema(UserCredentialSchema, data) + + def product_id_validation(self): + product = resolve_product(self.product_id.value, etcd_client) + if product: + product['quantity'] = float(product['quantity']) + self.product_id.value = product['uuid'] + self.objects['product'] = product + logging.debug('Got product {}'.format(product)) + + if not product['active']: + raise ValidationException('Product is not active at the moment.') + + if product['quantity'] <= 0: + raise ValidationException('Out of stock.') + else: + raise ValidationException('No such product exists.') + + def validation(self): + username = self.objects['user'].uid + customer_previous_orders = etcd_client.get_prefix('/v1/user/{}'.format(username), value_in_json=True) + customer_previous_orders = [o.value for o in customer_previous_orders] + membership = next(filter(lambda o: o['product'] == 'membership', customer_previous_orders), None) + if membership is None and self.objects['product']['usable-id'] != 'membership': + raise ValidationException('Please buy membership first to use this facility') + max_quantity_user_can_order = float(self.objects['product'].get('max_per_user', math.inf)) + previous_order_of_same_product = [ + o for o in customer_previous_orders if o['product'] == self.objects['product']['usable-id'] + ] + if len(previous_order_of_same_product) >= max_quantity_user_can_order: + raise ValidationException( + 'You cannot buy {} more than {} times'.format( + self.objects['product']['name'], int(max_quantity_user_can_order) + ) + ) + + +class OrderListSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.add_schema(UserCredentialSchema, data) + + +def make_return_message(err, status_code=200): + logging.debug('message: {}'.format(str(err))) + return {'message': str(err)}, status_code + + +def create_schema(specification, data): + fields = {} + for feature_name, feature_detail in specification['features'].items(): + if not feature_detail['constant']: + fields[feature_name] = Field( + feature_name, eval(feature_detail['unit']['type']), **BaseSchema.get(data, feature_name) + ) + + return type('{}Schema'.format(specification['name']), (BaseSchema,), fields) diff --git a/archive/uncloud_django_based/meow-payv1/stripe_hack.py b/archive/uncloud_django_based/meow-payv1/stripe_hack.py new file mode 100644 index 0000000..f436c62 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/stripe_hack.py @@ -0,0 +1,7 @@ +import stripe_utils +import os + + +if __name__ == '__main__': + s = stripe_utils.StripeUtils(os.environ['STRIPE_PRIVATE_KEY']) + print(s.get_stripe_customer_from_email('coder.purple+2002@gmail.com')) diff --git a/archive/uncloud_django_based/meow-payv1/stripe_utils.py b/archive/uncloud_django_based/meow-payv1/stripe_utils.py new file mode 100644 index 0000000..6a2cd29 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/stripe_utils.py @@ -0,0 +1,491 @@ +import re +import stripe +import stripe.error +import logging + +from config import etcd_client as client, config as config + +stripe.api_key = config.get('stripe', 'private_key') + + +def handle_stripe_error(f): + def handle_problems(*args, **kwargs): + response = { + 'paid': False, + 'response_object': None, + 'error': None + } + + common_message = "Currently it's not possible to make payments." + try: + response_object = f(*args, **kwargs) + response = { + 'response_object': response_object, + 'error': None + } + return response + except stripe.error.CardError as e: + # Since it's a decline, stripe.error.CardError will be caught + body = e.json_body + err = body['error'] + response.update({'error': err['message']}) + logging.error(str(e)) + return response + except stripe.error.RateLimitError: + response.update( + {'error': "Too many requests made to the API too quickly"}) + return response + except stripe.error.InvalidRequestError as e: + logging.error(str(e)) + response.update({'error': "Invalid parameters"}) + return response + except stripe.error.AuthenticationError as e: + # Authentication with Stripe's API failed + # (maybe you changed API keys recently) + logging.error(str(e)) + response.update({'error': common_message}) + return response + except stripe.error.APIConnectionError as e: + logging.error(str(e)) + response.update({'error': common_message}) + return response + except stripe.error.StripeError as e: + # maybe send email + logging.error(str(e)) + response.update({'error': common_message}) + return response + except Exception as e: + # maybe send email + logging.error(str(e)) + response.update({'error': common_message}) + return response + + return handle_problems + + +class StripeUtils(object): + CURRENCY = 'chf' + INTERVAL = 'month' + SUCCEEDED_STATUS = 'succeeded' + STRIPE_PLAN_ALREADY_EXISTS = 'Plan already exists' + STRIPE_NO_SUCH_PLAN = 'No such plan' + PLAN_EXISTS_ERROR_MSG = 'Plan {} exists already.\nCreating a local StripePlan now.' + PLAN_DOES_NOT_EXIST_ERROR_MSG = 'Plan {} does not exist.' + + def __init__(self, private_key): + self.stripe = stripe + stripe.api_key = private_key + + @handle_stripe_error + def card_exists(self, customer, cc_number, exp_month, exp_year, cvc): + token_obj = stripe.Token.create( + card={ + 'number': cc_number, + 'exp_month': exp_month, + 'exp_year': exp_year, + 'cvc': cvc, + }, + ) + cards = stripe.Customer.list_sources( + customer, + limit=20, + object='card' + ) + + for card in cards.data: + if (card.fingerprint == token_obj.card.fingerprint and + int(card.exp_month) == int(exp_month) and int(card.exp_year) == int(exp_year)): + return True + return False + + @staticmethod + def get_stripe_customer_from_email(email): + customer = stripe.Customer.list(limit=1, email=email) + return customer.data[0] if len(customer.data) == 1 else None + + @staticmethod + def update_customer_token(customer, token): + customer.source = token + customer.save() + + @handle_stripe_error + def get_token_from_card(self, cc_number, cvc, expiry_month, expiry_year): + token_obj = stripe.Token.create( + card={ + 'number': cc_number, + 'exp_month': expiry_month, + 'exp_year': expiry_year, + 'cvc': cvc, + }, + ) + return token_obj + + @handle_stripe_error + def associate_customer_card(self, stripe_customer_id, token, + set_as_default=False): + customer = stripe.Customer.retrieve(stripe_customer_id) + card = customer.sources.create(source=token) + if set_as_default: + customer.default_source = card.id + customer.save() + return True + + @handle_stripe_error + def dissociate_customer_card(self, stripe_customer_id, card_id): + customer = stripe.Customer.retrieve(stripe_customer_id) + card = customer.sources.retrieve(card_id) + card.delete() + + @handle_stripe_error + def update_customer_card(self, customer_id, token): + customer = stripe.Customer.retrieve(customer_id) + current_card_token = customer.default_source + customer.sources.retrieve(current_card_token).delete() + customer.source = token + customer.save() + credit_card_raw_data = customer.sources.data.pop() + new_card_data = { + 'last4': credit_card_raw_data.last4, + 'brand': credit_card_raw_data.brand + } + return new_card_data + + @handle_stripe_error + def get_card_details(self, customer_id): + customer = stripe.Customer.retrieve(customer_id) + credit_card_raw_data = customer.sources.data.pop() + card_details = { + 'last4': credit_card_raw_data.last4, + 'brand': credit_card_raw_data.brand, + 'exp_month': credit_card_raw_data.exp_month, + 'exp_year': credit_card_raw_data.exp_year, + 'fingerprint': credit_card_raw_data.fingerprint, + 'card_id': credit_card_raw_data.id + } + return card_details + + @handle_stripe_error + def get_all_invoices(self, customer_id, created_gt): + return_list = [] + has_more_invoices = True + starting_after = False + while has_more_invoices: + if starting_after: + invoices = stripe.Invoice.list( + limit=10, customer=customer_id, created={'gt': created_gt}, + starting_after=starting_after + ) + else: + invoices = stripe.Invoice.list( + limit=10, customer=customer_id, created={'gt': created_gt} + ) + has_more_invoices = invoices.has_more + for invoice in invoices.data: + sub_ids = [] + for line in invoice.lines.data: + if line.type == 'subscription': + sub_ids.append(line.id) + elif line.type == 'invoiceitem': + sub_ids.append(line.subscription) + else: + sub_ids.append('') + invoice_details = { + 'created': invoice.created, + 'receipt_number': invoice.receipt_number, + 'invoice_number': invoice.number, + 'paid_at': invoice.status_transitions.paid_at if invoice.paid else 0, + 'period_start': invoice.period_start, + 'period_end': invoice.period_end, + 'billing_reason': invoice.billing_reason, + 'discount': invoice.discount.coupon.amount_off if invoice.discount else 0, + 'total': invoice.total, + # to see how many line items we have in this invoice and + # then later check if we have more than 1 + 'lines_data_count': len(invoice.lines.data) if invoice.lines.data is not None else 0, + 'invoice_id': invoice.id, + 'lines_meta_data_csv': ','.join( + [line.metadata.VM_ID if hasattr(line.metadata, 'VM_ID') else '' for line in invoice.lines.data] + ), + 'subscription_ids_csv': ','.join(sub_ids), + 'line_items': invoice.lines.data + } + starting_after = invoice.id + return_list.append(invoice_details) + return return_list + + @handle_stripe_error + def get_cards_details_from_token(self, token): + stripe_token = stripe.Token.retrieve(token) + card_details = { + 'last4': stripe_token.card.last4, + 'brand': stripe_token.card.brand, + 'exp_month': stripe_token.card.exp_month, + 'exp_year': stripe_token.card.exp_year, + 'fingerprint': stripe_token.card.fingerprint, + 'card_id': stripe_token.card.id + } + return card_details + + def check_customer(self, stripe_cus_api_id, user, token): + try: + customer = stripe.Customer.retrieve(stripe_cus_api_id) + except stripe.error.InvalidRequestError: + customer = self.create_customer(token, user.email, user.name) + user.stripecustomer.stripe_id = customer.get( + 'response_object').get('id') + user.stripecustomer.save() + if type(customer) is dict: + customer = customer['response_object'] + return customer + + @handle_stripe_error + def get_customer(self, stripe_api_cus_id): + customer = stripe.Customer.retrieve(stripe_api_cus_id) + # data = customer.get('response_object') + return customer + + @handle_stripe_error + def create_customer(self, token, email, name=None, address=None): + if name is None or name.strip() == "": + name = email + customer = self.stripe.Customer.create( + source=token, + description=name, + email=email, + address=address + ) + return customer + + @handle_stripe_error + def make_charge(self, amount=None, customer=None): + _amount = float(amount) + amount = int(_amount * 100) # stripe amount unit, in cents + charge = self.stripe.Charge.create( + amount=amount, # in cents + currency=self.CURRENCY, + customer=customer + ) + return charge + + @staticmethod + def _get_all_stripe_plans(): + all_stripe_plans = client.get("/v1/stripe_plans") + all_stripe_plans_set = set() + if all_stripe_plans: + all_stripe_plans_obj = all_stripe_plans.value + if all_stripe_plans_obj and len(all_stripe_plans_obj['plans']) > 0: + all_stripe_plans_set = set(all_stripe_plans_obj["plans"]) + return all_stripe_plans_set + + @staticmethod + def _save_all_stripe_plans(stripe_plans): + client.put("/v1/stripe_plans", {"plans": list(stripe_plans)}) + + @handle_stripe_error + def get_or_create_stripe_plan(self, product_name, amount, stripe_plan_id, + interval=INTERVAL): + """ + This function checks if a StripePlan with the given + stripe_plan_id already exists. If it exists then the function + returns this object otherwise it creates a new StripePlan and + returns the new object. + + :param amount: The amount in CHF cents + :param product_name: The name of the Stripe plan (product) to be created. + :param stripe_plan_id: The id of the Stripe plan to be + created. Use get_stripe_plan_id_string function to + obtain the name of the plan to be created + :param interval: The interval for subscription {month, year}. Defaults + to month if not provided + :return: The StripePlan object if it exists else creates a + Plan object in Stripe and a local StripePlan and + returns it. Returns None in case of Stripe error + """ + _amount = float(amount) + amount = int(_amount * 100) # stripe amount unit, in cents + all_stripe_plans = self._get_all_stripe_plans() + if stripe_plan_id in all_stripe_plans: + logging.debug("{} plan exists in db.".format(stripe_plan_id)) + else: + logging.debug(("{} plan DOES NOT exist in db. " + "Creating").format(stripe_plan_id)) + try: + plan_obj = self.stripe.Plan.retrieve(id=stripe_plan_id) + logging.debug("{} plan exists in Stripe".format(stripe_plan_id)) + all_stripe_plans.add(stripe_plan_id) + except stripe.error.InvalidRequestError as e: + if "No such plan" in str(e): + logging.debug("Plan {} does not exist in Stripe, Creating") + plan_obj = self.stripe.Plan.create( + amount=amount, + product={'name': product_name}, + interval=interval, + currency=self.CURRENCY, + id=stripe_plan_id) + logging.debug(self.PLAN_EXISTS_ERROR_MSG.format(stripe_plan_id)) + all_stripe_plans.add(stripe_plan_id) + self._save_all_stripe_plans(all_stripe_plans) + return stripe_plan_id + + @handle_stripe_error + def delete_stripe_plan(self, stripe_plan_id): + """ + Deletes the Plan in Stripe and also deletes the local db copy + of the plan if it exists + + :param stripe_plan_id: The stripe plan id that needs to be + deleted + :return: True if the plan was deleted successfully from + Stripe, False otherwise. + """ + return_value = False + try: + plan = self.stripe.Plan.retrieve(stripe_plan_id) + plan.delete() + return_value = True + all_stripe_plans = self._get_all_stripe_plans() + all_stripe_plans.remove(stripe_plan_id) + self._save_all_stripe_plans(all_stripe_plans) + except stripe.error.InvalidRequestError as e: + if self.STRIPE_NO_SUCH_PLAN in str(e): + logging.debug( + self.PLAN_DOES_NOT_EXIST_ERROR_MSG.format(stripe_plan_id)) + return return_value + + @handle_stripe_error + def subscribe_customer_to_plan(self, customer, plans, trial_end=None): + """ + Subscribes the given customer to the list of given plans + + :param customer: The stripe customer identifier + :param plans: A list of stripe plans. + :param trial_end: An integer representing when the Stripe subscription + is supposed to end + Ref: https://stripe.com/docs/api/python#create_subscription-items + e.g. + plans = [ + { + "plan": "dcl-v1-cpu-2-ram-5gb-ssd-10gb", + }, + ] + :return: The subscription StripeObject + """ + + subscription_result = self.stripe.Subscription.create( + customer=customer, items=plans, trial_end=trial_end + ) + return subscription_result + + @handle_stripe_error + def set_subscription_metadata(self, subscription_id, metadata): + subscription = stripe.Subscription.retrieve(subscription_id) + subscription.metadata = metadata + subscription.save() + + @handle_stripe_error + def unsubscribe_customer(self, subscription_id): + """ + Cancels a given subscription + + :param subscription_id: The Stripe subscription id string + :return: + """ + sub = stripe.Subscription.retrieve(subscription_id) + return sub.delete() + + @handle_stripe_error + def make_payment(self, customer, amount, token): + charge = self.stripe.Charge.create( + amount=amount, # in cents + currency=self.CURRENCY, + customer=customer + ) + return charge + + @staticmethod + def get_stripe_plan_id(cpu, ram, ssd, version, app='dcl', hdd=None, + price=None): + """ + Returns the Stripe plan id string of the form + `dcl-v1-cpu-2-ram-5gb-ssd-10gb` based on the input parameters + + :param cpu: The number of cores + :param ram: The size of the RAM in GB + :param ssd: The size of ssd storage in GB + :param hdd: The size of hdd storage in GB + :param version: The version of the Stripe plans + :param app: The application to which the stripe plan belongs + to. By default it is 'dcl' + :param price: The price for this plan + :return: A string of the form `dcl-v1-cpu-2-ram-5gb-ssd-10gb` + """ + dcl_plan_string = 'cpu-{cpu}-ram-{ram}gb-ssd-{ssd}gb'.format(cpu=cpu, + ram=ram, + ssd=ssd) + if hdd is not None: + dcl_plan_string = '{dcl_plan_string}-hdd-{hdd}gb'.format( + dcl_plan_string=dcl_plan_string, hdd=hdd) + stripe_plan_id_string = '{app}-v{version}-{plan}'.format( + app=app, + version=version, + plan=dcl_plan_string + ) + if price is not None: + stripe_plan_id_string_with_price = '{}-{}chf'.format( + stripe_plan_id_string, + round(price, 2) + ) + return stripe_plan_id_string_with_price + else: + return stripe_plan_id_string + + @staticmethod + def get_vm_config_from_stripe_id(stripe_id): + """ + Given a string like "dcl-v1-cpu-2-ram-5gb-ssd-10gb" return different + configuration params as a dict + + :param stripe_id|str + :return: dict + """ + pattern = re.compile(r'^dcl-v(\d+)-cpu-(\d+)-ram-(\d+\.?\d*)gb-ssd-(\d+)gb-?(\d*\.?\d*)(chf)?$') + match_res = pattern.match(stripe_id) + if match_res is not None: + price = None + try: + price = match_res.group(5) + except IndexError: + logging.debug("Did not find price in {}".format(stripe_id)) + return { + 'version': match_res.group(1), + 'cores': match_res.group(2), + 'ram': match_res.group(3), + 'ssd': match_res.group(4), + 'price': price + } + + @staticmethod + def get_stripe_plan_name(cpu, memory, disk_size, price): + """ + Returns the Stripe plan name + :return: + """ + return "{cpu} Cores, {memory} GB RAM, {disk_size} GB SSD, " \ + "{price} CHF".format( + cpu=cpu, + memory=memory, + disk_size=disk_size, + price=round(price, 2) + ) + + @handle_stripe_error + def set_subscription_meta_data(self, subscription_id, meta_data): + """ + Adds VM metadata to a subscription + :param subscription_id: Stripe identifier for the subscription + :param meta_data: A dict of meta data to be added + :return: + """ + subscription = stripe.Subscription.retrieve(subscription_id) + subscription.metadata = meta_data + subscription.save() diff --git a/archive/uncloud_django_based/meow-payv1/ucloud_pay.py b/archive/uncloud_django_based/meow-payv1/ucloud_pay.py new file mode 100644 index 0000000..dbc0d2c --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/ucloud_pay.py @@ -0,0 +1,338 @@ +import logging + +from datetime import datetime +from uuid import uuid4 + +from flask import Flask, request +from flask_restful import Resource, Api +from werkzeug.exceptions import HTTPException +from config import etcd_client as client, config as config +from stripe_utils import StripeUtils +from schemas import ( + make_return_message, ValidationException, UserRegisterPaymentSchema, + AddProductSchema, ProductOrderSchema, OrderListSchema, create_schema +) +from helper import get_plan_id_from_product, calculate_charges + + +class ListProducts(Resource): + @staticmethod + def get(): + products = client.get_prefix('/v1/products/') + products = [ + product + for product in [p.value for p in products] + if product['active'] + ] + prod_dict = {} + for p in products: + prod_dict[p['usable-id']] = { + 'name': p['name'], + 'description': p['description'], + } + logger.debug('Products = {}'.format(prod_dict)) + return prod_dict, 200 + +class AddProduct(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + logger.debug('Got data: {}'.format(str(data))) + validator = AddProductSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + previous_product = cleaned_values.get('product', None) + if previous_product: + if not cleaned_values['update']: + return make_return_message('Product already exists. Pass --update to update the product.') + else: + product_uuid = previous_product.pop('uuid') + else: + product_uuid = uuid4().hex + + product_value = cleaned_values['specs'] + + product_key = '/v1/products/{}'.format(product_uuid) + product_value['uuid'] = product_uuid + + logger.debug('Adding product data: {}'.format(str(product_value))) + client.put(product_key, product_value) + if not previous_product: + return make_return_message('Product created.') + else: + return make_return_message('Product updated.') + +################################################################################ +# Nico-ok-marker + + +class UserRegisterPayment(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + logger.debug('Got data: {}'.format(str(data))) + validator = UserRegisterPaymentSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + last4 = data['card_number'].strip()[-4:] + + stripe_utils = StripeUtils() + + # Does customer already exist ? + stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail']) + + # Does customer already exist ? + if stripe_customer is not None: + logger.debug('Customer {}-{} exists already'.format( + cleaned_values['username'], cleaned_values['user']['mail']) + ) + + # Check if the card already exists + ce_response = stripe_utils.card_exists( + stripe_customer.id, cc_number=data['card_number'], + exp_month=int(data['expiry_month']), + exp_year=int(data['expiry_year']), + cvc=data['cvc']) + + if ce_response['response_object']: + message = 'The given card ending in {} exists already.'.format(last4) + return make_return_message(message, 400) + + elif ce_response['response_object'] is False: + # Associate card with user + logger.debug('Adding card ending in {}'.format(last4)) + token_response = stripe_utils.get_token_from_card( + data['card_number'], data['cvc'], data['expiry_month'], + data['expiry_year'] + ) + if token_response['response_object']: + logger.debug('Token {}'.format(token_response['response_object'].id)) + resp = stripe_utils.associate_customer_card( + stripe_customer.id, token_response['response_object'].id + ) + if resp['response_object']: + return make_return_message( + 'Card ending in {} registered as your payment source'.format(last4) + ) + else: + return make_return_message('Error with payment gateway. Contact support', 400) + else: + return make_return_message('Error: {}'.format(ce_response['error']), 400) + else: + # Stripe customer does not exist, create a new one + logger.debug( + 'Customer {} does not exist, creating new'.format(cleaned_values['user']['mail']) + ) + token_response = stripe_utils.get_token_from_card( + cleaned_values['card_number'], cleaned_values['cvc'], + cleaned_values['expiry_month'], cleaned_values['expiry_year'] + ) + if token_response['response_object']: + logger.debug('Token {}'.format(token_response['response_object'].id)) + + # Create stripe customer + stripe_customer_resp = stripe_utils.create_customer( + name=cleaned_values['card_holder_name'], + token=token_response['response_object'].id, + email=cleaned_values['user']['mail'], + address=cleaned_values['address'] + ) + stripe_customer = stripe_customer_resp['response_object'] + + if stripe_customer: + logger.debug('Created stripe customer {}'.format(stripe_customer.id)) + return make_return_message( + 'Card ending in {} registered as your payment source'.format(last4) + ) + else: + return make_return_message('Error with card. Contact support', 400) + else: + return make_return_message('Error with payment gateway. Contact support', 400) + + +class ProductOrder(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + validator = ProductOrderSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + stripe_utils = StripeUtils() + + product = cleaned_values['product'] + + # Check the user has a payment source added + stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail']) + + if not stripe_customer or len(stripe_customer.sources) == 0: + return make_return_message('Please register your payment method first.', 400) + + try: + product_schema = create_schema(product, data) + product_schema = product_schema() + product_schema.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + transformed_data = product_schema.get_cleaned_values() + logger.debug('Tranformed data: {}'.format(transformed_data)) + one_time_charge, recurring_charge = calculate_charges(product, transformed_data) + recurring_charge = int(recurring_charge) + + if not cleaned_values['pay']: + return make_return_message( + 'You would be charged {} CHF one time and {} CHF every {}. ' + 'Add --pay to command to order.'.format( + one_time_charge, recurring_charge, product['recurring_period'] + ) + ) + + with client.client.lock('product-order') as _: + # Initiate a one-time/subscription based on product type + if recurring_charge > 0: + logger.debug('Product {} is recurring payment'.format(product['name'])) + plan_id = get_plan_id_from_product(product) + res = stripe_utils.get_or_create_stripe_plan( + product_name=product['name'], + stripe_plan_id=plan_id, amount=recurring_charge, + interval=product['recurring_period'], + ) + if res['response_object']: + logger.debug('Obtained plan {}'.format(plan_id)) + subscription_res = stripe_utils.subscribe_customer_to_plan( + stripe_customer.id, + [{'plan': plan_id}] + ) + subscription_obj = subscription_res['response_object'] + if subscription_obj is None or subscription_obj.status != 'active': + return make_return_message( + 'Error subscribing to plan. Detail: {}'.format(subscription_res['error']), 400 + ) + else: + order_obj = { + 'order-id': uuid4().hex, + 'ordered-at': datetime.now().isoformat(), + 'product': product['usable-id'], + 'one-time-price': one_time_charge, + 'recurring-price': recurring_charge, + 'recurring-period': product['recurring_period'] + } + client.put( + '/v1/user/{}/orders/{}'.format( + cleaned_values['username'], order_obj['order-id'] + ), order_obj + ) + product['quantity'] -= 1 + client.put('/v1/products/{}'.format(product['uuid']), product) + + return { + 'message': 'Order Successful.', + **order_obj + } + else: + logger.error('Could not create plan {}'.format(plan_id)) + return make_return_message('Something wrong happened. Contact administrator', 400) + + elif recurring_charge == 0 and one_time_charge > 0: + logger.debug('Product {} is one-time payment'.format(product['name'])) + charge_response = stripe_utils.make_charge( + amount=one_time_charge, + customer=stripe_customer.id + ) + stripe_onetime_charge = charge_response.get('response_object') + + # Check if the payment was approved + if not stripe_onetime_charge: + msg = charge_response.get('error') + return make_return_message('Error subscribing to plan. Details: {}'.format(msg), 400) + + order_obj = { + 'order-id': uuid4().hex, + 'ordered-at': datetime.now().isoformat(), + 'product': product['usable-id'], + 'one-time-price': one_time_charge, + } + client.put( + '/v1/user/{}/orders/{}'.format(cleaned_values['username'], order_obj['order-id']), + order_obj + ) + product['quantity'] -= 1 + client.put('/v1/products/{}'.format(product['uuid']), product) + + return {'message': 'Order successful', **order_obj}, 200 + + +class OrderList(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + validator = OrderListSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + orders = client.get_prefix('/v1/user/{}/orders'.format(cleaned_values['username'])) + orders_dict = { + order.value['order-id']: { + **order.value + } + for order in orders + } + logger.debug('Orders = {}'.format(orders_dict)) + return {'orders': orders_dict}, 200 + + +if __name__ == '__main__': + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + log_formater = logging.Formatter('[%(filename)s:%(lineno)d] %(message)s') + + stream_logger = logging.StreamHandler() + stream_logger.setFormatter(log_formater) + + # file_logger = logging.FileHandler('log.txt') + # file_logger.setLevel(logging.DEBUG) + # file_logger.setFormatter(log_formater) + + logger.addHandler(stream_logger) + # logger.addHandler(file_logger) + + app = Flask(__name__) + + api = Api(app) + api.add_resource(ListProducts, '/product/list') + api.add_resource(AddProduct, '/product/add') + api.add_resource(ProductOrder, '/product/order') + api.add_resource(UserRegisterPayment, '/user/register_payment') + api.add_resource(OrderList, '/order/list') + + app.run(host='::', port=config.get('app', 'port', fallback=5000), debug=True) + + + @app.errorhandler(Exception) + def handle_exception(e): + app.logger.error(e) + # pass through HTTP errors + if isinstance(e, HTTPException): + return e + + # now you're handling non-HTTP exceptions only + return {'message': 'Server Error'}, 500 diff --git a/archive/uncloud_django_based/notes-abk.md b/archive/uncloud_django_based/notes-abk.md new file mode 100644 index 0000000..6d5c223 --- /dev/null +++ b/archive/uncloud_django_based/notes-abk.md @@ -0,0 +1,11 @@ +## TODO 2020-02-22 + +* ~~move the current rest api to /opennebula~~ +* ~~make the /opennebula api only accessible by an admin account~~ +* ~~create a new filtered api on /vm/list that~~ + * ~~a) requires authentication~~ + * ~~b) only shows the VMs of the current user~~ +* ~~the new api should not contain all details, but: cpus (as read by the vcpu field), ram, ips, disks~~ +* ~~also make a (random) uuid the primary key for VMs - everything in this uncloud hack will use uuids as the id~~ +* ~~still expose the opennebula id as opennebula_id~~ +* ~~note put all secrets/configs into uncloud.secrets - I added a sample file into the repo~~ diff --git a/archive/uncloud_django_based/notes-nico.org b/archive/uncloud_django_based/notes-nico.org new file mode 100644 index 0000000..811fbff --- /dev/null +++ b/archive/uncloud_django_based/notes-nico.org @@ -0,0 +1,102 @@ +* snapshot feature +** product: vm-snapshot +** flow +*** list all my VMs +**** get the uuid of the VM I want to take a snapshot of +*** request a snapshot +``` +vmuuid=$(http nicocustomer +http -a nicocustomer:xxx http://uncloud.ch/vm/create_snapshot uuid= +password=... +``` +** backend realisation +*** list snapshots + - have them in the DB + - create an entry on create +*** creating snapshots + - vm sync / fsync? + - rbd snapshot + - host/cluster mapping? + - need image(s) + +* steps +** DONE authenticate via ldap + CLOSED: [2020-02-20 Thu 19:05] +** DONE Make classes / views require authentication + CLOSED: [2020-02-20 Thu 19:05] +** TODO register credit card +*** TODO find out what saving with us +*** Info +**** should not be fully saved in the DB +**** model needs to be a bit different +* Decide where to save sensitive data +** stripe access key, etc. +* python requirements (nicohack202002) + django djangorestframework django-auth-ldap stripe +* os package requirements (alpine) + openldap-dev +* VPN case +** put on /orders with uuid +** register cc +* CC +** TODO check whether we can register or not at stripe +* membership +** required for "smaller" / "shorter" products + +* TODO Membership missing +* Flows to be implemented - see https://redmine.ungleich.ch/issues/7609 +** Membership +*** 5 CHF +** Django Hosting +*** One time payment 35 CHF +*** Monthly payment depends on VM size +*** Parameters: same as IPv6 only VM +** IPv6 VPN +*** Parameters: none +*** Is for free if the customer has an active VM +** IPv6 only VM +*** Parameters: cores, ram, os_disk_size, OS +* Django rest framework +** viewset: .list and .create +** view: .get .post +* TODO register CC +* DONE list products + CLOSED: [2020-02-24 Mon 20:15] +* An ungleich account - can be registered for free on + https://account.ungleich.ch +* httpie installed (provides the http command) + +## Get a membership + + +## Registering a payment method + +To be able to pay for the membership, you will need to register a +credit card or apply for payment on bill (TO BE IMPLEMENTED). + +### Register credit card + +``` +http POST https://api.ungleich.ch/membership \ + username=nico password=yourpassword \ + cc_number=.. \ + cc_ + +``` + + + +### Request payment via bill + + + + +## Create the membership + + +``` +http POST https://api.ungleich.ch/membership username=nico password=yourpassword + +``` + +## List available products diff --git a/archive/uncloud_django_based/plan.org b/archive/uncloud_django_based/plan.org new file mode 100644 index 0000000..9f172c2 --- /dev/null +++ b/archive/uncloud_django_based/plan.org @@ -0,0 +1,6 @@ +* TODO register CC +* TODO list products +* ahmed +** schemas +*** field: is_valid? - used by schemas +*** definition of a "schema" diff --git a/archive/uncloud_django_based/uncloud/.gitignore b/archive/uncloud_django_based/uncloud/.gitignore new file mode 100644 index 0000000..71202e1 --- /dev/null +++ b/archive/uncloud_django_based/uncloud/.gitignore @@ -0,0 +1,4 @@ +db.sqlite3 +uncloud/secrets.py +debug.log +uncloud/local_settings.py \ No newline at end of file diff --git a/bin/gen-version b/archive/uncloud_etcd_based/bin/gen-version similarity index 63% rename from bin/gen-version rename to archive/uncloud_etcd_based/bin/gen-version index a2e2882..06c3e22 100755 --- a/bin/gen-version +++ b/archive/uncloud_etcd_based/bin/gen-version @@ -1,22 +1,22 @@ #!/bin/sh # -*- coding: utf-8 -*- # -# 2019 Nico Schottelius (nico-ucloud at schottelius.org) +# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org) # -# This file is part of ucloud. +# This file is part of uncloud. # -# ucloud is free software: you can redistribute it and/or modify +# uncloud is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# ucloud is distributed in the hope that it will be useful, +# uncloud is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with ucloud. If not, see . +# along with uncloud. If not, see . # # @@ -26,4 +26,4 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py +printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py diff --git a/bin/uncloud b/archive/uncloud_etcd_based/bin/uncloud similarity index 100% rename from bin/uncloud rename to archive/uncloud_etcd_based/bin/uncloud diff --git a/bin/uncloud-run-reinstall b/archive/uncloud_etcd_based/bin/uncloud-run-reinstall similarity index 92% rename from bin/uncloud-run-reinstall rename to archive/uncloud_etcd_based/bin/uncloud-run-reinstall index 18e95c0..b211613 100755 --- a/bin/uncloud-run-reinstall +++ b/archive/uncloud_etcd_based/bin/uncloud-run-reinstall @@ -24,6 +24,6 @@ dir=${0%/*} ${dir}/gen-version; -pip uninstall -y uncloud -python setup.py install +pip uninstall -y uncloud >/dev/null +python setup.py install >/dev/null ${dir}/uncloud "$@" diff --git a/archive/uncloud_etcd_based/conf/uncloud.conf b/archive/uncloud_etcd_based/conf/uncloud.conf new file mode 100644 index 0000000..6a1b500 --- /dev/null +++ b/archive/uncloud_etcd_based/conf/uncloud.conf @@ -0,0 +1,13 @@ +[etcd] +url = localhost +port = 2379 +base_prefix = / +ca_cert +cert_cert +cert_key + +[client] +name = replace_me +realm = replace_me +seed = replace_me +api_server = http://localhost:5000 \ No newline at end of file diff --git a/uncloud/docs/Makefile b/archive/uncloud_etcd_based/docs/Makefile similarity index 93% rename from uncloud/docs/Makefile rename to archive/uncloud_etcd_based/docs/Makefile index 5e7ea85..246b56c 100644 --- a/uncloud/docs/Makefile +++ b/archive/uncloud_etcd_based/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source/ BUILDDIR = build/ -DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/ +DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/ .PHONY: all build clean diff --git a/archive/uncloud_etcd_based/docs/README.md b/archive/uncloud_etcd_based/docs/README.md new file mode 100644 index 0000000..a5afbaa --- /dev/null +++ b/archive/uncloud_etcd_based/docs/README.md @@ -0,0 +1,12 @@ +# uncloud docs + +## Requirements +1. Python3 +2. Sphinx + +## Usage +Run `make build` to build docs. + +Run `make clean` to remove build directory. + +Run `make publish` to push build dir to https://ungleich.ch/ucloud/ \ No newline at end of file diff --git a/uncloud/configure/__init__.py b/archive/uncloud_etcd_based/docs/__init__.py similarity index 100% rename from uncloud/configure/__init__.py rename to archive/uncloud_etcd_based/docs/__init__.py diff --git a/uncloud/docs/__init__.py b/archive/uncloud_etcd_based/docs/source/__init__.py similarity index 100% rename from uncloud/docs/__init__.py rename to archive/uncloud_etcd_based/docs/source/__init__.py diff --git a/uncloud/docs/source/admin-guide b/archive/uncloud_etcd_based/docs/source/admin-guide.rst similarity index 72% rename from uncloud/docs/source/admin-guide rename to archive/uncloud_etcd_based/docs/source/admin-guide.rst index ec6597d..b62808d 100644 --- a/uncloud/docs/source/admin-guide +++ b/archive/uncloud_etcd_based/docs/source/admin-guide.rst @@ -56,40 +56,13 @@ To start host we created earlier, execute the following command ucloud host ungleich.ch -Create OS Image ---------------- +File & image scanners +-------------------------- -Create ucloud-init ready OS image (Optional) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This step is optional if you just want to test ucloud. However, sooner or later -you want to create OS images with ucloud-init to properly -contexualize VMs. - -1. Start a VM with OS image on which you want to install ucloud-init -2. Execute the following command on the started VM - - .. code-block:: sh - - apk add git - git clone https://code.ungleich.ch/ucloud/ucloud-init.git - cd ucloud-init - sh ./install.sh -3. Congratulations. Your image is now ucloud-init ready. - - -Upload Sample OS Image -~~~~~~~~~~~~~~~~~~~~~~ -Execute the following to get the sample OS image file. - -.. code-block:: sh - - mkdir /var/www/admin - (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download) - -Run File Scanner and Image Scanner -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make -images from tracked files. So, we need to track the file by running File Scanner +Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our +uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by +ucloud. We can only make images from tracked files. So, we need to track the +file by running File Scanner .. code-block:: sh diff --git a/uncloud/docs/source/conf.py b/archive/uncloud_etcd_based/docs/source/conf.py similarity index 100% rename from uncloud/docs/source/conf.py rename to archive/uncloud_etcd_based/docs/source/conf.py diff --git a/uncloud/docs/source/diagram-code/ucloud b/archive/uncloud_etcd_based/docs/source/diagram-code/ucloud similarity index 100% rename from uncloud/docs/source/diagram-code/ucloud rename to archive/uncloud_etcd_based/docs/source/diagram-code/ucloud diff --git a/archive/uncloud_etcd_based/docs/source/hacking.rst b/archive/uncloud_etcd_based/docs/source/hacking.rst new file mode 100644 index 0000000..1c750d6 --- /dev/null +++ b/archive/uncloud_etcd_based/docs/source/hacking.rst @@ -0,0 +1,36 @@ +Hacking +======= +Using uncloud in hacking (aka development) mode. + + +Get the code +------------ +.. code-block:: sh + :linenos: + + git clone https://code.ungleich.ch/uncloud/uncloud.git + + + +Install python requirements +--------------------------- +You need to have python3 installed. + +.. code-block:: sh + :linenos: + + cd uncloud! + python -m venv venv + . ./venv/bin/activate + ./bin/uncloud-run-reinstall + + + +Install os requirements +----------------------- +Install the following software packages: **dnsmasq**. + +If you already have a working IPv6 SLAAC and DNS setup, +this step can be skipped. + +Note that you need at least one /64 IPv6 network to run uncloud. diff --git a/uncloud/docs/source/images/ucloud.svg b/archive/uncloud_etcd_based/docs/source/images/ucloud.svg similarity index 100% rename from uncloud/docs/source/images/ucloud.svg rename to archive/uncloud_etcd_based/docs/source/images/ucloud.svg diff --git a/uncloud/docs/source/index.rst b/archive/uncloud_etcd_based/docs/source/index.rst similarity index 90% rename from uncloud/docs/source/index.rst rename to archive/uncloud_etcd_based/docs/source/index.rst index b31cff3..fad1f88 100644 --- a/uncloud/docs/source/index.rst +++ b/archive/uncloud_etcd_based/docs/source/index.rst @@ -11,14 +11,13 @@ Welcome to ucloud's documentation! :caption: Contents: introduction - user-guide setup-install + vm-images + user-guide admin-guide - user-guide/how-to-create-an-os-image-for-ucloud troubleshooting hacking - Indices and tables ================== diff --git a/uncloud/docs/source/introduction.rst b/archive/uncloud_etcd_based/docs/source/introduction.rst similarity index 100% rename from uncloud/docs/source/introduction.rst rename to archive/uncloud_etcd_based/docs/source/introduction.rst diff --git a/uncloud/docs/source/misc/todo.rst b/archive/uncloud_etcd_based/docs/source/misc/todo.rst similarity index 100% rename from uncloud/docs/source/misc/todo.rst rename to archive/uncloud_etcd_based/docs/source/misc/todo.rst diff --git a/uncloud/docs/source/setup-install.rst b/archive/uncloud_etcd_based/docs/source/setup-install.rst similarity index 100% rename from uncloud/docs/source/setup-install.rst rename to archive/uncloud_etcd_based/docs/source/setup-install.rst diff --git a/uncloud/docs/source/theory/summary.rst b/archive/uncloud_etcd_based/docs/source/theory/summary.rst similarity index 100% rename from uncloud/docs/source/theory/summary.rst rename to archive/uncloud_etcd_based/docs/source/theory/summary.rst diff --git a/uncloud/docs/source/troubleshooting.rst b/archive/uncloud_etcd_based/docs/source/troubleshooting.rst similarity index 100% rename from uncloud/docs/source/troubleshooting.rst rename to archive/uncloud_etcd_based/docs/source/troubleshooting.rst diff --git a/uncloud/docs/source/user-guide.rst b/archive/uncloud_etcd_based/docs/source/user-guide.rst similarity index 100% rename from uncloud/docs/source/user-guide.rst rename to archive/uncloud_etcd_based/docs/source/user-guide.rst diff --git a/uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst b/archive/uncloud_etcd_based/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst similarity index 100% rename from uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst rename to archive/uncloud_etcd_based/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst diff --git a/archive/uncloud_etcd_based/docs/source/vm-images.rst b/archive/uncloud_etcd_based/docs/source/vm-images.rst new file mode 100644 index 0000000..4b2758a --- /dev/null +++ b/archive/uncloud_etcd_based/docs/source/vm-images.rst @@ -0,0 +1,66 @@ +VM images +================================== + +Overview +--------- + +ucloud tries to be least invasise towards VMs and only require +strictly necessary changes for running in a virtualised +environment. This includes configurations for: + +* Configuring the network +* Managing access via ssh keys +* Resizing the attached disk(s) + +Upstream images +--------------- + +The 'official' uncloud images are defined in the `uncloud/images +`_ repository. + +How to make you own Uncloud images +---------------------------------- + +.. note:: + It is fairly easy to create your own images for uncloud, as the common + operations (which are detailed below) can be automatically handled by the + `uncloud/uncloud-init `_ tool. + +Network configuration +~~~~~~~~~~~~~~~~~~~~~ +All VMs in ucloud are required to support IPv6. The primary network +configuration is always done using SLAAC. A VM thus needs only to be +configured to + +* accept router advertisements on all network interfaces +* use the router advertisements to configure the network interfaces +* accept the DNS entries from the router advertisements + + +Configuring SSH keys +~~~~~~~~~~~~~~~~~~~~ + +To be able to access the VM, ucloud support provisioning SSH keys. + +To accept ssh keys in your VM, request the URL +*http://metadata/ssh_keys*. Add the content to the appropriate user's +**authorized_keys** file. Below you find sample code to accomplish +this task: + +.. code-block:: sh + + tmp=$(mktemp) + curl -s http://metadata/ssk_keys > "$tmp" + touch ~/.ssh/authorized_keys # ensure it exists + cat ~/.ssh/authorized_keys >> "$tmp" + sort "$tmp" | uniq > ~/.ssh/authorized_keys + + +Disk resize +~~~~~~~~~~~ +In virtualised environments, the disk sizes might grow. The operating +system should detect disks that are bigger than the existing partition +table and resize accordingly. This task is os specific. + +ucloud does not support shrinking disks due to the complexity and +intra OS dependencies. diff --git a/archive/uncloud_etcd_based/scripts/uncloud b/archive/uncloud_etcd_based/scripts/uncloud new file mode 100755 index 0000000..9517b01 --- /dev/null +++ b/archive/uncloud_etcd_based/scripts/uncloud @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +import logging +import sys +import importlib +import argparse +import os + +from etcd3.exceptions import ConnectionFailedError + +from uncloud.common import settings +from uncloud import UncloudException +from uncloud.common.cli import resolve_otp_credentials + +# Components that use etcd +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', + 'imagescanner', 'metadata', 'configure', 'hack'] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('oneshot') +#ALL_COMPONENTS.append('cli') + + +if __name__ == '__main__': + arg_parser = argparse.ArgumentParser() + subparsers = arg_parser.add_subparsers(dest='command') + + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument('--debug', '-d', action='store_true', default=False, + help='More verbose logging') + parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory', + default=os.path.expanduser('~/uncloud')) + + etcd_parser = argparse.ArgumentParser(add_help=False) + etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-port') + etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') + etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') + etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') + + for component in ALL_COMPONENTS: + mod = importlib.import_module('uncloud.{}.main'.format(component)) + parser = getattr(mod, 'arg_parser') + + if component in ETCD_COMPONENTS: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) + else: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) + + arguments = vars(arg_parser.parse_args()) + etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value] + etcd_arguments = { + 'etcd': { + key.replace('etcd_', ''): arguments[key] + for key in etcd_arguments + } + } + if not arguments['command']: + arg_parser.print_help() + else: + # Initializing Settings and resolving otp_credentials + # It is neccessary to resolve_otp_credentials after argument parsing is done because + # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and + # providing the default values for --name, --realm and --seed arguments from the values + # we read from file. But, now we are asking user about where the config file lives. So, + # to providing default value is not possible before parsing arguments. So, we are doing + # it after.. +# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) +# resolve_otp_credentials(arguments) + + name = arguments.pop('command') + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') + + if arguments['debug']: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + log = logging.getLogger() + + try: + main(arguments) + except UncloudException as err: + log.error(err) + sys.exit(1) +# except ConnectionFailedError as err: +# log.error('Cannot connect to etcd: {}'.format(err)) + except Exception as err: + log.exception(err) diff --git a/setup.py b/archive/uncloud_etcd_based/setup.py similarity index 98% rename from setup.py rename to archive/uncloud_etcd_based/setup.py index 0764d74..f5e0718 100644 --- a/setup.py +++ b/archive/uncloud_etcd_based/setup.py @@ -41,6 +41,7 @@ setup( "colorama", "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3", "marshmallow", + "ldap3" ], scripts=["scripts/uncloud"], data_files=[ diff --git a/uncloud/docs/source/__init__.py b/archive/uncloud_etcd_based/test/__init__.py similarity index 100% rename from uncloud/docs/source/__init__.py rename to archive/uncloud_etcd_based/test/__init__.py diff --git a/archive/uncloud_etcd_based/test/test_mac_local.py b/archive/uncloud_etcd_based/test/test_mac_local.py new file mode 100644 index 0000000..3a4ac3a --- /dev/null +++ b/archive/uncloud_etcd_based/test/test_mac_local.py @@ -0,0 +1,37 @@ +import unittest +from unittest.mock import Mock + +from uncloud.hack.mac import MAC +from uncloud import UncloudException + +class TestMacLocal(unittest.TestCase): + def setUp(self): + self.config = Mock() + self.config.arguments = {"no_db":True} + self.mac = MAC(self.config) + self.mac.create() + + def testMacInt(self): + self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong first MAC index") + + def testMacRepr(self): + self.assertEqual(self.mac.__repr__(), '420000000001', "wrong first MAC index") + + def testMacStr(self): + self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong first MAC index") + + def testValidationRaise(self): + with self.assertRaises(UncloudException): + self.mac.validate_mac("2") + + def testValidation(self): + self.assertTrue(self.mac.validate_mac("42:00:00:00:00:01"), "Validation of a given MAC not working properly") + + def testNextMAC(self): + self.mac.create() + self.assertEqual(self.mac.__repr__(), '420000000001', "wrong second MAC index") + self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong second MAC index") + self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong second MAC index") + +if __name__ == '__main__': + unittest.main() diff --git a/archive/uncloud_etcd_based/uncloud/__init__.py b/archive/uncloud_etcd_based/uncloud/__init__.py new file mode 100644 index 0000000..2920f47 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/__init__.py @@ -0,0 +1,2 @@ +class UncloudException(Exception): + pass diff --git a/uncloud/api/README.md b/archive/uncloud_etcd_based/uncloud/api/README.md similarity index 100% rename from uncloud/api/README.md rename to archive/uncloud_etcd_based/uncloud/api/README.md diff --git a/uncloud/api/__init__.py b/archive/uncloud_etcd_based/uncloud/api/__init__.py similarity index 100% rename from uncloud/api/__init__.py rename to archive/uncloud_etcd_based/uncloud/api/__init__.py diff --git a/uncloud/api/common_fields.py b/archive/uncloud_etcd_based/uncloud/api/common_fields.py similarity index 83% rename from uncloud/api/common_fields.py rename to archive/uncloud_etcd_based/uncloud/api/common_fields.py index 8bcf777..ba9fb37 100755 --- a/uncloud/api/common_fields.py +++ b/archive/uncloud_etcd_based/uncloud/api/common_fields.py @@ -1,7 +1,6 @@ import os -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared class Optional: @@ -54,9 +53,7 @@ class VmUUIDField(Field): def vm_uuid_validation(self): r = shared.etcd_client.get( - os.path.join(settings["etcd"]["vm_prefix"], self.uuid) + os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid) ) if not r: - self.add_error( - "VM with uuid {} does not exists".format(self.uuid) - ) + self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/archive/uncloud_etcd_based/uncloud/api/create_image_store.py b/archive/uncloud_etcd_based/uncloud/api/create_image_store.py new file mode 100755 index 0000000..90e0f92 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/api/create_image_store.py @@ -0,0 +1,19 @@ +import json +import os + +from uuid import uuid4 + +from uncloud.common.shared import shared + +data = { + 'is_public': True, + 'type': 'ceph', + 'name': 'images', + 'description': 'first ever public image-store', + 'attributes': {'list': [], 'key': [], 'pool': 'images'}, +} + +shared.etcd_client.put( + os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex), + json.dumps(data), +) diff --git a/uncloud/api/helper.py b/archive/uncloud_etcd_based/uncloud/api/helper.py similarity index 86% rename from uncloud/api/helper.py rename to archive/uncloud_etcd_based/uncloud/api/helper.py index c806814..8ceb3a6 100755 --- a/uncloud/api/helper.py +++ b/archive/uncloud_etcd_based/uncloud/api/helper.py @@ -1,14 +1,12 @@ import binascii import ipaddress import random -import subprocess as sp import logging import requests from pyotp import TOTP -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared logger = logging.getLogger(__name__) @@ -16,9 +14,9 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": settings["otp"]["auth_name"], - "auth_token": TOTP(settings["otp"]["auth_seed"]).now(), - "auth_realm": settings["otp"]["auth_realm"], + "auth_name": shared.settings["otp"]["auth_name"], + "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), + "auth_realm": shared.settings["otp"]["auth_realm"], "name": name, "realm": realm, "token": token, @@ -26,13 +24,13 @@ def check_otp(name, realm, token): except binascii.Error as err: logger.error( "Cannot compute OTP for seed: {}".format( - settings["otp"]["auth_seed"] + shared.settings["otp"]["auth_seed"] ) ) return 400 response = requests.post( - settings["otp"]["verification_controller_url"], json=data + shared.settings["otp"]["verification_controller_url"], json=data ) return response.status_code @@ -88,7 +86,7 @@ def resolve_image_name(name, etcd_client): ) images = etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) # Try to find image with name == image_name and store_name == store_name @@ -112,9 +110,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -149,3 +145,4 @@ def mac2ipv6(mac, prefix): lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) prefix = ipaddress.IPv6Address(prefix) return str(prefix + int(lower_part)) + diff --git a/uncloud/api/main.py b/archive/uncloud_etcd_based/uncloud/api/main.py similarity index 54% rename from uncloud/api/main.py rename to archive/uncloud_etcd_based/uncloud/api/main.py index 93bada7..73e8e21 100644 --- a/uncloud/api/main.py +++ b/archive/uncloud_etcd_based/uncloud/api/main.py @@ -1,6 +1,7 @@ import json import pynetbox import logging +import argparse from uuid import uuid4 from os.path import join as join_path @@ -9,14 +10,13 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException +from uncloud.common.shared import shared + from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType -from uncloud.settings import settings -from uncloud.shared import shared - -from . import schemas -from .helper import generate_mac, mac2ipv6 +from uncloud.api import schemas +from uncloud.api.helper import generate_mac, mac2ipv6 from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -25,6 +25,9 @@ app = Flask(__name__) api = Api(app) app.logger.handlers.clear() +arg_parser = argparse.ArgumentParser('api', add_help=False) +arg_parser.add_argument('--port', '-p') + @app.errorhandler(Exception) def handle_exception(e): @@ -34,7 +37,7 @@ def handle_exception(e): return e # now you're handling non-HTTP exceptions only - return {"message": "Server Error"}, 500 + return {'message': 'Server Error'}, 500 class CreateVM(Resource): @@ -46,33 +49,33 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(settings["etcd"]["vm_prefix"], vm_uuid) + vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) specs = { - "cpu": validator.specs["cpu"], - "ram": validator.specs["ram"], - "os-ssd": validator.specs["os-ssd"], - "hdd": validator.specs["hdd"], + 'cpu': validator.specs['cpu'], + 'ram': validator.specs['ram'], + 'os-ssd': validator.specs['os-ssd'], + 'hdd': validator.specs['hdd'], } - macs = [generate_mac() for _ in range(len(data["network"]))] + macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, "/v1/counter/tap" + shared.etcd_client, shared.settings['etcd']['tap_counter'] ) - for _ in range(len(data["network"])) + for _ in range(len(data['network'])) ] vm_entry = { - "name": data["vm_name"], - "owner": data["name"], - "owner_realm": data["realm"], - "specs": specs, - "hostname": "", - "status": VMStatus.stopped, - "image_uuid": validator.image_uuid, - "log": [], - "vnc_socket": "", - "network": list(zip(data["network"], macs, tap_ids)), - "metadata": {"ssh-keys": []}, - "in_migration": False, + 'name': data['vm_name'], + 'owner': data['name'], + 'owner_realm': data['realm'], + 'specs': specs, + 'hostname': '', + 'status': VMStatus.stopped, + 'image_uuid': validator.image_uuid, + 'log': [], + 'vnc_socket': '', + 'network': list(zip(data['network'], macs, tap_ids)), + 'metadata': {'ssh-keys': []}, + 'in_migration': False, } shared.etcd_client.put(vm_key, vm_entry, value_in_json=True) @@ -80,39 +83,39 @@ class CreateVM(Resource): r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) - return {"message": "VM Creation Queued"}, 200 + return {'message': 'VM Creation Queued'}, 200 return validator.get_errors(), 400 class VmStatus(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], data["uuid"]) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) vm_value = vm.value.copy() - vm_value["ip"] = [] + vm_value['ip'] = [] for network_mac_and_tap in vm.network: network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], - data["name"], + shared.settings['etcd']['network_prefix'], + data['name'], network_name, ), value_in_json=True, ) ipv6_addr = ( - network.value.get("ipv6").split("::")[0] + "::" + network.value.get('ipv6').split('::')[0] + '::' ) - vm_value["ip"].append(mac2ipv6(mac, ipv6_addr)) + vm_value['ip'].append(mac2ipv6(mac, ipv6_addr)) vm.value = vm_value return vm.value else: @@ -126,26 +129,26 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = shared.etcd_client.get( - join_path(settings["etcd"]["file_prefix"], data["uuid"]) + join_path(shared.settings['etcd']['file_prefix'], data['uuid']) ) file_entry_value = json.loads(file_entry.value) image_entry_json = { - "status": "TO_BE_CREATED", - "owner": file_entry_value["owner"], - "filename": file_entry_value["filename"], - "name": data["name"], - "store_name": data["image_store"], - "visibility": "public", + 'status': 'TO_BE_CREATED', + 'owner': file_entry_value['owner'], + 'filename': file_entry_value['filename'], + 'name': data['name'], + 'store_name': data['image_store'], + 'visibility': 'public', } shared.etcd_client.put( join_path( - settings["etcd"]["image_prefix"], data["uuid"] + shared.settings['etcd']['image_prefix'], data['uuid'] ), json.dumps(image_entry_json), ) - return {"message": "Image queued for creation."} + return {'message': 'Image queued for creation.'} return validator.get_errors(), 400 @@ -153,15 +156,15 @@ class ListPublicImages(Resource): @staticmethod def get(): images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings['etcd']['image_prefix'], value_in_json=True ) - r = {"images": []} + r = {'images': []} for image in images: - image_key = "{}:{}".format( - image.value["store_name"], image.value["name"] + image_key = '{}:{}'.format( + image.value['store_name'], image.value['name'] ) - r["images"].append( - {"name": image_key, "status": image.value["status"]} + r['images'].append( + {'name': image_key, 'status': image.value['status']} ) return r, 200 @@ -174,14 +177,14 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], data["uuid"]) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) - action = data["action"] + action = data['action'] - if action == "start": - action = "schedule" + if action == 'start': + action = 'schedule' - if action == "delete" and vm_entry.hostname == "": + if action == 'delete' and vm_entry.hostname == '': if shared.storage_handler.is_vm_image_exists( vm_entry.uuid ): @@ -190,25 +193,25 @@ class VMAction(Resource): ) if r_status: shared.etcd_client.client.delete(vm_entry.key) - return {"message": "VM successfully deleted"} + return {'message': 'VM successfully deleted'} else: logger.error( - "Some Error Occurred while deleting VM" + 'Some Error Occurred while deleting VM' ) - return {"message": "VM deletion unsuccessfull"} + return {'message': 'VM deletion unsuccessfull'} else: shared.etcd_client.client.delete(vm_entry.key) - return {"message": "VM successfully deleted"} + return {'message': 'VM successfully deleted'} r = RequestEntry.from_scratch( - type="{}VM".format(action.title()), - uuid=data["uuid"], + type='{}VM'.format(action.title()), + uuid=data['uuid'], hostname=vm_entry.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( - {"message": "VM {} Queued".format(action.title())}, + {'message': 'VM {} Queued'.format(action.title())}, 200, ) else: @@ -222,20 +225,20 @@ class VMMigration(Resource): validator = schemas.VmMigrationSchema(data) if validator.is_valid(): - vm = shared.vm_pool.get(data["uuid"]) + vm = shared.vm_pool.get(data['uuid']) r = RequestEntry.from_scratch( type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( - settings["etcd"]["host_prefix"], + shared.settings['etcd']['host_prefix'], validator.destination.value, ), - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( - {"message": "VM Migration Initialization Queued"}, + {'message': 'VM Migration Initialization Queued'}, 200, ) else: @@ -244,32 +247,32 @@ class VMMigration(Resource): class ListUserVM(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) if validator.is_valid(): vms = shared.etcd_client.get_prefix( - settings["etcd"]["vm_prefix"], value_in_json=True + shared.settings['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter( - lambda v: v.value["owner"] == data["name"], vms + lambda v: v.value['owner'] == data['name'], vms ) for vm in user_vms: return_vms.append( { - "name": vm.value["name"], - "vm_uuid": vm.key.split("/")[-1], - "specs": vm.value["specs"], - "status": vm.value["status"], - "hostname": vm.value["hostname"], - "vnc_socket": vm.value.get("vnc_socket", None), + 'name': vm.value['name'], + 'vm_uuid': vm.key.split('/')[-1], + 'specs': vm.value['specs'], + 'status': vm.value['status'], + 'hostname': vm.value['hostname'], + 'vnc_socket': vm.value.get('vnc_socket', None), } ) if return_vms: - return {"message": return_vms}, 200 - return {"message": "No VM found"}, 404 + return {'message': return_vms}, 200 + return {'message': 'No VM found'}, 404 else: return validator.get_errors(), 400 @@ -277,28 +280,26 @@ class ListUserVM(Resource): class ListUserFiles(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) if validator.is_valid(): files = shared.etcd_client.get_prefix( - settings["etcd"]["file_prefix"], value_in_json=True + shared.settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] - user_files = list( - filter( - lambda f: f.value["owner"] == data["name"], files - ) - ) + user_files = [f for f in files if f.value['owner'] == data['name']] for file in user_files: - return_files.append( - { - "filename": file.value["filename"], - "uuid": file.key.split("/")[-1], - } - ) - return {"message": return_files}, 200 + file_uuid = file.key.split('/')[-1] + file = file.value + file['uuid'] = file_uuid + + file.pop('sha512sum', None) + file.pop('owner', None) + + return_files.append(file) + return {'message': return_files}, 200 else: return validator.get_errors(), 400 @@ -310,19 +311,19 @@ class CreateHost(Resource): validator = schemas.CreateHostSchema(data) if validator.is_valid(): host_key = join_path( - settings["etcd"]["host_prefix"], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { - "specs": data["specs"], - "hostname": data["hostname"], - "status": "DEAD", - "last_heartbeat": "", + 'specs': data['specs'], + 'hostname': data['hostname'], + 'status': 'DEAD', + 'last_heartbeat': '', } shared.etcd_client.put( host_key, host_entry, value_in_json=True ) - return {"message": "Host Created"}, 200 + return {'message': 'Host Created'}, 200 return validator.get_errors(), 400 @@ -333,9 +334,9 @@ class ListHost(Resource): hosts = shared.host_pool.hosts r = { host.key: { - "status": host.status, - "specs": host.specs, - "hostname": host.hostname, + 'status': host.status, + 'specs': host.specs, + 'hostname': host.hostname, } for host in hosts } @@ -344,7 +345,7 @@ class ListHost(Resource): class GetSSHKeys(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.GetSSHSchema(data) if validator.is_valid(): @@ -352,29 +353,29 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', ) etcd_entry = shared.etcd_client.get_prefix( etcd_key, value_in_json=True ) keys = { - key.key.split("/")[-1]: key.value + key.key.split('/')[-1]: key.value for key in etcd_entry } - return {"keys": keys} + return {'keys': keys} else: # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True @@ -382,14 +383,14 @@ class GetSSHKeys(Resource): if etcd_entry: return { - "keys": { - etcd_entry.key.split("/")[ + 'keys': { + etcd_entry.key.split('/')[ -1 ]: etcd_entry.value } } else: - return {"keys": {}} + return {'keys': {}} else: return validator.get_errors(), 400 @@ -403,56 +404,56 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True ) if etcd_entry: return { - "message": "Key with name '{}' already exists".format( - data["key_name"] + 'message': 'Key with name "{}" already exists'.format( + data['key_name'] ) } else: # Key Not Found. It implies user' haven't added any key yet. shared.etcd_client.put( - etcd_key, data["key"], value_in_json=True + etcd_key, data['key'], value_in_json=True ) - return {"message": "Key added successfully"} + return {'message': 'Key added successfully'} else: return validator.get_errors(), 400 class RemoveSSHKey(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.RemoveSSHSchema(data) if validator.is_valid(): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings["etcd"]["user_prefix"], - data["realm"], - data["name"], - "key", - data["key_name"], + shared.settings['etcd']['user_prefix'], + data['realm'], + data['name'], + 'key', + data['key_name'], ) etcd_entry = shared.etcd_client.get( etcd_key, value_in_json=True ) if etcd_entry: shared.etcd_client.client.delete(etcd_key) - return {"message": "Key successfully removed."} + return {'message': 'Key successfully removed.'} else: return { - "message": "No Key with name '{}' Exists at all.".format( - data["key_name"] + 'message': 'No Key with name "{}" Exists at all.'.format( + data['key_name'] ) } else: @@ -468,104 +469,107 @@ class CreateNetwork(Resource): if validator.is_valid(): network_entry = { - "id": counters.increment_etcd_counter( - shared.etcd_client, "/v1/counter/vxlan" + 'id': counters.increment_etcd_counter( + shared.etcd_client, shared.settings['etcd']['vxlan_counter'] ), - "type": data["type"], + 'type': data['type'], } if validator.user.value: try: nb = pynetbox.api( - url=settings["netbox"]["url"], - token=settings["netbox"]["token"], + url=shared.settings['netbox']['url'], + token=shared.settings['netbox']['token'], ) nb_prefix = nb.ipam.prefixes.get( - prefix=settings["network"]["prefix"] + prefix=shared.settings['network']['prefix'] ) prefix = nb_prefix.available_prefixes.create( data={ - "prefix_length": int( - settings["network"]["prefix_length"] + 'prefix_length': int( + shared.settings['network']['prefix_length'] ), - "description": '{}\'s network "{}"'.format( - data["name"], data["network_name"] + 'description': '{}\'s network "{}"'.format( + data['name'], data['network_name'] ), - "is_pool": True, + 'is_pool': True, } ) except Exception as err: app.logger.error(err) return { - "message": "Error occured while creating network." + 'message': 'Error occured while creating network.' } else: - network_entry["ipv6"] = prefix["prefix"] + network_entry['ipv6'] = prefix['prefix'] else: - network_entry["ipv6"] = "fd00::/64" + network_entry['ipv6'] = 'fd00::/64' network_key = join_path( - settings["etcd"]["network_prefix"], - data["name"], - data["network_name"], + shared.settings['etcd']['network_prefix'], + data['name'], + data['network_name'], ) shared.etcd_client.put( network_key, network_entry, value_in_json=True ) - return {"message": "Network successfully added."} + return {'message': 'Network successfully added.'} else: return validator.get_errors(), 400 class ListUserNetwork(Resource): @staticmethod - def get(): + def post(): data = request.json validator = schemas.OTPSchema(data) if validator.is_valid(): prefix = join_path( - settings["etcd"]["network_prefix"], data["name"] + shared.settings['etcd']['network_prefix'], data['name'] ) networks = shared.etcd_client.get_prefix( prefix, value_in_json=True ) user_networks = [] for net in networks: - net.value["name"] = net.key.split("/")[-1] + net.value['name'] = net.key.split('/')[-1] user_networks.append(net.value) - return {"networks": user_networks}, 200 + return {'networks': user_networks}, 200 else: return validator.get_errors(), 400 -api.add_resource(CreateVM, "/vm/create") -api.add_resource(VmStatus, "/vm/status") +api.add_resource(CreateVM, '/vm/create') +api.add_resource(VmStatus, '/vm/status') -api.add_resource(VMAction, "/vm/action") -api.add_resource(VMMigration, "/vm/migrate") +api.add_resource(VMAction, '/vm/action') +api.add_resource(VMMigration, '/vm/migrate') -api.add_resource(CreateImage, "/image/create") -api.add_resource(ListPublicImages, "/image/list-public") +api.add_resource(CreateImage, '/image/create') +api.add_resource(ListPublicImages, '/image/list-public') -api.add_resource(ListUserVM, "/user/vms") -api.add_resource(ListUserFiles, "/user/files") -api.add_resource(ListUserNetwork, "/user/networks") +api.add_resource(ListUserVM, '/user/vms') +api.add_resource(ListUserFiles, '/user/files') +api.add_resource(ListUserNetwork, '/user/networks') -api.add_resource(AddSSHKey, "/user/add-ssh") -api.add_resource(RemoveSSHKey, "/user/remove-ssh") -api.add_resource(GetSSHKeys, "/user/get-ssh") +api.add_resource(AddSSHKey, '/user/add-ssh') +api.add_resource(RemoveSSHKey, '/user/remove-ssh') +api.add_resource(GetSSHKeys, '/user/get-ssh') -api.add_resource(CreateHost, "/host/create") -api.add_resource(ListHost, "/host/list") +api.add_resource(CreateHost, '/host/create') +api.add_resource(ListHost, '/host/list') -api.add_resource(CreateNetwork, "/network/create") +api.add_resource(CreateNetwork, '/network/create') -def main(debug=False, port=None): +def main(arguments): + debug = arguments['debug'] + port = arguments['port'] + try: image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], value_in_json=True + shared.settings['etcd']['image_store_prefix'], value_in_json=True ) ) except KeyError: @@ -576,27 +580,21 @@ def main(debug=False, port=None): # # if not image_stores: # data = { - # "is_public": True, - # "type": "ceph", - # "name": "images", - # "description": "first ever public image-store", - # "attributes": {"list": [], "key": [], "pool": "images"}, + # 'is_public': True, + # 'type': 'ceph', + # 'name': 'images', + # 'description': 'first ever public image-store', + # 'attributes': {'list': [], 'key': [], 'pool': 'images'}, # } # shared.etcd_client.put( # join_path( - # settings["etcd"]["image_store_prefix"], uuid4().hex + # shared.settings['etcd']['image_store_prefix'], uuid4().hex # ), # json.dumps(data), # ) try: - app.run(host="::", - port=port, - debug=debug) + app.run(host='::', port=port, debug=debug) except OSError as e: - raise UncloudException("Failed to start Flask: {}".format(e)) - - -if __name__ == "__main__": - main() + raise UncloudException('Failed to start Flask: {}'.format(e)) diff --git a/uncloud/api/schemas.py b/archive/uncloud_etcd_based/uncloud/api/schemas.py similarity index 95% rename from uncloud/api/schemas.py rename to archive/uncloud_etcd_based/uncloud/api/schemas.py index 65055c4..87f20c9 100755 --- a/uncloud/api/schemas.py +++ b/archive/uncloud_etcd_based/uncloud/api/schemas.py @@ -21,8 +21,7 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema): def file_uuid_validation(self): file_entry = shared.etcd_client.get( os.path.join( - settings["etcd"]["file_prefix"], self.uuid.value + shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value ) ) if file_entry is None: @@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema): def image_store_name_validation(self): image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"] + shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] ) ) @@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema): for net in _network: network = shared.etcd_client.get( os.path.join( - settings["etcd"]["network_prefix"], + shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, net, ), @@ -322,7 +321,7 @@ class CreateVMSchema(OTPSchema): "Your specified OS-SSD is not in correct units" ) - if _cpu < 1: + if int(_cpu) < 1: self.add_error("CPU must be atleast 1") if parsed_ram < bitmath.GB(1): @@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema): self.add_error("Can't migrate non-running VM") if vm.hostname == os.path.join( - settings["etcd"]["host_prefix"], self.destination.value + shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value ): self.add_error( "Destination host couldn't be same as Source Host" @@ -528,9 +527,7 @@ class GetSSHSchema(OTPSchema): class CreateNetwork(OTPSchema): def __init__(self, data): - self.network_name = Field( - "network_name", str, data.get("network_name", KeyError) - ) + self.network_name = Field("network_name", str, data.get("network_name", KeyError)) self.type = Field("type", str, data.get("type", KeyError)) self.user = Field("user", bool, bool(data.get("user", False))) @@ -541,14 +538,8 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - network = shared.etcd_client.get( - os.path.join( - settings["etcd"]["network_prefix"], - self.name.value, - self.network_name.value, - ), - value_in_json=True, - ) + key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) + network = shared.etcd_client.get(key, value_in_json=True) if network: self.add_error( "Network with name {} already exists".format( diff --git a/uncloud/network/__init__.py b/archive/uncloud_etcd_based/uncloud/cli/__init__.py similarity index 100% rename from uncloud/network/__init__.py rename to archive/uncloud_etcd_based/uncloud/cli/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/cli/helper.py b/archive/uncloud_etcd_based/uncloud/cli/helper.py new file mode 100644 index 0000000..51a4355 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/helper.py @@ -0,0 +1,46 @@ +import requests +import json +import argparse +import binascii + +from pyotp import TOTP +from os.path import join as join_path +from uncloud.common.shared import shared + + +def get_otp_parser(): + otp_parser = argparse.ArgumentParser('otp') + otp_parser.add_argument('--name') + otp_parser.add_argument('--realm') + otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') + + return otp_parser + + +def load_dump_pretty(content): + if isinstance(content, bytes): + content = content.decode('utf-8') + parsed = json.loads(content) + return json.dumps(parsed, indent=4, sort_keys=True) + + +def make_request(*args, data=None, request_method=requests.post): + try: + r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) + except requests.exceptions.RequestException: + print('Error occurred while connecting to API server.') + else: + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except binascii.Error: + raise argparse.ArgumentTypeError('Invalid seed') + else: + return token diff --git a/archive/uncloud_etcd_based/uncloud/cli/host.py b/archive/uncloud_etcd_based/uncloud/cli/host.py new file mode 100644 index 0000000..e912567 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/host.py @@ -0,0 +1,45 @@ +import requests + +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class HostParser(BaseParser): + def __init__(self): + super().__init__('host') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs) + p.add_argument('--hostname', required=True) + p.add_argument('--cpu', required=True, type=int) + p.add_argument('--ram', required=True) + p.add_argument('--os-ssd', required=True) + p.add_argument('--hdd', default=list()) + + def list(self, **kwargs): + self.subparser.add_parser('list', **kwargs) + + +parser = HostParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('host_subcommand') + if not subcommand: + arg_parser.print_help() + else: + request_method = requests.post + data = None + if subcommand == 'create': + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + data = kwargs + elif subcommand == 'list': + request_method = requests.get + + make_request('host', subcommand, data=data, request_method=request_method) diff --git a/archive/uncloud_etcd_based/uncloud/cli/image.py b/archive/uncloud_etcd_based/uncloud/cli/image.py new file mode 100644 index 0000000..2f59c32 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/image.py @@ -0,0 +1,38 @@ +import requests + +from uncloud.cli.helper import make_request +from uncloud.common.parser import BaseParser + + +class ImageParser(BaseParser): + def __init__(self): + super().__init__('image') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', **kwargs) + p.add_argument('--name', required=True) + p.add_argument('--uuid', required=True) + p.add_argument('--image-store', required=True, dest='image_store') + + def list(self, **kwargs): + self.subparser.add_parser('list', **kwargs) + + +parser = ImageParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('image_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = None + request_method = requests.post + if subcommand == 'list': + subcommand = 'list-public' + request_method = requests.get + elif subcommand == 'create': + data = kwargs + + make_request('image', subcommand, data=data, request_method=request_method) diff --git a/archive/uncloud_etcd_based/uncloud/cli/main.py b/archive/uncloud_etcd_based/uncloud/cli/main.py new file mode 100644 index 0000000..9a42497 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/main.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import argparse +import importlib + +arg_parser = argparse.ArgumentParser('cli', add_help=False) +subparser = arg_parser.add_subparsers(dest='subcommand') + +for component in ['user', 'host', 'image', 'network', 'vm']: + module = importlib.import_module('uncloud.cli.{}'.format(component)) + parser = getattr(module, 'arg_parser') + subparser.add_parser(name=parser.prog, parents=[parser]) + + +def main(arguments): + if not arguments['subcommand']: + arg_parser.print_help() + else: + name = arguments.pop('subcommand') + arguments.pop('debug') + mod = importlib.import_module('uncloud.cli.{}'.format(name)) + _main = getattr(mod, 'main') + _main(**arguments) diff --git a/archive/uncloud_etcd_based/uncloud/cli/network.py b/archive/uncloud_etcd_based/uncloud/cli/network.py new file mode 100644 index 0000000..55798bf --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/network.py @@ -0,0 +1,32 @@ +import requests + +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class NetworkParser(BaseParser): + def __init__(self): + super().__init__('network') + + def create(self, **kwargs): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs) + p.add_argument('--network-name', required=True) + p.add_argument('--network-type', required=True, dest='type') + p.add_argument('--user', action='store_true') + + +parser = NetworkParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('network_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = None + request_method = requests.post + if subcommand == 'create': + data = kwargs + + make_request('network', subcommand, data=data, request_method=request_method) diff --git a/archive/uncloud_etcd_based/uncloud/cli/user.py b/archive/uncloud_etcd_based/uncloud/cli/user.py new file mode 100755 index 0000000..3a4cc4e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/user.py @@ -0,0 +1,41 @@ +from uncloud.cli.helper import make_request, get_otp_parser +from uncloud.common.parser import BaseParser + + +class UserParser(BaseParser): + def __init__(self): + super().__init__('user') + + def files(self, **kwargs): + self.subparser.add_parser('files', parents=[get_otp_parser()], **kwargs) + + def vms(self, **kwargs): + self.subparser.add_parser('vms', parents=[get_otp_parser()], **kwargs) + + def networks(self, **kwargs): + self.subparser.add_parser('networks', parents=[get_otp_parser()], **kwargs) + + def add_ssh(self, **kwargs): + p = self.subparser.add_parser('add-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', required=True) + p.add_argument('--key', required=True) + + def get_ssh(self, **kwargs): + p = self.subparser.add_parser('get-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', default='') + + def remove_ssh(self, **kwargs): + p = self.subparser.add_parser('remove-ssh', parents=[get_otp_parser()], **kwargs) + p.add_argument('--key-name', required=True) + + +parser = UserParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('user_subcommand') + if not subcommand: + arg_parser.print_help() + else: + make_request('user', subcommand, data=kwargs) diff --git a/archive/uncloud_etcd_based/uncloud/cli/vm.py b/archive/uncloud_etcd_based/uncloud/cli/vm.py new file mode 100644 index 0000000..396530e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/vm.py @@ -0,0 +1,62 @@ +from uncloud.common.parser import BaseParser +from uncloud.cli.helper import make_request, get_otp_parser + + +class VMParser(BaseParser): + def __init__(self): + super().__init__('vm') + + def start(self, **args): + p = self.subparser.add_parser('start', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def stop(self, **args): + p = self.subparser.add_parser('stop', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def status(self, **args): + p = self.subparser.add_parser('status', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def delete(self, **args): + p = self.subparser.add_parser('delete', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + + def migrate(self, **args): + p = self.subparser.add_parser('migrate', parents=[get_otp_parser()], **args) + p.add_argument('--vm-name', required=True) + p.add_argument('--destination', required=True) + + def create(self, **args): + p = self.subparser.add_parser('create', parents=[get_otp_parser()], **args) + p.add_argument('--cpu', required=True) + p.add_argument('--ram', required=True) + p.add_argument('--os-ssd', required=True) + p.add_argument('--hdd', action='append', default=list()) + p.add_argument('--image', required=True) + p.add_argument('--network', action='append', default=[]) + p.add_argument('--vm-name', required=True) + + +parser = VMParser() +arg_parser = parser.arg_parser + + +def main(**kwargs): + subcommand = kwargs.pop('vm_subcommand') + if not subcommand: + arg_parser.print_help() + else: + data = kwargs + endpoint = subcommand + if subcommand in ['start', 'stop', 'delete']: + endpoint = 'action' + data['action'] = subcommand + elif subcommand == 'create': + kwargs['specs'] = { + 'cpu': kwargs.pop('cpu'), + 'ram': kwargs.pop('ram'), + 'os-ssd': kwargs.pop('os_ssd'), + 'hdd': kwargs.pop('hdd') + } + make_request('vm', endpoint, data=data) diff --git a/uncloud/scheduler/tests/__init__.py b/archive/uncloud_etcd_based/uncloud/client/__init__.py similarity index 100% rename from uncloud/scheduler/tests/__init__.py rename to archive/uncloud_etcd_based/uncloud/client/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/client/main.py b/archive/uncloud_etcd_based/uncloud/client/main.py new file mode 100644 index 0000000..062308c --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/client/main.py @@ -0,0 +1,23 @@ +import argparse +import etcd3 +from uncloud.common.etcd_wrapper import Etcd3Wrapper + +arg_parser = argparse.ArgumentParser('client', add_help=False) +arg_parser.add_argument('--dump-etcd-contents-prefix', help="Dump contents below the given prefix") + +def dump_etcd_contents(prefix): + etcd = Etcd3Wrapper() + for k,v in etcd.get_prefix_raw(prefix): + k = k.decode('utf-8') + v = v.decode('utf-8') + print("{} = {}".format(k,v)) +# print("{} = {}".format(k,v)) + +# for k,v in etcd.get_prefix(prefix): +# + print("done") + + +def main(arguments): + if 'dump_etcd_contents_prefix' in arguments: + dump_etcd_contents(prefix=arguments['dump_etcd_contents_prefix']) diff --git a/uncloud/common/__init__.py b/archive/uncloud_etcd_based/uncloud/common/__init__.py similarity index 100% rename from uncloud/common/__init__.py rename to archive/uncloud_etcd_based/uncloud/common/__init__.py diff --git a/uncloud/common/classes.py b/archive/uncloud_etcd_based/uncloud/common/classes.py similarity index 100% rename from uncloud/common/classes.py rename to archive/uncloud_etcd_based/uncloud/common/classes.py diff --git a/archive/uncloud_etcd_based/uncloud/common/cli.py b/archive/uncloud_etcd_based/uncloud/common/cli.py new file mode 100644 index 0000000..3d3c248 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/cli.py @@ -0,0 +1,26 @@ +from uncloud.common.shared import shared +from pyotp import TOTP + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except Exception: + raise Exception('Invalid seed') + else: + return token + + +def resolve_otp_credentials(kwargs): + d = { + 'name': shared.settings['client']['name'], + 'realm': shared.settings['client']['realm'], + 'token': get_token(shared.settings['client']['seed']) + } + + for k, v in d.items(): + if k in kwargs and kwargs[k] is None: + kwargs.update({k: v}) + + return d diff --git a/uncloud/common/counters.py b/archive/uncloud_etcd_based/uncloud/common/counters.py similarity index 100% rename from uncloud/common/counters.py rename to archive/uncloud_etcd_based/uncloud/common/counters.py diff --git a/archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py b/archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py new file mode 100644 index 0000000..38471ab --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py @@ -0,0 +1,75 @@ +import etcd3 +import json + +from functools import wraps + +from uncloud import UncloudException +from uncloud.common import logger + + +class EtcdEntry: + def __init__(self, meta_or_key, value, value_in_json=False): + if hasattr(meta_or_key, 'key'): + # if meta has attr 'key' then get it + self.key = meta_or_key.key.decode('utf-8') + else: + # otherwise meta is the 'key' + self.key = meta_or_key + self.value = value.decode('utf-8') + + if value_in_json: + self.value = json.loads(self.value) + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError: + raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') + except etcd3.exceptions.ConnectionTimeoutError as err: + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err + except Exception: + logger.exception('Some etcd error occured. See syslog for details.') + + return wrapper + + +class Etcd3Wrapper: + @readable_errors + def __init__(self, *args, **kwargs): + self.client = etcd3.client(*args, **kwargs) + + @readable_errors + def get(self, *args, value_in_json=False, **kwargs): + _value, _key = self.client.get(*args, **kwargs) + if _key is None or _value is None: + return None + return EtcdEntry(_key, _value, value_in_json=value_in_json) + + @readable_errors + def put(self, *args, value_in_json=False, **kwargs): + _key, _value = args + if value_in_json: + _value = json.dumps(_value) + + if not isinstance(_key, str): + _key = _key.decode('utf-8') + + return self.client.put(_key, _value, **kwargs) + + @readable_errors + def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs): + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) + + @readable_errors + def watch_prefix(self, key, raise_exception=True, value_in_json=False): + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = e._event + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) diff --git a/uncloud/common/host.py b/archive/uncloud_etcd_based/uncloud/common/host.py similarity index 100% rename from uncloud/common/host.py rename to archive/uncloud_etcd_based/uncloud/common/host.py diff --git a/uncloud/common/network.py b/archive/uncloud_etcd_based/uncloud/common/network.py similarity index 97% rename from uncloud/common/network.py rename to archive/uncloud_etcd_based/uncloud/common/network.py index adba108..32f6951 100644 --- a/uncloud/common/network.py +++ b/archive/uncloud_etcd_based/uncloud/common/network.py @@ -1,8 +1,6 @@ import subprocess as sp import random import logging -import socket -from contextlib import closing logger = logging.getLogger(__name__) diff --git a/archive/uncloud_etcd_based/uncloud/common/parser.py b/archive/uncloud_etcd_based/uncloud/common/parser.py new file mode 100644 index 0000000..576f0e7 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/parser.py @@ -0,0 +1,13 @@ +import argparse + + +class BaseParser: + def __init__(self, command): + self.arg_parser = argparse.ArgumentParser(command, add_help=False) + self.subparser = self.arg_parser.add_subparsers(dest='{}_subcommand'.format(command)) + self.common_args = {'add_help': False} + + methods = [attr for attr in dir(self) if not attr.startswith('__') + and type(getattr(self, attr)).__name__ == 'method'] + for method in methods: + getattr(self, method)(**self.common_args) diff --git a/uncloud/common/request.py b/archive/uncloud_etcd_based/uncloud/common/request.py similarity index 81% rename from uncloud/common/request.py rename to archive/uncloud_etcd_based/uncloud/common/request.py index a8c2d0a..cb0add5 100644 --- a/uncloud/common/request.py +++ b/archive/uncloud_etcd_based/uncloud/common/request.py @@ -2,8 +2,8 @@ import json from os.path import join from uuid import uuid4 -from .etcd_wrapper import PsuedoEtcdEntry -from .classes import SpecificEtcdEntryBase +from uncloud.common.etcd_wrapper import EtcdEntry +from uncloud.common.classes import SpecificEtcdEntryBase class RequestType: @@ -29,11 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase): @classmethod def from_scratch(cls, request_prefix, **kwargs): - e = PsuedoEtcdEntry( - join(request_prefix, uuid4().hex), - value=json.dumps(kwargs).encode("utf-8"), - value_in_json=True, - ) + e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex), + value=json.dumps(kwargs).encode('utf-8'), value_in_json=True) return cls(e) diff --git a/uncloud/common/schemas.py b/archive/uncloud_etcd_based/uncloud/common/schemas.py similarity index 100% rename from uncloud/common/schemas.py rename to archive/uncloud_etcd_based/uncloud/common/schemas.py diff --git a/archive/uncloud_etcd_based/uncloud/common/settings.py b/archive/uncloud_etcd_based/uncloud/common/settings.py new file mode 100644 index 0000000..8503f42 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/settings.py @@ -0,0 +1,136 @@ +import configparser +import logging +import sys +import os + +from datetime import datetime +from uncloud.common.etcd_wrapper import Etcd3Wrapper +from os.path import join as join_path + +logger = logging.getLogger(__name__) +settings = None + + +class CustomConfigParser(configparser.RawConfigParser): + def __getitem__(self, key): + try: + result = super().__getitem__(key) + except KeyError as err: + raise KeyError( + 'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format( + key + ) + ) from err + else: + return result + + +class Settings(object): + def __init__(self, conf_dir, seed_value=None): + conf_name = 'uncloud.conf' + self.config_file = join_path(conf_dir, conf_name) + + # this is used to cache config from etcd for 1 minutes. Without this we + # would make a lot of requests to etcd which slows down everything. + self.last_config_update = datetime.fromtimestamp(0) + + self.config_parser = CustomConfigParser(allow_no_value=True) + self.config_parser.add_section('etcd') + self.config_parser.set('etcd', 'base_prefix', '/') + + if os.access(self.config_file, os.R_OK): + self.config_parser.read(self.config_file) + else: + raise FileNotFoundError('Config file %s not found!', self.config_file) + self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') + + self.read_internal_values() + + if seed_value is None: + seed_value = dict() + + self.config_parser.read_dict(seed_value) + + def get_etcd_client(self): + args = tuple() + try: + kwargs = { + 'host': self.config_parser.get('etcd', 'url'), + 'port': self.config_parser.get('etcd', 'port'), + 'ca_cert': self.config_parser.get('etcd', 'ca_cert'), + 'cert_cert': self.config_parser.get('etcd', 'cert_cert'), + 'cert_key': self.config_parser.get('etcd', 'cert_key'), + } + except configparser.Error as err: + raise configparser.Error( + '{} in config file {}'.format( + err.message, self.config_file + ) + ) from err + else: + try: + wrapper = Etcd3Wrapper(*args, **kwargs) + except Exception as err: + logger.error( + 'etcd connection not successfull. Please check your config file.' + '\nDetails: %s\netcd connection parameters: %s', + err, + kwargs, + ) + sys.exit(1) + else: + return wrapper + + def read_internal_values(self): + base_prefix = self['etcd']['base_prefix'] + self.config_parser.read_dict( + { + 'etcd': { + 'file_prefix': join_path(base_prefix, 'files/'), + 'host_prefix': join_path(base_prefix, 'hosts/'), + 'image_prefix': join_path(base_prefix, 'images/'), + 'image_store_prefix': join_path(base_prefix, 'imagestore/'), + 'network_prefix': join_path(base_prefix, 'networks/'), + 'request_prefix': join_path(base_prefix, 'requests/'), + 'user_prefix': join_path(base_prefix, 'users/'), + 'vm_prefix': join_path(base_prefix, 'vms/'), + 'vxlan_counter': join_path(base_prefix, 'counters/vxlan'), + 'tap_counter': join_path(base_prefix, 'counters/tap') + } + } + ) + + def read_config_file_values(self, config_file): + try: + # Trying to read configuration file + with open(config_file) as config_file_handle: + self.config_parser.read_file(config_file_handle) + except FileNotFoundError: + sys.exit('Configuration file {} not found!'.format(config_file)) + except Exception as err: + logger.exception(err) + sys.exit('Error occurred while reading configuration file') + + def read_values_from_etcd(self): + etcd_client = self.get_etcd_client() + if (datetime.utcnow() - self.last_config_update).total_seconds() > 60: + config_from_etcd = etcd_client.get(self.config_key, value_in_json=True) + if config_from_etcd: + self.config_parser.read_dict(config_from_etcd.value) + self.last_config_update = datetime.utcnow() + else: + raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key)) + + def __getitem__(self, key): + # Allow failing to read from etcd if we have + # it locally + if key not in self.config_parser.sections(): + try: + self.read_values_from_etcd() + except KeyError: + pass + return self.config_parser[key] + + +def get_settings(): + return settings diff --git a/archive/uncloud_etcd_based/uncloud/common/shared.py b/archive/uncloud_etcd_based/uncloud/common/shared.py new file mode 100644 index 0000000..aea7cbc --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/shared.py @@ -0,0 +1,34 @@ +from uncloud.common.settings import get_settings +from uncloud.common.vm import VmPool +from uncloud.common.host import HostPool +from uncloud.common.request import RequestPool +import uncloud.common.storage_handlers as storage_handlers + + +class Shared: + @property + def settings(self): + return get_settings() + + @property + def etcd_client(self): + return self.settings.get_etcd_client() + + @property + def host_pool(self): + return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) + + @property + def vm_pool(self): + return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) + + @property + def request_pool(self): + return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) + + @property + def storage_handler(self): + return storage_handlers.get_storage_handler() + + +shared = Shared() diff --git a/uncloud/common/storage_handlers.py b/archive/uncloud_etcd_based/uncloud/common/storage_handlers.py similarity index 92% rename from uncloud/common/storage_handlers.py rename to archive/uncloud_etcd_based/uncloud/common/storage_handlers.py index 06751c4..58c2dc2 100644 --- a/uncloud/common/storage_handlers.py +++ b/archive/uncloud_etcd_based/uncloud/common/storage_handlers.py @@ -6,8 +6,7 @@ import stat from abc import ABC from . import logger from os.path import join as join_path - -from uncloud.settings import settings as config +import uncloud.common.shared as shared class ImageStorageHandler(ABC): @@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): def get_storage_handler(): - __storage_backend = config["storage"]["storage_backend"] + __storage_backend = shared.shared.settings["storage"]["storage_backend"] if __storage_backend == "filesystem": return FileSystemBasedImageStorageHandler( - vm_base=config["storage"]["vm_dir"], - image_base=config["storage"]["image_dir"], + vm_base=shared.shared.settings["storage"]["vm_dir"], + image_base=shared.shared.settings["storage"]["image_dir"], ) elif __storage_backend == "ceph": return CEPHBasedImageStorageHandler( - vm_base=config["storage"]["ceph_vm_pool"], - image_base=config["storage"]["ceph_image_pool"], + vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], + image_base=shared.shared.settings["storage"]["ceph_image_pool"], ) else: - raise Exception("Unknown Image Storage Handler") + raise Exception("Unknown Image Storage Handler") \ No newline at end of file diff --git a/uncloud/common/vm.py b/archive/uncloud_etcd_based/uncloud/common/vm.py similarity index 100% rename from uncloud/common/vm.py rename to archive/uncloud_etcd_based/uncloud/common/vm.py diff --git a/archive/uncloud_etcd_based/uncloud/configure/__init__.py b/archive/uncloud_etcd_based/uncloud/configure/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/archive/uncloud_etcd_based/uncloud/configure/main.py b/archive/uncloud_etcd_based/uncloud/configure/main.py new file mode 100644 index 0000000..87f5752 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/configure/main.py @@ -0,0 +1,57 @@ +import os +import argparse + +from uncloud.common.shared import shared + +arg_parser = argparse.ArgumentParser('configure', add_help=False) +configure_subparsers = arg_parser.add_subparsers(dest='subcommand') + +otp_parser = configure_subparsers.add_parser('otp') +otp_parser.add_argument('--verification-controller-url', required=True, metavar='URL') +otp_parser.add_argument('--auth-name', required=True, metavar='OTP-NAME') +otp_parser.add_argument('--auth-realm', required=True, metavar='OTP-REALM') +otp_parser.add_argument('--auth-seed', required=True, metavar='OTP-SEED') + +network_parser = configure_subparsers.add_parser('network') +network_parser.add_argument('--prefix-length', required=True, type=int) +network_parser.add_argument('--prefix', required=True) +network_parser.add_argument('--vxlan-phy-dev', required=True) + +netbox_parser = configure_subparsers.add_parser('netbox') +netbox_parser.add_argument('--url', required=True) +netbox_parser.add_argument('--token', required=True) + +ssh_parser = configure_subparsers.add_parser('ssh') +ssh_parser.add_argument('--username', default='root') +ssh_parser.add_argument('--private-key-path', default=os.path.expanduser('~/.ssh/id_rsa'),) + +storage_parser = configure_subparsers.add_parser('storage') +storage_parser.add_argument('--file-dir', required=True) +storage_parser_subparsers = storage_parser.add_subparsers(dest='storage_backend') + +filesystem_storage_parser = storage_parser_subparsers.add_parser('filesystem') +filesystem_storage_parser.add_argument('--vm-dir', required=True) +filesystem_storage_parser.add_argument('--image-dir', required=True) + +ceph_storage_parser = storage_parser_subparsers.add_parser('ceph') +ceph_storage_parser.add_argument('--ceph-vm-pool', required=True) +ceph_storage_parser.add_argument('--ceph-image-pool', required=True) + + +def update_config(section, kwargs): + uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) + if not uncloud_config: + uncloud_config = {} + else: + uncloud_config = uncloud_config.value + + uncloud_config[section] = kwargs + shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) + + +def main(arguments): + subcommand = arguments['subcommand'] + if not subcommand: + arg_parser.print_help() + else: + update_config(subcommand, arguments) diff --git a/uncloud/filescanner/__init__.py b/archive/uncloud_etcd_based/uncloud/filescanner/__init__.py similarity index 100% rename from uncloud/filescanner/__init__.py rename to archive/uncloud_etcd_based/uncloud/filescanner/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/filescanner/main.py b/archive/uncloud_etcd_based/uncloud/filescanner/main.py new file mode 100755 index 0000000..046f915 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/filescanner/main.py @@ -0,0 +1,85 @@ +import glob +import os +import pathlib +import subprocess as sp +import time +import argparse +import bitmath + +from uuid import uuid4 + +from . import logger +from uncloud.common.shared import shared + +arg_parser = argparse.ArgumentParser('filescanner', add_help=False) +arg_parser.add_argument('--hostname', required=True) + + +def sha512sum(file: str): + """Use sha512sum utility to compute sha512 sum of arg:file + + IF arg:file does not exists: + raise FileNotFoundError exception + ELSE IF sum successfully computer: + return computed sha512 sum + ELSE: + return None + """ + if not isinstance(file, str): + raise TypeError + try: + output = sp.check_output(['sha512sum', file], stderr=sp.PIPE) + except sp.CalledProcessError as e: + error = e.stderr.decode('utf-8') + if 'No such file or directory' in error: + raise FileNotFoundError from None + else: + output = output.decode('utf-8').strip() + output = output.split(' ') + return output[0] + return None + + +def track_file(file, base_dir, host): + file_path = file.relative_to(base_dir) + file_str = str(file) + # Get Username + try: + owner = file_path.parts[0] + except IndexError: + pass + else: + file_path = file_path.relative_to(owner) + creation_date = time.ctime(os.stat(file_str).st_ctime) + + entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) + entry_value = { + 'filename': str(file_path), + 'owner': owner, + 'sha512sum': sha512sum(file_str), + 'creation_date': creation_date, + 'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()), + 'host': host + } + + logger.info('Tracking %s', file_str) + + shared.etcd_client.put(entry_key, entry_value, value_in_json=True) + + +def main(arguments): + hostname = arguments['hostname'] + base_dir = shared.settings['storage']['file_dir'] + # Recursively Get All Files and Folder below BASE_DIR + files = glob.glob('{}/**'.format(base_dir), recursive=True) + files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] + + # Files that are already tracked + tracked_files = [ + pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) + for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) + if f.value['host'] == hostname + ] + untracked_files = set(files) - set(tracked_files) + for file in untracked_files: + track_file(file, base_dir, hostname) diff --git a/uncloud/hack/README.org b/archive/uncloud_etcd_based/uncloud/hack/README.org similarity index 100% rename from uncloud/hack/README.org rename to archive/uncloud_etcd_based/uncloud/hack/README.org diff --git a/archive/uncloud_etcd_based/uncloud/hack/__init__.py b/archive/uncloud_etcd_based/uncloud/hack/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/__init__.py @@ -0,0 +1 @@ + diff --git a/uncloud/hack/conf.d/ucloud-host b/archive/uncloud_etcd_based/uncloud/hack/conf.d/ucloud-host similarity index 100% rename from uncloud/hack/conf.d/ucloud-host rename to archive/uncloud_etcd_based/uncloud/hack/conf.d/ucloud-host diff --git a/archive/uncloud_etcd_based/uncloud/hack/config.py b/archive/uncloud_etcd_based/uncloud/hack/config.py new file mode 100644 index 0000000..7e2655d --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/config.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +class Config(object): + def __init__(self, arguments): + """ read arguments dicts as a base """ + + self.arguments = arguments + + # Split them so *etcd_args can be used and we can + # iterate over etcd_hosts + self.etcd_hosts = [ arguments['etcd_host'] ] + self.etcd_args = { + 'ca_cert': arguments['etcd_ca_cert'], + 'cert_cert': arguments['etcd_cert_cert'], + 'cert_key': arguments['etcd_cert_key'], +# 'user': None, +# 'password': None + } + self.etcd_prefix = '/nicohack/' diff --git a/archive/uncloud_etcd_based/uncloud/hack/db.py b/archive/uncloud_etcd_based/uncloud/hack/db.py new file mode 100644 index 0000000..3d5582e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/db.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +import etcd3 +import json +import logging +import datetime +import re + +from functools import wraps +from uncloud import UncloudException + +log = logging.getLogger(__name__) + +def db_logentry(message): + timestamp = datetime.datetime.now() + return { + "timestamp": str(timestamp), + "message": message + } + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + + +class DB(object): + def __init__(self, config, prefix="/"): + self.config = config + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + + try: + self.connect() + except FileNotFoundError as e: + raise UncloudException("Is the path to the etcd certs correct? {}".format(e)) + + @readable_errors + def connect(self): + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) + + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) + + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + @readable_errors + def get_prefix(self, key, as_json=False, **kwargs): + for value, meta in self._db_clients[0].get_prefix(self.realkey(key), **kwargs): + k = meta.key.decode("utf-8") + value = value.decode("utf-8") + if as_json: + value = json.loads(value) + + yield (k, value) + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: + value = json.dumps(value) + + log.debug("Setting {} = {}".format(self.realkey(key), value)) + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + + @readable_errors + def list_and_filter(self, key, filter_key=None, filter_regexp=None): + for k,v in self.get_prefix(key, as_json=True): + + if filter_key and filter_regexp: + if filter_key in v: + if re.match(filter_regexp, v[filter_key]): + yield v + else: + yield v + + + @readable_errors + def increment(self, key, **kwargs): + print(self.realkey(key)) + + + print("prelock") + lock = self._db_clients[0].lock('/nicohack/foo') + print("prelockacq") + lock.acquire() + print("prelockrelease") + lock.release() + + with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock: + print("in lock") + pass + +# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs)) +# self.set(self.realkey(key), str(value + 1), **kwargs) + + +if __name__ == '__main__': + endpoints = [ "https://etcd1.ungleich.ch:2379", + "https://etcd2.ungleich.ch:2379", + "https://etcd3.ungleich.ch:2379" ] + + db = DB(url=endpoints) diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore new file mode 100644 index 0000000..0ad647b --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore @@ -0,0 +1,3 @@ +*.iso +radvdpid +foo diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py @@ -0,0 +1 @@ + diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh new file mode 100644 index 0000000..ab102a5 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \ + --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \ + --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \ + --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@" diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh new file mode 100755 index 0000000..5753099 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo $@ diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh new file mode 100755 index 0000000..e0a3ca0 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +dev=$1; shift + +# bridge is setup from outside +ip link set dev "$dev" master ${bridge} +ip link set dev "$dev" up diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last new file mode 100644 index 0000000..8c5f254 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last @@ -0,0 +1 @@ +000000000252 diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix new file mode 100644 index 0000000..5084a2f --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix @@ -0,0 +1 @@ +02:00 diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/net.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/net.sh new file mode 100755 index 0000000..4e2bfa1 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/net.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -x + +netid=100 +dev=wlp2s0 +dev=wlp0s20f3 +#dev=wlan0 + +ip=2a0a:e5c1:111:888::48/64 +vxlandev=vxlan${netid} +bridgedev=br${netid} + +ip -6 link add ${vxlandev} type vxlan \ + id ${netid} \ + dstport 4789 \ + group ff05::${netid} \ + dev ${dev} \ + ttl 5 + +ip link set ${vxlandev} up + + +ip link add ${bridgedev} type bridge +ip link set ${bridgedev} up + +ip link set ${vxlandev} master ${bridgedev} up + +ip addr add ${ip} dev ${bridgedev} diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules new file mode 100644 index 0000000..636c63d --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules @@ -0,0 +1,31 @@ +flush ruleset + +table bridge filter { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; + + ibrname br100 jump br100 + } + + chain br100 { + # Allow all incoming traffic from outside + iifname vxlan100 accept + + # Default blocks: router advertisements, dhcpv6, dhcpv4 + icmpv6 type nd-router-advert drop + ip6 version 6 udp sport 547 drop + ip version 4 udp sport 67 drop + + jump br100_vmlist + drop + } + chain br100_vmlist { + # VM1 + iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept + + # VM2 + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept + } +} diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules-v2 b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules-v2 new file mode 100644 index 0000000..b6d4cf3 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules-v2 @@ -0,0 +1,64 @@ +flush ruleset + +table bridge filter { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; + + ibrname br100 jump netpublic + } + + chain netpublic { + iifname vxlan100 jump from_uncloud + + # Default blocks: router advertisements, dhcpv6, dhcpv4 + icmpv6 type nd-router-advert drop + ip6 version 6 udp sport 547 drop + ip version 4 udp sport 67 drop + + # Individual blocks +# iifname tap1 jump vm1 + } + + chain vm1 { + ether saddr != 02:00:f0:a9:c4:4e drop + ip6 saddr != 2a0a:e5c1:111:888:0:f0ff:fea9:c44e drop + } + + chain from_uncloud { + accept + } +} + +# table ip6 filter { +# chain forward { +# type filter hook forward priority 0; + +# # policy drop; + +# ct state established,related accept; + +# } + +# } + +# table ip filter { +# chain input { +# type filter hook input priority filter; policy drop; +# iif "lo" accept +# icmp type { echo-reply, destination-unreachable, source-quench, redirect, echo-request, router-advertisement, router-solicitation, time-exceeded, parameter-problem, timestamp-request, timestamp-reply, info-request, info-reply, address-mask-request, address-mask-reply } accept +# ct state established,related accept +# tcp dport { 22 } accept +# log prefix "firewall-ipv4: " +# udp sport 67 drop +# } + +# chain forward { +# type filter hook forward priority filter; policy drop; +# log prefix "firewall-ipv4: " +# } + +# chain output { +# type filter hook output priority filter; policy accept; +# } +# } diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.conf b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.conf new file mode 100644 index 0000000..3d8ce4d --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.conf @@ -0,0 +1,13 @@ +interface br100 +{ + AdvSendAdvert on; + MinRtrAdvInterval 3; + MaxRtrAdvInterval 5; + AdvDefaultLifetime 3600; + + prefix 2a0a:e5c1:111:888::/64 { + }; + + RDNSS 2a0a:e5c0::3 2a0a:e5c0::4 { AdvRDNSSLifetime 6000; }; + DNSSL place7.ungleich.ch { AdvDNSSLLifetime 6000; } ; +}; diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.sh new file mode 100644 index 0000000..9d0e7d1 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +radvd -C ./radvd.conf -n -p ./radvdpid diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm-2.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm-2.sh new file mode 100755 index 0000000..af9dec7 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm-2.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +vmid=$1; shift + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +#accel=tcg + +memory=1024 +cores=2 +uuid=732e08c7-84f8-4d43-9571-263db4f80080 + +export bridge=br100 + +$qemu -name uc${vmid} \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=02:00:f0:a9:c4:4e diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh new file mode 100755 index 0000000..dd9be84 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# if [ $# -ne 1 ]; then +# echo "$0: owner" +# exit 1 +# fi + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +#accel=tcg + +memory=1024 +cores=2 +uuid=$(uuidgen) +mac=$(./mac-gen.py) +owner=nico + +export bridge=br100 + +set -x +$qemu -name "uncloud-${uuid}" \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac} diff --git a/archive/uncloud_etcd_based/uncloud/hack/host.py b/archive/uncloud_etcd_based/uncloud/hack/host.py new file mode 100644 index 0000000..06ccf98 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/host.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +import uuid + +from uncloud.hack.db import DB +from uncloud import UncloudException + +class Host(object): + def __init__(self, config, db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/hosts") + + if db_entry: + self.db_entry = db_entry + + + def list_hosts(self, filter_key=None, filter_regexp=None): + """ Return list of all hosts """ + for entry in self.db.list_and_filter("", filter_key, filter_regexp): + yield self.__class__(self.config, db_entry=entry) + + def cmdline_add_host(self): + """ FIXME: make this a bit smarter and less redundant """ + + for required_arg in [ + 'add_vm_host', + 'max_cores_per_vm', + 'max_cores_total', + 'max_memory_in_gb' ]: + if not required_arg in self.config.arguments: + raise UncloudException("Missing argument: {}".format(required_arg)) + + return self.add_host( + self.config.arguments['add_vm_host'], + self.config.arguments['max_cores_per_vm'], + self.config.arguments['max_cores_total'], + self.config.arguments['max_memory_in_gb']) + + + def add_host(self, + hostname, + max_cores_per_vm, + max_cores_total, + max_memory_in_gb): + + db_entry = {} + db_entry['uuid'] = str(uuid.uuid4()) + db_entry['hostname'] = hostname + db_entry['max_cores_per_vm'] = max_cores_per_vm + db_entry['max_cores_total'] = max_cores_total + db_entry['max_memory_in_gb'] = max_memory_in_gb + db_entry["db_version"] = 1 + db_entry["log"] = [] + + self.db.set(db_entry['uuid'], db_entry, as_json=True) + + return self.__class__(self.config, db_entry) diff --git a/archive/uncloud_etcd_based/uncloud/hack/mac.py b/archive/uncloud_etcd_based/uncloud/hack/mac.py new file mode 100755 index 0000000..e35cd9f --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/mac.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2012 Nico Schottelius (nico-cinv at schottelius.org) +# +# This file is part of cinv. +# +# cinv is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cinv is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cinv. If not, see . +# +# + +import argparse +import logging +import os.path +import os +import re +import json + +from uncloud import UncloudException +from uncloud.hack.db import DB + +log = logging.getLogger(__name__) + + +class MAC(object): + def __init__(self, config): + self.config = config + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(config, prefix="/mac") + + self.prefix = 0x420000000000 + self._number = 0 # Not set by default + + @staticmethod + def validate_mac(mac): + if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): + raise UncloudException("Not a valid mac address: %s" % mac) + else: + return True + + def last_used_index(self): + if not self.no_db: + value = self.db.get("last_used_index") + if not value: + self.db.set("last_used_index", "0") + value = self.db.get("last_used_index") + + else: + value = "0" + + return int(value) + + def last_used_mac(self): + return self.int_to_mac(self.prefix + self.last_used_index()) + + def to_colon_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ':'.join(format(s, '02x') for s in b) + + def to_str_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ''.join(format(s, '02x') for s in b) + + def create(self): + last_number = self.last_used_index() + + if last_number == int('0xffffffff', 16): + raise UncloudException("Exhausted all possible mac addresses - try to free some") + + next_number = last_number + 1 + self._number = self.prefix + next_number + + #next_number_string = "{:012x}".format(next_number) + #next_mac = self.int_to_mac(next_mac_number) + # db_entry = {} + # db_entry['vm_uuid'] = vmuuid + # db_entry['index'] = next_number + # db_entry['mac_address'] = next_mac + + # should be one transaction + # self.db.increment("last_used_index") + # self.db.set("used/{}".format(next_mac), + # db_entry, as_json=True) + + def __int__(self): + return self._number + + def __repr__(self): + return self.to_str_format() + + def __str__(self): + return self.to_colon_format() diff --git a/archive/uncloud_etcd_based/uncloud/hack/main.py b/archive/uncloud_etcd_based/uncloud/hack/main.py new file mode 100644 index 0000000..0ddd8fb --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/main.py @@ -0,0 +1,186 @@ +import argparse +import logging +import re + +import ldap3 + + +from uncloud.hack.vm import VM +from uncloud.hack.host import Host +from uncloud.hack.config import Config +from uncloud.hack.mac import MAC +from uncloud.hack.net import VXLANBridge, DNSRA + +from uncloud import UncloudException +from uncloud.hack.product import ProductOrder + +arg_parser = argparse.ArgumentParser('hack', add_help=False) + #description="Commands that are unfinished - use at own risk") +arg_parser.add_argument('--last-used-mac', action='store_true') +arg_parser.add_argument('--get-new-mac', action='store_true') + +arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true') +arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true') +arg_parser.add_argument('--network', help="/64 IPv6 network") +arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0") +arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) +arg_parser.add_argument('--run-dns-ra', action='store_true', + help="Provide router advertisements and DNS resolution via dnsmasq") +arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') + +arg_parser.add_argument('--create-vm', action='store_true') +arg_parser.add_argument('--destroy-vm', action='store_true') +arg_parser.add_argument('--get-vm-status', action='store_true') +arg_parser.add_argument('--get-vm-vnc', action='store_true') +arg_parser.add_argument('--list-vms', action='store_true') +arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int, default=2) +arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int, default=1) +arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") + +arg_parser.add_argument('--image-format', help="Image format: qcow2 or raw", choices=['raw', 'qcow2']) +arg_parser.add_argument('--uuid', help="VM UUID") + +arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') +arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") + +# order based commands => later to be shifted below "order" +arg_parser.add_argument('--order', action='store_true') +arg_parser.add_argument('--list-orders', help="List all orders", action='store_true') +arg_parser.add_argument('--filter-order-key', help="Which key to filter on") +arg_parser.add_argument('--filter-order-regexp', help="Which regexp the value should match") + +arg_parser.add_argument('--process-orders', help="Process all (pending) orders", action='store_true') + +arg_parser.add_argument('--product', choices=["dualstack-vm"]) +arg_parser.add_argument('--os-image-name', help="Name of OS image (successor to --image)") +arg_parser.add_argument('--os-image-size', help="Size of OS image in GB", type=int, default=10) + +arg_parser.add_argument('--username') +arg_parser.add_argument('--password') + +arg_parser.add_argument('--api', help="Run the API") +arg_parser.add_argument('--mode', + choices=["direct", "api", "client"], + default="client", + help="Directly manipulate etcd, spawn the API server or behave as a client") + + +arg_parser.add_argument('--add-vm-host', help="Add a host that can run VMs") +arg_parser.add_argument('--list-vm-hosts', action='store_true') + +arg_parser.add_argument('--max-cores-per-vm') +arg_parser.add_argument('--max-cores-total') +arg_parser.add_argument('--max-memory-in-gb') + + +log = logging.getLogger(__name__) + +def authenticate(username, password, totp_token=None): + server = ldap3.Server("ldaps://ldap1.ungleich.ch") + dn = "uid={},ou=customer,dc=ungleich,dc=ch".format(username) + + log.debug("LDAP: connecting to {} as {}".format(server, dn)) + + try: + conn = ldap3.Connection(server, dn, password, auto_bind=True) + except ldap3.core.exceptions.LDAPBindError as e: + raise UncloudException("Credentials not verified by LDAP server: {}".format(e)) + + + +def order(config): + for required_arg in [ 'product', 'username', 'password' ]: + if not config.arguments[required_arg]: + raise UncloudException("Missing required argument: {}".format(required_arg)) + + if config.arguments['product'] == 'dualstack-vm': + for required_arg in [ 'cores', 'memory', 'os_image_name', 'os_image_size' ]: + if not config.arguments[required_arg]: + raise UncloudException("Missing required argument: {}".format(required_arg)) + + log.debug(config.arguments) + authenticate(config.arguments['username'], config.arguments['password']) + + # create DB entry for VM + vm = VM(config) + return vm.product.place_order(owner=config.arguments['username']) + + + + + +def main(arguments): + config = Config(arguments) + + if arguments['add_vm_host']: + h = Host(config) + h.cmdline_add_host() + + if arguments['list_vm_hosts']: + h = Host(config) + + for host in h.list_hosts(filter_key=arguments['filter_order_key'], + filter_regexp=arguments['filter_order_regexp']): + print("Host {}: {}".format(host.db_entry['uuid'], host.db_entry)) + + if arguments['order']: + print("Created order: {}".format(order(config))) + + if arguments['list_orders']: + p = ProductOrder(config) + for product_order in p.list_orders(filter_key=arguments['filter_order_key'], + filter_regexp=arguments['filter_order_regexp']): + print("Order {}: {}".format(product_order.db_entry['uuid'], product_order.db_entry)) + + if arguments['process_orders']: + p = ProductOrder(config) + p.process_orders() + + if arguments['create_vm']: + vm = VM(config) + vm.create() + + if arguments['destroy_vm']: + vm = VM(config) + vm.stop() + + if arguments['get_vm_status']: + vm = VM(config) + vm.status() + + if arguments['get_vm_vnc']: + vm = VM(config) + vm.vnc_addr() + + if arguments['list_vms']: + vm = VM(config) + vm.list() + + if arguments['last_used_mac']: + m = MAC(config) + print(m.last_used_mac()) + + if arguments['get_new_mac']: + print(MAC(config).get_next()) + + #if arguments['init_network']: + if arguments['create_vxlan']: + if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: + raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + vb = VXLANBridge(vni=arguments['vni'], + route=arguments['network'], + uplinkdev=arguments['vxlan_uplink_device'], + use_sudo=arguments['use_sudo']) + vb._setup_vxlan() + vb._setup_bridge() + vb._add_vxlan_to_bridge() + vb._route_network() + + if arguments['run_dns_ra']: + if not arguments['network'] or not arguments['vni']: + raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + + dnsra = DNSRA(route=arguments['network'], + vni=arguments['vni'], + use_sudo=arguments['use_sudo']) + dnsra._setup_dnsmasq() diff --git a/archive/uncloud_etcd_based/uncloud/hack/net.py b/archive/uncloud_etcd_based/uncloud/hack/net.py new file mode 100644 index 0000000..4887e04 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/net.py @@ -0,0 +1,116 @@ +import subprocess +import ipaddress +import logging + + +from uncloud import UncloudException + +log = logging.getLogger(__name__) + + +class VXLANBridge(object): + cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" + cmd_up_dev = "{sudo}ip link set {dev} up" + cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge" + cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up" + cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}" + cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}" + + # VXLAN ids are at maximum 24 bit - use a /104 + multicast_network = ipaddress.IPv6Network("ff05::/104") + max_vni = (2**24)-1 + + def __init__(self, + vni, + uplinkdev, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' + + self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + self.config['multicast_address'] = self.multicast_network[vni] + + self.config['route_network'] = ipaddress.IPv6Network(route) + self.config['route'] = route + + self.config['uplinkdev'] = uplinkdev + self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + + def setup_networking(self): + pass + + def _setup_vxlan(self): + self._execute_cmd(self.cmd_create_vxlan) + self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev']) + + def _setup_bridge(self): + self._execute_cmd(self.cmd_create_bridge) + self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev']) + + def _route_network(self): + self._execute_cmd(self.cmd_add_route_dev) + + def _add_vxlan_to_bridge(self): + self._execute_cmd(self.cmd_add_to_bridge) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + +class ManagementBridge(VXLANBridge): + pass + + +class DNSRA(object): + # VXLAN ids are at maximum 24 bit + max_vni = (2**24)-1 + + + # Command to start dnsmasq + cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra --no-daemon" + + def __init__(self, + vni, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' + + #TODO: remove if not needed + #self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + + # dnsmasq only wants the network without the prefix, therefore, cut it off + self.config['route'] = ipaddress.IPv6Network(route).network_address + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + def _setup_dnsmasq(self): + self._execute_cmd(self.cmd_start_dnsmasq) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + print("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + +class Firewall(object): + pass diff --git a/uncloud/hack/nftables.conf b/archive/uncloud_etcd_based/uncloud/hack/nftables.conf similarity index 100% rename from uncloud/hack/nftables.conf rename to archive/uncloud_etcd_based/uncloud/hack/nftables.conf diff --git a/archive/uncloud_etcd_based/uncloud/hack/product.py b/archive/uncloud_etcd_based/uncloud/hack/product.py new file mode 100755 index 0000000..f979268 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/product.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +import json +import uuid +import logging +import re +import importlib + +from uncloud import UncloudException +from uncloud.hack.db import DB, db_logentry + +log = logging.getLogger(__name__) + +class ProductOrder(object): + def __init__(self, config, product_entry=None, db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/orders") + self.db_entry = {} + self.db_entry["product"] = product_entry + + # Overwrite if we are loading an existing product order + if db_entry: + self.db_entry = db_entry + + # FIXME: this should return a list of our class! + def list_orders(self, filter_key=None, filter_regexp=None): + for entry in self.db.list_and_filter("", filter_key, filter_regexp): + yield self.__class__(self.config, db_entry=entry) + + + def set_required_values(self): + """Set values that are required to make the db entry valid""" + if not "uuid" in self.db_entry: + self.db_entry["uuid"] = str(uuid.uuid4()) + if not "status" in self.db_entry: + self.db_entry["status"] = "NEW" + if not "owner" in self.db_entry: + self.db_entry["owner"] = "UNKNOWN" + if not "log" in self.db_entry: + self.db_entry["log"] = [] + if not "db_version" in self.db_entry: + self.db_entry["db_version"] = 1 + + def validate_status(self): + if "status" in self.db_entry: + if self.db_entry["status"] in [ "NEW", + "SCHEDULED", + "CREATED_ACTIVE", + "CANCELLED", + "REJECTED" ]: + return False + return True + + def order(self): + self.set_required_values() + if not self.db_entry["status"] == "NEW": + raise UncloudException("Cannot re-order same order. Status: {}".format(self.db_entry["status"])) + self.db.set(self.db_entry["uuid"], self.db_entry, as_json=True) + + return self.db_entry["uuid"] + + def process_orders(self): + """processing orders can be done stand alone on server side""" + for order in self.list_orders(): + if order.db_entry["status"] == "NEW": + log.info("Handling new order: {}".format(order)) + + # FIXME: these all should be a transactions! -> fix concurrent access! ! + if not "log" in order.db_entry: + order.db_entry['log'] = [] + + is_valid = True + # Verify the order entry + for must_attribute in [ "owner", "product" ]: + if not must_attribute in order.db_entry: + message = "Missing {} entry in order, rejecting order".format(must_attribute) + log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message)) + + order.db_entry['log'].append(db_logentry(message)) + order.db_entry['status'] = "REJECTED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + is_valid = False + + # Rejected the order + if not is_valid: + continue + + # Verify the product entry + for must_attribute in [ "python_product_class", "python_product_module" ]: + if not must_attribute in order.db_entry['product']: + message = "Missing {} entry in product of order, rejecting order".format(must_attribute) + log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message)) + + order.db_entry['log'].append(db_logentry(message)) + order.db_entry['status'] = "REJECTED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + is_valid = False + + # Rejected the order + if not is_valid: + continue + + print(order.db_entry["product"]["python_product_class"]) + + # Create the product + m = importlib.import_module(order.db_entry["product"]["python_product_module"]) + c = getattr(m, order.db_entry["product"]["python_product_class"]) + + product = c(config, db_entry=order.db_entry["product"]) + + # STOPPED + product.create_product() + + order.db_entry['status'] = "SCHEDULED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + + + def __str__(self): + return str(self.db_entry) + +class Product(object): + def __init__(self, + config, + product_name, + product_class, + db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/orders") + + self.db_entry = {} + self.db_entry["product_name"] = product_name + self.db_entry["python_product_class"] = product_class.__qualname__ + self.db_entry["python_product_module"] = product_class.__module__ + self.db_entry["db_version"] = 1 + self.db_entry["log"] = [] + self.db_entry["features"] = {} + + # Existing product? Read in db_entry + if db_entry: + self.db_entry = db_entry + + self.valid_periods = [ "per_year", "per_month", "per_week", + "per_day", "per_hour", + "per_minute", "per_second" ] + + def define_feature(self, + name, + one_time_price, + recurring_price, + recurring_period, + minimum_period): + + self.db_entry['features'][name] = {} + self.db_entry['features'][name]['one_time_price'] = one_time_price + self.db_entry['features'][name]['recurring_price'] = recurring_price + + if not recurring_period in self.valid_periods: + raise UncloudException("Invalid recurring period: {}".format(recurring_period)) + + self.db_entry['features'][name]['recurring_period'] = recurring_period + + if not minimum_period in self.valid_periods: + raise UncloudException("Invalid recurring period: {}".format(recurring_period)) + + recurring_index = self.valid_periods.index(recurring_period) + minimum_index = self.valid_periods.index(minimum_period) + + if minimum_index < recurring_index: + raise UncloudException("Minimum period for product '{}' feature '{}' must be shorter or equal than/as recurring period: {} > {}".format(self.db_entry['product_name'], name, minimum_period, recurring_period)) + + self.db_entry['features'][name]['minimum_period'] = minimum_period + + + def validate_product(self): + for feature in self.db_entry['features']: + pass + + def place_order(self, owner): + """ Schedule creating the product in etcd """ + order = ProductOrder(self.config, product_entry=self.db_entry) + order.db_entry["owner"] = owner + return order.order() + + def __str__(self): + return json.dumps(self.db_entry) diff --git a/uncloud/hack/rc-scripts/ucloud-api b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-api similarity index 100% rename from uncloud/hack/rc-scripts/ucloud-api rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-api diff --git a/uncloud/hack/rc-scripts/ucloud-host b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-host similarity index 100% rename from uncloud/hack/rc-scripts/ucloud-host rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-host diff --git a/uncloud/hack/rc-scripts/ucloud-metadata b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-metadata similarity index 100% rename from uncloud/hack/rc-scripts/ucloud-metadata rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-metadata diff --git a/uncloud/hack/rc-scripts/ucloud-scheduler b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-scheduler similarity index 100% rename from uncloud/hack/rc-scripts/ucloud-scheduler rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-scheduler diff --git a/archive/uncloud_etcd_based/uncloud/hack/uncloud-hack-init-host b/archive/uncloud_etcd_based/uncloud/hack/uncloud-hack-init-host new file mode 100644 index 0000000..787ff80 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/uncloud-hack-init-host @@ -0,0 +1,26 @@ +id=100 +rawdev=eth0 + +# create vxlan +ip -6 link add vxlan${id} type vxlan \ + id ${id} \ + dstport 4789 \ + group ff05::${id} \ + dev ${rawdev} \ + ttl 5 + +ip link set vxlan${id} up + +# create bridge +ip link set vxlan${id} up +ip link set br${id} up + +# Add vxlan into bridge +ip link set vxlan${id} master br${id} + + +# useradd -m uncloud +# [18:05] tablett.place10:~# id uncloud +# uid=1000(uncloud) gid=1000(uncloud) groups=1000(uncloud),34(kvm),36(qemu) +# apk add qemu-system-x86_64 +# also needs group netdev diff --git a/archive/uncloud_etcd_based/uncloud/hack/uncloud-run-vm b/archive/uncloud_etcd_based/uncloud/hack/uncloud-run-vm new file mode 100644 index 0000000..33e5860 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/uncloud-run-vm @@ -0,0 +1,25 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo $0 vmid + exit 1 +fi + +id=$1; shift + +memory=512 +macaddress=02:00:b9:cb:70:${id} +netname=net${id}-1 + +qemu-system-x86_64 \ + -name uncloud-${id} \ + -accel kvm \ + -m ${memory} \ + -smp 2,sockets=2,cores=1,threads=1 \ + -device virtio-net-pci,netdev=net0,mac=$macaddress \ + -netdev tap,id=net0,ifname=${netname},script=no,downscript=no \ + -vnc [::]:0 + +# To be changed: +# -vnc to unix path +# or -spice diff --git a/archive/uncloud_etcd_based/uncloud/hack/vm.py b/archive/uncloud_etcd_based/uncloud/hack/vm.py new file mode 100755 index 0000000..4b0ca14 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/vm.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +# This module is directly called from the hack module, and can be used as follow: +# +# Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix. +# `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2` +# +# List running VMs (returns a list of UUIDs). +# `uncloud hack --hackprefix /tmp/hackcloud --list-vms +# +# Get VM status: +# `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid` +# +# Stop a VM: +# `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` +# `` + +import subprocess +import uuid +import os +import logging + +from uncloud.hack.db import DB +from uncloud.hack.mac import MAC +from uncloud.vmm import VMM +from uncloud.hack.product import Product + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +class VM(object): + def __init__(self, config, db_entry=None): + self.config = config + + #TODO: Enable etcd lookup + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(self.config, prefix="/vm") + + if db_entry: + self.db_entry = db_entry + + # General CLI arguments. + self.hackprefix = self.config.arguments['hackprefix'] + self.uuid = self.config.arguments['uuid'] + self.memory = self.config.arguments['memory'] or '1024M' + self.cores = self.config.arguments['cores'] or 1 + + if self.config.arguments['image']: + self.image = os.path.join(self.hackprefix, self.config.arguments['image']) + else: + self.image = None + + if self.config.arguments['image_format']: + self.image_format=self.config.arguments['image_format'] + else: + self.image_format='qcow2' + + # External components. + + # This one is broken: + # TypeError: expected str, bytes or os.PathLike object, not NoneType + # Fix before re-enabling + # self.vmm = VMM(vmm_backend=self.hackprefix) + self.mac = MAC(self.config) + + # Harcoded & generated values. + self.owner = 'uncloud' + self.accel = 'kvm' + self.threads = 1 + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + self.ifname = "uc{}".format(self.mac.to_str_format()) + + self.vm = {} + + self.product = Product(config, product_name="dualstack-vm", + product_class=self.__class__) + self.product.define_feature(name="base", + one_time_price=0, + recurring_price=9, + recurring_period="per_month", + minimum_period="per_hour") + + + self.features = [] + + + def get_qemu_args(self): + command = ( + "-name {owner}-{name}" + " -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}" + " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" + ).format( + owner=self.owner, name=self.uuid, + accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname, + mac=self.mac + ) + + return command.split(" ") + + def create_product(self): + """Find a VM host and schedule on it""" + pass + + def create(self): + # New VM: new UUID, new MAC. + self.uuid = str(uuid.uuid4()) + self.mac=MAC(self.config) + self.mac.create() + + qemu_args = self.get_qemu_args() + log.debug("QEMU args passed to VMM: {}".format(qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + + self.mac.create() + self.vm['mac'] = self.mac + self.vm['ifname'] = "uc{}".format(self.mac.__repr__()) + + # FIXME: TODO: turn this into a string and THEN + # .split() it later -- easier for using .format() + #self.vm['commandline'] = [ "{}".format(self.sudo), + self.vm['commandline'] = "{sudo}{qemu} -name uncloud-{uuid} -machine pc,accel={accel} -m {memory} -smp {cores} -uuid {uuid} -drive file={os_image},media=cdrom -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname} -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" +# self.vm['commandline'] = [ "{}".format(self.sudo), +# "{}".format(self.qemu), +# "-name", "uncloud-{}".format(self.vm['uuid']), +# "-machine", "pc,accel={}".format(self.accel), +# "-m", "{}".format(self.vm['memory']), +# "-smp", "{}".format(self.vm['cores']), +# "-uuid", "{}".format(self.vm['uuid']), +# "-drive", "file={},media=cdrom".format(self.vm['os_image']), +# "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), +# "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.vm['mac']) +# ] + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.vm, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + + def stop(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + self.vmm.stop(self.uuid) + + def status(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_status(self.uuid)) + + def vnc_addr(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_vnc(self.uuid)) + + def list(self): + print(self.vmm.discover()) diff --git a/uncloud/host/__init__.py b/archive/uncloud_etcd_based/uncloud/host/__init__.py similarity index 100% rename from uncloud/host/__init__.py rename to archive/uncloud_etcd_based/uncloud/host/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/host/main.py b/archive/uncloud_etcd_based/uncloud/host/main.py new file mode 100755 index 0000000..f680991 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/host/main.py @@ -0,0 +1,123 @@ +import argparse +import multiprocessing as mp +import time + +from uuid import uuid4 + +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.shared import shared +from uncloud.common.vm import VMStatus +from uncloud.vmm import VMM +from os.path import join as join_path + +from . import virtualmachine, logger + +arg_parser = argparse.ArgumentParser('host', add_help=False) +arg_parser.add_argument('--hostname', required=True) + + +def update_heartbeat(hostname): + """Update Last HeartBeat Time for :param hostname: in etcd""" + host_pool = shared.host_pool + this_host = next( + filter(lambda h: h.hostname == hostname, host_pool.hosts), None + ) + while True: + this_host.update_heartbeat() + host_pool.put(this_host) + time.sleep(10) + + +def maintenance(host): + vmm = VMM() + running_vms = vmm.discover() + for vm_uuid in running_vms: + if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': + logger.debug('VM {} is running on {}'.format(vm_uuid, host)) + vm = shared.vm_pool.get( + join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) + ) + vm.status = VMStatus.running + vm.vnc_socket = vmm.get_vnc(vm_uuid) + vm.hostname = host + shared.vm_pool.put(vm) + + +def main(arguments): + hostname = arguments['hostname'] + host_pool = shared.host_pool + host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + + # Does not yet exist, create it + if not host: + host_key = join_path( + shared.settings['etcd']['host_prefix'], uuid4().hex + ) + host_entry = { + 'specs': '', + 'hostname': hostname, + 'status': 'DEAD', + 'last_heartbeat': '', + } + shared.etcd_client.put( + host_key, host_entry, value_in_json=True + ) + + # update, get ourselves now for sure + host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) + + try: + heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) + heartbeat_updating_process.start() + except Exception as e: + raise Exception('uncloud-host heartbeat updating mechanism is not working') from e + + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for events_iterator in [ + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False) + ]: + for request_event in events_iterator: + request_event = RequestEntry(request_event) + + maintenance(host.key) + + if request_event.hostname == host.key: + logger.debug('VM Request: %s on Host %s', request_event, host.hostname) + + shared.request_pool.client.client.delete(request_event.key) + vm_entry = shared.etcd_client.get( + join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) + ) + + logger.debug('VM hostname: {}'.format(vm_entry.value)) + + vm = virtualmachine.VM(vm_entry) + if request_event.type == RequestType.StartVM: + vm.start() + + elif request_event.type == RequestType.StopVM: + vm.stop() + + elif request_event.type == RequestType.DeleteVM: + vm.delete() + + elif request_event.type == RequestType.InitVMMigration: + vm.start(destination_host_key=host.key) + + elif request_event.type == RequestType.TransferVM: + destination_host = host_pool.get(request_event.destination_host_key) + if destination_host: + vm.migrate( + destination_host=destination_host.hostname, + destination_sock_path=request_event.destination_sock_path, + ) + else: + logger.error('Host %s not found!', request_event.destination_host_key) diff --git a/uncloud/host/virtualmachine.py b/archive/uncloud_etcd_based/uncloud/host/virtualmachine.py similarity index 95% rename from uncloud/host/virtualmachine.py rename to archive/uncloud_etcd_based/uncloud/host/virtualmachine.py index 0bd20bf..a592efc 100755 --- a/uncloud/host/virtualmachine.py +++ b/archive/uncloud_etcd_based/uncloud/host/virtualmachine.py @@ -16,8 +16,7 @@ from uncloud.common.vm import VMStatus, declare_stopped from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared from uncloud.vmm import VMM from marshmallow import ValidationError @@ -42,7 +41,7 @@ class VM: def get_qemu_args(self): command = ( - "-drive file={file},format=raw,if=virtio,cache=none" + "-drive file={file},format=raw,if=virtio" " -device virtio-rng-pci" " -m {memory} -smp cores={cores},threads={threads}" " -name {owner}_{name}" @@ -91,7 +90,7 @@ class VM: self.vmm.socket_dir, self.uuid ), destination_host_key=destination_host_key, # Where source host transfer VM - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) else: @@ -119,7 +118,7 @@ class VM: network_name, mac, tap = network_mac_and_tap _key = os.path.join( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], self.vm["owner"], network_name, ) @@ -133,13 +132,13 @@ class VM: if network["type"] == "vxlan": tap = create_vxlan_br_tap( _id=network["id"], - _dev=settings["network"]["vxlan_phy_dev"], + _dev=shared.settings["network"]["vxlan_phy_dev"], tap_id=tap, ip=network["ipv6"], ) all_networks = shared.etcd_client.get_prefix( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], value_in_json=True, ) @@ -153,7 +152,10 @@ class VM: ) ) - return command.split(" ") + if command: + command = command.split(' ') + + return command def delete_network_dev(self): try: @@ -226,7 +228,7 @@ class VM: def resolve_network(network_name, network_owner): network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], network_owner, network_name, ), diff --git a/uncloud/imagescanner/__init__.py b/archive/uncloud_etcd_based/uncloud/imagescanner/__init__.py similarity index 100% rename from uncloud/imagescanner/__init__.py rename to archive/uncloud_etcd_based/uncloud/imagescanner/__init__.py diff --git a/uncloud/imagescanner/main.py b/archive/uncloud_etcd_based/uncloud/imagescanner/main.py similarity index 90% rename from uncloud/imagescanner/main.py rename to archive/uncloud_etcd_based/uncloud/imagescanner/main.py index 93e4dd5..ee9da2e 100755 --- a/uncloud/imagescanner/main.py +++ b/archive/uncloud_etcd_based/uncloud/imagescanner/main.py @@ -1,13 +1,16 @@ import json import os +import argparse import subprocess as sp from os.path import join as join_path -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared from uncloud.imagescanner import logger +arg_parser = argparse.ArgumentParser('imagescanner', add_help=False) + + def qemu_img_type(path): qemu_img_info_command = [ "qemu-img", @@ -26,10 +29,10 @@ def qemu_img_type(path): return qemu_img_info["format"] -def main(): +def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) images_to_be_created = list( filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) @@ -42,13 +45,13 @@ def main(): image_filename = image.value["filename"] image_store_name = image.value["store_name"] image_full_path = join_path( - settings["storage"]["file_dir"], + shared.settings["storage"]["file_dir"], image_owner, image_filename, ) image_stores = shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], + shared.settings["etcd"]["image_store_prefix"], value_in_json=True, ) user_image_store = next( diff --git a/uncloud/metadata/__init__.py b/archive/uncloud_etcd_based/uncloud/metadata/__init__.py similarity index 100% rename from uncloud/metadata/__init__.py rename to archive/uncloud_etcd_based/uncloud/metadata/__init__.py diff --git a/uncloud/metadata/main.py b/archive/uncloud_etcd_based/uncloud/metadata/main.py similarity index 59% rename from uncloud/metadata/main.py rename to archive/uncloud_etcd_based/uncloud/metadata/main.py index da993ae..374260e 100644 --- a/uncloud/metadata/main.py +++ b/archive/uncloud_etcd_based/uncloud/metadata/main.py @@ -1,17 +1,22 @@ import os +import argparse from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from uncloud.settings import settings -from uncloud.shared import shared +from uncloud.common.shared import shared app = Flask(__name__) api = Api(app) app.logger.handlers.clear() +DEFAULT_PORT=1234 + +arg_parser = argparse.ArgumentParser('metadata', add_help=False) +arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT)) + @app.errorhandler(Exception) def handle_exception(e): @@ -68,7 +73,7 @@ class Root(Resource): ) else: etcd_key = os.path.join( - settings["etcd"]["user_prefix"], + shared.settings["etcd"]["user_prefix"], data.value["owner_realm"], data.value["owner"], "key", @@ -80,40 +85,11 @@ class Root(Resource): data.value["metadata"]["ssh-keys"] += user_personal_ssh_keys return data.value["metadata"], 200 - @staticmethod - def post(): - return {"message": "Previous Implementation is deprecated."} - # data = etcd_client.get("/v1/metadata/{}".format(request.remote_addr), value_in_json=True) - # print(data) - # if data: - # for k in request.json: - # if k not in data.value: - # data.value[k] = request.json[k] - # if k.endswith("-list"): - # data.value[k] = [request.json[k]] - # else: - # if k.endswith("-list"): - # data.value[k].append(request.json[k]) - # else: - # data.value[k] = request.json[k] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data.value, value_in_json=True) - # else: - # data = {} - # for k in request.json: - # data[k] = request.json[k] - # if k.endswith("-list"): - # data[k] = [request.json[k]] - # etcd_client.put("/v1/metadata/{}".format(request.remote_addr), - # data, value_in_json=True) - api.add_resource(Root, "/") -def main(): - app.run(debug=True, host="::", port="80") - - -if __name__ == "__main__": - main() +def main(arguments): + port = arguments['port'] + debug = arguments['debug'] + app.run(debug=debug, host="::", port=port) diff --git a/uncloud/network/README b/archive/uncloud_etcd_based/uncloud/network/README similarity index 100% rename from uncloud/network/README rename to archive/uncloud_etcd_based/uncloud/network/README diff --git a/archive/uncloud_etcd_based/uncloud/network/__init__.py b/archive/uncloud_etcd_based/uncloud/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/network/create-bridge.sh b/archive/uncloud_etcd_based/uncloud/network/create-bridge.sh similarity index 100% rename from uncloud/network/create-bridge.sh rename to archive/uncloud_etcd_based/uncloud/network/create-bridge.sh diff --git a/uncloud/network/create-tap.sh b/archive/uncloud_etcd_based/uncloud/network/create-tap.sh similarity index 100% rename from uncloud/network/create-tap.sh rename to archive/uncloud_etcd_based/uncloud/network/create-tap.sh diff --git a/uncloud/network/create-vxlan.sh b/archive/uncloud_etcd_based/uncloud/network/create-vxlan.sh similarity index 100% rename from uncloud/network/create-vxlan.sh rename to archive/uncloud_etcd_based/uncloud/network/create-vxlan.sh diff --git a/uncloud/network/radvd-template.conf b/archive/uncloud_etcd_based/uncloud/network/radvd-template.conf similarity index 100% rename from uncloud/network/radvd-template.conf rename to archive/uncloud_etcd_based/uncloud/network/radvd-template.conf diff --git a/uncloud/scheduler/__init__.py b/archive/uncloud_etcd_based/uncloud/oneshot/__init__.py similarity index 100% rename from uncloud/scheduler/__init__.py rename to archive/uncloud_etcd_based/uncloud/oneshot/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/oneshot/main.py b/archive/uncloud_etcd_based/uncloud/oneshot/main.py new file mode 100644 index 0000000..dbb3b32 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/oneshot/main.py @@ -0,0 +1,123 @@ +import argparse +import os + + +from pathlib import Path +from uncloud.vmm import VMM +from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap + +from . import virtualmachine, logger + +### +# Argument parser loaded by scripts/uncloud. +arg_parser = argparse.ArgumentParser('oneshot', add_help=False) + +# Actions. +arg_parser.add_argument('--list', action='store_true', + help='list UUID and name of running VMs') +arg_parser.add_argument('--start', nargs=4, + metavar=('NAME', 'IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'), + help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix') +arg_parser.add_argument('--stop', metavar='UUID', + help='stop a VM') +arg_parser.add_argument('--get-status', metavar='UUID', + help='return the status of the VM') +arg_parser.add_argument('--get-vnc', metavar='UUID', + help='return the path of the VNC socket of the VM') +arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK', + help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix') + +# Arguments. +arg_parser.add_argument('--workdir', default=Path.home(), + help='Working directory, defaulting to $HOME') +arg_parser.add_argument('--mac', + help='MAC address of the VM to create (--start)') +arg_parser.add_argument('--memory', type=int, + help='Memory (MB) to allocate (--start)') +arg_parser.add_argument('--cores', type=int, + help='Number of cores to allocate (--start)') +arg_parser.add_argument('--threads', type=int, + help='Number of threads to allocate (--start)') +arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], + help='Format of OS image (--start)') +arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm', + help='QEMU acceleration to use (--start)') +arg_parser.add_argument('--upstream-interface', default='eth0', + help='Name of upstream interface (--start)') + +### +# Helpers. + +# XXX: check if it is possible to use the type returned by ETCD queries. +class UncloudEntryWrapper: + def __init__(self, value): + self.value = value + + def value(self): + return self.value + +def status_line(vm): + return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status()) + +### +# Entrypoint. + +def main(arguments): + # Initialize VMM. + workdir = arguments['workdir'] + vmm = VMM(vmm_backend=workdir) + + # Harcoded debug values. + net_id = 0 + + # Build VM configuration. + vm_config = {} + vm_options = [ + 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', + '--upstream_interface', 'upstream_interface', 'network', 'accel' + ] + for option in vm_options: + if arguments.get(option): + vm_config[option] = arguments[option] + + vm_config['net_id'] = net_id + + # Execute requested VM action. + if arguments['reconfigure_radvd']: + # TODO: check that RADVD is available. + prefix = arguments['reconfigure_radvd'] + network = UncloudEntryWrapper({ + 'id': net_id, + 'ipv6': prefix + }) + + # Make use of uncloud.host.virtualmachine for network configuration. + update_radvd_conf([network]) + elif arguments['start']: + # Extract from --start positional arguments. Quite fragile. + vm_config['name'] = arguments['start'][0] + vm_config['image'] = arguments['start'][1] + vm_config['network'] = arguments['start'][2] + vm_config['upstream_interface'] = arguments['start'][3] + + vm_config['tap_interface'] = "uc{}".format(len(vmm.discover())) + vm = virtualmachine.VM(vmm, vm_config) + vm.start() + elif arguments['stop']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']}) + vm.stop() + elif arguments['get_status']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']}) + print(status_line(vm)) + elif arguments['get_vnc']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']}) + print(vm.get_vnc_addr()) + elif arguments['list']: + vms = vmm.discover() + print("Found {} VMs.".format(len(vms))) + for uuid in vms: + vm = virtualmachine.VM(vmm, {'uuid': uuid}) + print(status_line(vm)) + else: + print('Please specify an action: --start, --stop, --list,\ +--get-status, --get-vnc, --reconfigure-radvd') diff --git a/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py b/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py new file mode 100644 index 0000000..5749bee --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py @@ -0,0 +1,81 @@ +import uuid +import os + +from uncloud.host.virtualmachine import create_vxlan_br_tap +from uncloud.oneshot import logger + +class VM(object): + def __init__(self, vmm, config): + self.config = config + self.vmm = vmm + + # Extract VM specs/metadata from configuration. + self.name = config.get('name', 'no-name') + self.memory = config.get('memory', 1024) + self.cores = config.get('cores', 1) + self.threads = config.get('threads', 1) + self.image_format = config.get('image_format', 'qcow2') + self.image = config.get('image') + self.uuid = config.get('uuid', str(uuid.uuid4())) + self.mac = config.get('mac') + self.accel = config.get('accel', 'kvm') + + self.net_id = config.get('net_id', 0) + self.upstream_interface = config.get('upstream_interface', 'eth0') + self.tap_interface = config.get('tap_interface', 'uc0') + self.network = config.get('network') + + def get_qemu_args(self): + command = ( + "-uuid {uuid} -name {name} -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" + ).format( + uuid=self.uuid, name=self.name, accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + net_id=self.net_id, tap=self.tap_interface, mac=self.mac + ) + + return command.split(" ") + + def start(self): + # Check that VM image is available. + if not os.path.isfile(self.image): + logger.error("Image {} does not exist. Aborting.".format(self.image)) + + # Create Bridge, VXLAN and tap interface for VM. + create_vxlan_br_tap( + self.net_id, self.upstream_interface, self.tap_interface, self.network + ) + + # Generate config for and run QEMU. + qemu_args = self.get_qemu_args() + logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + self.vmm.stop(self.uuid) + + def get_status(self): + return self.vmm.get_status(self.uuid) + + def get_vnc_addr(self): + return self.vmm.get_vnc(self.uuid) + + def get_uuid(self): + return self.uuid + + def get_name(self): + success, json = self.vmm.execute_command(self.uuid, 'query-name') + if success: + return json['return']['name'] + + return None diff --git a/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py b/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py new file mode 100644 index 0000000..eea436a --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/uncloud/scheduler/helper.py b/archive/uncloud_etcd_based/uncloud/scheduler/helper.py similarity index 96% rename from uncloud/scheduler/helper.py rename to archive/uncloud_etcd_based/uncloud/scheduler/helper.py index 7edf623..79db322 100755 --- a/uncloud/scheduler/helper.py +++ b/archive/uncloud_etcd_based/uncloud/scheduler/helper.py @@ -6,8 +6,7 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus -from uncloud.shared import shared -from uncloud.settings import settings +from uncloud.common.shared import shared def accumulated_specs(vms_specs): @@ -130,7 +129,7 @@ def assign_host(vm): type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) diff --git a/archive/uncloud_etcd_based/uncloud/scheduler/main.py b/archive/uncloud_etcd_based/uncloud/scheduler/main.py new file mode 100755 index 0000000..38c07bf --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/scheduler/main.py @@ -0,0 +1,51 @@ +# TODO +# 1. send an email to an email address defined by env['admin-email'] +# if resources are finished +# 2. Introduce a status endpoint of the scheduler - +# maybe expose a prometheus compatible output + +import argparse + +from uncloud.common.request import RequestEntry, RequestType +from uncloud.common.shared import shared +from uncloud.scheduler import logger +from uncloud.scheduler.helper import (dead_host_mitigation, dead_host_detection, + assign_host, NoSuitableHostFound) + +arg_parser = argparse.ArgumentParser('scheduler', add_help=False) + + +def main(arguments): + # The below while True is neccessary for gracefully handling leadership transfer and temporary + # unavailability in etcd. Why does it work? It works because the get_prefix,watch_prefix return + # iter([]) that is iterator of empty list on exception (that occur due to above mentioned reasons) + # which ends the loop immediately. So, having it inside infinite loop we try again and again to + # get prefix until either success or deamon death comes. + while True: + for request_iterator in [ + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, + raise_exception=False), + ]: + for request_event in request_iterator: + dead_host_mitigation(dead_host_detection()) + request_entry = RequestEntry(request_event) + + if request_entry.type == RequestType.ScheduleVM: + logger.debug('%s, %s', request_entry.key, request_entry.value) + + vm_entry = shared.vm_pool.get(request_entry.uuid) + if vm_entry is None: + logger.info('Trying to act on {} but it is deleted'.format(request_entry.uuid)) + continue + + shared.etcd_client.client.delete(request_entry.key) # consume Request + + try: + assign_host(vm_entry) + except NoSuitableHostFound: + vm_entry.add_log('Can\'t schedule VM. No Resource Left.') + shared.vm_pool.put(vm_entry) + + logger.info('No Resource Left. Emailing admin....') diff --git a/archive/uncloud_etcd_based/uncloud/scheduler/tests/__init__.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/scheduler/tests/test_basics.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/test_basics.py similarity index 100% rename from uncloud/scheduler/tests/test_basics.py rename to archive/uncloud_etcd_based/uncloud/scheduler/tests/test_basics.py diff --git a/uncloud/scheduler/tests/test_dead_host_mechanism.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/test_dead_host_mechanism.py similarity index 100% rename from uncloud/scheduler/tests/test_dead_host_mechanism.py rename to archive/uncloud_etcd_based/uncloud/scheduler/tests/test_dead_host_mechanism.py diff --git a/archive/uncloud_etcd_based/uncloud/version.py b/archive/uncloud_etcd_based/uncloud/version.py new file mode 100644 index 0000000..ccf3980 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/version.py @@ -0,0 +1 @@ +VERSION = "0.0.5-30-ge91fd9e" diff --git a/uncloud/vmm/__init__.py b/archive/uncloud_etcd_based/uncloud/vmm/__init__.py similarity index 90% rename from uncloud/vmm/__init__.py rename to archive/uncloud_etcd_based/uncloud/vmm/__init__.py index 6cdd938..6db61eb 100644 --- a/uncloud/vmm/__init__.py +++ b/archive/uncloud_etcd_based/uncloud/vmm/__init__.py @@ -100,9 +100,9 @@ class TransferVM(Process): class VMM: # Virtual Machine Manager def __init__( - self, - qemu_path="/usr/bin/qemu-system-x86_64", - vmm_backend=os.path.expanduser("~/uncloud/vmm/"), + self, + qemu_path="/usr/bin/qemu-system-x86_64", + vmm_backend=os.path.expanduser("~/uncloud/vmm/"), ): self.qemu_path = qemu_path self.vmm_backend = vmm_backend @@ -125,7 +125,7 @@ class VMM: os.makedirs(self.socket_dir, exist_ok=True) def is_running(self, uuid): - sock_path = os.path.join(self.vmm_backend, uuid) + sock_path = os.path.join(self.socket_dir, uuid) try: sock = socket.socket(socket.AF_UNIX) sock.connect(sock_path) @@ -163,7 +163,7 @@ class VMM: qmp_arg = ( "-qmp", "unix:{},server,nowait".format( - join_path(self.vmm_backend, uuid) + join_path(self.socket_dir, uuid) ), ) vnc_arg = ( @@ -190,18 +190,10 @@ class VMM: err.stderr.decode("utf-8"), ) else: - with suppress(sp.CalledProcessError): - sp.check_output( - [ - "sudo", - "-p", - "Enter password to correct permission for uncloud-vmm's directory", - "chmod", - "-R", - "o=rwx,g=rwx", - self.vmm_backend, - ] - ) + sp.check_output( + ["sudo", "-p", "Enter password to correct permission for uncloud-vmm's directory", + "chmod", "-R", "o=rwx,g=rwx", self.vmm_backend] + ) # TODO: Find some good way to check whether the virtual machine is up and # running without relying on non-guarenteed ways. @@ -220,7 +212,7 @@ class VMM: def execute_command(self, uuid, command, **kwargs): # execute_command -> sucess?, output try: - with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as ( + with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as ( sock_handle, file_handle, ): @@ -263,8 +255,8 @@ class VMM: def discover(self): vms = [ uuid - for uuid in os.listdir(self.vmm_backend) - if not isdir(join_path(self.vmm_backend, uuid)) + for uuid in os.listdir(self.socket_dir) + if not isdir(join_path(self.socket_dir, uuid)) ] return vms diff --git a/bin/deploy.sh b/bin/deploy.sh new file mode 100755 index 0000000..99f7ba0 --- /dev/null +++ b/bin/deploy.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# Nico Schottelius, 2021-01-17 + +set -e + +if [ $# -ne 1 ]; then + echo "$0 target-host" + exit 1 +fi + +target_host=$1; shift +user=app + +dir=${0%/*} +uncloud_base=$(cd ${dir}/.. && pwd -P) +conf_name=local_settings-${target_host}.py +conf_file=${uncloud_base}/uncloud/${conf_name} + +if [ ! -e ${conf_file} ]; then + echo "No settings for ${target_host}." + echo "Create ${conf_file} before using this script." + exit 1 +fi + +# Deploy +rsync -av \ + --exclude venv/ \ + --exclude '*.pyc' \ + --exclude uncloud/local_settings.py \ + --delete \ + ${uncloud_base}/ ${user}@${target_host}:app/ + +ssh "${user}@${target_host}" ". ~/pyvenv/bin/activate; cd ~/app; pip install -r requirements.txt" + +# Config +ssh "${user}@${target_host}" "cd ~/app/uncloud; ln -sf ${conf_name} local_settings.py" + +# Restart / Apply +ssh "${user}@${target_host}" "sudo /etc/init.d/uwsgi restart" diff --git a/bin/make-migrations-from-scratch.sh b/bin/make-migrations-from-scratch.sh new file mode 100755 index 0000000..8baccfa --- /dev/null +++ b/bin/make-migrations-from-scratch.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# For undoing/redoing everything +# Needed in special cases and needs to be avoided as soon as +# uncloud.version >= 1 +for a in */migrations; do rm ${a}/*.py; done +for a in */migrations; do python manage.py makemigrations ${a%%/migrations}; done diff --git a/conf/uncloud.conf b/conf/uncloud.conf deleted file mode 100644 index 334bbeb..0000000 --- a/conf/uncloud.conf +++ /dev/null @@ -1,7 +0,0 @@ -[etcd] -url = localhost -port = 2379 - -ca_cert -cert_cert -cert_key diff --git a/doc/.gitignore b/doc/.gitignore new file mode 100644 index 0000000..b51a70d --- /dev/null +++ b/doc/.gitignore @@ -0,0 +1,2 @@ +*.pdf +*.tex diff --git a/doc/README-billing.org b/doc/README-billing.org new file mode 100644 index 0000000..50b26fa --- /dev/null +++ b/doc/README-billing.org @@ -0,0 +1,85 @@ +* How to handle billing in general +** Manual test flow / setting up bills + - Needs orders + - +** Orders + - Orders are the heart of uncloud billing + - Have a starting date + - Have an ending date + - Orders are immutable + - Can usually not be cancelled / cancellation is not a refund + - Customer/user commits on a certain period -> gets discount + based on it + - Can be upgraded + - Create a new order + - We link the new order to the old order and say this one + replaces it + - If the price of the new order is HIGHER than the OLD order, + then we charge the difference until the end of the order period + - In the next billing run we set the OLD order to not to bill anymore + - And only the NEW order will be billed afterwards + - Can be downgraded in the next period (but not for this period) + - We create a new order, same as for upgrade + - The new order starts directly after the OLD order + - As the amount is LOWER than the OLD order, no additional charge is done + during this order period + - We might need to have an activate datetime + - When to implement this + - Order periods can be +*** Statuses + - CREATING/PREPARING + - INACTIVE (?) + - TO_BILL + - NOT_TO_BILL: we use this to accelerate queries to the DB +*** Updating status of orders + - If has succeeding order and billing date is last month -> set inactive +** Bills + - Are always for a month + - Can be preliminary +*** Which orders to include + - Not the cancelled ones / not active ones +** Flows / Approach +*** Finding all orders for a bill + - Get all orders, state != NOT_TO_BILL; for each order do: + - is it a one time order? + - has it a bill assigned? + - yes: set to NOT_TO_BILL + - no: + - get_or_create_bill_for_this_month + - assign bill to this order + - set to NOT_TO_BILL + - is it a recurring order? + - if it has a REPLACING order: + - + - First of month + - Last of month +*** Handling replacement of orders + - The OLD order will appear in the month that it was cancelled on + the bill + - The OLD order needs to be set to NOT_TO_BILL after it was billed + the last time + - The NEW order will be added pro rata if the amount is higher in + the same month + - The NEW order will be used next month +**** Disabling the old order + - On billing run + - If order.replacement_order (naming!) is set + - if the order.replacement_order starts during THIS_MONTH + - add order to bill + - if NOT: + - the order was already replaced in a previous billing period + - set the order to NOT_TO_BILL +**** Billing the new order + - If order.previous_order +*** Handling multiple times a recurring order + - For each recurring order check the order.period + - Find out when it was billed last + - lookup latest bill + - Calculate how many times it has been used until 2359, last day + of month + - For preliminary bill: until datetime.now() + - Call the bill_end_datetime + - Getting duration: bill_end_datetime - order.last_billed + - Amount in seconds; duration_in_seconds + - Divide duration_in_seconds by order.period; amount_used: + - If >= 1: add amount_used * order.recurring_amount to bill diff --git a/doc/README-how-to-configure-remote-uncloud-clients.org b/doc/README-how-to-configure-remote-uncloud-clients.org new file mode 100644 index 0000000..b48886b --- /dev/null +++ b/doc/README-how-to-configure-remote-uncloud-clients.org @@ -0,0 +1,28 @@ +* What is a remote uncloud client? +** Systems that configure themselves for the use with uncloud +** Examples are VMHosts, VPN Servers, cdist control server, etc. +* Which access do these clients need? +** They need read / write access to the database +* Possible methods +** Overview +| | pros | cons | +| SSL based | Once setup, can access all django parts natively, locally | X.509 infrastructure | +| SSH -L tunnel | All nodes can use [::1]:5432 | SSH setup can be fragile | +| ssh djangohost manage.py | All DB ops locally | Code is only executed on django host | +| https + token | Rest alike / consistent access | Code is only executed on django host | +| from_django | Everything is on the django host | main host can become bottleneck | +** remote vs. local Django code execution + - If manage.py is executed locally (= on the client), it can + check/modify local configs + - However local execution requires a pyvenv + packages + db access + - Local execution also *could* make use of postgresql notify for + triggering actions (which is quite neat) + - Remote execution (= on the primary django host) can acess the db + via unix socket + - However remote execution cannot check local state +** from_django + - might reuse existing methods like celery + - reduces the amount of things to be installed on the client to + almost zero + - follows the opennebula model + - has a single point of failurebin diff --git a/doc/uncloud-manual-2020-08-01.org b/doc/uncloud-manual-2020-08-01.org new file mode 100644 index 0000000..b997600 --- /dev/null +++ b/doc/uncloud-manual-2020-08-01.org @@ -0,0 +1,485 @@ +* Bootstrap / Installation / Deployment +** Pre-requisites by operating system +*** General + To run uncloud you need: + - ldap development libraries + - libxml2-dev libxslt-dev + - gcc / libc headers: for compiling things + - python3-dev + - wireguard: wg (for checking keys) +*** Alpine + #+BEGIN_SRC sh +apk add openldap-dev postgresql-dev libxml2-dev libxslt-dev gcc python3-dev musl-dev wireguard-tools-wg +#+END_SRC +*** Debian/Devuan: + #+BEGIN_SRC sh +apt install postgresql-server-dev-all +#+END_SRC +** Creating a virtual environment / installing python requirements +*** Virtual env + To separate uncloud requirements, you can use a python virtual + env as follows: + #+BEGIN_SRC sh +python3 -m venv venv +. ./venv/bin/activate +#+END_SRC + Then install the requirements + #+BEGIN_SRC sh +pip install -r requirements.txt +#+END_SRC +** Setting up the the database +*** Install the database service + The database can run on the same host as uncloud, but can also run + a different server. Consult the usual postgresql documentation for + a secure configuration. + + The database needs to be accessible from all worker nodes. +**** Alpine + #+BEGIN_SRC sh +apk add postgresql-server +rc-update add postgresql +rc-service postgresql start` +#+END_SRC + +**** Debian/Devuan: + #+BEGIN_SRC sh + apt install postgresql + #+END_SRC +*** Create the database + Due to the use of the JSONField, postgresql is required. + To get started, + create a database and have it owned by the user that runs uncloud + (usually "uncloud"): + + #+BEGIN_SRC sh +bridge:~# su - postgres +bridge:~$ psql +postgres=# create role uncloud login; +postgres=# create database uncloud owner nico; +#+END_SRC +*** Creating the schema + #+BEGIN_SRC sh +python manage.py migrate +#+END_SRC + +*** Configuring remote access + - Get a letsencrypt certificate + - Expose SSL ports + - Create a user + + #+BEGIN_SRC sh + certbot certonly --standalone \ + -d -m your@email.come \ + --agree-tos --no-eff-email + #+END_SRC + + - Configuring postgresql.conf: + #+BEGIN_SRC sh +listen_addresses = '*' # what IP address(es) to listen on; +ssl = on +ssl_cert_file = '/etc/postgresql/server.crt' +ssl_key_file = '/etc/postgresql/server.key' + + #+END_SRC + + - Cannot load directly due to permission error: +2020-12-26 13:01:55.235 CET [27805] FATAL: could not load server +certificate file +"/etc/letsencrypt/live/2a0a-e5c0-0013-0000-9f4b-e619-efe5-a4ac.has-a.name/fullchain.pem": +Permission denied + + - hook + #+BEGIN_SRC sh +bridge:/etc/letsencrypt/renewal-hooks/deploy# cat /etc/letsencrypt/renewal-hooks/deploy/postgresql +#!/bin/sh + +umask 0177 +export DOMAIN=2a0a-e5c0-0013-0000-9f4b-e619-efe5-a4ac.has-a.name +export DATA_DIR=/etc/postgresql + +cp /etc/letsencrypt/live/$DOMAIN/fullchain.pem $DATA_DIR/server.crt +cp /etc/letsencrypt/live/$DOMAIN/privkey.pem $DATA_DIR/server.key +chown postgres:postgres $DATA_DIR/server.crt $DATA_DIR/server.key + #+END_SRC + + - Allowing access with md5 encrypted password encrypted via TLS + #+BEGIN_SRC sh +hostssl all all ::/0 md5 + #+END_SRC + + #+BEGIN_SRC sh + +postgres=# create role uncloud password '...'; +CREATE ROLE +postgres=# alter role uncloud login ; +ALTER ROLE + #+END_SRC + + Testing the connection: + + #+BEGIN_SRC sh +psql postgresql://uncloud@2a0a-e5c0-0013-0000-9f4b-e619-efe5-a4ac.has-a.name/uncloud?sslmode +=require +g #+END_SRC + +** Bootstrap + - Login via a user so that the user object gets created + - Run the following (replace nicocustomer with the username) + #+BEGIN_SRC sh + python manage.py bootstrap-user --username nicocustomer + #+END_SRC + +** Initialise the database + While it is not strictly required to add default values to the + database, it might significantly reduce the starting time with + uncloud. + + To add the default database values run: + + #+BEGIN_SRC shell + # Add local objects + python manage.py db-add-defaults + + # Import VAT rates + python manage.py import-vat-rates + #+END_SRC + +** Worker nodes + Nodes that realise services (VMHosts, VPNHosts, etc.) need to be + accessible from the main node and also need access to the database. + + Workers usually should have an "uncloud" user account, even though + strictly speaking the username can be any. +*** WireGuardVPN Server + - Allow write access to /etc/wireguard for uncloud user + - Allow sudo access to "ip" and "wg" + + #+BEGIN_SRC sh + chown uncloud /etc/wireguard/ + [14:30] vpn-2a0ae5c1200:/etc/sudoers.d# cat uncloud + app ALL=(ALL) NOPASSWD:/sbin/ip + app ALL=(ALL) NOPASSWD:/usr/bin/wg + #+END_SRC +** Typical source code based deployment + - Deploy using bin/deploy.sh on a remote server + - Remote server should have + - postgresql running, accessible via TLS from outside + - rabbitmq-configured [in progress] + +* Testing / CLI Access + Access via the commandline (CLI) can be done using curl or + httpie. In our examples we will use httpie. +** Checkout out the API + #+BEGIN_SRC sh + http localhost:8000/api/ + #+END_SRC +** Authenticate via ldap user in password store + #+BEGIN_SRC sh + http --auth nicocustomer:$(pass ldap/nicocustomer) localhost:8000/api/ + #+END_SRC +* Database +** uncloud clients access the data base from a variety of outside hosts +** So the postgresql data base needs to be remotely accessible +** Instead of exposing the tcp socket, we make postgresql bind to localhost via IPv6 +*** ::1, port 5432 +** Then we remotely connect to the database server with ssh tunneling +*** ssh -L5432:localhost:5432 uncloud-database-host +** Configuring your database for SSH based remote access +*** host all all ::1/128 trust + +* URLs + - api/ - the rest API +* uncloud Products +** Product features + - Dependencies on other products + - Minimum parameters (min cpu, min ram, etc). + - Can also realise the dcl vm + - dualstack vm = VM + IPv4 + SSD + - Need to have a non-misguiding name for the "bare VM" + - Should support network boot (?) + +** VPN +*** How to add a new VPN Host +**** Install wireguard to the host +**** Install uncloud to the host +**** Add `python manage.py vpn --hostname fqdn-of-this-host` to the crontab +**** Use the CLI to configure one or more VPN Networks for this host +*** Example of adding a VPN host at ungleich +**** Create a new dual stack alpine VM +**** Add it to DNS as vpn-XXX.ungleich.ch +**** Route a /40 network to its IPv6 address +**** Install wireguard on it +**** TODO [#C] Enable wireguard on boot +**** TODO [#C] Create a new VPNPool on uncloud with +***** the network address (selecting from our existing pool) +***** the network size (/...) +***** the vpn host that provides the network (selecting the created VM) +***** the wireguard private key of the vpn host (using wg genkey) +***** http command + ``` + http -a nicoschottelius:$(pass + ungleich.ch/nico.schottelius@ungleich.ch) + http://localhost:8000/admin/vpnpool/ network=2a0a:e5c1:200:: \ + network_size=40 subnetwork_size=48 + vpn_hostname=vpn-2a0ae5c1200.ungleich.ch + wireguard_private_key=... + ``` +*** Example http commands / REST calls +**** creating a new vpn pool + http -a nicoschottelius:$(pass + ungleich.ch/nico.schottelius@ungleich.ch) + http://localhost:8000/admin/vpnpool/ network_size=40 + subnetwork_size=48 network=2a0a:e5c1:200:: + vpn_hostname=vpn-2a0ae5c1200.ungleich.ch wireguard_private_key=$(wg + genkey) +**** Creating a new vpn network +*** Creating a VPN pool + + #+BEGIN_SRC sh +http -a uncloudadmin:$(pass uncloudadmin) https://localhost:8000/v1/admin/vpnpool/ \ + network=2a0a:e5c1:200:: network_size=40 subnetwork_size=48 \ + vpn_hostname=vpn-2a0ae5c1200.ungleich.ch wireguard_private_key=$(wg genkey) + #+END_SRC + +This will create the VPNPool 2a0a:e5c1:200::/40 from which /48 +networks will be used for clients. + +VPNPools can only be managed by staff. + +*** Managing VPNNetworks + +To request a network as a client, use the following call: + + #+BEGIN_SRC sh + http -a user:$(pass user) https://localhost:8000/v1/net/vpn/ \ + network_size=48 \ + wireguard_public_key=$(wg genkey | tee privatekey | wg pubkey) +``` + +VPNNetworks can be managed by all authenticated users. + +* Developer Handbook + The following section describe decisions / architecture of + uncloud. These chapters are intended to be read by developers. +** This Documentation + This documentation is written in org-mode. To compile it to + html/pdf, just open emacs and press *C-c C-e l p*. +** Models +*** Bill + Bills are summarising usage in a specific timeframe. Bills usually + spawn one month. +*** BillRecord + Bill records are used to model the usage of one order during the + timeframe. +*** Order + Orders register the intent of a user to buy something. They might + refer to a product. (???) + Order register the one time price and the recurring price. These + fields should be treated as immutable. If they need to be modified, + a new order that replaces the current order should be created. +**** Replacing orders + If an order is updated, a new order is created and points to the + old order. The old order stops one second before the new order + starts. + + If a order has been replaced can be seen by its replaced_by count: + #+BEGIN_SRC sh + >>> Order.objects.get(id=1).replaced_by.count() + 1 + #+END_SRC + +*** Product and Product Children + - A product describes something a user can buy + - A product inherits from the uncloud_pay.models.Product model to + get basic attributes +** Identifiers +*** Problem description + Identifiers can be integers, strings or other objects. They should + be unique. +*** Approach 1: integers + Integers are somewhat easy to remember, but also include + predictable growth, which might allow access to guessed hacking + (obivously proper permissions should prevent this). +*** Approach 2: random uuids + UUIDs are 128 bit integers. Python supports uuid.uuid4() for random + uuids. +*** Approach 3: IPv6 addresses + uncloud heavily depends on IPv6 in the first place. uncloud could + use a /48 to identify all objects. Objects that have IPv6 addresses + on their own, don't need to draw from the system /48. +**** Possible Subnetworks + Assuming uncloud uses a /48 to represent all resources. + + | Network | Name | Description | + |-----------------+-----------------+----------------------------------------------| + | 2001:db8::/48 | uncloud network | All identifiers drawn from here | + | 2001:db8:1::/64 | VM network | Every VM has an IPv6 address in this network | + | 2001:db8:2::/64 | Bill network | Every bill has an IPv6 address | + | 2001:db8:3::/64 | Order network | Every order has an IPv6 address | + | 2001:db8:5::/64 | Product network | Every product (?) has an IPv6 address | + | 2001:db8:4::/64 | Disk network | Every disk is identified | + +**** Tests + [15:47:37] black3.place6:~# rbd create -s 10G ssd/2a0a:e5c0:1::8 + +*** Decision + We use integers, because they are easy. + +** Distributing/Dispatching/Orchestrating +*** Variant 1: using cdist + - The uncloud server can git commit things + - The uncloud server loads cdist and configures the server + - Advantages + - Fully integrated into normal flow + - Disadvantage + - web frontend has access to more data than it needs + - On compromise of the machine, more data leaks + - Some cdist usual delay +*** Variant 2: via celery + - The uncloud server dispatches via celery + - Every decentral node also runs celery/connects to the broker + - Summary brokers: + - If local only celery -> good to use redis - Broker + - If remote: probably better to use rabbitmq + - redis + - simpler + - rabbitmq + - more versatile + - made for remote connections + - quorom queues would be nice, but not clear if supported + - https://github.com/celery/py-amqp/issues/302 + - https://github.com/celery/celery/issues/6067 + - Cannot be installed on alpine Linux at the moment + - Advantage + - Very python / django integrated + - Rather instant + - Disadvantages + - Every decentral node needs to have the uncloud code available + - Decentral nodes *might* need to access the database + - Tasks can probably be written to work without that + (i.e. only strings/bytes) + +**** log/tests + (venv) [19:54] vpn-2a0ae5c1200:~/uncloud$ celery -A uncloud -b redis://bridge.place7.ungleich.ch worker -n worker1@%h --logfile ~/celery.log - +Q vpn-2a0ae5c1200.ungleich.ch + + +*** Variant 3: dedicated cdist instance via message broker + - A separate VM/machine + - Has Checkout of ~/.cdist + - Has cdist checkout + - Tiny API for management + - Not directly web accessible + - "cdist" queue + +** Milestones :uncloud: +*** 1.1 (cleanup 1) +**** TODO [#C] Unify ValidationError, FieldError - define proper Exception + - What do we use for model errors +**** TODO [#C] Cleanup the results handling in celery + - Remove the results broker? + - Setup app to ignore results? + - Actually use results? +*** 1.0 (initial release) +**** TODO [#C] Initial Generic product support + - Product +***** TODO [#C] Recurring product support +****** TODO [#C] Support replacing orders for updates +****** DONE [#A] Finish split of bill creation + CLOSED: [2020-09-11 Fri 23:19] +****** TODO [#C] Test the new functions in the Order class +****** Define the correct order replacement logic + Assumption: + - recurringperiods are 30days +******* Case 1: downgrading + - User commits to 10 CHF for 30 days + - Wants to downgrade after 15 days to 5 CHF product + - Expected result: + - order 1: 10 CHF until +30days + - order 2: 5 CHF starting 30days + 1s + - Sum of the two orders is 15 CHF + - Question is + - when is the VM shutdown? + - a) instantly + - b) at the end of the cycle + - best solution + - user can choose between a ... b any time +******* Duration + - You cannot cancel the duration + - You can upgrade and with that cancel the duration + - The idea of a duration is that you commit for it + - If you want to commit lower (daily basis for instance) you + have higher per period prices +******* Case X + - User has VM with 2 Core / 2 GB RAM + - User modifies with to 1 core / 3 GB RAM + - We treat it as down/upgrade independent of the modifications + +******* Case 2: upgrading after 1 day + - committed for 30 days + - upgrade after 1 day + - so first order will be charged for 1/30ths + +******* Case 2: upgrading + - User commits to 10 CHF for 30 days + - Wants to upgrade after 15 days to 20 CHF product + - Order 1 : 1 VM with 2 Core / 2 GB / 10 SSD -- 10 CHF + - 30days period, stopped after 15, so quantity is 0.5 = 5 CHF + - Order 2 : 1 VM with 2 Core / 6 GB / 10 SSD -- 20 CHF + - after 15 days + - VM is upgraded instantly + - Expected result: + - order 1: 10 CHF until +15days = 0.5 units = 5 CHF + - order 2: 20 CHF starting 15days + 1s ... +30 days after + the 15 days -> 45 days = 1 unit = 20 CHF + - Total on bill: 25 CHF + +******* Case 2: upgrading + - User commits to 10 CHF for 30 days + - Wants to upgrade after 15 days to 20 CHF product + - Expected result: + - order 1: 10 CHF until +30days = 1 units = 10 CHF + + - order 2: 20 CHF starting 15days + 1s = 1 unit = 20 CHF + - Total on bill: 30 CHF + + +****** TODO [#C] Note: ending date not set if replaced by default (implicit!) + - Should the new order modify the old order on save()? +****** DONE Fix totally wrong bill dates in our test case + CLOSED: [2020-09-09 Wed 01:00] + - 2020 used instead of 2019 + - Was due to existing test data ... +***** DONE Bill logic is still wrong + CLOSED: [2020-11-05 Thu 18:58] + - Bill starting_date is the date of the first order + - However first encountered order does not have to be the + earliest in the bill! + - Bills should not have a duration + - Bills should only have a (unique) issue date + - We charge based on bill_records + - Last time charged issue date of the bill OR earliest date + after that + - Every bill generation checks all (relevant) orders + - add a flag "not_for_billing" or "closed" + - query on that flag + - verify it every time + +***** TODO Generating bill for admins/staff + - + + + + +**** Bill fixes needed +***** TODO Double bill in bill id +***** TODO Name the currency +***** TODO Maybe remove the chromium pdf rendering artefacts + - date on the top + - title on the top + - filename bottom left + - page number could even stay +***** TODO Try to shorten the timestamp (remove time zone?) +***** TODO Bill date might be required +***** TODO Total and VAT are empty +***** TODO Line below detail/ heading diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..b050590 --- /dev/null +++ b/manage.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/matrixhosting/__init__.py b/matrixhosting/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/matrixhosting/admin.py b/matrixhosting/admin.py new file mode 100644 index 0000000..c33589b --- /dev/null +++ b/matrixhosting/admin.py @@ -0,0 +1,4 @@ +from django.contrib import admin +from .models import VMInstance + +admin.site.register(VMInstance) diff --git a/matrixhosting/apps.py b/matrixhosting/apps.py new file mode 100644 index 0000000..ad02796 --- /dev/null +++ b/matrixhosting/apps.py @@ -0,0 +1,9 @@ +from django.apps import AppConfig + + +class MatrixhostingConfig(AppConfig): + default_auto_field = 'django.db.models.BigAutoField' + name = 'matrixhosting' + + def ready(self): + from . import signals diff --git a/matrixhosting/forms.py b/matrixhosting/forms.py new file mode 100644 index 0000000..793aeeb --- /dev/null +++ b/matrixhosting/forms.py @@ -0,0 +1,48 @@ +import tldextract + +from django import forms +from django.forms import ModelForm +from django.utils.translation import get_language, ugettext_lazy as _ +from django.core.exceptions import ValidationError +from .validators import domain_name_validator +from uncloud_pay.models import BillingAddress + + +class DomainNameField(forms.CharField): + description = 'Domain name form field' + default_validators = [domain_name_validator, ] + + def __init__(self, *args, **kwargs): + super(DomainNameField, self).__init__(*args, **kwargs) + +class RequestHostedVMForm(forms.Form): + cores = forms.IntegerField(label='CPU', min_value=1, max_value=48, initial=1) + memory = forms.IntegerField(label='RAM', min_value=2, max_value=200, initial=2) + storage = forms.IntegerField(label='Storage', min_value=100, max_value=10000, initial=100) + matrix_domain = DomainNameField(required=True) + homeserver_domain = DomainNameField(required=True) + webclient_domain = DomainNameField(required=True) + is_open_registration = forms.BooleanField(required=False, initial=False) + pricing_name = forms.CharField(required=True) + + def clean(self): + homeserver_domain = self.cleaned_data.get('homeserver_domain', False) + webclient_domain = self.cleaned_data.get('webclient_domain', False) + if homeserver_domain and webclient_domain: + # Homserver-Domain and Webclient-Domain cannot be below the same second level domain (i.e. homeserver.abc.ch and webclient.def.cloud are ok, + # homeserver.abc.ch and webclient.abc.ch are not ok + homeserver_base = tldextract.extract(homeserver_domain).domain + webclient_base = tldextract.extract(webclient_domain).domain + if homeserver_base == webclient_base: + self._errors['webclient_domain'] = self.error_class([ + 'Homserver-Domain and Webclient-Domain cannot be below the same second level domain']) + return self.cleaned_data + + +class BillingAddressForm(ModelForm): + class Meta: + model = BillingAddress + fields = ['full_name', 'street', + 'city', 'postal_code', 'country', 'vat_number', 'active', 'owner'] + + diff --git a/matrixhosting/migrations/0001_initial.py b/matrixhosting/migrations/0001_initial.py new file mode 100644 index 0000000..dce49c3 --- /dev/null +++ b/matrixhosting/migrations/0001_initial.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.4 on 2021-06-30 07:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='VMPricing', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ('vat_inclusive', models.BooleanField(default=True)), + ('vat_percentage', models.DecimalField(blank=True, decimal_places=5, default=0, max_digits=7)), + ('set_up_fees', models.DecimalField(decimal_places=5, default=0, max_digits=7)), + ('cores_unit_price', models.DecimalField(decimal_places=5, default=0, max_digits=7)), + ('ram_unit_price', models.DecimalField(decimal_places=5, default=0, max_digits=7)), + ('storage_unit_price', models.DecimalField(decimal_places=5, default=0, max_digits=7)), + ('discount_name', models.CharField(blank=True, max_length=255, null=True)), + ('discount_amount', models.DecimalField(decimal_places=2, default=0, max_digits=6)), + ('stripe_coupon_id', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + ] diff --git a/matrixhosting/migrations/0002_rename_vmpricing_matrixvmpricing.py b/matrixhosting/migrations/0002_rename_vmpricing_matrixvmpricing.py new file mode 100644 index 0000000..f21241d --- /dev/null +++ b/matrixhosting/migrations/0002_rename_vmpricing_matrixvmpricing.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-07-01 08:48 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('matrixhosting', '0001_initial'), + ] + + operations = [ + migrations.RenameModel( + old_name='VMPricing', + new_name='MatrixVMPricing', + ), + ] diff --git a/matrixhosting/migrations/0003_auto_20210703_1523.py b/matrixhosting/migrations/0003_auto_20210703_1523.py new file mode 100644 index 0000000..fe45ab0 --- /dev/null +++ b/matrixhosting/migrations/0003_auto_20210703_1523.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.4 on 2021-07-03 15:23 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('matrixhosting', '0002_rename_vmpricing_matrixvmpricing'), + ] + + operations = [ + migrations.AlterField( + model_name='matrixvmpricing', + name='cores_unit_price', + field=models.DecimalField(decimal_places=2, default=0, max_digits=7), + ), + migrations.AlterField( + model_name='matrixvmpricing', + name='ram_unit_price', + field=models.DecimalField(decimal_places=2, default=0, max_digits=7), + ), + migrations.AlterField( + model_name='matrixvmpricing', + name='set_up_fees', + field=models.DecimalField(decimal_places=2, default=0, max_digits=7), + ), + migrations.AlterField( + model_name='matrixvmpricing', + name='storage_unit_price', + field=models.DecimalField(decimal_places=2, default=0, max_digits=7), + ), + ] diff --git a/matrixhosting/migrations/0004_matrixhostingorder_vmspecs.py b/matrixhosting/migrations/0004_matrixhostingorder_vmspecs.py new file mode 100644 index 0000000..0259c51 --- /dev/null +++ b/matrixhosting/migrations/0004_matrixhostingorder_vmspecs.py @@ -0,0 +1,43 @@ +# Generated by Django 3.2.4 on 2021-07-05 06:52 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0014_auto_20210703_1747'), + ('matrixhosting', '0003_auto_20210703_1523'), + ] + + operations = [ + migrations.CreateModel( + name='VMSpecs', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('cores', models.IntegerField(default=1)), + ('memory', models.IntegerField(default=2)), + ('storage', models.IntegerField(default=100)), + ('matrix_domain', models.CharField(max_length=255)), + ('homeserver_domain', models.CharField(max_length=255)), + ('webclient_domain', models.CharField(max_length=255)), + ('is_open_registration', models.BooleanField(default=False, null=True)), + ], + ), + migrations.CreateModel( + name='MatrixHostingOrder', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('vm_id', models.IntegerField(default=0)), + ('created_at', models.DateTimeField(auto_now_add=True)), + ('status', models.CharField(choices=[('draft', 'Draft'), ('declined', 'Declined'), ('approved', 'Approved')], default='draft', max_length=100)), + ('stripe_charge_id', models.CharField(max_length=100, null=True)), + ('price', models.FloatField()), + ('billing_address', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='uncloud_pay.billingaddress')), + ('customer', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.stripecustomer')), + ('specs', models.ForeignKey(blank=True, default=None, null=True, on_delete=django.db.models.deletion.SET_NULL, to='matrixhosting.vmspecs')), + ('vm_pricing', models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, to='matrixhosting.matrixvmpricing')), + ], + ), + ] diff --git a/matrixhosting/migrations/0005_auto_20210705_0849.py b/matrixhosting/migrations/0005_auto_20210705_0849.py new file mode 100644 index 0000000..742a63f --- /dev/null +++ b/matrixhosting/migrations/0005_auto_20210705_0849.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2021-07-05 08:49 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('matrixhosting', '0004_matrixhostingorder_vmspecs'), + ] + + operations = [ + migrations.DeleteModel( + name='MatrixHostingOrder', + ), + migrations.DeleteModel( + name='VMSpecs', + ), + ] diff --git a/matrixhosting/migrations/0006_delete_matrixvmpricing.py b/matrixhosting/migrations/0006_delete_matrixvmpricing.py new file mode 100644 index 0000000..f6b0f01 --- /dev/null +++ b/matrixhosting/migrations/0006_delete_matrixvmpricing.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.4 on 2021-07-06 13:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('matrixhosting', '0005_auto_20210705_0849'), + ] + + operations = [ + migrations.DeleteModel( + name='MatrixVMPricing', + ), + ] diff --git a/matrixhosting/migrations/0007_vminstance.py b/matrixhosting/migrations/0007_vminstance.py new file mode 100644 index 0000000..2990e10 --- /dev/null +++ b/matrixhosting/migrations/0007_vminstance.py @@ -0,0 +1,31 @@ +# Generated by Django 3.2.4 on 2021-07-09 09:14 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('uncloud_pay', '0021_auto_20210709_0914'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('matrixhosting', '0006_delete_matrixvmpricing'), + ] + + operations = [ + migrations.CreateModel( + name='VMInstance', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ip', models.TextField(default='')), + ('config', models.JSONField()), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('termination_date', models.DateTimeField(blank=True, null=True)), + ('order', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, related_name='instance_id', to='uncloud_pay.order')), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/matrixhosting/migrations/0008_remove_vminstance_ip.py b/matrixhosting/migrations/0008_remove_vminstance_ip.py new file mode 100644 index 0000000..054359b --- /dev/null +++ b/matrixhosting/migrations/0008_remove_vminstance_ip.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-07-10 14:29 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('matrixhosting', '0007_vminstance'), + ] + + operations = [ + migrations.RemoveField( + model_name='vminstance', + name='ip', + ), + ] diff --git a/matrixhosting/migrations/0009_vminstance_vm_id.py b/matrixhosting/migrations/0009_vminstance_vm_id.py new file mode 100644 index 0000000..2771f58 --- /dev/null +++ b/matrixhosting/migrations/0009_vminstance_vm_id.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2021-07-13 10:20 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('matrixhosting', '0008_remove_vminstance_ip'), + ] + + operations = [ + migrations.AddField( + model_name='vminstance', + name='vm_id', + field=models.UUIDField(default=uuid.uuid4, editable=False, unique=True), + ), + ] diff --git a/matrixhosting/migrations/__init__.py b/matrixhosting/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/matrixhosting/models.py b/matrixhosting/models.py new file mode 100644 index 0000000..8945f69 --- /dev/null +++ b/matrixhosting/models.py @@ -0,0 +1,77 @@ +import logging +import uuid +import os +import sys +import gitlab +from jinja2 import Environment, FileSystemLoader + +from django.db import models +from django.conf import settings +from django.contrib.auth import get_user_model +from django.template.loader import render_to_string + +from uncloud_pay.models import Order + + +# Initialize logger. +logger = logging.getLogger(__name__) + +class VMInstance(models.Model): + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE, + editable=True) + + vm_id = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) + + config = models.JSONField(null=False, blank=False) + + order = models.OneToOneField(Order, on_delete=models.CASCADE, related_name='instance_id') + + creation_date = models.DateTimeField(auto_now_add=True) + + termination_date = models.DateTimeField(blank=True, null=True) + + def save(self, *args, **kwargs): + # Read the deployment yaml file and render the template + # Then save it as new yaml file and push it to github repo + if 'test' in sys.argv: + return super().save(*args, **kwargs) + template_dir = os.path.join(os.path.dirname(__file__), 'yaml') + env = Environment(loader = FileSystemLoader(template_dir),autoescape = True) + tmpl = env.get_template('deployment.yaml.tmpl') + result = tmpl.render( + name=self.vm_id + ) + gl = gitlab.Gitlab(settings.GITLAB_SERVER, oauth_token=settings.GITLAB_OAUTH_TOKEN) + project = gl.projects.get(settings.GITLAB_PROJECT_ID) + project.files.create({'file_path': settings.GITLAB_YAML_DIR + f'{self.vm_id}.yaml', + 'branch': 'master', + 'content': result, + 'author_email': settings.GITLAB_AUTHOR_EMAIL, + 'author_name': settings.GITLAB_AUTHOR_NAME, + 'commit_message': f'Add New Deployment for {self.vm_id}'}) + super().save(*args, **kwargs) + + def delete(self, *args, **kwargs): + # Delete the deployment yaml file first then + # Then delete it + if 'test' in sys.argv: + return super().delete(*args, **kwargs) + gl = gitlab.Gitlab(settings.GITLAB_SERVER, oauth_token=settings.GITLAB_OAUTH_TOKEN) + project = gl.projects.get(settings.GITLAB_PROJECT_ID) + f_path = settings.GITLAB_YAML_DIR + f'{self.vm_id}.yaml' + file = project.files.get(file_path=f_path, ref='master') + if file: + project.files.delete(file_path=f_path, + commit_message=f'Delete {self.vm_id}', branch='master', + author_email=settings.GITLAB_AUTHOR_EMAIL, + author_name=settings.GITLAB_AUTHOR_NAME) + + super().delete(*args, **kwargs) + + def __str__(self): + return f"{self.id}-{self.order}" + + def delete_for_bill(self, bill): + #TODO delete related instances + return True \ No newline at end of file diff --git a/matrixhosting/serializers.py b/matrixhosting/serializers.py new file mode 100644 index 0000000..7711612 --- /dev/null +++ b/matrixhosting/serializers.py @@ -0,0 +1,8 @@ +from rest_framework import serializers + +from .models import * + +class VMInstanceSerializer(serializers.ModelSerializer): + class Meta: + model = VMInstance + fields = '__all__' \ No newline at end of file diff --git a/matrixhosting/signals.py b/matrixhosting/signals.py new file mode 100644 index 0000000..494a1fc --- /dev/null +++ b/matrixhosting/signals.py @@ -0,0 +1,10 @@ +from matrixhosting.models import VMInstance +from uncloud_pay.models import Order +from django.db.models.signals import post_save +from django.dispatch import receiver + +@receiver(post_save, sender=Order) +def create_instance(sender, instance, created, **kwargs): + machine = VMInstance.objects.filter(order=instance).first() + if not machine: + VMInstance.objects.create(owner=instance.owner, order=instance, config=instance.config) \ No newline at end of file diff --git a/matrixhosting/static/matrixhosting/css/common.css b/matrixhosting/static/matrixhosting/css/common.css new file mode 100644 index 0000000..6ef2b64 --- /dev/null +++ b/matrixhosting/static/matrixhosting/css/common.css @@ -0,0 +1,1346 @@ +body, +html { + width: 100%; + height: 100%; +} + +body, +h1, +h2, +h3, +h4, +h5, +h6 { + font-family: 'Lato', sans-serif; +} + + +/* bootstrap danger color override from #a94442 */ + +.text-danger, +.has-error .help-block, +.has-error .control-label, +.has-error .radio, +.has-error .checkbox, +.has-error .radio-inline, +.has-error .checkbox-inline, +.has-error.radio label, +.has-error.checkbox label, +.has-error.radio-inline label, +.has-error.checkbox-inline label, +.has-error .form-control-feedback, +.alert-danger, +.list-group-item-danger, +a.list-group-item-danger, +a.list-group-item-danger:hover, +a.list-group-item-danger:focus, +.panel-danger>.panel-heading { + color: #eb4d5c; +} + +.alert-danger { + background: rgba(235, 204, 209, 0.2); +} + +.has-error .form-control, +.has-error .form-control:focus, +.has-error .form-control:active, +.has-error .input-group-addon { + color: #eb4d5c; + border-color: #eb4d5c; +} + +a.list-group-item-danger.active, +a.list-group-item-danger.active:hover, +a.list-group-item-danger.active:focus { + background-color: #eb4d5c; + border-color: #eb4d5c; +} + +.panel-danger>.panel-heading .badge { + background-color: #eb4d5c; +} + +.topnav { + font-size: 14px; +} + +.navbar-default { + background: #fff; + padding: 5px; +} + +.navbar-brand { + padding: 10px; +} + +.navbar-brand > img { + height: 100%; +} + +#logoWhite, +.navbar-transparent #logoBlack { + display: none; +} + +#logoBlack, +.navbar-transparent #logoWhite { + display: block; +} + +@media (min-width: 768px) { + .navbar-right { + margin-right: 10px; + } + .navbar-brand { + padding-right: 15px; + padding-left: 15px; + } +} + +.navbar .dcl-link { + display: block; + padding: 15px; + color: #777; +} + +.navbar .dcl-link:focus, +.navbar .dcl-link:active, +.navbar .dcl-link:hover { + text-decoration: none; +} + +.navbar .dropdown-menu .dcl-link { + padding: 1px 10px; +} + +p.copyright { + margin: 0; +} + +footer { + font-weight: 300; + padding: 25px 0; + background-color: #f8f8f8; +} + +footer .list-inline { + margin-bottom: 15px; +} + +footer a { + color: #777; +} + +footer .dcl-link-separator { + position: relative; + padding-left: 10px; +} + +footer .dcl-link-separator::before { + content: ""; + position: absolute; + display: inline-block; + top: 9px; + bottom: 0; + left: -2px; + right: 0; + width: 2px; + height: 2px; + border-radius: 100%; + background: #777; +} + +.mb-0 { + margin-bottom: 0; +} + +.thin-hr { + margin-top: 10px; + margin-bottom: 10px; +} + +.payment-container .credit-card-info { + padding-bottom: 15px; + border-bottom: 1px solid #eee; +} +.credit-card-info { + display: flex; +} + +.credit-card-info .align-bottom { + align-self: flex-end; + padding-right: 0 !important; +} + +.new-card-head { + margin-top: 10px; +} +.new-card-button-margin button{ + margin-top: 5px; + margin-bottom: 5px; +} + +.input-no-border { + border: none !important; + background: transparent !important; + resize: none; +} + +.existing-keys-title { + font-weight: bold; + font-size: 14px; +} + +@media(max-width:767px) { + .vspace-top { + margin-top: 35px; + } +} + +/* index */ +.btn { + box-shadow: 0 1px 4px rgba(0, 0, 0, .6); +} + +.fa-li.fa-lg { + color: #29427A; + margin-top: 6px; +} + +.btn-transparent { + background: transparent; + border: 2px solid #fff; + color: #fff; + transition: all .2s ease-in; +} + +.btn-primary { + background: #29427A; + border-color: #29427A; + color: #fff; + width: auto; +} + +.btn-primary:hover { + background: rgba(41, 66, 122, 0.8); + border-color: #29427A; +} + +.btn-transparent:hover { + background: #fff; + border: 2px solid #fff; + color: #000; + transition: all .2s ease-in; +} + +.btn-lg { + min-width: 180px; +} + +.lead { + font-size: 18px; +} + +@media (min-width: 768px) { + .lead { + font-size: 21px; + } +} + + +/* Top navbar */ + +.navbar { + transition: all .3s ease-in; + font-weight: 400; +} + +.navbar-default .navbar-nav>.open>a, +.navbar-default .navbar-nav>.open>a:focus, +.navbar-default .navbar-nav>.open>a:hover { + background: transparent; +} + +.navbar-default .navbar-nav>.active>a, +.navbar-default .navbar-nav>.active>a:focus, +.navbar-default .navbar-nav>.active>a:hover { + background: #2D457A; + color: #fff; + border-radius: 6px; +} + +@media (max-width: 767px) { + .navbar-default .navbar-nav>li>a{ + font-weight: 400; + } +} + +.navbar-transparent .navbar-nav>li a, +.navbar-transparent .navbar-nav>.open>a, +.navbar-transparent .navbar-nav>.open>a:focus, +.navbar-transparent .navbar-nav>.open>a:hover { + color: #fff; +} + + +.navbar-transparent .navbar-nav>li a:focus, +.navbar-transparent .navbar-nav>li a:active, +.navbar-transparent .navbar-nav>li a:hover { + color: #fff; + background-color: transparent; + text-decoration: none; +} + +.topnav .nav .open>a, +.topnav .nav .open>a:focus, +.topnav .nav .open>a:hover { + background: transparent; +} + +.navbar-transparent .navbar-nav>li>.on-hover-border { + transition: all 0.3s linear; + box-shadow: none; +} + +.navbar-transparent .navbar-nav>li>.on-hover-border:hover { + box-shadow: 0 0 0 1px #eee; + border-radius: 5px; +} + +.navbar-transparent { + background: transparent; + border: none; + padding: 20px; +} + +.navbar-transparent .nav-language .select-language { + color: #fff; +} + +.nav-language { + position: relative; +} + +.nav-language .select-language { + padding: 15px 10px; + color: #777; +} + +.nav-language .select-language span { + margin-left: 5px; + margin-right: 5px; + font-weight: normal; +} + +.nav-language .drop-language { + top: 45px; + left: auto !important; + width: 100px; + min-width: 100px; + height: 40px; + padding: 9px 10px; + -webkit-box-shadow: -8px 13px 31px -8px rgba(77, 77, 77, 1); + -moz-box-shadow: -8px 13px 31px -8px rgba(77, 77, 77, 1); + box-shadow: -8px 13px 31px -8px rgba(77, 77, 77, 1); + z-index: 100; + text-align: center; + border-radius: 4px; +} + +.nav-language .drop-language a { + cursor: pointer; + padding: 5px 10px !important; +} + +.nav-language .open .drop-language { + width: 100px; + min-width: 100px; +} + +.dropdown-menu { + border: 1px solid #fff; + -webkit-box-shadow: -8px 14px 20px -5px rgba(77, 77, 77, 0.5); + -moz-box-shadow: -8px 14px 20px -5px rgba(77, 77, 77, 0.5); + box-shadow: -8px 14px 20px -5px rgba(77, 77, 77, 0.5); + border-radius: 4px !important; + left: 0 !important; + min-width: 155px; + padding: 5px; + margin-left: 15px; +} + +.dropdown-menu>li a:focus, +.dropdown-menu>li a:hover { + background: transparent; + text-decoration: underline !important; +} + +@media (min-width: 768px) { + .dropdown-menu>li>a { + font-weight: 300; + } +} + +.highlights-dropdown .dropdown-menu>li>a { + font-size: 13px; + padding: 1px 10px; +} + + +/* Show the dropdown menu on hover */ + +@media (min-width: 768px) { + .nav-language .dropdown:hover .dropdown-menu { + display: block; + } +} + +@media (max-width: 767px) { + .nav-language .open .dropdown-menu>li>a { + line-height: 1.42857143; + } +} + +.navbar-transparent .nav-language .drop-language { + background: transparent; + border: 1px solid #fff; +} + +.navbar-transparent .nav-language .drop-language a { + color: #fff; + padding: 5px 10px !important; +} + + +/* dcl header */ +.dcl-header { + padding: 150px 0 150px 0; + text-align: center; + color: #f8f8f8; + background: url(../img/pattern.jpg) no-repeat center center; + background-size: cover; + position: relative; + background-attachment: fixed; +} + +.dcl-header::before { + content: ""; + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + background: rgba(90, 116, 175, 0.85); +} + +.dcl-header .container { + position: relative; +} + +.dcl-header h1 { + font-size: 65px; + margin: 0; + padding: 0; +} + +@media(max-width:767px) { + .dcl-header h1 { + font-size: 50px; + } +} + +.intro-header { + min-height: 100vh; + text-align: center; + color: #fff; + background: url(../img/configure.jpg) no-repeat center center; + background-size: cover; + position: relative; + display: flex; + justify-content: center; + align-items: center; +} + +.intro-header::before { + content: ""; + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + background: rgba(38, 59, 107, 0.7); +} + +.intro-header-2 { + padding-top: 50px; + /* If you're making other pages, make sure there is 50px of padding to make sure the navbar doesn't overlap content! */ + padding-bottom: 50px; + color: #f8f8f8; + background: url(../img/pattern.jpg) no-repeat center center; + background-size: cover; + position: relative; +} + +.intro-header-2::before { + content: ""; + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + background: rgba(41, 66, 122, 0.59); +} + +.intro-message { + position: relative; + width: 80%; + margin: 0 auto; +} + +.intro-message>h1 { + margin: 0; + font-size: 6em; +} + +.intro-divider { + width: 400px; + border-top: 1px solid #f8f8f8; + border-bottom: 1px solid rgba(0, 0, 0, 0.2); +} + +.intro-pricing { + text-align: center; + color: #fff; + background: url(../img/pattern.jpg) no-repeat center center; + background-size: cover; + height: 70vh; + max-height: 400px; + display: flex; + justify-content: center; + align-items: center; + position: relative; +} + +.intro-pricing.success-pricing { + height: 100vh; + max-height: 100vh; +} + +.intro-pricing::before { + content: ""; + position: absolute; + top: 0; + bottom: 0; + left: 0; + right: 0; + background: rgba(90, 116, 175, 0.7); +} + +.intro-pricing .intro-message .section-heading { + font-size: 45px; + width: 80%; + margin: 0 auto; +} + +.split-section { + padding: 70px 0; + border-top: 1px solid #f6f7f8; +} + +.split-section .icon-section { + position: relative; + min-height: 330px; +} + +.split-section .icon-section i { + position: absolute; + left: 50%; + top: 50%; + transform: translate(-50%, -50%); + font-size: 216px; + color: #5A74AF; +} + +.split-section h2 { + font-size: 36px; + font-weight: 400; +} + +.split-section .split-title-plain h2 { + font-size: 40px; + font-weight: 300; + line-height: 50px; + color: #3a3a3a; +} + +.split-section .split-title { + position: relative; + margin-bottom: 25px; +} + +.split-section .split-title h2 { + font-size: 50px; + font-weight: 300; + padding-bottom: 25px; + letter-spacing: 2px; +} + +.section-gradient { + background: -webkit-linear-gradient(#f0f4f7, #fff) no-repeat; + background: -o-linear-gradient(#f0f4f7, #fff) no-repeat; + background: linear-gradient(#f0f4f7, #fff) no-repeat; +} + +.split-section.left .split-description { + margin-right: auto; +} + +.split-section .split-description .lead { + color: #3a3a3a; +} + +@media (min-width: 768px) { + .split-section .split-description .lead { + font-size: 21px; + } + .split-section .space .split-description .lead { + font-size: 20px; + } +} + +.split-section.right .split-description { + width: 90%; + margin-left: auto; +} + +.split-section.right .split-description.title p { + font-size: 27px; + margin-bottom: 10px; + text-align: left; +} + +.split-section.right .split-text ul, +.split-section.left .split-text, +.split-section.left .space { + text-align: left; +} + +.split-section.right .split-text, +.split-section.right .space { + text-align: right; +} + +.split-section .split-title::before { + content: ""; + position: absolute; + bottom: 0; + background: #29427A; + height: 7px; + width: 70px; + left: auto; +} + +.split-section.right .split-title::before { + right: 0; +} + +.split-section.left .split-title::before { + left: 0; +} + +.section-figure { + display: flex; + flex-wrap: wrap; + justify-content: center; + text-align: center; +} + +.section-figure .section-image { + padding: 20px 40px 30px; + flex-basis: 50%; + flex-grow: 1; + display: flex; + flex-direction: column; + justify-content: space-between; +} + +@media (max-width: 767px) { + .section-figure .section-image { + flex-basis: 100%; + } +} + +.split-section-plain .section-figure .section-image { + flex-grow: 0; + padding: 50px 15px 0; +} + +.split-section-plain .section-figure { + justify-content: flex-start; +} + +@media (min-width: 768px) { + .split-section-plain .split-figure { + width: 41.66666667%; + } + .split-section-plain .split-figure.col-sm-pull-6 { + right: 58.33333333%; + } + .split-section-plain .split-text { + width: 58.33333333%; + } + .split-section-plain .split-text.col-sm-push-6 { + left: 41.66666667%; + } +} + +.section-image img { + margin: auto; +} + +.section-image-caption { + padding-top: 20px; + display: inline-block; + color: #999 !important; + word-break: break-all; +} + +.price-calc-section .card { + width: 350px; + margin: 0 auto; + background: #fff; + box-shadow: 0 10px 20px rgba(0, 0, 0, 0.19), 0 6px 6px rgba(0, 0, 0, 0.23); + padding-bottom: 40px; + border-radius: 7px; + position: relative; +} + +.price-calc-section .card .title { + padding: 15px 40px; +} + +.price-calc-section .card .price { + background: #5A74AF; + padding: 22px; + color: #fff; + font-size: 32px; +} + +.price-calc-section .card .description { + padding: 12px; +} + +.price-calc-section .card .descriptions { + padding: 10px 30px; +} + +.price-calc-section .card .description p { + margin: 0; +} + +.price-calc-section .card .btn { + margin-top: 20px; +} + +@keyframes sending { + 0% { + content: '.'; + } + 50% { + content: '..'; + } + 100% { + content: '...'; + } +} +/*Why DCL*/ + +#tech_stack { + background: #fff; +} + +#tech_stack h3 { + font-size: 42px; + width: 70%; +} + +.space { + max-width: 660px; + margin: auto; +} + +.percent-text { + font-size: 50px; + color: #999; +} + +.space-middle { + /* padding: 45px 0; */ + display: inline-block; +} + +.ssdimg { + margin: 0 15px; +} + +@media (max-width: 767px) { + .ssdimg img { + max-height: 120px; + } +} + +.padding-vertical { + padding: 30px 2px 20px; +} + + +/*Pricing page*/ + +.price-calc-section { + display: flex; + margin-top: 25px; + margin-bottom: 25px; +} + +.price-calc-section .card { + width: 100%; + margin: 0 auto; + background: #fff; + box-shadow: 0 5px 15px rgba(0, 0, 0, 0.1), 0 0 6px rgba(0, 0, 0, 0.15); + padding-bottom: 40px; + border-radius: 7px; + text-align: center; + max-width: 400px; + position: relative; +} + +.price-calc-section .card .title { + padding: 15px 40px; +} + +.price-calc-section .card .title h3 { + font-weight: normal; +} + +.price-calc-section .card .price { + background: #5A74AF; + padding: 22px; + color: #fff; + font-size: 32px; +} + +.price-calc-section .card .price .price-text { + font-size: 14px; +} + +.price-calc-section .card .description { + padding: 12px; + position: relative; + display: flex; + justify-content: space-around !important; + align-items: center !important; +} + +.price-calc-section .card .description span { + font-size: 16px; + margin-left: 0px; + width: 30%; + text-align: left; +} + +.price-calc-section .card .description .select-number { + font-size: 20px; + text-align: center; + width: 85px; +} + +.price-calc-section .card .description i { + color: #29427A; + cursor: pointer; + font-size: 24px; +} + +.price-calc-section .card .description .left { + margin-right: 7px; +} + +.price-calc-section .card .description .right { + margin-left: 7px; +} + +.price-calc-section .card .descriptions { + padding: 10px 30px; +} + +.price-calc-section .card .description p { + margin: 0; +} + +.price-calc-section .card .btn { + margin-top: 20px; + font-size: 20px; + width: 200px; + border: none; +} + +.price-calc-section .card .select-configuration select { + outline: none; + background: #fff; + border-color: #d0d0d0; + height: 40px; + width: 200px; + text-align: center; + font-size: 16px; + margin-left: 10px; +} + +.price-calc-section .card .check-ip { + font-size: 18px; +} + +.price-calc-section .card .justify-center { + justify-content: center !important; +} + +.price-calc-section .card .description.input label { + font-size: 15px; + font-weight: 700; + margin-bottom: 0; + width: 40px; +} + + +/*Changed class****.price-calc-section .card .description.input input*/ + +.price-calc-section .card .description input { + width: 200px; + font-size: 14px; + text-align: left; + padding: 5px 10px; + border-radius: 4px; + border: 1px solid #d0d0d0; + background: #fff; + margin-left: 10px; +} + +.price-calc-section .card .check-ip input[type=checkbox] { + font-size: 17px; + margin: 0 8px; +} + +.help-block.with-errors { + text-align: center; + margin: 0; + padding: 0; +} + +.form-group { + margin: 0; + border-bottom: 1px solid rgba(128, 128, 128, 0.3); +} + +@media(max-width:767px) { + #tech_stack h3 { + font-size: 30px; + line-height: 40px; + width: 100%; + } + .navbar-nav .open .dropdown-menu { + text-align: left; + font-size: 12px; + } + + .navbar-default .navbar-nav>.open>a, + .navbar-default .navbar-nav>.open>a:focus, + .navbar-default .navbar-nav>.open>a:hover { + background: transparent; + color: #777 !important; + } +} + + +@media(max-width:767px) { + .section-sm-center .split-text, + .section-sm-center .space { + text-align: center !important; + margin-bottom: 40px; + } + .section-sm-center .split-title::before { + left: 50% !important; + transform: translate(-50%, 0); + } + .section-sm-center .split-description { + width: 100% !important; + } +} + +@media(max-width:767px) { + .navbar-transparent li a { + color: #777 !important; + } + .intro-message { + padding-bottom: 15%; + } + .intro-message>h1 { + font-size: 3em; + } + ul.intro-social-buttons>li { + display: block; + margin-bottom: 20px; + padding: 0; + } + .intro-pricing .intro-message .section-heading { + font-size: 35px; + width: 80%; + margin: 0 auto; + } + .intro-pricing .intro-message { + padding-bottom: 0; + } + ul.intro-social-buttons>li:last-child { + margin-bottom: 0; + } + .intro-divider { + width: 100%; + } + .navbar-transparent { + background: #fff; + border: none; + padding: 5px; + } + .navbar-transparent #logoBlack { + display: block; + } + .navbar-transparent #logoWhite { + display: none; + } + .navbar-transparent .nav-language .select-language { + color: #777; + } + .navbar-transparent .nav-language .drop-language a { + color: #777; + } + .navbar-transparent .nav-language .drop-language { + background: #fff; + z-index: 100000; + left: 9px; + border: 1px solid rgba(119, 119, 119, 0.4); + box-shadow: none; + } + .navbar-default .nav-language .drop-language { + background: #fff; + z-index: 100000; + left: 9px; + border: 1px solid rgba(119, 119, 119, 0.4); + box-shadow: none; + } + .navbar-default .nav-language .select-language { + color: #777; + } + .navbar-default .nav-language .drop-language a { + color: #777; + } + .navbar-transparent .navbar-nav>li>a:focus, + .navbar-transparent .navbar-nav>li>a:hover { + color: #333; + background-color: transparent; + } + .navbar-default .navbar-nav>li>a:focus, + .navbar-default .navbar-nav>li>a:hover { + color: #333; + background-color: transparent; + } + .split-section { + padding: 20px 0; + } + .split-section .icon-section { + min-height: 160px; + } + .split-section .icon-section i { + font-size: 120px; + } + .split-section h2 { + font-size: 28px; + } + .split-section .split-title-plain h2 { + font-size: 30px; + line-height: 35px; + } + .split-section .split-title h2 { + font-size: 32px; + line-height: 34px; + } + .contact-section .title { + margin: 0 auto; + } + .contact-section .title h2 { + font-size: 45px; + line-height: 40px; + margin-top: 35px; + } + .contact-section .title h2::before { + left: 50%; + transform: translate(-50%, 0); + } + .contact-section .card .social a { + color: #29427A; + font-size: 30px; + } + .intro-pricing .intro-message .section-heading { + font-size: 30px; + } + .price-calc-section { + flex-direction: column; + /* padding: 60px 10px !important; */ + } + .price-calc-section .card { + width: 90%; + } + .price-calc-section .text { + width: 80%; + text-align: center; + margin: 0 auto; + margin-top: 20px; + } + .price-calc-section .text .section-heading { + font-size: 35px; + line-height: 35px; + padding-bottom: 15px; + text-align: center; + } + .price-calc-section .text .section-heading::before { + left: 50%; + transform: translate(-50%, 0); + } + .price-calc-section .text .description { + font-size: 18px; + text-align: center; + } + .price-calc-section .card .description .select-number { + font-size: 17px; + text-align: center; + width: 60px; + } +} + +@media(max-width:575px) { + .percent-text { + font-weight: normal; + font-size: 37px; + } + .contact-section .card { + width: 90%; + } + .form-beta { + width: 90%; + padding: 25px 10px; + } + .intro-message>h1 { + font-size: 2em; + } + .price-calc-section .text .section-heading { + font-size: 24px; + line-height: 25px; + } + .price-calc-section .card .description span { + font-size: 15px; + } +} + +.network-name { + text-transform: uppercase; + font-size: 14px; + font-weight: 300; + letter-spacing: 2px; + line-height: 24px; + display: block; +} + +.section-heading { + margin-bottom: 30px; +} + +footer { + padding: 50px 20px; +} + +.topnav a:focus { + outline: none; + outline-offset: 0; +} + +.topnav .btn:focus { + outline: none !important; + outline-offset: 0; +} + +.flex-row-rev { + margin-top: 25px; +} + +.flex-row .percent-text { + display: flex; + align-items: center; +} + +@media (min-width: 768px) { + .flex-row { + display: flex; + align-items: center; + justify-content: space-between; + } + .flex-row .percent-text { + flex-shrink: 0; + padding: 0 15px; + } + .flex-row .desc-text { + text-align: right; + } + .flex-row .desc-text, + .flex-row .percent-text { + max-width: 430px; + } + .flex-row-rev .desc-text { + max-width: 600px; + text-align: left; + } + .flex-row-rev .percent-text { + order: 2; + } + .flex-row-rev { + margin-bottom: 25px; + } +} + +.checkmark { + display: inline-block; +} + +.checkmark:after { + /*Add another block-level blank space*/ + content: ''; + display: block; + /*Make it a small rectangle so the border will create an L-shape*/ + width: 25px; + height: 60px; + /*Add a white border on the bottom and left, creating that 'L' */ + border: solid #777; + border-width: 0 3px 3px 0; + /*Rotate the L 45 degrees to turn it into a checkmark*/ + transform: rotate(45deg); +} + + +/* new styles for whydcl section cms plugin (to replace older style) */ + +.banner-list { + border-top: 2px solid #eee; + padding: 50px 0; +} + +.banner-list-heading h2 { + font-size: 42px; +} + +@media (max-width: 767px) { + .banner-list-heading h2 { + font-size: 30px; + } +} + + +/* cms section promo */ + +.promo-section { + padding: 75px 15px; +} + +.promo-section.promo-with-bg { + color: #fff; + background-size: cover; + background-position: center; +} + +.promo-section.promo-with-bg a { + color: #87B6EA; +} + +.promo-section.promo-with-bg a:hover, +.promo-section.promo-with-bg a:focus { + color: #77a6da; +} + +.promo-section h3 { + font-weight: 700; + font-size: 36px; + text-transform: uppercase; + letter-spacing: 0.5px; + margin-top: 10px; + margin-bottom: 25px; +} + +.promo-section h4 { + font-size: 24px; + margin-bottom: 20px; +} + +.promo-section p { + font-size: 18px; + line-height: 1.5; +} + +.promo-section.text-center p { + max-width: 720px; + margin: auto; +} + +.promo-section.text-center h3, +.promo-section.text-center h4 { + margin-bottom: 35px; +} + +.split-text .split-subsection { + margin-top: 25px; + margin-bottom: 25px; +} + +.split-text .promo-section { + padding: 20px 15px; + margin-top: 30px; + margin-bottom: 30px; +} + +.split-text .promo-section .container { + width: auto; +} + +.split-text .promo-section h3, +.split-text .promo-section h4 { + margin-bottom: 15px; +} + +@media (max-width: 767px) { + .split-text .split-subsection { + margin-left: -15px; + margin-right: -15px; + } + .promo-section h3 { + font-size: 29px; + } + .split-text .promo-section { + padding-left: 0; + padding-right: 0; + } +} + +ul.errorlist { + padding-left: 0px; +} +ul.errorlist > li { + color: red; + list-style-type: none; +} +div.domain { + flex-direction: column; +} \ No newline at end of file diff --git a/matrixhosting/static/matrixhosting/css/hosting.css b/matrixhosting/static/matrixhosting/css/hosting.css new file mode 100644 index 0000000..b3e0bbb --- /dev/null +++ b/matrixhosting/static/matrixhosting/css/hosting.css @@ -0,0 +1,618 @@ +.navbar-transparent #logoWhite { + display: none; + } + + .navbar-transparent #logoBlack { + display: block; + width: 220px; + } + + .topnav .navbar-fixed-top .navbar-collapse { + max-height: 740px; + } + + .navbar-default .navbar-header { + position: relative; + z-index: 1; + } + + .navbar-right .highlights-dropdown .dropdown-menu { + left: 0 !important; + min-width: 155px; + margin-left: 15px; + padding: 0 5px 8px !important; + } + + @media(min-width: 768px) { + .navbar-default .navbar-nav>li a, + .navbar-right .highlights-dropdown .dropdown-menu>li a { + font-weight: 300; + } + .navbar-right .highlights-dropdown .dropdown-menu { + border-width: 0 0 1px 0; + border-color: #e7e7e7; + box-shadow: -8px 14px 20px -5px rgba(77, 77, 77, 0.5); + } + } + + .navbar-right .highlights-dropdown .dropdown-menu>li a { + font-size: 13px; + font-family: 'Lato', sans-serif; + padding: 1px 10px 1px 18px !important; + background: transparent; + color: #333; + } + + .navbar-right .highlights-dropdown .dropdown-menu>li a:hover, + .navbar-right .highlights-dropdown .dropdown-menu>li a:focus, + .navbar-right .highlights-dropdown .dropdown-menu>li a:active { + background: transparent; + text-decoration: underline !important; + } + + .un-icon { + width: 15px; + height: 15px; + opacity: 0.5; + margin-top: -1px; + } + + + /***** DCL payment page **********/ + + .dcl-order-container { + font-weight: 300; + } + + .dcl-place-order-text { + color: #808080; + } + + .card-warning-content { + font-weight: 300; + border: 1px solid #a1a1a1; + border-radius: 3px; + padding: 5px; + margin-bottom: 15px; + } + + .card-warning-error { + border: 1px solid #EB4D5C; + color: #EB4D5C; + } + + .card-warning-addtional-margin { + margin-top: 15px; + } + + .card-cvc-element label { + padding-left: 10px; + } + + .card-element { + margin-bottom: 10px; + } + + .card-element label { + width: 100%; + margin-bottom: 0px; + } + + .my-input { + border-bottom: 1px solid #ccc; + } + + .card-cvc-element .my-input { + padding-left: 10px; + } + + #card-errors { + clear: both; + padding: 0 0 10px; + color: #eb4d5c; + } + + .credit-card-goup { + padding: 0; + } + + @media (max-width: 767px) { + .card-expiry-element { + padding-right: 10px; + } + + .card-cvc-element { + padding-left: 10px; + } + + #billing-form .form-control { + box-shadow: none !important; + font-weight: 400; + } + } + + @media (min-width: 1200px) { + .dcl-order-container { + width: 990px; + padding: 0 15px; + margin: 0 auto; + } + } + + .footer-vm p.copyright { + margin-top: 4px; + } + + .navbar-default .navbar-nav>.open>a, + .navbar-default .navbar-nav>.open>a:focus, + .navbar-default .navbar-nav>.open>a:hover, + .navbar-default .navbar-nav>.active>a, + .navbar-default .navbar-nav>.active>a:focus, + .navbar-default .navbar-nav>.active>a:hover { + background-color: transparent; + } + + @media (max-width: 767px) { + .navbar-default .navbar-nav .open .dropdown-menu>.active a, + .navbar-default .navbar-nav .open .dropdown-menu>.active a:focus, + .navbar-default .navbar-nav .open .dropdown-menu>.active a:hover { + background-color: transparent; + } + } + + + + /* bootstrap input box-shadow disable */ + + .has-error .form-control:focus, + .has-error .form-control:active, + .has-success .form-control:focus, + .has-success .form-control:active { + box-shadow: inset 0 0 1px rgba(0, 0, 0, 0.25); + } + + .content-dashboard { + min-height: calc(100vh - 96px); + width: 100%; + margin: 0 auto; + max-width: 1120px; + } + + @media (max-width: 767px) { + .content-dashboard { + padding: 0 15px; + } + } + + @media (max-width: 575px) { + select { + width: 280px; + } + } + + .btn:focus, + .btn:active:focus { + outline: 0; + } + + + + + /***********Styles for Model********************/ + + .modal-content { + border-radius: 0px; + font-family: Lato, "Helvetica Neue", Helvetica, Arial, sans-serif; + width: 100%; + float: left; + border-radius: 0; + font-weight: 300; + } + + .modal-header { + min-height: 30px; + border-bottom: 0px solid #e5e5e5; + padding: 0px 15px; + width: 100%; + } + + .modal-header .close { + font-size: 75px; + font-weight: 300; + margin-top: 0; + position: absolute; + top: 0; + right: 11px; + z-index: 10; + line-height: 60px; + } + + .modal-header .close span { + display: block; + } + + .modal-header .close:focus { + outline: 0; + } + + .modal-body { + text-align: center; + width: 100%; + float: left; + padding: 0px 30px 15px 30px; + } + + .modal-body .modal-icon i { + font-size: 80px; + font-weight: 100; + color: #999; + } + + .modal-body .modal-icon { + margin-bottom: 15px; + } + + .modal-title { + margin: 0; + line-height: 1.42857143; + font-size: 25px; + padding: 0; + font-weight: 300; + } + + .modal-text { + padding-top: 5px; + font-size: 16px; + } + + .modal-text p:not(:last-of-type) { + margin-bottom: 5px; + } + + .modal-title+.modal-footer { + margin-top: 5px; + } + + .modal-footer { + border-top: 0px solid #e5e5e5; + width: 100%; + float: left; + text-align: center; + padding: 15px 15px; + } + + .modal { + text-align: center; + } + + .modal-dialog { + display: inline-block; + text-align: left; + vertical-align: middle; + width: 40%; + margin: 15px auto; + } + + @media (min-width: 768px) and (max-width: 991px) { + .modal-dialog { + width: 50%; + } + } + + @media (max-width: 767px) { + .modal-dialog { + width: 95%; + } + } + + @media(min-width: 576px) { + .modal:before { + content: ''; + display: inline-block; + height: 100%; + vertical-align: middle; + margin-right: -4px; + } + } + + + + /* ========= */ + + .btn-wide { + min-width: 100px; + } + + .choice-btn { + min-width: 110px; + background-color: #3C5480; + color: #fff; + border: 2px solid #3C5480; + padding: 4px 10px; + transition: 0.3s all ease-out; + } + + .choice-btn:focus, + .choice-btn:hover, + .choice-btn:active { + color: #3C5480; + background-color: #fff; + } + + @media (max-width: 767px) { + .choice-btn { + margin-top: 15px; + } + } + + .payment-container { + padding-top: 70px; + padding-bottom: 11%; + } + + .last-p { + margin-bottom: 0; + } + + .dcl-payment-section { + max-width: 391px; + margin: 0 auto 30px; + padding: 0 10px 30px; + border-bottom: 1px solid #edebeb; + height: 100%; + } + + .dcl-payment-section hr { + margin-top: 15px; + margin-bottom: 15px; + } + + .dcl-payment-section .top-hr { + margin-left: -10px; + } + + .dcl-payment-section h3 { + font-weight: 600; + } + + .dcl-payment-section p { + font-weight: 400; + } + + .dcl-payment-section .card-warning-content { + padding: 8px 10px; + font-weight: 300; + } + + .dcl-payment-order strong { + font-size: 17px; + } + + .dcl-payment-order p { + font-weight: 300; + } + + .dcl-payment-section .form-group { + margin-bottom: 10px; + } + + .dcl-payment-section .form-control { + box-shadow: none; + padding: 6px 12px; + height: 32px; + } + + .dcl-payment-user { + height: 100%; + display: flex; + flex-direction: column; + justify-content: center; + } + + .dcl-payment-user h4 { + font-weight: 600; + font-size: 17px; + } + + @media (min-width: 768px) { + .dcl-payment-grid { + display: flex; + align-items: stretch; + flex-wrap: wrap; + } + .dcl-payment-box { + width: 50%; + position: relative; + padding: 0 30px; + } + .dcl-payment-box:nth-child(2) { + order: 1; + } + .dcl-payment-box:nth-child(4) { + order: 2; + } + .dcl-payment-section { + padding-top: 15px; + padding-bottom: 15px; + margin-bottom: 0; + border-bottom-width: 5px; + } + .dcl-payment-box:nth-child(2n) .dcl-payment-section { + border-bottom: none; + } + .dcl-payment-box:nth-child(1):after, + .dcl-payment-box:nth-child(2):after { + content: ' '; + display: block; + background: #eee; + width: 1px; + position: absolute; + right: 0; + z-index: 2; + top: 20px; + bottom: 20px; + } + } + + #virtual_machine_create_form { + padding: 15px 0; + } + + .btn-vm-contact { + color: #fff; + background: #A3C0E2; + border: 2px solid #A3C0E2; + padding: 5px 25px; + font-size: 12px; + letter-spacing: 1.3px; + } + + .btn-vm-contact:hover, + .btn-vm-contact:focus { + background: #fff; + color: #a3c0e2; + } + + + + /* hosting-order */ + + .order-detail-container { + max-width: 600px; + margin: 100px auto 40px; + border: 1px solid #ccc; + padding: 30px 30px 20px; + color: #595959; + } + + .order-detail-container .dashboard-title-thin { + margin-top: 0; + margin-left: -3px; + } + + .order-detail-container .dashboard-title-thin .un-icon { + margin-top: -6px; + } + + .order-detail-container .dashboard-container-head { + position: relative; + padding: 0; + margin-bottom: 38px; + } + + .order-detail-container .order-details { + margin-bottom: 15px; + } + + .order-detail-container h4 { + font-size: 16px; + font-weight: bold; + margin-bottom: 10px; + } + + .order-detail-container p { + margin-bottom: 5px; + } + + .order-detail-container hr { + margin: 15px 0; + } + + .order-detail-container .thin-hr { + margin: 10px 0; + } + + .order-detail-container .subtotal-price { + font-size: 16px; + } + + .order-detail-container .subtotal-price .text-primary { + font-size: 17px; + } + + .order-detail-container .total-price { + font-size: 18px; + line-height: 20px; + } + + @media (max-width: 767px) { + .order-detail-container { + padding: 15px; + } + .order-confirm-btn { + text-align: center; + margin-top: 10px; + } + .order-detail-container .dashboard-container-options { + position: absolute; + top: 4px; + right: -4px; + } + .order-detail-container .dashboard-container-options .svg-img { + height: 16px; + width: 16px; + } + } + + .order_detail_footer { + font-size: 9px; + letter-spacing: 1px; + color: #333333; + } + + .order_detail_footer strong { + font-size: 11px; + } + + .order_detail_footer small { + font-size: 8px; + } + + .dashboard-title-thin { + font-weight: 300; + font-size: 32px; + } + + .dashboard-title-thin .un-icon { + height: 34px; + margin-right: 5px; + margin-top: -2px; + width: 34px; + vertical-align: middle; + } + + @media (max-width:767px) { + .dashboard-title-thin { + font-size: 22px; + } + .dashboard-title-thin .un-icon { + height: 22px; + width: 22px; + margin-top: -3px; + } + } + + .locale_date { + opacity: 0; + } + + .locale_date.done { + opacity: 1; + } + + .btn-vm-back { + color: #fff; + background: #C4CEDA; + border: 2px solid #C4CEDA; + padding: 5px 25px; + font-size: 12px; + letter-spacing: 1.3px; + } + + .btn-vm-back:hover, + .btn-vm-back:focus { + color: #fff; + background: #8da4c0; + border-color: #8da4c0; + } + \ No newline at end of file diff --git a/matrixhosting/static/matrixhosting/js/main.js b/matrixhosting/static/matrixhosting/js/main.js new file mode 100644 index 0000000..52ae598 --- /dev/null +++ b/matrixhosting/static/matrixhosting/js/main.js @@ -0,0 +1,46 @@ +(function($) { + "use strict"; // Start of use strict + + $(document).ready(function() { + function fetch_pricing() { + var url = '/matrix/pricing/' + $('#pricing_name').val() + '/calculate/'; + var cores = $('#cores').val(); + var memory = $('#memory').val(); + var storage = $('#storage').val(); + $.ajax({ + type: 'GET', + url: url, + data: { cores: cores, memory: memory, storage: storage}, + dataType: 'json', + success: function (data) { + if (data && data['price']) { + $('#total').text(data['price']); + } + } + }); + }; + + function incrementValue(e) { + var valueElement = $(e.target).parent().parent().find('input'); + var step = $(valueElement).attr('step'); + var min = parseInt($(valueElement).attr('min')); + var max = parseInt($(valueElement).attr('max')); + var new_value = 0; + if (e.data.inc == 1) { + new_value = Math.min(parseInt($(valueElement).val()) + parseInt(step) * e.data.inc, max); + } else { + new_value = Math.max(parseInt($(valueElement).val()) + parseInt(step) * e.data.inc, min); + } + $(valueElement).val(new_value); + fetch_pricing(); + return false; + }; + if ($('#pricing_name') != undefined) { + fetch_pricing(); + } + + $('.fa-plus-circle.right').bind('click', {inc: 1}, incrementValue); + + $('.fa-minus-circle.left').bind('click', {inc: -1}, incrementValue); + }); +})(jQuery); diff --git a/matrixhosting/static/matrixhosting/js/order.js b/matrixhosting/static/matrixhosting/js/order.js new file mode 100644 index 0000000..af733bf --- /dev/null +++ b/matrixhosting/static/matrixhosting/js/order.js @@ -0,0 +1,36 @@ +$( document ).ready(function() { + var create_vm_form = $('#virtual_machine_create_form'); + create_vm_form.submit(placeOrderPayment); + function placeOrderPayment(e) { + e.preventDefault(); + $.ajax({ + url: create_vm_form.attr('action'), + type: 'POST', + data: create_vm_form.serialize(), + init: function () { + ok_btn = $('#createvm-modal-done-btn'); + close_btn = $('#createvm-modal-close-btn'); + ok_btn.addClass('btn btn-success btn-ok btn-wide hide'); + close_btn.addClass('btn btn-danger btn-ok btn-wide hide'); + }, + success: function (data) { + fa_icon = $('.modal-icon').find('.fa-cog'); + modal_btn = $('#createvm-modal-done-btn'); + if (data.error) { + // Display error.message in your UI. + modal_btn.attr('href', error_url).removeClass('visually-hidden'); + fa_icon.attr('class', 'fa fa-close'); + modal_btn.attr('class', '').addClass('btn btn-danger btn-ok btn-wide'); + $('#createvm-modal-title').text("Error Occurred"); + $('#createvm-modal-body').html(data.error.message); + } else { + // The payment has succeeded + // Display a success message + modal_btn.attr('href', data.redirect).removeClass('visually-hidden'); + $('#createvm-modal-title').text("Order Succeeded"); + $('#createvm-modal-body').html("Order has been added and the instance will be ready soon"); + } + } + }); + } +}); \ No newline at end of file diff --git a/matrixhosting/static/matrixhosting/js/payment.js b/matrixhosting/static/matrixhosting/js/payment.js new file mode 100644 index 0000000..d009b78 --- /dev/null +++ b/matrixhosting/static/matrixhosting/js/payment.js @@ -0,0 +1,204 @@ +var cardBrandToPfClass = { + 'visa': 'pf-visa', + 'mastercard': 'pf-mastercard', + 'amex': 'pf-american-express', + 'discover': 'pf-discover', + 'diners': 'pf-diners', + 'jcb': 'pf-jcb', + 'unknown': 'pf-credit-card' +}; +function setBrandIcon(brand) { + var brandIconElement = document.getElementById('brand-icon'); + var pfClass = 'pf-credit-card'; + if (brand in cardBrandToPfClass) { + pfClass = cardBrandToPfClass[brand]; + } + for (var i = brandIconElement.classList.length - 1; i >= 0; i--) { + brandIconElement.classList.remove(brandIconElement.classList[i]); + } + brandIconElement.classList.add('pf'); + brandIconElement.classList.add(pfClass); +} + + +$(document).ready(function () { + $.ajaxSetup({ + beforeSend: function (xhr, settings) { + function getCookie(name) { + var cookieValue = null; + if (document.cookie && document.cookie != '') { + var cookies = document.cookie.split(';'); + for (var i = 0; i < cookies.length; i++) { + var cookie = jQuery.trim(cookies[i]); + // Does this cookie string begin with the name we want? + if (cookie.substring(0, name.length + 1) == (name + '=')) { + cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); + break; + } + } + } + return cookieValue; + } + + if (!(/^http:.*/.test(settings.url) || /^https:.*/.test(settings.url))) { + // Only send the token to relative URLs i.e. locally. + xhr.setRequestHeader("X-CSRFToken", getCookie('csrftoken')); + } + } + }); + + + var hasCreditcard = window.hasCreditcard || false; + if (!hasCreditcard && window.stripeKey) { + var stripe = Stripe(window.stripeKey); + if (window.pm_id) { + + } else { + var element_style = { + fonts: [{ + family: 'lato-light', + src: 'url(https://cdn.jsdelivr.net/font-lato/2.0/Lato/Lato-Light.woff) format("woff2")' + }, { + family: 'lato-regular', + src: 'url(https://cdn.jsdelivr.net/font-lato/2.0/Lato/Lato-Regular.woff) format("woff2")' + } + ], + locale: window.current_lan + }; + var elements = stripe.elements(element_style); + var credit_card_text_style = { + base: { + iconColor: '#666EE8', + color: '#31325F', + lineHeight: '25px', + fontWeight: 300, + fontFamily: "'lato-light', sans-serif", + fontSize: '14px', + '::placeholder': { + color: '#777' + } + }, + invalid: { + iconColor: '#eb4d5c', + color: '#eb4d5c', + lineHeight: '25px', + fontWeight: 300, + fontFamily: "'lato-regular', sans-serif", + fontSize: '14px', + '::placeholder': { + color: '#eb4d5c', + fontWeight: 400 + } + } + }; + + var enter_ccard_text = "Enter your credit card number"; + if (typeof window.enter_your_card_text !== 'undefined') { + enter_ccard_text = window.enter_your_card_text; + } + var cardNumberElement = elements.create('cardNumber', { + style: credit_card_text_style, + placeholder: enter_ccard_text + }); + cardNumberElement.mount('#card-number-element'); + + var cardExpiryElement = elements.create('cardExpiry', { + style: credit_card_text_style + }); + cardExpiryElement.mount('#card-expiry-element'); + + var cardCvcElement = elements.create('cardCvc', { + style: credit_card_text_style + }); + cardCvcElement.mount('#card-cvc-element'); + cardNumberElement.on('change', function (event) { + if (event.brand) { + setBrandIcon(event.brand); + } + }); + } + } + + function submitBillingForm(pmId) { + var billing_form = $('#billing-form'); + billing_form.append(''); + billing_form.submit(); + } + + var $form_new = $('#payment-form-new'); + $form_new.submit(payWithPaymentIntent); + window.result = ""; + window.card = ""; + function payWithPaymentIntent(e) { + e.preventDefault(); + function stripePMHandler(paymentMethod) { + // Insert the token ID into the form so it gets submitted to the server + console.log(paymentMethod); + $('#id_payment_method').val(paymentMethod.id); + submitBillingForm(paymentMethod.id); + } + stripe.createPaymentMethod({ + type: 'card', + card: cardNumberElement, + }) + .then(function(result) { + // Handle result.error or result.paymentMethod + window.result = result; + if(result.error) { + var errorElement = document.getElementById('card-errors'); + errorElement.textContent = result.error.message; + } else { + console.log("created paymentMethod " + result.paymentMethod.id); + stripePMHandler(result.paymentMethod); + } + }); + window.card = cardNumberElement; + } + + /* Form validation */ + $.validator.addMethod("month", function (value, element) { + return this.optional(element) || /^(01|02|03|04|05|06|07|08|09|10|11|12)$/.test(value); + }, "Please specify a valid 2-digit month."); + + $.validator.addMethod("year", function (value, element) { + return this.optional(element) || /^[0-9]{2}$/.test(value); + }, "Please specify a valid 2-digit year."); + + validator = $form_new.validate({ + rules: { + cardNumber: { + required: true, + creditcard: true, + digits: true + }, + expMonth: { + required: true, + month: true + }, + expYear: { + required: true, + year: true + }, + cvCode: { + required: true, + digits: true + } + }, + highlight: function (element) { + $(element).closest('.form-control').removeClass('success').addClass('error'); + }, + unhighlight: function (element) { + $(element).closest('.form-control').removeClass('error').addClass('success'); + }, + errorPlacement: function (error, element) { + $(element).closest('.form-group').append(error); + } + }); + + $('.credit-card-info .btn.choice-btn').click(function () { + var id = this.dataset['id_card']; + $('#id_card').val(id); + submitBillingForm(id); + }); + +}); diff --git a/matrixhosting/tasks.py b/matrixhosting/tasks.py new file mode 100644 index 0000000..c681e8c --- /dev/null +++ b/matrixhosting/tasks.py @@ -0,0 +1,64 @@ +import logging +from datetime import date, timedelta, timezone +from django.conf import settings +from django.template.loader import render_to_string +from django_q.tasks import async_task, schedule +from django_q.models import Schedule +from django.db.models import Q +from uncloud_pay.models import Bill, Payment +from uncloud_pay.selectors import has_enough_balance, get_balance_for_user +from .models import VMInstance + +log = logging.getLogger(__name__) + +def send_warning_email(bill, html_message): + schedule('django.core.mail.send_mail', + 'Renewal Warning', + None, + settings.RENEWAL_FROM_EMAIL, + [bill.owner.email], + html_message, + schedule_type=Schedule.ONCE, + next_run=timezone.now() + timedelta(hours=1)) + +def charge_open_bills(): + un_paid_bills = Bill.objects.filter(is_closed=False) + for bill in un_paid_bills: + date_diff = (date.today() - bill.due_date.date()).days + # If there is not enough money in the account 7 days before renewal, the system sends a warning + # If there is not enough money in the account 3 days before renewal, the system sends a 2nd warning + # If on renewal date there is not enough money in the account, delete the instance + if date_diff == 7: + if not has_enough_balance(bill.owner): + context = {'name': bill.owner.name, 'message': "You don't have enough balance for renewal... upload to your account _here"} + html_message = render_to_string('matrixhosting/emails/renewal_warning.html', context) + send_warning_email(bill, html_message) + elif date_diff == 3: + if not has_enough_balance(bill.owner): + context = {'name': bill.owner.name, 'message': "You don't have enough balance for renewal... Your instance will be deleted in 3 days"} + html_message = render_to_string('matrixhosting/emails/renewal_warning.html', context) + send_warning_email(bill, html_message) + elif date_diff <= 0: + if not has_enough_balance(bill.owner): + VMInstance.delete_for_bill(bill) + else: + try: + balance = get_balance_for_user(bill.owner) + if balance < 0: + payment = Payment.objects.create(owner=bill.owner, amount=balance, source='stripe') + if payment: + bill.close() + bill.close() + except Exception as e: + log.error(f"It seems that there is issue in payment for {bill.owner.name}", e) + # do nothing + + +def process_recurring_orders(): + """ + Check for pending recurring and charge it and generate bills or send the customer warning + """ + Bill.create_bills_for_all_users() + +def delete_instance(instance_id): + VMInstance.objects.delete(instance_id) \ No newline at end of file diff --git a/matrixhosting/templates/matrixhosting/base.html b/matrixhosting/templates/matrixhosting/base.html new file mode 100644 index 0000000..3abc7a2 --- /dev/null +++ b/matrixhosting/templates/matrixhosting/base.html @@ -0,0 +1,60 @@ +{% load static i18n %} +{% get_current_language as LANGUAGE_CODE %} +{% load bootstrap5 %} + + + + + + + + + + + Matrix Hosting - {% block title %} made in Switzerland{% endblock %} + + + + {% bootstrap_css %} + + + + + {% block css_extra %} + {% endblock css_extra %} + + + + + + + + + + + + + + {% block navbar %} + {% include "matrixhosting/includes/_navbar.html" %} + {% endblock navbar %} + + {% block content %} + {% endblock %} + + {% include "matrixhosting/includes/_footer.html" %} + + + + + + {% bootstrap_javascript %} + + + {% block js_extra %} + {% endblock js_extra %} + + diff --git a/matrixhosting/templates/matrixhosting/dashboard.html b/matrixhosting/templates/matrixhosting/dashboard.html new file mode 100644 index 0000000..d183665 --- /dev/null +++ b/matrixhosting/templates/matrixhosting/dashboard.html @@ -0,0 +1,127 @@ +{% extends "matrixhosting/base.html" %} {% load static i18n %} +{% block content%} + +{% csrf_token %} +
+
+
+
+ + + + + + + + + + + + + + + + {% for object in object_list %} + + + + + + + + + + {% if object.ending_date %} + + {% else %} + + {% endif %} + + {% endfor %} + +
#DescriptionStarting AtConfigPricing PlanOneTime PriceRecurring PriceEnding At
{{ object.id }}{{ object.description }}{{ object.starting_date }}{{ object.config }}{{ object.pricing_plan}}{{ object.one_time_price }}{{ object.recurring_price }}{{ object.ending_date }} + +
+
+
+
+
+ + + + + +{% endblock %} + +{% block js_extra %} + +{% endblock %} diff --git a/matrixhosting/templates/matrixhosting/emails/renewal_warning.html b/matrixhosting/templates/matrixhosting/emails/renewal_warning.html new file mode 100644 index 0000000..75a4782 --- /dev/null +++ b/matrixhosting/templates/matrixhosting/emails/renewal_warning.html @@ -0,0 +1,13 @@ + + + + + + + Renewal Warning + + + hello {{name}}, + {{message}} + + \ No newline at end of file diff --git a/matrixhosting/templates/matrixhosting/includes/_calculator_form.html b/matrixhosting/templates/matrixhosting/includes/_calculator_form.html new file mode 100644 index 0000000..fb4ce57 --- /dev/null +++ b/matrixhosting/templates/matrixhosting/includes/_calculator_form.html @@ -0,0 +1,101 @@ +{% load static i18n %} + +
+ {% csrf_token %} +
+

{% trans "Matrix Chat hosting" %}

+
+
+ {{ matrix_vm_pricing.name }} + CHF/{% trans "month" %} +
+

+ {% if matrix_vm_pricing.set_up_fees %}{{ matrix_vm_pricing.set_up_fees }} CHF Setup
{% endif %} + {% if matrix_vm_pricing.vat_inclusive %}{% trans "VAT included" %}
{% endif %} + {% if matrix_vm_pricing.discount_amount %} + {% trans "You save" %} {{ matrix_vm_pricing.discount_amount }} CHF + {% endif %} +

+
+
+
+
+

{% trans "Hosted in Switzerland" %}

+
+
+
+ + + Core + +
+
+ {% for message in messages %} + {% if 'cores' in message.tags %} +
    +
  • {{ message|safe }}
    • +
+ {% endif %} + {% endfor %} +
+
+
+
+ + + GB RAM + +
+
+ {% for message in messages %} + {% if 'memory' in message.tags %} +
  • + {{ message|safe }} +
+ {% endif %} + {% endfor %} +
+
+
+
+ + + {% trans "GB Storage (SSD)" %} + +
+
+ {% for message in messages %} + {% if 'storage' in message.tags %} +
  • + {{ message|safe }} +
+ {% endif %} + {% endfor %} +
+
+
+ +

{{ form.matrix_domain.errors }}

+
+
+ +

{{ form.homeserver_domain.errors }}

+
+
+ +

{{ form.webclient_domain.errors }}

+
+
+
+ Is open registration possible: + {{ form.is_open_registration }} +
+
+ +
+ + +
\ No newline at end of file diff --git a/matrixhosting/templates/matrixhosting/includes/_card.html b/matrixhosting/templates/matrixhosting/includes/_card.html new file mode 100644 index 0000000..ec30a7d --- /dev/null +++ b/matrixhosting/templates/matrixhosting/includes/_card.html @@ -0,0 +1,43 @@ +{% load i18n %} +
+ + +
+
+
+ +
+
+
+
+ +
+
+
+ +
+
+
+
+ + +
+
+
+
+
+ {% for message in messages %} + {% if 'failed_payment' in message.tags or 'make_charge_error' in message.tags or 'error' in message.tags %} +
    +

  • {{ message|safe }}

    • +
+ {% endif %} + {% endfor %} +
+
+ +
+
+

+
+
\ No newline at end of file diff --git a/matrixhosting/templates/matrixhosting/includes/_footer.html b/matrixhosting/templates/matrixhosting/includes/_footer.html new file mode 100644 index 0000000..07d9fb6 --- /dev/null +++ b/matrixhosting/templates/matrixhosting/includes/_footer.html @@ -0,0 +1,18 @@ +{% load i18n %} + diff --git a/matrixhosting/templates/matrixhosting/includes/_navbar.html b/matrixhosting/templates/matrixhosting/includes/_navbar.html new file mode 100644 index 0000000..599c6b9 --- /dev/null +++ b/matrixhosting/templates/matrixhosting/includes/_navbar.html @@ -0,0 +1,33 @@ +{% load static i18n %} +{% get_current_language as LANGUAGE_CODE %} + \ No newline at end of file diff --git a/matrixhosting/templates/matrixhosting/index.html b/matrixhosting/templates/matrixhosting/index.html new file mode 100644 index 0000000..ab9085c --- /dev/null +++ b/matrixhosting/templates/matrixhosting/index.html @@ -0,0 +1,21 @@ +{% extends "matrixhosting/base.html" %} +{% load static i18n %} + +{% block content %} + + +
+
+
+
+
+
+ {% include "matrixhosting/includes/_calculator_form.html" %} +
+
+
+
+
+
+ +{% endblock %} diff --git a/matrixhosting/templates/matrixhosting/order_detail.html b/matrixhosting/templates/matrixhosting/order_detail.html new file mode 100644 index 0000000..b6942e3 --- /dev/null +++ b/matrixhosting/templates/matrixhosting/order_detail.html @@ -0,0 +1,268 @@ +{% load static i18n %} +{% load bootstrap5 %} + + + + + + + + + + Matrix Hosting - {% block title %} made in Switzerland{% endblock %} + + + + {% bootstrap_css %} + + + + + + + + + + + + + + +
+ {% if messages %} +
+ {% for message in messages %} + {{ message }} + {% endfor %} +
+ {% endif %} + {% if not error %} +
+

+ {% blocktrans with page_header_text=page_header_text|default:"Order" %}{{page_header_text}}{% endblocktrans %} +

+
+
+
+
+
+

{% trans "Billed to" %}:

+

+ {% with request.session.billing_address_data as billing_address %} + {{billing_address.full_name}}
+ {{billing_address.street}}, {{billing_address.postal_code}}
+ {{billing_address.city}}, {{billing_address.country}} + {% if billing_address.vat_number %} +
{% trans "VAT Number" %} {{billing_address.vat_number}} + {% if pricing.vat_country != "ch" and pricing.vat_validation_status != "not_needed" %} + {% if pricing.vat_validation_status == "verified" %} + + {% else %} + + {% endif %} + {% endif %} + {% endif %} + {% endwith %} +

+
+
+
+
+

{% trans "Payment method" %}:

+

+ {{card.brand|default:_('Credit Card')}} {% trans "ending in" %} ****{{card.last4}}
+ {% trans "Expiry" %} {{card.exp_year}}/{{card.exp_month}}
+ {{request.user.email}} +

+
+
+
+

{% trans "Order summary" %}

+ +

+ {% trans "Product" %}:  + Matrix Chat Hosting +

+
+
+

+ {% trans "Cores" %}: + {{order.cores}} +

+

+ {% trans "Memory" %}: + {{order.memory}} GB +

+

+ {% trans "Disk space" %}: + {{order.storage}} GB +

+
+
+
+
+
+

+ {% trans "Price Before VAT" %} + {{pricing.subtotal|floatformat:2}} CHF +

+
+
+
+
+
+
+
+

+
+
+

{% trans "Pre VAT" %}

+
+
+

{% trans "With VAT for" %} {{pricing.vat_country}} ({{pricing.vat_percent}}%)

+
+
+
+
+

Subtotal

+
+
+

{{pricing.subtotal|floatformat:2}} CHF

+
+
+

{{pricing.price_with_vat|floatformat:2}} CHF

+
+
+ {% if pricing.discount.amount > 0 %} +
+
+

{{pricing.discount.name}}

+
+
+

-{{pricing.discount.amount|floatformat:2}} CHF

+
+
+

-{{pricing.discount.amount_with_vat|floatformat:2}} CHF

+
+
+ {% endif %} +
+
+
+
+
+
+
+

Total

+
+
+

{{pricing.subtotal_after_discount|floatformat:2}} CHF

+
+
+

{{pricing.price_after_discount_with_vat|floatformat:2}} CHF

+
+
+
+
+
+
+
+ {% trans "Your Price in Total" %} + {{pricing.total_price|floatformat:2}} CHF +
+
+
+
+
+
+ {% csrf_token %} +
+
+
{% blocktrans with vm_total_price=vm.total_price|floatformat:2 %}By clicking "Place order" you agree to our Terms of Service and this plan will charge your credit card account with {{ vm_total_price }} CHF/month{% endblocktrans %}.
+
+
+ +
+
+
+ {% endif %} +
+ + + + + + + + + + + + + + + {% bootstrap_javascript %} + + + + diff --git a/matrixhosting/templates/matrixhosting/payment.html b/matrixhosting/templates/matrixhosting/payment.html new file mode 100644 index 0000000..e296d0c --- /dev/null +++ b/matrixhosting/templates/matrixhosting/payment.html @@ -0,0 +1,169 @@ +{% load static i18n %} +{% load bootstrap5 %} + + + + + + + + + + + Matrix Hosting - {% block title %} made in Switzerland{% endblock %} + + + + {% bootstrap_css %} + + + + + + + + + + + + + +
+
+
+
+
+

{%trans "Your Order" %}

+
+
+

{% trans "Cores"%} {{request.session.order.cores|floatformat}}

+
+

{% trans "Memory"%} {{request.session.order.memory|floatformat}} GB

+
+

{% trans "Disk space"%} {{request.session.order.storage|floatformat}} GB

+
+

+ {%trans "Total" %}   + + ({% if matrix_vm_pricing.vat_inclusive %}{%trans "including VAT" %}{% else %}{%trans "excluding VAT" %}{% endif %}) + + {{request.session.order.subtotal|floatformat}} CHF / {% trans "Month" %} +

+
+ {% if matrix_vm_pricing.discount_amount %} +

+ {{ request.session.order.discount.name }}   + - {{ request.session.order.discount.amount }} CHF / {% trans "Month" %} +

+ {% endif %} +
+
+
+
+

{%trans "Billing Address"%}

+
+ {% for message in messages %} + {% if 'vat_error' in message.tags %} +
  • + {{ message|safe }} +
+ {% endif %} + {% endfor %} +
+ {% csrf_token %} + {% for field in billing_address_form %} + {% if field.html_name in 'active,owner' %} + {{ field.as_hidden }} + {%else %} + {% bootstrap_field field show_label=False type='fields'%} + {% endif %} + {% endfor %} +
+
+
+
+
+
+
+ {% with cards_len=cards|length %} +

{%trans "Credit Card"%}

+
+

+ {% if cards_len > 0 %} + {% blocktrans %}Please select one of the cards that you used before or fill in your credit card information below. We are using Stripe for payment and do not store your information in our database.{% endblocktrans %} + {% else %} + {% blocktrans %}Please fill in your credit card information below. We are using Stripe for payment and do not store your information in our database.{% endblocktrans %} + {% endif %} +

+
+ {% for card in cards %} +
+
+
{% trans "Credit Card" %}
+
{% trans "Last" %} 4: ***** {{card.last4}}
+
{% trans "Type" %}: {{card.brand}}
+
{% trans "Expiry" %}: {{card.month}}/{{card.year}}
+
+
+ {% trans "SELECT" %} +
+
+ {% endfor %} + {% if cards_len > 0 %} +
+
+
+

{% trans "Add a new credit card" %}

+
+
+ +
+
+
+
+
+
+

{%trans "New Credit Card" %}

+
+ {% include "matrixhosting/includes/_card.html" %} +
+
+ {% else%} + {% include "matrixhosting/includes/_card.html" %} + {% endif %} +
+ {% endwith %} +
+
+
+
+ {% if stripe_key %} + {% get_current_language as LANGUAGE_CODE %} + + {%endif%} + + + + + + + + + {% bootstrap_javascript %} + + + + diff --git a/matrixhosting/tests.py b/matrixhosting/tests.py new file mode 100644 index 0000000..27468b3 --- /dev/null +++ b/matrixhosting/tests.py @@ -0,0 +1,67 @@ +import datetime +import json + +from django.test import TestCase +from django.contrib.auth import get_user_model +from django.utils import timezone + +from .models import VMInstance +from uncloud_pay.models import Order, PricingPlan, BillingAddress, Product, RecurringPeriod + + +vm_product_config = { + 'features': { + 'cores': + { 'min': 1, + 'max': 48 + }, + 'ram_gb': + { 'min': 2, + 'max': 200 + }, + }, +} + +class VMInstanceTestCase(TestCase): + + def setUp(self): + RecurringPeriod.populate_db_defaults() + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + self.config = json.dumps({ + 'cores': 1, + 'memory': 2, + 'storage': 100, + 'homeserver_domain': '', + 'webclient_domain': '', + 'matrix_domain': '', + }) + self.pricing_plan = PricingPlan.objects.create(name="PricingSample", set_up_fees=35, cores_unit_price=3, + ram_unit_price=4, storage_unit_price=0.02) + self.ba = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="somewhere else", + active=True) + + self.product = Product.objects.create(name="Testproduct", + description="Only for testing", + config=vm_product_config) + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + self.product.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + def test_create_matrix_vm(self): + order = Order.objects.create(owner=self.user, + recurring_period=self.default_recurring_period, + billing_address=self.ba, + pricing_plan = self.pricing_plan, + product=self.product, + config=self.config) + instances = VMInstance.objects.filter(order=order) + self.assertEqual(len(instances), 1) + + diff --git a/matrixhosting/urls.py b/matrixhosting/urls.py new file mode 100644 index 0000000..318ef8b --- /dev/null +++ b/matrixhosting/urls.py @@ -0,0 +1,15 @@ +from django.urls import path, include +from django.conf import settings +from django.conf.urls.static import static + +from .views import IndexView, PricingView, OrderPaymentView, OrderDetailsView, Dashboard + +app_name = 'matrixhosting' + +urlpatterns = [ + path('pricing//calculate/', PricingView.as_view(), name='pricing_calculator'), + path('payment/', OrderPaymentView.as_view(), name='payment'), + path('order/details/', OrderDetailsView.as_view(), name='order_details'), + path('dashboard/', Dashboard.as_view(), name='dashboard'), + path('', IndexView.as_view(), name='index'), +] diff --git a/matrixhosting/validators.py b/matrixhosting/validators.py new file mode 100644 index 0000000..08cc818 --- /dev/null +++ b/matrixhosting/validators.py @@ -0,0 +1,34 @@ +from django.core.validators import RegexValidator + + +def _validator(): + + ul = '\u00a1-\uffff' # unicode letters range (must not be a raw string) + + # IP patterns + ipv4_re = r'(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)(?:\.(?:25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}' + ipv6_re = r'\[[0-9a-f:\.]+\]' # (simple regex, validated later) + + # Host patterns + hostname_re = r'[a-z' + ul + \ + r'0-9](?:[a-z' + ul + r'0-9-]{0,61}[a-z' + ul + r'0-9])?' + # Max length for domain name labels is 63 characters per RFC 1034 sec. 3.1 + domain_re = r'(?:\.(?!-)[a-z' + ul + r'0-9-]{1,63}(? + + + LogEntry + + + + + id + + AutoField + + + + + + content_type + + ForeignKey (id) + + + + + + user + + ForeignKey (id) + + + + + + action_flag + + PositiveSmallIntegerField + + + + + + action_time + + DateTimeField + + + + + + change_message + + TextField + + + + + + object_id + + TextField + + + + + + object_repr + + CharField + + + + + >] + + + + + django_contrib_auth_models_Permission [label=< + + + + + + + + + + + + + + + + + + + +
+ + Permission +
+ id + + AutoField +
+ content_type + + ForeignKey (id) +
+ codename + + CharField +
+ name + + CharField +
+ >] + + django_contrib_auth_models_Group [label=< + + + + + + + + + + + +
+ + Group +
+ id + + AutoField +
+ name + + CharField +
+ >] + + + + + django_contrib_contenttypes_models_ContentType [label=< + + + + + + + + + + + + + + + +
+ + ContentType +
+ id + + AutoField +
+ app_label + + CharField +
+ model + + CharField +
+ >] + + + + + django_contrib_sessions_base_session_AbstractBaseSession [label=< + + + + + + + + + + + +
+ + AbstractBaseSession +
+ expire_date + + DateTimeField +
+ session_data + + TextField +
+ >] + + django_contrib_sessions_models_Session [label=< + + + + + + + + + + + + + + + +
+ + Session
<AbstractBaseSession> +
+ session_key + + CharField +
+ expire_date + + DateTimeField +
+ session_data + + TextField +
+ >] + + + + + uncloud_pay_models_StripeCustomer [label=< + + + + + + + + + + + +
+ + StripeCustomer +
+ owner + + OneToOneField (id) +
+ stripe_id + + CharField +
+ >] + + uncloud_pay_models_Payment [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + Payment +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ amount + + DecimalField +
+ source + + CharField +
+ timestamp + + DateTimeField +
+ >] + + uncloud_pay_models_PaymentMethod [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + PaymentMethod +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ description + + TextField +
+ primary + + BooleanField +
+ source + + CharField +
+ stripe_card_id + + CharField +
+ >] + + uncloud_pay_models_Bill [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Bill +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ creation_date + + DateTimeField +
+ due_date + + DateField +
+ ending_date + + DateTimeField +
+ starting_date + + DateTimeField +
+ valid + + BooleanField +
+ >] + + uncloud_pay_models_Order [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Order +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ creation_date + + DateTimeField +
+ ending_date + + DateTimeField +
+ recurring_period + + CharField +
+ starting_date + + DateTimeField +
+ >] + + uncloud_pay_models_OrderRecord [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + OrderRecord +
+ id + + AutoField +
+ order + + ForeignKey (uuid) +
+ description + + TextField +
+ one_time_price + + DecimalField +
+ recurring_price + + DecimalField +
+ >] + + + + + django_contrib_auth_models_AbstractUser [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + AbstractUser
<AbstractBaseUser,PermissionsMixin> +
+ date_joined + + DateTimeField +
+ email + + EmailField +
+ first_name + + CharField +
+ is_active + + BooleanField +
+ is_staff + + BooleanField +
+ is_superuser + + BooleanField +
+ last_login + + DateTimeField +
+ last_name + + CharField +
+ password + + CharField +
+ username + + CharField +
+ >] + + uncloud_auth_models_User [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + User
<AbstractUser> +
+ id + + AutoField +
+ date_joined + + DateTimeField +
+ email + + EmailField +
+ first_name + + CharField +
+ is_active + + BooleanField +
+ is_staff + + BooleanField +
+ is_superuser + + BooleanField +
+ last_login + + DateTimeField +
+ last_name + + CharField +
+ password + + CharField +
+ username + + CharField +
+ >] + + + + + uncloud_pay_models_Product [label=< + + + + + + + + + + + + + + + +
+ + Product +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ status + + CharField +
+ >] + + uncloud_vm_models_VMHost [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMHost +
+ uuid + + UUIDField +
+ hostname + + CharField +
+ physical_cores + + IntegerField +
+ status + + CharField +
+ usable_cores + + IntegerField +
+ usable_ram_in_gb + + FloatField +
+ >] + + uncloud_vm_models_VMProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vmhost + + ForeignKey (uuid) +
+ cores + + IntegerField +
+ name + + CharField +
+ ram_in_gb + + FloatField +
+ status + + CharField +
+ vmid + + IntegerField +
+ >] + + uncloud_vm_models_VMWithOSProduct [label=< + + + + + + + +
+ + VMWithOSProduct +
+ vmproduct_ptr + + OneToOneField (uuid) +
+ >] + + uncloud_vm_models_VMDiskImageProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMDiskImageProduct +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ image_source + + CharField +
+ image_source_type + + CharField +
+ import_url + + URLField +
+ is_os_image + + BooleanField +
+ is_public + + BooleanField +
+ name + + CharField +
+ size_in_gb + + FloatField +
+ status + + CharField +
+ storage_class + + CharField +
+ >] + + uncloud_vm_models_VMDiskProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMDiskProduct +
+ uuid + + UUIDField +
+ image + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ size_in_gb + + FloatField +
+ >] + + uncloud_vm_models_VMNetworkCard [label=< + + + + + + + + + + + + + + + + + + + +
+ + VMNetworkCard +
+ id + + AutoField +
+ vm + + ForeignKey (uuid) +
+ ip_address + + GenericIPAddressField +
+ mac_address + + BigIntegerField +
+ >] + + uncloud_vm_models_VMSnapshotProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMSnapshotProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ gb_hdd + + FloatField +
+ gb_ssd + + FloatField +
+ status + + CharField +
+ >] + + + + + uncloud_pay_models_Product [label=< + + + + + + + + + + + + + + + +
+ + Product +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ status + + CharField +
+ >] + + ungleich_service_models_MatrixServiceProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + MatrixServiceProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ domain + + CharField +
+ status + + CharField +
+ >] + + + + + opennebula_models_VM [label=< + + + + + + + + + + + + + + + +
+ + VM +
+ vmid + + IntegerField +
+ owner + + ForeignKey (id) +
+ data + + JSONField +
+ >] + + + + + // Relations + + django_contrib_admin_models_LogEntry -> uncloud_auth_models_User + [label=" user (logentry)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_admin_models_LogEntry -> django_contrib_contenttypes_models_ContentType + [label=" content_type (logentry)"] [arrowhead=none, arrowtail=dot, dir=both]; + + + django_contrib_auth_models_Permission -> django_contrib_contenttypes_models_ContentType + [label=" content_type (permission)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_auth_models_Group -> django_contrib_auth_models_Permission + [label=" permissions (group)"] [arrowhead=dot arrowtail=dot, dir=both]; + + + + django_contrib_sessions_models_Session -> django_contrib_sessions_base_session_AbstractBaseSession + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_StripeCustomer -> uncloud_auth_models_User + [label=" owner (stripecustomer)"] [arrowhead=none, arrowtail=none, dir=both]; + + uncloud_pay_models_Payment -> uncloud_auth_models_User + [label=" owner (payment)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_PaymentMethod -> uncloud_auth_models_User + [label=" owner (paymentmethod)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Bill -> uncloud_auth_models_User + [label=" owner (bill)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Order -> uncloud_auth_models_User + [label=" owner (order)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Order -> uncloud_pay_models_Bill + [label=" bill (order)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_pay_models_OrderRecord -> uncloud_pay_models_Order + [label=" order (orderrecord)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_auth_base_user_AbstractBaseUser [label=< + + +
+ AbstractBaseUser +
+ >] + django_contrib_auth_models_AbstractUser -> django_contrib_auth_base_user_AbstractBaseUser + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + django_contrib_auth_models_PermissionsMixin [label=< + + +
+ PermissionsMixin +
+ >] + django_contrib_auth_models_AbstractUser -> django_contrib_auth_models_PermissionsMixin + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_Group + [label=" groups (user)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_Permission + [label=" user_permissions (user)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_AbstractUser + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_Product -> uncloud_auth_models_User + [label=" owner (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Product -> uncloud_pay_models_Order + [label=" order (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMProduct -> uncloud_vm_models_VMHost + [label=" vmhost (vmproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_vm_models_VMWithOSProduct -> uncloud_vm_models_VMProduct + [label=" multi-table\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_vm_models_VMDiskImageProduct -> uncloud_auth_models_User + [label=" owner (vmdiskimageproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_auth_models_User + [label=" owner (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_vm_models_VMProduct + [label=" vm (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_vm_models_VMDiskImageProduct + [label=" image (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMNetworkCard -> uncloud_vm_models_VMProduct + [label=" vm (vmnetworkcard)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMSnapshotProduct -> uncloud_vm_models_VMProduct + [label=" vm (vmsnapshotproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMSnapshotProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_Product -> uncloud_auth_models_User + [label=" owner (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Product -> uncloud_pay_models_Order + [label=" order (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + ungleich_service_models_MatrixServiceProduct -> uncloud_vm_models_VMProduct + [label=" vm (matrixserviceproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + ungleich_service_models_MatrixServiceProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + opennebula_models_VM -> uncloud_auth_models_User + [label=" owner (vm)"] [arrowhead=none, arrowtail=dot, dir=both]; + + +} diff --git a/models.png b/models.png new file mode 100644 index 0000000..f9d0c2e Binary files /dev/null and b/models.png differ diff --git a/opennebula/__init__.py b/opennebula/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/opennebula/admin.py b/opennebula/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/opennebula/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/opennebula/apps.py b/opennebula/apps.py new file mode 100644 index 0000000..0750576 --- /dev/null +++ b/opennebula/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class OpennebulaConfig(AppConfig): + name = 'opennebula' diff --git a/opennebula/management/commands/opennebula-synchosts.py b/opennebula/management/commands/opennebula-synchosts.py new file mode 100644 index 0000000..29f9ac1 --- /dev/null +++ b/opennebula/management/commands/opennebula-synchosts.py @@ -0,0 +1,74 @@ +import json + +import uncloud.secrets as secrets + +from xmlrpc.client import ServerProxy as RPCClient + +from django.core.management.base import BaseCommand +from xmltodict import parse +from enum import IntEnum +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost +from django_auth_ldap.backend import LDAPBackend + + +class HostStates(IntEnum): + """ + The following flags are copied from + https://docs.opennebula.org/5.8/integration/system_interfaces/api.html#schemas-for-host + """ + INIT = 0 # Initial state for enabled hosts + MONITORING_MONITORED = 1 # Monitoring the host (from monitored) + MONITORED = 2 # The host has been successfully monitored + ERROR = 3 # An error ocurrer while monitoring the host + DISABLED = 4 # The host is disabled + MONITORING_ERROR = 5 # Monitoring the host (from error) + MONITORING_INIT = 6 # Monitoring the host (from init) + MONITORING_DISABLED = 7 # Monitoring the host (from disabled) + OFFLINE = 8 # The host is totally offline + + +class Command(BaseCommand): + help = 'Syncronize Host information from OpenNebula' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + with RPCClient(secrets.OPENNEBULA_URL) as rpc_client: + success, response, *_ = rpc_client.one.hostpool.info(secrets.OPENNEBULA_USER_PASS) + if success: + response = json.loads(json.dumps(parse(response))) + host_pool = response.get('HOST_POOL', {}).get('HOST', {}) + for host in host_pool: + host_share = host.get('HOST_SHARE', {}) + + host_name = host.get('NAME') + state = int(host.get('STATE', HostStates.OFFLINE.value)) + + if state == HostStates.MONITORED: + status = 'active' + elif state == HostStates.DISABLED: + status = 'disabled' + else: + status = 'unusable' + + usable_cores = host_share.get('TOTAL_CPU') + usable_ram_in_kb = int(host_share.get('TOTAL_MEM', 0)) + usable_ram_in_gb = int(usable_ram_in_kb / 2 ** 20) + + # vms cannot be created like this -- Nico, 2020-03-17 + # vms = host.get('VMS', {}) or {} + # vms = vms.get('ID', []) or [] + # vms = ','.join(vms) + + VMHost.objects.update_or_create( + hostname=host_name, + defaults={ + 'usable_cores': usable_cores, + 'usable_ram_in_gb': usable_ram_in_gb, + 'status': status + } + ) + else: + print(response) diff --git a/opennebula/management/commands/opennebula-syncvms.py b/opennebula/management/commands/opennebula-syncvms.py new file mode 100644 index 0000000..3c12fa9 --- /dev/null +++ b/opennebula/management/commands/opennebula-syncvms.py @@ -0,0 +1,44 @@ +import json + +from xmlrpc.client import ServerProxy as RPCClient +from django_auth_ldap.backend import LDAPBackend +from django.core.management.base import BaseCommand +from django.conf import settings +from xmltodict import parse + +from opennebula.models import VM as VMModel + + +class Command(BaseCommand): + help = 'Syncronize VM information from OpenNebula' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + with RPCClient(settings.OPENNEBULA_URL) as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + settings.OPENNEBULA_USER_PASS, -2, -1, -1, -1 + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + unknown_user = set() + + backend = LDAPBackend() + + for vm in vms: + vm_id = vm['ID'] + vm_owner = vm['UNAME'] + + user = backend.populate_user(username=vm_owner) + + if not user: + unknown_user.add(vm_owner) + else: + VMModel.objects.update_or_create( + vmid=vm_id, + defaults={'data': vm, 'owner': user} + ) + print('User not found in ldap:', unknown_user) + else: + print(response) diff --git a/opennebula/management/commands/opennebula-to-uncloud.py b/opennebula/management/commands/opennebula-to-uncloud.py new file mode 100644 index 0000000..230159a --- /dev/null +++ b/opennebula/management/commands/opennebula-to-uncloud.py @@ -0,0 +1,193 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand +from django.utils import timezone +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct + +from uncloud_pay.models import Order + +import logging + +log = logging.getLogger(__name__) + +def convert_mac_to_int(mac_address: str): + # Remove octet connecting characters + mac_address = mac_address.replace(':', '') + mac_address = mac_address.replace('.', '') + mac_address = mac_address.replace('-', '') + mac_address = mac_address.replace(' ', '') + + # Parse the resulting number as hexadecimal + mac_address = int(mac_address, base=16) + + return mac_address + + +def get_vm_price(core, ram, ssd_size, hdd_size, n_of_ipv4, n_of_ipv6): + total = 3 * core + 4 * ram + (3.5 * ssd_size/10.) + (1.5 * hdd_size/100.) + 8 * n_of_ipv4 + 0 * n_of_ipv6 + + # TODO: Find some reason about the following magical subtraction. + total -= 8 + + return total + + +def create_nics(one_vm, vm_product): + for nic in one_vm.nics: + mac_address = convert_mac_to_int(nic.get('MAC')) + ip_address = nic.get('IP', None) or nic.get('IP6_GLOBAL', None) + + VMNetworkCard.objects.update_or_create( + mac_address=mac_address, vm=vm_product, defaults={'ip_address': ip_address} + ) + + +def sync_disk_and_image(one_vm, vm_product, disk_owner): + """ + a) Check all opennebula disk if they are in the uncloud VM, if not add + b) Check all uncloud disks and remove them if they are not in the opennebula VM + """ + + vmdisknum = 0 + + one_disks_extra_data = [] + + for disk in one_vm.disks: + vmowner = one_vm.owner + name = disk.get('image') + vmdisknum += 1 + + log.info("Checking disk {} for VM {}".format(name, one_vm)) + + is_os_image, is_public, status = True, False, 'active' + + image_size_in_gb = disk.get('image_size_in_gb') + disk_size_in_gb = disk.get('size_in_gb') + storage_class = disk.get('storage_class') + image_source = disk.get('source') + image_source_type = disk.get('source_type') + + image, _ = VMDiskImageProduct.objects.update_or_create( + name=name, + defaults={ + 'owner': disk_owner, + 'is_os_image': is_os_image, + 'is_public': is_public, + 'size_in_gb': image_size_in_gb, + 'storage_class': storage_class, + 'image_source': image_source, + 'image_source_type': image_source_type, + 'status': status + } + ) + + # identify vmdisk from opennebula - primary mapping key + extra_data = { + 'opennebula_vm': one_vm.vmid, + 'opennebula_size_in_gb': disk_size_in_gb, + 'opennebula_source': disk.get('opennebula_source'), + 'opennebula_disk_num': vmdisknum + } + # Save for comparing later + one_disks_extra_data.append(extra_data) + + try: + vm_disk = VMDiskProduct.objects.get(extra_data=extra_data) + except VMDiskProduct.DoesNotExist: + vm_disk = VMDiskProduct.objects.create( + owner=vmowner, + vm=vm_product, + image=image, + size_in_gb=disk_size_in_gb, + extra_data=extra_data + ) + + # Now remove all disks that are not in above extra_data list + for disk in VMDiskProduct.objects.filter(vm=vm_product): + extra_data = disk.extra_data + if not extra_data in one_disks_extra_data: + log.info("Removing disk {} from VM {}".format(disk, vm_product)) + disk.delete() + + disks = [ disk.extra_data for disk in VMDiskProduct.objects.filter(vm=vm_product) ] + log.info("VM {} has disks: {}".format(vm_product, disks)) + +class Command(BaseCommand): + help = 'Migrate Opennebula VM to regular (uncloud) vm' + + def add_arguments(self, parser): + parser.add_argument('--disk-owner', required=True, help="The user who owns the the opennebula disks") + + def handle(self, *args, **options): + log.debug("{} {}".format(args, options)) + + disk_owner = get_user_model().objects.get(username=options['disk_owner']) + + for one_vm in VMModel.objects.all(): + + if not one_vm.last_host: + log.warning("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid)) + continue + + try: + vmhost = VMHost.objects.get(hostname=one_vm.last_host) + except VMHost.DoesNotExist: + log.error("VMHost {} does not exist, aborting".format(one_vm.last_host)) + raise + + cores = one_vm.cores + ram_in_gb = one_vm.ram_in_gb + owner = one_vm.owner + status = 'active' + + ssd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['ssd', 'one'] ]) + hdd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['hdd'] ]) + + # List of IPv4 addresses and Global IPv6 addresses + ipv4, ipv6 = one_vm.ips + + # TODO: Insert actual/real creation_date, starting_date, ending_date + # instead of pseudo one we are putting currently + creation_date = starting_date = datetime.now(tz=timezone.utc) + + # Price calculation based on datacenterlight.ch + one_time_price = 0 + recurring_period = 'per_month' + recurring_price = get_vm_price(cores, ram_in_gb, + ssd_size, hdd_size, + len(ipv4), len(ipv6)) + + try: + vm_product = VMProduct.objects.get(extra_data__opennebula_id=one_vm.vmid) + except VMProduct.DoesNotExist: + order = Order.objects.create( + owner=owner, + creation_date=creation_date, + starting_date=starting_date + ) + vm_product = VMProduct( + extra_data={ 'opennebula_id': one_vm.vmid }, + name=one_vm.uncloud_name, + order=order + ) + + # we don't use update_or_create, as filtering by json AND setting json + # at the same time does not work + + vm_product.vmhost = vmhost + vm_product.owner = owner + vm_product.cores = cores + vm_product.ram_in_gb = ram_in_gb + vm_product.status = status + + vm_product.save() + + # Create VMNetworkCards + create_nics(one_vm, vm_product) + + # Create VMDiskImageProduct and VMDiskProduct + sync_disk_and_image(one_vm, vm_product, disk_owner=disk_owner) diff --git a/opennebula/migrations/0001_initial.py b/opennebula/migrations/0001_initial.py new file mode 100644 index 0000000..9a135c6 --- /dev/null +++ b/opennebula/migrations/0001_initial.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='VM', + fields=[ + ('vmid', models.IntegerField(primary_key=True, serialize=False)), + ('data', models.JSONField()), + ], + ), + ] diff --git a/opennebula/migrations/__init__.py b/opennebula/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/opennebula/models.py b/opennebula/models.py new file mode 100644 index 0000000..f15b845 --- /dev/null +++ b/opennebula/models.py @@ -0,0 +1,90 @@ +import uuid +from django.db import models +from django.contrib.auth import get_user_model +from uncloud_pay.models import Product + +# ungleich specific +storage_class_mapping = { + 'one': 'ssd', + 'ssd': 'ssd', + 'hdd': 'hdd' +} + +class VM(models.Model): + vmid = models.IntegerField(primary_key=True) + data = models.JSONField() + + @property + def uncloud_name(self): + return "opennebula-{}".format(self.vmid) + + @property + def cores(self): + return int(self.data['TEMPLATE']['VCPU']) + + @property + def ram_in_gb(self): + return int(self.data['TEMPLATE']['MEMORY'])/1024 + + @property + def disks(self): + """ + If there is no disk then the key DISK does not exist. + + If there is only one disk, we have a dictionary in the database. + + If there are multiple disks, we have a list of dictionaries in the database. + """ + + disks = [] + + if 'DISK' in self.data['TEMPLATE']: + if type(self.data['TEMPLATE']['DISK']) is dict: + disks = [self.data['TEMPLATE']['DISK']] + else: + disks = self.data['TEMPLATE']['DISK'] + + disks = [ + { + 'size_in_gb': int(d['SIZE'])/1024, + 'opennebula_source': d['SOURCE'], + 'opennebula_name': d['IMAGE'], + 'image_size_in_gb': int(d['ORIGINAL_SIZE'])/1024, + 'pool_name': d['POOL_NAME'], + 'image': d['IMAGE'], + 'source': d['SOURCE'], + 'source_type': d['TM_MAD'], + 'storage_class': storage_class_mapping[d['POOL_NAME']] + + } + for d in disks + ] + + return disks + + @property + def last_host(self): + return ((self.data.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None) + + @property + def graphics(self): + return self.data.get('TEMPLATE', {}).get('GRAPHICS', {}) + + @property + def nics(self): + _nics = self.data.get('TEMPLATE', {}).get('NIC', {}) + if isinstance(_nics, dict): + _nics = [_nics] + return _nics + + @property + def ips(self): + ipv4, ipv6 = [], [] + for nic in self.nics: + ip = nic.get('IP') + ip6 = nic.get('IP6_GLOBAL') + if ip: + ipv4.append(ip) + if ip6: + ipv6.append(ip6) + return ipv4, ipv6 diff --git a/opennebula/serializers.py b/opennebula/serializers.py new file mode 100644 index 0000000..cd00622 --- /dev/null +++ b/opennebula/serializers.py @@ -0,0 +1,10 @@ +from rest_framework import serializers +from opennebula.models import VM + + +class OpenNebulaVMSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = VM + fields = [ 'vmid', 'owner', 'data', + 'uncloud_name', 'cores', 'ram_in_gb', + 'disks', 'nics', 'ips' ] diff --git a/opennebula/tests.py b/opennebula/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/opennebula/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/opennebula/views.py b/opennebula/views.py new file mode 100644 index 0000000..688f0b4 --- /dev/null +++ b/opennebula/views.py @@ -0,0 +1,16 @@ +from rest_framework import viewsets, permissions + +#from .models import VM +# from .serializers import OpenNebulaVMSerializer + +# class VMViewSet(viewsets.ModelViewSet): +# permission_classes = [permissions.IsAuthenticated] +# serializer_class = OpenNebulaVMSerializer + +# def get_queryset(self): +# if self.request.user.is_superuser: +# obj = VM.objects.all() +# else: +# obj = VM.objects.filter(owner=self.request.user) + +# return obj diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..8231fd0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,34 @@ +# Django basics +Django==3.2.4 +djangorestframework +django-auth-ldap +django-bootstrap-v5 +fontawesome-free + +psycopg2 +ldap3 +django-allauth +xmltodict +parsedatetime +# Follow are for creating graph models +pyparsing +pydot +django-extensions + +# PDF creating +django-hardcopy + +# schema support +pyyaml +uritemplate +tldextract +# Payment & VAT +vat-validator +stripe + +#Jobs +django-q +redis + +jinja2 +python-gitlab diff --git a/resources/ci/.lock b/resources/ci/.lock new file mode 100644 index 0000000..e69de29 diff --git a/resources/ci/Dockerfile b/resources/ci/Dockerfile new file mode 100644 index 0000000..020b66e --- /dev/null +++ b/resources/ci/Dockerfile @@ -0,0 +1,3 @@ +FROM fedora:latest + +RUN dnf install -y python3-devel python3-pip python3-coverage libpq-devel openldap-devel gcc chromium diff --git a/resources/vat-rates.csv b/resources/vat-rates.csv new file mode 100644 index 0000000..17bdb99 --- /dev/null +++ b/resources/vat-rates.csv @@ -0,0 +1,325 @@ +start_date,stop_date,territory_codes,currency_code,rate,rate_type,description +2011-01-04,,AI,XCD,0,standard,Anguilla (British overseas territory) is exempted of VAT. +1984-01-01,,AT,EUR,0.2,standard,Austria (member state) standard VAT rate. +1976-01-01,1984-01-01,AT,EUR,0.18,standard, +1973-01-01,1976-01-01,AT,EUR,0.16,standard, +1984-01-01,,"AT-6691 +DE-87491",EUR,0.19,standard,Jungholz (Austrian town) special VAT rate. +1984-01-01,,"AT-6991 +AT-6992 +AT-6993 +DE-87567 +DE-87568 +DE-87569",EUR,0.19,standard,Mittelberg (Austrian town) special VAT rate. +1996-01-01,,BE,EUR,0.21,standard,Belgium (member state) standard VAT rate. +1994-01-01,1996-01-01,BE,EUR,0.205,standard, +1992-04-01,1994-01-01,BE,EUR,0.195,standard, +1983-01-01,1992-04-01,BE,EUR,0.19,standard, +1981-07-01,1983-01-01,BE,EUR,0.17,standard, +1978-07-01,1981-07-01,BE,EUR,0.16,standard, +1971-07-01,1978-07-01,BE,EUR,0.18,standard, +1999-01-01,,BG,BGN,0.2,standard,Bulgaria (member state) standard VAT rate. +1996-07-01,1999-01-01,BG,BGN,0.22,standard, +1994-04-01,1996-07-01,BG,BGN,0.18,standard, +2011-01-04,,BM,BMD,0,standard,Bermuda (British overseas territory) is exempted of VAT. +2014-01-13,,"CY +GB-BFPO 57 +GB-BFPO 58 +GB-BFPO 59 +UK-BFPO 57 +UK-BFPO 58 +UK-BFPO 59",EUR,0.19,standard,"Cyprus (member state) standard VAT rate. +Akrotiri and Dhekelia (British overseas territory) is subjected to Cyprus' standard VAT rate." +2013-01-14,2014-01-13,CY,EUR,0.18,standard, +2012-03-01,2013-01-14,CY,EUR,0.17,standard, +2003-01-01,2012-03-01,CY,EUR,0.15,standard, +2002-07-01,2003-01-01,CY,EUR,0.13,standard, +2000-07-01,2002-07-01,CY,EUR,0.1,standard, +1993-10-01,2000-07-01,CY,EUR,0.08,standard, +1992-07-01,1993-10-01,CY,EUR,0.05,standard, +2013-01-01,,CZ,CZK,0.21,standard,Czech Republic (member state) standard VAT rate. +2010-01-01,2013-01-01,CZ,CZK,0.2,standard, +2004-05-01,2010-01-01,CZ,CZK,0.19,standard, +1995-01-01,2004-05-01,CZ,CZK,0.22,standard, +1993-01-01,1995-01-01,CZ,CZK,0.23,standard, +2007-01-01,,DE,EUR,0.19,standard,Germany (member state) standard VAT rate. +1998-04-01,2007-01-01,DE,EUR,0.16,standard, +1993-01-01,1998-04-01,DE,EUR,0.15,standard, +1983-07-01,1993-01-01,DE,EUR,0.14,standard, +1979-07-01,1983-07-01,DE,EUR,0.13,standard, +1978-01-01,1979-07-01,DE,EUR,0.12,standard, +1968-07-01,1978-01-01,DE,EUR,0.11,standard, +1968-01-01,1968-07-01,DE,EUR,0.1,standard, +2007-01-01,,DE-27498,EUR,0,standard,Heligoland (German island) is exempted of VAT. +2007-01-01,,"DE-78266 +CH-8238",EUR,0,standard,Busingen am Hochrhein (German territory) is exempted of VAT. +1992-01-01,,DK,DKK,0.25,standard,Denmark (member state) standard VAT rate. +1980-06-30,1992-01-01,DK,DKK,0.22,standard, +1978-10-30,1980-06-30,DK,DKK,0.2025,standard, +1977-10-03,1978-10-30,DK,DKK,0.18,standard, +1970-06-29,1977-10-03,DK,DKK,0.15,standard, +1968-04-01,1970-06-29,DK,DKK,0.125,standard, +1967-07-03,1968-04-01,DK,DKK,0.1,standard, +2009-07-01,,EE,EUR,0.2,standard,Estonia (member state) standard VAT rate. +1993-01-01,2009-07-01,EE,EUR,0.18,standard, +1991-01-01,1993-01-01,EE,EUR,0.1,standard, +2016-06-01,,"GR +EL",EUR,0.24,standard,Greece (member state) standard VAT rate. +2010-07-01,2016-06-01,"GR +EL",EUR,0.23,standard, +2010-03-15,2010-07-01,"GR +EL",EUR,0.21,standard, +2005-04-01,2010-03-15,"GR +EL",EUR,0.19,standard, +1990-04-28,2005-04-01,"GR +EL",EUR,0.18,standard, +1988-01-01,1990-04-28,"GR +EL",EUR,0.16,standard, +1987-01-01,1988-01-01,"GR +EL",EUR,0.18,standard, +2012-09-01,,ES,EUR,0.21,standard,Spain (member state) standard VAT rate. +2010-07-01,2012-09-01,ES,EUR,0.18,standard, +1995-01-01,2010-07-01,ES,EUR,0.16,standard, +1992-08-01,1995-01-01,ES,EUR,0.15,standard, +1992-01-01,1992-08-01,ES,EUR,0.13,standard, +1986-01-01,1992-01-01,ES,EUR,0.12,standard, +2012-09-01,,"ES-CN +ES-GC +ES-TF +IC",EUR,0,standard,Canary Islands (Spanish autonomous community) is exempted of VAT. +2012-09-01,,"ES-ML +ES-CE +EA",EUR,0,standard,Ceuta and Melilla (Spanish autonomous cities) is exempted of VAT. +2013-01-01,,FI,EUR,0.24,standard,Finland (member state) standard VAT rate. +2010-07-01,2013-01-01,FI,EUR,0.23,standard, +1994-06-01,2010-07-01,FI,EUR,0.22,standard, +2013-01-01,,"FI-01 +AX",EUR,0,standard,Aland Islands (Finish autonomous region) is exempted of VAT. +2011-01-04,,FK,FKP,0,standard,Falkland Islands (British overseas territory) is exempted of VAT. +1992-01-01,,FO,DKK,0,standard,Faroe Islands (Danish autonomous country) is exempted of VAT. +2014-01-01,,"FR +MC",EUR,0.2,standard,"France (member state) standard VAT rate. +Monaco (sovereign city-state) is member of the EU VAT area and subjected to France's standard VAT rate." +2000-04-01,2014-01-01,"FR +MC",EUR,0.196,standard, +1995-08-01,2000-04-01,"FR +MC",EUR,0.206,standard, +1982-07-01,1995-08-01,"FR +MC",EUR,0.186,standard, +1977-01-01,1982-07-01,"FR +MC",EUR,0.176,standard, +1973-01-01,1977-01-01,"FR +MC",EUR,0.2,standard, +1970-01-01,1973-01-01,"FR +MC",EUR,0.23,standard, +1968-12-01,1970-01-01,"FR +MC",EUR,0.19,standard, +1968-01-01,1968-12-01,"FR +MC",EUR,0.1666,standard, +2014-01-01,,"FR-BL +BL",EUR,0,standard,Saint Barthelemy (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-GF +GF",EUR,0,standard,Guiana (French overseas department) is exempted of VAT. +2014-01-01,,"FR-GP +GP",EUR,0.085,standard,Guadeloupe (French overseas department) special VAT rate. +2014-01-01,,"FR-MF +MF",EUR,0,standard,Saint Martin (French overseas collectivity) is subjected to France's standard VAT rate. +2014-01-01,,"FR-MQ +MQ",EUR,0.085,standard,Martinique (French overseas department) special VAT rate. +2014-01-01,,"FR-NC +NC",XPF,0,standard,New Caledonia (French special collectivity) is exempted of VAT. +2014-01-01,,"FR-PF +PF",XPF,0,standard,French Polynesia (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-PM +PM",EUR,0,standard,Saint Pierre and Miquelon (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-RE +RE",EUR,0.085,standard,Reunion (French overseas department) special VAT rate. +2014-01-01,,"FR-TF +TF",EUR,0,standard,French Southern and Antarctic Lands (French overseas territory) is exempted of VAT. +2014-01-01,,"FR-WF +WF",XPF,0,standard,Wallis and Futuna (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-YT +YT",EUR,0,standard,Mayotte (French overseas department) is exempted of VAT. +2011-01-04,,GG,GBP,0,standard,Guernsey (British Crown dependency) is exempted of VAT. +2011-01-04,,GI,GIP,0,standard,Gibraltar (British overseas territory) is exempted of VAT. +1992-01-01,,GL,DKK,0,standard,Greenland (Danish autonomous country) is exempted of VAT. +2010-07-01,2016-06-01,"GR-34007 +EL-34007",EUR,0.16,standard,Skyros (Greek island) special VAT rate. +2010-07-01,2016-06-01,"GR-37002 +GR-37003 +GR-37005 +EL-37002 +EL-37003 +EL-37005",EUR,0.16,standard,Northern Sporades (Greek islands) special VAT rate. +2010-07-01,2016-06-01,"GR-64004 +EL-64004",EUR,0.16,standard,Thasos (Greek island) special VAT rate. +2010-07-01,2016-06-01,"GR-68002 +EL-68002",EUR,0.16,standard,Samothrace (Greek island) special VAT rate. +2010-07-01,,"GR-69 +EL-69",EUR,0,standard,Mount Athos (Greek self-governed part) is exempted of VAT. +2010-07-01,2016-06-01,"GR-81 +EL-81",EUR,0.16,standard,Dodecanese (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-82 +EL-82",EUR,0.16,standard,Cyclades (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-83 +EL-83",EUR,0.16,standard,Lesbos (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-84 +EL-84",EUR,0.16,standard,Samos (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-85 +EL-85",EUR,0.16,standard,Chios (Greek department) special VAT rate. +2011-01-04,,GS,GBP,0,standard,South Georgia and the South Sandwich Islands (British overseas territory) is exempted of VAT. +2012-03-01,,HR,HRK,0.25,standard,Croatia (member state) standard VAT rate. +2009-08-01,2012-03-01,HR,HRK,0.23,standard, +1998-08-01,2009-08-01,HR,HRK,0.22,standard, +2012-01-01,,HU,HUF,0.27,standard,Hungary (member state) standard VAT rate. +2009-07-01,2012-01-01,HU,HUF,0.25,standard, +2006-01-01,2009-07-01,HU,HUF,0.2,standard, +1988-01-01,2006-01-01,HU,HUF,0.25,standard, +2012-01-01,,IE,EUR,0.23,standard,Republic of Ireland (member state) standard VAT rate. +2010-01-01,2012-01-01,IE,EUR,0.21,standard, +2008-12-01,2010-01-01,IE,EUR,0.215,standard, +2002-03-01,2008-12-01,IE,EUR,0.21,standard, +2001-01-01,2002-03-01,IE,EUR,0.2,standard, +1991-03-01,2001-01-01,IE,EUR,0.21,standard, +1990-03-01,1991-03-01,IE,EUR,0.23,standard, +1986-03-01,1990-03-01,IE,EUR,0.25,standard, +1983-05-01,1986-03-01,IE,EUR,0.23,standard, +1983-03-01,1983-05-01,IE,EUR,0.35,standard, +1982-05-01,1983-03-01,IE,EUR,0.3,standard, +1980-05-01,1982-05-01,IE,EUR,0.25,standard, +1976-03-01,1980-05-01,IE,EUR,0.2,standard, +1973-09-03,1976-03-01,IE,EUR,0.195,standard, +1972-11-01,1973-09-03,IE,EUR,0.1637,standard, +2011-01-04,,IO,GBP,0,standard,British Indian Ocean Territory (British overseas territory) is exempted of VAT. +2013-10-01,,IT,EUR,0.22,standard,Italy (member state) standard VAT rate. +2011-09-17,2013-10-01,IT,EUR,0.21,standard, +1997-10-01,2011-09-17,IT,EUR,0.2,standard, +1988-08-01,1997-10-01,IT,EUR,0.19,standard, +1982-08-05,1988-08-01,IT,EUR,0.18,standard, +1981-01-01,1982-08-05,IT,EUR,0.15,standard, +1980-11-01,1981-01-01,IT,EUR,0.14,standard, +1980-07-03,1980-11-01,IT,EUR,0.15,standard, +1977-02-08,1980-07-03,IT,EUR,0.14,standard, +1973-01-01,1977-02-08,IT,EUR,0.12,standard, +2013-10-01,,"IT-22060 +CH-6911",CHF,0,standard,Campione (Italian town) is exempted of VAT. +2013-10-01,,IT-23030,EUR,0,standard,Livigno (Italian town) is exempted of VAT. +2011-01-04,,JE,GBP,0,standard,Jersey (British Crown dependency) is exempted of VAT. +2011-01-04,,KY,KYD,0,standard,Cayman Islands (British overseas territory) is exempted of VAT. +2009-09-01,,LT,EUR,0.21,standard,Lithuania (member state) standard VAT rate. +2009-01-01,2009-09-01,LT,EUR,0.19,standard, +1994-05-01,2009-01-01,LT,EUR,0.18,standard, +2015-01-01,,LU,EUR,0.17,standard,Luxembourg (member state) standard VAT rate. +1992-01-01,2015-01-01,LU,EUR,0.15,standard, +1983-07-01,1992-01-01,LU,EUR,0.12,standard, +1971-01-01,1983-07-01,LU,EUR,0.1,standard, +1970-01-01,1971-01-01,LU,EUR,0.8,standard, +2012-07-01,,LV,EUR,0.21,standard,Latvia (member state) standard VAT rate. +2011-01-01,2012-07-01,LV,EUR,0.22,standard, +2009-01-01,2011-01-01,LV,EUR,0.21,standard, +1995-05-01,2009-01-01,LV,EUR,0.18,standard, +2011-01-04,,MS,XCD,0,standard,Montserrat (British overseas territory) is exempted of VAT. +2004-01-01,,MT,EUR,0.18,standard,Malta (member state) standard VAT rate. +1995-01-01,2004-01-01,MT,EUR,0.15,standard, +2012-10-01,,NL,EUR,0.21,standard,Netherlands (member state) standard VAT rate. +2001-01-01,2012-10-01,NL,EUR,0.19,standard, +1992-10-01,2001-01-01,NL,EUR,0.175,standard, +1989-01-01,1992-10-01,NL,EUR,0.185,standard, +1986-10-01,1989-01-01,NL,EUR,0.2,standard, +1984-01-01,1986-10-01,NL,EUR,0.19,standard, +1976-01-01,1984-01-01,NL,EUR,0.18,standard, +1973-01-01,1976-01-01,NL,EUR,0.16,standard, +1971-01-01,1973-01-01,NL,EUR,0.14,standard, +1969-01-01,1971-01-01,NL,EUR,0.12,standard, +2012-10-01,,"NL-AW +AW",AWG,0,standard,Aruba (Dutch country) are exempted of VAT. +2012-10-01,,"NL-CW +NL-SX +CW +SX",ANG,0,standard,Curacao and Sint Maarten (Dutch countries) are exempted of VAT. +2012-10-01,,"NL-BQ1 +NL-BQ2 +NL-BQ3 +BQ +BQ-BO +BQ-SA +BQ-SE",USD,0,standard,"Bonaire, Saba and Sint Eustatius (Dutch special municipalities) are exempted of VAT." +2011-01-01,,PL,PLN,0.23,standard,Poland (member state) standard VAT rate. +1993-01-08,2011-01-01,PL,PLN,0.22,standard, +2011-01-04,,PN,NZD,0,standard,Pitcairn Islands (British overseas territory) is exempted of VAT. +2011-01-01,,PT,EUR,0.23,standard,Portugal (member state) standard VAT rate. +2010-07-01,2011-01-01,PT,EUR,0.21,standard, +2008-07-01,2010-07-01,PT,EUR,0.2,standard, +2005-07-01,2008-07-01,PT,EUR,0.21,standard, +2002-06-05,2005-07-01,PT,EUR,0.19,standard, +1995-01-01,2002-06-05,PT,EUR,0.17,standard, +1992-03-24,1995-01-01,PT,EUR,0.16,standard, +1988-02-01,1992-03-24,PT,EUR,0.17,standard, +1986-01-01,1988-02-01,PT,EUR,0.16,standard, +2011-01-01,,PT-20,EUR,0.18,standard,Azores (Portuguese autonomous region) special VAT rate. +2011-01-01,,PT-30,EUR,0.22,standard,Madeira (Portuguese autonomous region) special VAT rate. +2017-01-01,,RO,RON,0.19,standard,Romania (member state) standard VAT rate. +2016-01-01,2017-01-01,RO,RON,0.2,standard,Romania (member state) standard VAT rate. +2010-07-01,2016-01-01,RO,RON,0.24,standard, +2000-01-01,2010-07-01,RO,RON,0.19,standard, +1998-02-01,2000-01-01,RO,RON,0.22,standard, +1993-07-01,1998-02-01,RO,RON,0.18,standard, +1990-07-01,,SE,SEK,0.25,standard,Sweden (member state) standard VAT rate. +1983-01-01,1990-07-01,SE,SEK,0.2346,standard, +1981-11-16,1983-01-01,SE,SEK,0.2151,standard, +1980-09-08,1981-11-16,SE,SEK,0.2346,standard, +1977-06-01,1980-09-08,SE,SEK,0.2063,standard, +1971-01-01,1977-06-01,SE,SEK,0.1765,standard, +1969-01-01,1971-01-01,SE,SEK,0.1111,standard, +2011-01-04,,"AC +SH +SH-AC +SH-HL",SHP,0,standard,Ascension and Saint Helena (British overseas territory) is exempted of VAT. +2011-01-04,,"TA +SH-TA",GBP,0,standard,Tristan da Cunha (British oversea territory) is exempted of VAT. +2013-07-01,,SI,EUR,0.22,standard,Slovenia (member state) standard VAT rate. +2002-01-01,2013-07-01,SI,EUR,0.2,standard, +1999-07-01,2002-01-01,SI,EUR,0.19,standard, +2011-01-01,,SK,EUR,0.2,standard,Slovakia (member state) standard VAT rate. +2004-01-01,2011-01-01,SK,EUR,0.19,standard, +2003-01-01,2004-01-01,SK,EUR,0.2,standard, +1996-01-01,2003-01-01,SK,EUR,0.23,standard, +1993-08-01,1996-01-01,SK,EUR,0.25,standard, +1993-01-01,1993-08-01,SK,EUR,0.23,standard, +2011-01-04,,TC,USD,0,standard,Turks and Caicos Islands (British overseas territory) is exempted of VAT. +2011-01-04,,"GB +UK +IM",GBP,0.2,standard,"United Kingdom (member state) standard VAT rate. +Isle of Man (British self-governing dependency) is member of the EU VAT area and subjected to UK's standard VAT rate." +2010-01-01,2011-01-04,"GB +UK +IM",GBP,0.175,standard, +2008-12-01,2010-01-01,"GB +UK +IM",GBP,0.15,standard, +1991-04-01,2008-12-01,"GB +UK +IM",GBP,0.175,standard, +1979-06-18,1991-04-01,"GB +UK +IM",GBP,0.15,standard, +1974-07-29,1979-06-18,"GB +UK +IM",GBP,0.08,standard, +1973-04-01,1974-07-29,"GB +UK +IM",GBP,0.1,standard, +2011-01-04,,VG,USD,0,standard,British Virgin Islands (British overseas territory) is exempted of VAT. +2014-01-01,,CP,EUR,0,standard,Clipperton Island (French overseas possession) is exempted of VAT. +2019-11-15,,CH,CHF,0.077,standard,Switzerland standard VAT (added manually) +2019-11-15,,MC,EUR,0.196,standard,Monaco standard VAT (added manually) +2019-11-15,,FR,EUR,0.2,standard,France standard VAT (added manually) +2019-11-15,,GR,EUR,0.24,standard,Greece standard VAT (added manually) +2019-11-15,,GB,EUR,0.2,standard,UK standard VAT (added manually) +2019-12-17,,AD,EUR,0.045,standard,Andorra standard VAT (added manually) +2019-12-17,,TK,EUR,0.18,standard,Turkey standard VAT (added manually) +2019-12-17,,IS,EUR,0.24,standard,Iceland standard VAT (added manually) +2019-12-17,,FX,EUR,0.20,standard,France metropolitan standard VAT (added manually) +2020-01-04,,CY,EUR,0.19,standard,Cyprus standard VAT (added manually) +2019-01-04,,IL,EUR,0.23,standard,Ireland standard VAT (added manually) +2019-01-04,,LI,EUR,0.077,standard,Liechtenstein standard VAT (added manually) diff --git a/scripts/uncloud b/scripts/uncloud deleted file mode 100755 index 28d8344..0000000 --- a/scripts/uncloud +++ /dev/null @@ -1,70 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import logging -import importlib -import multiprocessing as mp -import sys - -from logging.handlers import SysLogHandler -from uncloud.configure.main import configure_parser - -from uncloud import UncloudException - -def exception_hook(exc_type, exc_value, exc_traceback): - logging.getLogger(__name__).error( - 'Uncaught exception', - exc_info=(exc_type, exc_value, exc_traceback) - ) - - -sys.excepthook = exception_hook - - -if __name__ == '__main__': - # Setting up root logger - logger = logging.getLogger() - - logger.setLevel(logging.DEBUG) - - parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument("--debug", "-d", action='store_true') - - arg_parser = argparse.ArgumentParser() - - subparsers = arg_parser.add_subparsers(dest="command") - - api_parser = subparsers.add_parser("api", parents=[parent_parser]) - api_parser.add_argument("--port", "-p") - - host_parser = subparsers.add_parser("host") - host_parser.add_argument("--hostname", required=True) - - scheduler_parser = subparsers.add_parser("scheduler") - filescanner_parser = subparsers.add_parser("filescanner") - imagescanner_parser = subparsers.add_parser("imagescanner") - metadata_parser = subparsers.add_parser("metadata") - config_parser = subparsers.add_parser("configure") - - configure_parser(config_parser) - args = arg_parser.parse_args() - - if not args.command: - arg_parser.print_help() - else: - - # if we start etcd in seperate process with default settings - # i.e inheriting few things from parent process etcd3 module - # errors out, so the following command configure multiprocessing - # module to not inherit anything from parent. - mp.set_start_method('spawn') - - arguments = vars(args) - try: - name = arguments.pop('command') - mod = importlib.import_module("uncloud.{}.main".format(name)) - main = getattr(mod, "main") - main(**arguments) - except UncloudException as err: - logger.error(err) - except Exception as err: - logger.exception(err) diff --git a/uncloud/.gitignore b/uncloud/.gitignore new file mode 100644 index 0000000..b03e0a5 --- /dev/null +++ b/uncloud/.gitignore @@ -0,0 +1,2 @@ +local_settings.py +ldap_max_uid_file \ No newline at end of file diff --git a/uncloud/__init__.py b/uncloud/__init__.py index 2920f47..3ce2d95 100644 --- a/uncloud/__init__.py +++ b/uncloud/__init__.py @@ -1,2 +1,253 @@ -class UncloudException(Exception): - pass +from django.utils.translation import gettext_lazy as _ +import decimal + +# Define DecimalField properties, used to represent amounts of money. +AMOUNT_MAX_DIGITS=10 +AMOUNT_DECIMALS=2 + +decimal.getcontext().prec = AMOUNT_DECIMALS + +# http://xml.coverpages.org/country3166.html +COUNTRIES = ( + ('AD', _('Andorra')), + ('AE', _('United Arab Emirates')), + ('AF', _('Afghanistan')), + ('AG', _('Antigua & Barbuda')), + ('AI', _('Anguilla')), + ('AL', _('Albania')), + ('AM', _('Armenia')), + ('AN', _('Netherlands Antilles')), + ('AO', _('Angola')), + ('AQ', _('Antarctica')), + ('AR', _('Argentina')), + ('AS', _('American Samoa')), + ('AT', _('Austria')), + ('AU', _('Australia')), + ('AW', _('Aruba')), + ('AZ', _('Azerbaijan')), + ('BA', _('Bosnia and Herzegovina')), + ('BB', _('Barbados')), + ('BD', _('Bangladesh')), + ('BE', _('Belgium')), + ('BF', _('Burkina Faso')), + ('BG', _('Bulgaria')), + ('BH', _('Bahrain')), + ('BI', _('Burundi')), + ('BJ', _('Benin')), + ('BM', _('Bermuda')), + ('BN', _('Brunei Darussalam')), + ('BO', _('Bolivia')), + ('BR', _('Brazil')), + ('BS', _('Bahama')), + ('BT', _('Bhutan')), + ('BV', _('Bouvet Island')), + ('BW', _('Botswana')), + ('BY', _('Belarus')), + ('BZ', _('Belize')), + ('CA', _('Canada')), + ('CC', _('Cocos (Keeling) Islands')), + ('CF', _('Central African Republic')), + ('CG', _('Congo')), + ('CH', _('Switzerland')), + ('CI', _('Ivory Coast')), + ('CK', _('Cook Iislands')), + ('CL', _('Chile')), + ('CM', _('Cameroon')), + ('CN', _('China')), + ('CO', _('Colombia')), + ('CR', _('Costa Rica')), + ('CU', _('Cuba')), + ('CV', _('Cape Verde')), + ('CX', _('Christmas Island')), + ('CY', _('Cyprus')), + ('CZ', _('Czech Republic')), + ('DE', _('Germany')), + ('DJ', _('Djibouti')), + ('DK', _('Denmark')), + ('DM', _('Dominica')), + ('DO', _('Dominican Republic')), + ('DZ', _('Algeria')), + ('EC', _('Ecuador')), + ('EE', _('Estonia')), + ('EG', _('Egypt')), + ('EH', _('Western Sahara')), + ('ER', _('Eritrea')), + ('ES', _('Spain')), + ('ET', _('Ethiopia')), + ('FI', _('Finland')), + ('FJ', _('Fiji')), + ('FK', _('Falkland Islands (Malvinas)')), + ('FM', _('Micronesia')), + ('FO', _('Faroe Islands')), + ('FR', _('France')), + ('FX', _('France, Metropolitan')), + ('GA', _('Gabon')), + ('GB', _('United Kingdom (Great Britain)')), + ('GD', _('Grenada')), + ('GE', _('Georgia')), + ('GF', _('French Guiana')), + ('GH', _('Ghana')), + ('GI', _('Gibraltar')), + ('GL', _('Greenland')), + ('GM', _('Gambia')), + ('GN', _('Guinea')), + ('GP', _('Guadeloupe')), + ('GQ', _('Equatorial Guinea')), + ('GR', _('Greece')), + ('GS', _('South Georgia and the South Sandwich Islands')), + ('GT', _('Guatemala')), + ('GU', _('Guam')), + ('GW', _('Guinea-Bissau')), + ('GY', _('Guyana')), + ('HK', _('Hong Kong')), + ('HM', _('Heard & McDonald Islands')), + ('HN', _('Honduras')), + ('HR', _('Croatia')), + ('HT', _('Haiti')), + ('HU', _('Hungary')), + ('ID', _('Indonesia')), + ('IE', _('Ireland')), + ('IL', _('Israel')), + ('IN', _('India')), + ('IO', _('British Indian Ocean Territory')), + ('IQ', _('Iraq')), + ('IR', _('Islamic Republic of Iran')), + ('IS', _('Iceland')), + ('IT', _('Italy')), + ('JM', _('Jamaica')), + ('JO', _('Jordan')), + ('JP', _('Japan')), + ('KE', _('Kenya')), + ('KG', _('Kyrgyzstan')), + ('KH', _('Cambodia')), + ('KI', _('Kiribati')), + ('KM', _('Comoros')), + ('KN', _('St. Kitts and Nevis')), + ('KP', _('Korea, Democratic People\'s Republic of')), + ('KR', _('Korea, Republic of')), + ('KW', _('Kuwait')), + ('KY', _('Cayman Islands')), + ('KZ', _('Kazakhstan')), + ('LA', _('Lao People\'s Democratic Republic')), + ('LB', _('Lebanon')), + ('LC', _('Saint Lucia')), + ('LI', _('Liechtenstein')), + ('LK', _('Sri Lanka')), + ('LR', _('Liberia')), + ('LS', _('Lesotho')), + ('LT', _('Lithuania')), + ('LU', _('Luxembourg')), + ('LV', _('Latvia')), + ('LY', _('Libyan Arab Jamahiriya')), + ('MA', _('Morocco')), + ('MC', _('Monaco')), + ('MD', _('Moldova, Republic of')), + ('MG', _('Madagascar')), + ('MH', _('Marshall Islands')), + ('ML', _('Mali')), + ('MN', _('Mongolia')), + ('MM', _('Myanmar')), + ('MO', _('Macau')), + ('MP', _('Northern Mariana Islands')), + ('MQ', _('Martinique')), + ('MR', _('Mauritania')), + ('MS', _('Monserrat')), + ('MT', _('Malta')), + ('MU', _('Mauritius')), + ('MV', _('Maldives')), + ('MW', _('Malawi')), + ('MX', _('Mexico')), + ('MY', _('Malaysia')), + ('MZ', _('Mozambique')), + ('NA', _('Namibia')), + ('NC', _('New Caledonia')), + ('NE', _('Niger')), + ('NF', _('Norfolk Island')), + ('NG', _('Nigeria')), + ('NI', _('Nicaragua')), + ('NL', _('Netherlands')), + ('NO', _('Norway')), + ('NP', _('Nepal')), + ('NR', _('Nauru')), + ('NU', _('Niue')), + ('NZ', _('New Zealand')), + ('OM', _('Oman')), + ('PA', _('Panama')), + ('PE', _('Peru')), + ('PF', _('French Polynesia')), + ('PG', _('Papua New Guinea')), + ('PH', _('Philippines')), + ('PK', _('Pakistan')), + ('PL', _('Poland')), + ('PM', _('St. Pierre & Miquelon')), + ('PN', _('Pitcairn')), + ('PR', _('Puerto Rico')), + ('PT', _('Portugal')), + ('PW', _('Palau')), + ('PY', _('Paraguay')), + ('QA', _('Qatar')), + ('RE', _('Reunion')), + ('RO', _('Romania')), + ('RU', _('Russian Federation')), + ('RW', _('Rwanda')), + ('SA', _('Saudi Arabia')), + ('SB', _('Solomon Islands')), + ('SC', _('Seychelles')), + ('SD', _('Sudan')), + ('SE', _('Sweden')), + ('SG', _('Singapore')), + ('SH', _('St. Helena')), + ('SI', _('Slovenia')), + ('SJ', _('Svalbard & Jan Mayen Islands')), + ('SK', _('Slovakia')), + ('SL', _('Sierra Leone')), + ('SM', _('San Marino')), + ('SN', _('Senegal')), + ('SO', _('Somalia')), + ('SR', _('Suriname')), + ('ST', _('Sao Tome & Principe')), + ('SV', _('El Salvador')), + ('SY', _('Syrian Arab Republic')), + ('SZ', _('Swaziland')), + ('TC', _('Turks & Caicos Islands')), + ('TD', _('Chad')), + ('TF', _('French Southern Territories')), + ('TG', _('Togo')), + ('TH', _('Thailand')), + ('TJ', _('Tajikistan')), + ('TK', _('Tokelau')), + ('TM', _('Turkmenistan')), + ('TN', _('Tunisia')), + ('TO', _('Tonga')), + ('TP', _('East Timor')), + ('TR', _('Turkey')), + ('TT', _('Trinidad & Tobago')), + ('TV', _('Tuvalu')), + ('TW', _('Taiwan, Province of China')), + ('TZ', _('Tanzania, United Republic of')), + ('UA', _('Ukraine')), + ('UG', _('Uganda')), + ('UM', _('United States Minor Outlying Islands')), + ('US', _('United States of America')), + ('UY', _('Uruguay')), + ('UZ', _('Uzbekistan')), + ('VA', _('Vatican City State (Holy See)')), + ('VC', _('St. Vincent & the Grenadines')), + ('VE', _('Venezuela')), + ('VG', _('British Virgin Islands')), + ('VI', _('United States Virgin Islands')), + ('VN', _('Viet Nam')), + ('VU', _('Vanuatu')), + ('WF', _('Wallis & Futuna Islands')), + ('WS', _('Samoa')), + ('YE', _('Yemen')), + ('YT', _('Mayotte')), + ('YU', _('Yugoslavia')), + ('ZA', _('South Africa')), + ('ZM', _('Zambia')), + ('ZR', _('Zaire')), + ('ZW', _('Zimbabwe')), +) + + +__all__ = () diff --git a/uncloud/admin.py b/uncloud/admin.py new file mode 100644 index 0000000..38f8cce --- /dev/null +++ b/uncloud/admin.py @@ -0,0 +1,6 @@ +from django.contrib import admin + +from .models import * + +for m in [ UncloudProvider, UncloudNetwork ]: + admin.site.register(m) diff --git a/uncloud/api/create_image_store.py b/uncloud/api/create_image_store.py deleted file mode 100755 index 73b92f1..0000000 --- a/uncloud/api/create_image_store.py +++ /dev/null @@ -1,20 +0,0 @@ -import json -import os - -from uuid import uuid4 - -from uncloud.shared import shared -from uncloud.settings import settings - -data = { - "is_public": True, - "type": "ceph", - "name": "images", - "description": "first ever public image-store", - "attributes": {"list": [], "key": [], "pool": "images"}, -} - -shared.etcd_client.put( - os.path.join(settings["etcd"]["image_store_prefix"], uuid4().hex), - json.dumps(data), -) diff --git a/uncloud/asgi.py b/uncloud/asgi.py new file mode 100644 index 0000000..2b5a7a3 --- /dev/null +++ b/uncloud/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for uncloud project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + +application = get_asgi_application() diff --git a/uncloud/common/etcd_wrapper.py b/uncloud/common/etcd_wrapper.py deleted file mode 100644 index 6a979ba..0000000 --- a/uncloud/common/etcd_wrapper.py +++ /dev/null @@ -1,118 +0,0 @@ -import etcd3 -import json -import queue -import copy -from uncloud import UncloudException - -from collections import namedtuple -from functools import wraps - -from . import logger - -PseudoEtcdMeta = namedtuple("PseudoEtcdMeta", ["key"]) - - -class EtcdEntry: - # key: str - # value: str - - def __init__(self, meta, value, value_in_json=False): - self.key = meta.key.decode("utf-8") - self.value = value.decode("utf-8") - - if value_in_json: - self.value = json.loads(self.value) - - -def readable_errors(func): - @wraps(func) - def wrapper(*args, **kwargs): - try: - return func(*args, **kwargs) - except etcd3.exceptions.ConnectionFailedError as err: - raise UncloudException( - "Cannot connect to etcd: is etcd running as configured in uncloud.conf?" - ) - except etcd3.exceptions.ConnectionTimeoutError as err: - raise etcd3.exceptions.ConnectionTimeoutError( - "etcd connection timeout." - ) from err - except Exception: - logger.exception( - "Some etcd error occured. See syslog for details." - ) - - return wrapper - - -class Etcd3Wrapper: - @readable_errors - def __init__(self, *args, **kwargs): - self.client = etcd3.client(*args, **kwargs) - - @readable_errors - def get(self, *args, value_in_json=False, **kwargs): - _value, _key = self.client.get(*args, **kwargs) - if _key is None or _value is None: - return None - return EtcdEntry(_key, _value, value_in_json=value_in_json) - - @readable_errors - def put(self, *args, value_in_json=False, **kwargs): - _key, _value = args - if value_in_json: - _value = json.dumps(_value) - - if not isinstance(_key, str): - _key = _key.decode("utf-8") - - return self.client.put(_key, _value, **kwargs) - - @readable_errors - def get_prefix(self, *args, value_in_json=False, **kwargs): - r = self.client.get_prefix(*args, **kwargs) - for entry in r: - e = EtcdEntry(*entry[::-1], value_in_json=value_in_json) - if e.value: - yield e - - @readable_errors - def watch_prefix(self, key, timeout=0, value_in_json=False): - timeout_event = EtcdEntry( - PseudoEtcdMeta(key=b"TIMEOUT"), - value=str.encode( - json.dumps({"status": "TIMEOUT", "type": "TIMEOUT"}) - ), - value_in_json=value_in_json, - ) - - event_queue = queue.Queue() - - def add_event_to_queue(event): - if hasattr(event, "events"): - for e in event.events: - if e.value: - event_queue.put( - EtcdEntry( - e, e.value, value_in_json=value_in_json - ) - ) - - self.client.add_watch_prefix_callback(key, add_event_to_queue) - - while True: - try: - while True: - v = event_queue.get(timeout=timeout) - yield v - except queue.Empty: - event_queue.put(copy.deepcopy(timeout_event)) - - -class PsuedoEtcdEntry(EtcdEntry): - def __init__(self, key, value, value_in_json=False): - super().__init__( - PseudoEtcdMeta(key=key.encode("utf-8")), - value, - value_in_json=value_in_json, - ) diff --git a/uncloud/configure/main.py b/uncloud/configure/main.py deleted file mode 100644 index a9b4901..0000000 --- a/uncloud/configure/main.py +++ /dev/null @@ -1,79 +0,0 @@ -import os - -from uncloud.settings import settings -from uncloud.shared import shared - - -def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get( - settings.config_key, value_in_json=True - ) - if not uncloud_config: - uncloud_config = {} - else: - uncloud_config = uncloud_config.value - - uncloud_config[section] = kwargs - shared.etcd_client.put( - settings.config_key, uncloud_config, value_in_json=True - ) - - -def configure_parser(parser): - configure_subparsers = parser.add_subparsers(dest="subcommand") - - otp_parser = configure_subparsers.add_parser("otp") - otp_parser.add_argument( - "--verification-controller-url", required=True, metavar="URL" - ) - otp_parser.add_argument( - "--auth-name", required=True, metavar="OTP-NAME" - ) - otp_parser.add_argument( - "--auth-realm", required=True, metavar="OTP-REALM" - ) - otp_parser.add_argument( - "--auth-seed", required=True, metavar="OTP-SEED" - ) - - network_parser = configure_subparsers.add_parser("network") - network_parser.add_argument( - "--prefix-length", required=True, type=int - ) - network_parser.add_argument("--prefix", required=True) - network_parser.add_argument("--vxlan-phy-dev", required=True) - - netbox_parser = configure_subparsers.add_parser("netbox") - netbox_parser.add_argument("--url", required=True) - netbox_parser.add_argument("--token", required=True) - - ssh_parser = configure_subparsers.add_parser("ssh") - ssh_parser.add_argument("--username", default="root") - ssh_parser.add_argument( - "--private-key-path", - default=os.path.expanduser("~/.ssh/id_rsa"), - ) - - storage_parser = configure_subparsers.add_parser("storage") - storage_parser.add_argument("--file-dir", required=True) - storage_parser_subparsers = storage_parser.add_subparsers( - dest="storage_backend" - ) - - filesystem_storage_parser = storage_parser_subparsers.add_parser( - "filesystem" - ) - filesystem_storage_parser.add_argument("--vm-dir", required=True) - filesystem_storage_parser.add_argument("--image-dir", required=True) - - ceph_storage_parser = storage_parser_subparsers.add_parser("ceph") - ceph_storage_parser.add_argument("--ceph-vm-pool", required=True) - ceph_storage_parser.add_argument("--ceph-image-pool", required=True) - - -def main(**kwargs): - subcommand = kwargs.pop("subcommand") - if not subcommand: - pass - else: - update_config(subcommand, kwargs) diff --git a/uncloud/docs/source/hacking.rst b/uncloud/docs/source/hacking.rst deleted file mode 100644 index 2df42a7..0000000 --- a/uncloud/docs/source/hacking.rst +++ /dev/null @@ -1,17 +0,0 @@ -Hacking -======= -How to hack on the code. - -[ to be done by Balazs: - -* make nice -* indent with shell script mode - -] - -* git clone the repo -* cd to the repo -* Setup your venv: python -m venv venv -* . ./venv/bin/activate # you need the leading dot for sourcing! -* Run ./bin/ucloud-run-reinstall - it should print you an error - message on how to use ucloud diff --git a/uncloud/filescanner/main.py b/uncloud/filescanner/main.py deleted file mode 100755 index 7ce8654..0000000 --- a/uncloud/filescanner/main.py +++ /dev/null @@ -1,89 +0,0 @@ -import glob -import os -import pathlib -import subprocess as sp -import time - -from uuid import uuid4 - -from . import logger -from uncloud.settings import settings -from uncloud.shared import shared - - -def sha512sum(file: str): - """Use sha512sum utility to compute sha512 sum of arg:file - - IF arg:file does not exists: - raise FileNotFoundError exception - ELSE IF sum successfully computer: - return computed sha512 sum - ELSE: - return None - """ - if not isinstance(file, str): - raise TypeError - try: - output = sp.check_output(["sha512sum", file], stderr=sp.PIPE) - except sp.CalledProcessError as e: - error = e.stderr.decode("utf-8") - if "No such file or directory" in error: - raise FileNotFoundError from None - else: - output = output.decode("utf-8").strip() - output = output.split(" ") - return output[0] - return None - - -def track_file(file, base_dir): - file_id = uuid4() - - # Get Username - owner = pathlib.Path(file).parts[len(pathlib.Path(base_dir).parts)] - - # Get Creation Date of File - # Here, we are assuming that ctime is creation time - # which is mostly not true. - creation_date = time.ctime(os.stat(file).st_ctime) - - file_path = pathlib.Path(file).parts[-1] - - # Create Entry - entry_key = os.path.join( - settings["etcd"]["file_prefix"], str(file_id) - ) - entry_value = { - "filename": file_path, - "owner": owner, - "sha512sum": sha512sum(file), - "creation_date": creation_date, - "size": os.path.getsize(file), - } - - logger.info("Tracking %s", file) - - shared.etcd_client.put(entry_key, entry_value, value_in_json=True) - os.setxattr(file, "user.utracked", b"True") - - -def main(): - base_dir = settings["storage"]["file_dir"] - - # Recursively Get All Files and Folder below BASE_DIR - files = glob.glob("{}/**".format(base_dir), recursive=True) - - # Retain only Files - files = [file for file in files if os.path.isfile(file)] - - untracked_files = [] - for file in files: - try: - os.getxattr(file, "user.utracked") - except OSError: - track_file(file, base_dir) - untracked_files.append(file) - - -if __name__ == "__main__": - main() diff --git a/uncloud/forms.py b/uncloud/forms.py new file mode 100644 index 0000000..153a49a --- /dev/null +++ b/uncloud/forms.py @@ -0,0 +1,8 @@ +from django import forms +from django.contrib.auth.models import User + + +class UserDeleteForm(forms.ModelForm): + class Meta: + model = User + fields = [] diff --git a/uncloud/host/main.py b/uncloud/host/main.py deleted file mode 100755 index d1e7c9a..0000000 --- a/uncloud/host/main.py +++ /dev/null @@ -1,116 +0,0 @@ -import argparse -import multiprocessing as mp -import time -from uuid import uuid4 - -from uncloud.common.request import RequestEntry, RequestType -from uncloud.shared import shared -from uncloud.settings import settings -from uncloud.common.vm import VMStatus -from uncloud.vmm import VMM -from os.path import join as join_path - -from . import virtualmachine, logger - - -def update_heartbeat(hostname): - """Update Last HeartBeat Time for :param hostname: in etcd""" - host_pool = shared.host_pool - this_host = next( - filter(lambda h: h.hostname == hostname, host_pool.hosts), None - ) - while True: - this_host.update_heartbeat() - host_pool.put(this_host) - time.sleep(10) - - -def maintenance(host): - vmm = VMM() - running_vms = vmm.discover() - for vm_uuid in running_vms: - if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == "running": - logger.debug('VM {} is running on {}'.format(vm_uuid, host)) - vm = shared.vm_pool.get( - join_path(settings["etcd"]["vm_prefix"], vm_uuid) - ) - vm.status = VMStatus.running - vm.vnc_socket = vmm.get_vnc(vm_uuid) - vm.hostname = host - shared.vm_pool.put(vm) - - -def main(hostname): - host_pool = shared.host_pool - host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None) - - # Does not yet exist, create it - if not host: - host_key = join_path( - settings["etcd"]["host_prefix"], uuid4().hex - ) - host_entry = { - "specs": "", - "hostname": hostname, - "status": "DEAD", - "last_heartbeat": "", - } - shared.etcd_client.put( - host_key, host_entry, value_in_json=True - ) - - try: - heartbeat_updating_process = mp.Process(target=update_heartbeat, args=(hostname,)) - heartbeat_updating_process.start() - except Exception as e: - raise Exception("uncloud-host heartbeat updating mechanism is not working") from e - - for events_iterator in [ - shared.etcd_client.get_prefix(settings["etcd"]["request_prefix"], value_in_json=True), - shared.etcd_client.watch_prefix(settings["etcd"]["request_prefix"], timeout=10, value_in_json=True) - ]: - for request_event in events_iterator: - request_event = RequestEntry(request_event) - - if request_event.type == "TIMEOUT": - maintenance(host.key) - - elif request_event.hostname == host.key: - logger.debug("VM Request: %s on Host %s", request_event, host.hostname) - shared.request_pool.client.client.delete(request_event.key) - vm_entry = shared.etcd_client.get( - join_path(settings["etcd"]["vm_prefix"], request_event.uuid) - ) - logger.debug("VM hostname: {}".format(vm_entry.value)) - vm = virtualmachine.VM(vm_entry) - if request_event.type == RequestType.StartVM: - vm.start() - - elif request_event.type == RequestType.StopVM: - vm.stop() - - elif request_event.type == RequestType.DeleteVM: - vm.delete() - - elif request_event.type == RequestType.InitVMMigration: - vm.start(destination_host_key=host.key) - - elif request_event.type == RequestType.TransferVM: - destination_host = host_pool.get(request_event.destination_host_key) - if destination_host: - vm.migrate( - destination_host=destination_host.hostname, - destination_sock_path=request_event.destination_sock_path, - ) - else: - logger.error("Host %s not found!", request_event.destination_host_key) - - -if __name__ == "__main__": - argparser = argparse.ArgumentParser() - argparser.add_argument( - "hostname", help="Name of this host. e.g uncloud1.ungleich.ch" - ) - args = argparser.parse_args() - mp.set_start_method("spawn") - main(args.hostname) diff --git a/uncloud/management/commands/db-add-defaults.py b/uncloud/management/commands/db-add-defaults.py new file mode 100644 index 0000000..605c8f5 --- /dev/null +++ b/uncloud/management/commands/db-add-defaults.py @@ -0,0 +1,43 @@ +import random +import string + +from django.core.management.base import BaseCommand +from django.core.exceptions import ObjectDoesNotExist +from django.contrib.auth import get_user_model +from django.conf import settings + +from uncloud_pay.models import BillingAddress, RecurringPeriod, Product +from uncloud.models import UncloudProvider, UncloudNetwork + + +class Command(BaseCommand): + help = 'Add standard uncloud values' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + # Order matters, objects can be dependent on each other + + admin_username="uncloud-admin" + pw_length = 32 + + # Only set password if the user did not exist before + try: + admin_user = get_user_model().objects.get(username=settings.UNCLOUD_ADMIN_NAME) + except ObjectDoesNotExist: + random_password = ''.join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(pw_length)) + + admin_user = get_user_model().objects.create_user(username=settings.UNCLOUD_ADMIN_NAME, password=random_password) + admin_user.is_superuser=True + admin_user.is_staff=True + admin_user.save() + + print(f"Created admin user '{admin_username}' with password '{random_password}'") + + BillingAddress.populate_db_defaults() + RecurringPeriod.populate_db_defaults() + Product.populate_db_defaults() + + UncloudNetwork.populate_db_defaults() + UncloudProvider.populate_db_defaults() diff --git a/uncloud/management/commands/uncloud.py b/uncloud/management/commands/uncloud.py new file mode 100644 index 0000000..bd47c6b --- /dev/null +++ b/uncloud/management/commands/uncloud.py @@ -0,0 +1,28 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand + +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster + +import logging +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'General uncloud commands' + + def add_arguments(self, parser): + parser.add_argument('--bootstrap', action='store_true', help='Bootstrap a typical uncloud installation') + + def handle(self, *args, **options): + + if options['bootstrap']: + self.bootstrap() + + def bootstrap(self): + default_cluster = VMCluster.objects.get_or_create(name="default") +# local_host = diff --git a/uncloud/migrations/0001_initial.py b/uncloud/migrations/0001_initial.py new file mode 100644 index 0000000..10d1144 --- /dev/null +++ b/uncloud/migrations/0001_initial.py @@ -0,0 +1,46 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import uncloud.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ] + + operations = [ + migrations.CreateModel( + name='UncloudNetwork', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('network_address', models.GenericIPAddressField(unique=True)), + ('network_mask', models.IntegerField(validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(128)])), + ('description', models.CharField(max_length=256)), + ], + ), + migrations.CreateModel( + name='UncloudProvider', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('full_name', models.CharField(max_length=256)), + ('organization', models.CharField(blank=True, max_length=256, null=True)), + ('street', models.CharField(max_length=256)), + ('city', models.CharField(max_length=256)), + ('postal_code', models.CharField(max_length=64)), + ('country', uncloud.models.CountryField(blank=True, choices=[('AD', 'Andorra'), ('AE', 'United Arab Emirates'), ('AF', 'Afghanistan'), ('AG', 'Antigua & Barbuda'), ('AI', 'Anguilla'), ('AL', 'Albania'), ('AM', 'Armenia'), ('AN', 'Netherlands Antilles'), ('AO', 'Angola'), ('AQ', 'Antarctica'), ('AR', 'Argentina'), ('AS', 'American Samoa'), ('AT', 'Austria'), ('AU', 'Australia'), ('AW', 'Aruba'), ('AZ', 'Azerbaijan'), ('BA', 'Bosnia and Herzegovina'), ('BB', 'Barbados'), ('BD', 'Bangladesh'), ('BE', 'Belgium'), ('BF', 'Burkina Faso'), ('BG', 'Bulgaria'), ('BH', 'Bahrain'), ('BI', 'Burundi'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BN', 'Brunei Darussalam'), ('BO', 'Bolivia'), ('BR', 'Brazil'), ('BS', 'Bahama'), ('BT', 'Bhutan'), ('BV', 'Bouvet Island'), ('BW', 'Botswana'), ('BY', 'Belarus'), ('BZ', 'Belize'), ('CA', 'Canada'), ('CC', 'Cocos (Keeling) Islands'), ('CF', 'Central African Republic'), ('CG', 'Congo'), ('CH', 'Switzerland'), ('CI', 'Ivory Coast'), ('CK', 'Cook Iislands'), ('CL', 'Chile'), ('CM', 'Cameroon'), ('CN', 'China'), ('CO', 'Colombia'), ('CR', 'Costa Rica'), ('CU', 'Cuba'), ('CV', 'Cape Verde'), ('CX', 'Christmas Island'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DE', 'Germany'), ('DJ', 'Djibouti'), ('DK', 'Denmark'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('DZ', 'Algeria'), ('EC', 'Ecuador'), ('EE', 'Estonia'), ('EG', 'Egypt'), ('EH', 'Western Sahara'), ('ER', 'Eritrea'), ('ES', 'Spain'), ('ET', 'Ethiopia'), ('FI', 'Finland'), ('FJ', 'Fiji'), ('FK', 'Falkland Islands (Malvinas)'), ('FM', 'Micronesia'), ('FO', 'Faroe Islands'), ('FR', 'France'), ('FX', 'France, Metropolitan'), ('GA', 'Gabon'), ('GB', 'United Kingdom (Great Britain)'), ('GD', 'Grenada'), ('GE', 'Georgia'), ('GF', 'French Guiana'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GL', 'Greenland'), ('GM', 'Gambia'), ('GN', 'Guinea'), ('GP', 'Guadeloupe'), ('GQ', 'Equatorial Guinea'), ('GR', 'Greece'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GT', 'Guatemala'), ('GU', 'Guam'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HK', 'Hong Kong'), ('HM', 'Heard & McDonald Islands'), ('HN', 'Honduras'), ('HR', 'Croatia'), ('HT', 'Haiti'), ('HU', 'Hungary'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('IL', 'Israel'), ('IN', 'India'), ('IO', 'British Indian Ocean Territory'), ('IQ', 'Iraq'), ('IR', 'Islamic Republic of Iran'), ('IS', 'Iceland'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JO', 'Jordan'), ('JP', 'Japan'), ('KE', 'Kenya'), ('KG', 'Kyrgyzstan'), ('KH', 'Cambodia'), ('KI', 'Kiribati'), ('KM', 'Comoros'), ('KN', 'St. Kitts and Nevis'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('KW', 'Kuwait'), ('KY', 'Cayman Islands'), ('KZ', 'Kazakhstan'), ('LA', "Lao People's Democratic Republic"), ('LB', 'Lebanon'), ('LC', 'Saint Lucia'), ('LI', 'Liechtenstein'), ('LK', 'Sri Lanka'), ('LR', 'Liberia'), ('LS', 'Lesotho'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('LV', 'Latvia'), ('LY', 'Libyan Arab Jamahiriya'), ('MA', 'Morocco'), ('MC', 'Monaco'), ('MD', 'Moldova, Republic of'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('ML', 'Mali'), ('MN', 'Mongolia'), ('MM', 'Myanmar'), ('MO', 'Macau'), ('MP', 'Northern Mariana Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MS', 'Monserrat'), ('MT', 'Malta'), ('MU', 'Mauritius'), ('MV', 'Maldives'), ('MW', 'Malawi'), ('MX', 'Mexico'), ('MY', 'Malaysia'), ('MZ', 'Mozambique'), ('NA', 'Namibia'), ('NC', 'New Caledonia'), ('NE', 'Niger'), ('NF', 'Norfolk Island'), ('NG', 'Nigeria'), ('NI', 'Nicaragua'), ('NL', 'Netherlands'), ('NO', 'Norway'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('NU', 'Niue'), ('NZ', 'New Zealand'), ('OM', 'Oman'), ('PA', 'Panama'), ('PE', 'Peru'), ('PF', 'French Polynesia'), ('PG', 'Papua New Guinea'), ('PH', 'Philippines'), ('PK', 'Pakistan'), ('PL', 'Poland'), ('PM', 'St. Pierre & Miquelon'), ('PN', 'Pitcairn'), ('PR', 'Puerto Rico'), ('PT', 'Portugal'), ('PW', 'Palau'), ('PY', 'Paraguay'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SA', 'Saudi Arabia'), ('SB', 'Solomon Islands'), ('SC', 'Seychelles'), ('SD', 'Sudan'), ('SE', 'Sweden'), ('SG', 'Singapore'), ('SH', 'St. Helena'), ('SI', 'Slovenia'), ('SJ', 'Svalbard & Jan Mayen Islands'), ('SK', 'Slovakia'), ('SL', 'Sierra Leone'), ('SM', 'San Marino'), ('SN', 'Senegal'), ('SO', 'Somalia'), ('SR', 'Suriname'), ('ST', 'Sao Tome & Principe'), ('SV', 'El Salvador'), ('SY', 'Syrian Arab Republic'), ('SZ', 'Swaziland'), ('TC', 'Turks & Caicos Islands'), ('TD', 'Chad'), ('TF', 'French Southern Territories'), ('TG', 'Togo'), ('TH', 'Thailand'), ('TJ', 'Tajikistan'), ('TK', 'Tokelau'), ('TM', 'Turkmenistan'), ('TN', 'Tunisia'), ('TO', 'Tonga'), ('TP', 'East Timor'), ('TR', 'Turkey'), ('TT', 'Trinidad & Tobago'), ('TV', 'Tuvalu'), ('TW', 'Taiwan, Province of China'), ('TZ', 'Tanzania, United Republic of'), ('UA', 'Ukraine'), ('UG', 'Uganda'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VA', 'Vatican City State (Holy See)'), ('VC', 'St. Vincent & the Grenadines'), ('VE', 'Venezuela'), ('VG', 'British Virgin Islands'), ('VI', 'United States Virgin Islands'), ('VN', 'Viet Nam'), ('VU', 'Vanuatu'), ('WF', 'Wallis & Futuna Islands'), ('WS', 'Samoa'), ('YE', 'Yemen'), ('YT', 'Mayotte'), ('YU', 'Yugoslavia'), ('ZA', 'South Africa'), ('ZM', 'Zambia'), ('ZR', 'Zaire'), ('ZW', 'Zimbabwe')], default='CH', max_length=2)), + ('starting_date', models.DateField()), + ('ending_date', models.DateField(blank=True, null=True)), + ('billing_network', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='uncloudproviderbill', to='uncloud.uncloudnetwork')), + ('coupon_network', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='uncloudprovidercoupon', to='uncloud.uncloudnetwork')), + ('referral_network', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='uncloudproviderreferral', to='uncloud.uncloudnetwork')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/uncloud/migrations/0002_uncloudtasks.py b/uncloud/migrations/0002_uncloudtasks.py new file mode 100644 index 0000000..9c69606 --- /dev/null +++ b/uncloud/migrations/0002_uncloudtasks.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2020-12-20 17:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='UncloudTasks', + fields=[ + ('task_id', models.UUIDField(primary_key=True, serialize=False)), + ], + ), + ] diff --git a/uncloud/migrations/0003_auto_20201220_1728.py b/uncloud/migrations/0003_auto_20201220_1728.py new file mode 100644 index 0000000..2ec0eec --- /dev/null +++ b/uncloud/migrations/0003_auto_20201220_1728.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1 on 2020-12-20 17:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud', '0002_uncloudtasks'), + ] + + operations = [ + migrations.RenameModel( + old_name='UncloudTasks', + new_name='UncloudTask', + ), + ] diff --git a/uncloud/migrations/0004_auto_20210101_1308.py b/uncloud/migrations/0004_auto_20210101_1308.py new file mode 100644 index 0000000..8385b16 --- /dev/null +++ b/uncloud/migrations/0004_auto_20210101_1308.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2021-01-01 13:08 + +from django.db import migrations +import uncloud.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud', '0003_auto_20201220_1728'), + ] + + operations = [ + migrations.AlterField( + model_name='uncloudprovider', + name='country', + field=uncloud.models.CountryField(choices=[('AD', 'Andorra'), ('AE', 'United Arab Emirates'), ('AF', 'Afghanistan'), ('AG', 'Antigua & Barbuda'), ('AI', 'Anguilla'), ('AL', 'Albania'), ('AM', 'Armenia'), ('AN', 'Netherlands Antilles'), ('AO', 'Angola'), ('AQ', 'Antarctica'), ('AR', 'Argentina'), ('AS', 'American Samoa'), ('AT', 'Austria'), ('AU', 'Australia'), ('AW', 'Aruba'), ('AZ', 'Azerbaijan'), ('BA', 'Bosnia and Herzegovina'), ('BB', 'Barbados'), ('BD', 'Bangladesh'), ('BE', 'Belgium'), ('BF', 'Burkina Faso'), ('BG', 'Bulgaria'), ('BH', 'Bahrain'), ('BI', 'Burundi'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BN', 'Brunei Darussalam'), ('BO', 'Bolivia'), ('BR', 'Brazil'), ('BS', 'Bahama'), ('BT', 'Bhutan'), ('BV', 'Bouvet Island'), ('BW', 'Botswana'), ('BY', 'Belarus'), ('BZ', 'Belize'), ('CA', 'Canada'), ('CC', 'Cocos (Keeling) Islands'), ('CF', 'Central African Republic'), ('CG', 'Congo'), ('CH', 'Switzerland'), ('CI', 'Ivory Coast'), ('CK', 'Cook Iislands'), ('CL', 'Chile'), ('CM', 'Cameroon'), ('CN', 'China'), ('CO', 'Colombia'), ('CR', 'Costa Rica'), ('CU', 'Cuba'), ('CV', 'Cape Verde'), ('CX', 'Christmas Island'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DE', 'Germany'), ('DJ', 'Djibouti'), ('DK', 'Denmark'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('DZ', 'Algeria'), ('EC', 'Ecuador'), ('EE', 'Estonia'), ('EG', 'Egypt'), ('EH', 'Western Sahara'), ('ER', 'Eritrea'), ('ES', 'Spain'), ('ET', 'Ethiopia'), ('FI', 'Finland'), ('FJ', 'Fiji'), ('FK', 'Falkland Islands (Malvinas)'), ('FM', 'Micronesia'), ('FO', 'Faroe Islands'), ('FR', 'France'), ('FX', 'France, Metropolitan'), ('GA', 'Gabon'), ('GB', 'United Kingdom (Great Britain)'), ('GD', 'Grenada'), ('GE', 'Georgia'), ('GF', 'French Guiana'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GL', 'Greenland'), ('GM', 'Gambia'), ('GN', 'Guinea'), ('GP', 'Guadeloupe'), ('GQ', 'Equatorial Guinea'), ('GR', 'Greece'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GT', 'Guatemala'), ('GU', 'Guam'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HK', 'Hong Kong'), ('HM', 'Heard & McDonald Islands'), ('HN', 'Honduras'), ('HR', 'Croatia'), ('HT', 'Haiti'), ('HU', 'Hungary'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('IL', 'Israel'), ('IN', 'India'), ('IO', 'British Indian Ocean Territory'), ('IQ', 'Iraq'), ('IR', 'Islamic Republic of Iran'), ('IS', 'Iceland'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JO', 'Jordan'), ('JP', 'Japan'), ('KE', 'Kenya'), ('KG', 'Kyrgyzstan'), ('KH', 'Cambodia'), ('KI', 'Kiribati'), ('KM', 'Comoros'), ('KN', 'St. Kitts and Nevis'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('KW', 'Kuwait'), ('KY', 'Cayman Islands'), ('KZ', 'Kazakhstan'), ('LA', "Lao People's Democratic Republic"), ('LB', 'Lebanon'), ('LC', 'Saint Lucia'), ('LI', 'Liechtenstein'), ('LK', 'Sri Lanka'), ('LR', 'Liberia'), ('LS', 'Lesotho'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('LV', 'Latvia'), ('LY', 'Libyan Arab Jamahiriya'), ('MA', 'Morocco'), ('MC', 'Monaco'), ('MD', 'Moldova, Republic of'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('ML', 'Mali'), ('MN', 'Mongolia'), ('MM', 'Myanmar'), ('MO', 'Macau'), ('MP', 'Northern Mariana Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MS', 'Monserrat'), ('MT', 'Malta'), ('MU', 'Mauritius'), ('MV', 'Maldives'), ('MW', 'Malawi'), ('MX', 'Mexico'), ('MY', 'Malaysia'), ('MZ', 'Mozambique'), ('NA', 'Namibia'), ('NC', 'New Caledonia'), ('NE', 'Niger'), ('NF', 'Norfolk Island'), ('NG', 'Nigeria'), ('NI', 'Nicaragua'), ('NL', 'Netherlands'), ('NO', 'Norway'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('NU', 'Niue'), ('NZ', 'New Zealand'), ('OM', 'Oman'), ('PA', 'Panama'), ('PE', 'Peru'), ('PF', 'French Polynesia'), ('PG', 'Papua New Guinea'), ('PH', 'Philippines'), ('PK', 'Pakistan'), ('PL', 'Poland'), ('PM', 'St. Pierre & Miquelon'), ('PN', 'Pitcairn'), ('PR', 'Puerto Rico'), ('PT', 'Portugal'), ('PW', 'Palau'), ('PY', 'Paraguay'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SA', 'Saudi Arabia'), ('SB', 'Solomon Islands'), ('SC', 'Seychelles'), ('SD', 'Sudan'), ('SE', 'Sweden'), ('SG', 'Singapore'), ('SH', 'St. Helena'), ('SI', 'Slovenia'), ('SJ', 'Svalbard & Jan Mayen Islands'), ('SK', 'Slovakia'), ('SL', 'Sierra Leone'), ('SM', 'San Marino'), ('SN', 'Senegal'), ('SO', 'Somalia'), ('SR', 'Suriname'), ('ST', 'Sao Tome & Principe'), ('SV', 'El Salvador'), ('SY', 'Syrian Arab Republic'), ('SZ', 'Swaziland'), ('TC', 'Turks & Caicos Islands'), ('TD', 'Chad'), ('TF', 'French Southern Territories'), ('TG', 'Togo'), ('TH', 'Thailand'), ('TJ', 'Tajikistan'), ('TK', 'Tokelau'), ('TM', 'Turkmenistan'), ('TN', 'Tunisia'), ('TO', 'Tonga'), ('TP', 'East Timor'), ('TR', 'Turkey'), ('TT', 'Trinidad & Tobago'), ('TV', 'Tuvalu'), ('TW', 'Taiwan, Province of China'), ('TZ', 'Tanzania, United Republic of'), ('UA', 'Ukraine'), ('UG', 'Uganda'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VA', 'Vatican City State (Holy See)'), ('VC', 'St. Vincent & the Grenadines'), ('VE', 'Venezuela'), ('VG', 'British Virgin Islands'), ('VI', 'United States Virgin Islands'), ('VN', 'Viet Nam'), ('VU', 'Vanuatu'), ('WF', 'Wallis & Futuna Islands'), ('WS', 'Samoa'), ('YE', 'Yemen'), ('YT', 'Mayotte'), ('YU', 'Yugoslavia'), ('ZA', 'South Africa'), ('ZM', 'Zambia'), ('ZR', 'Zaire'), ('ZW', 'Zimbabwe')], default='CH', max_length=2), + ), + ] diff --git a/uncloud/migrations/0005_delete_uncloudtask.py b/uncloud/migrations/0005_delete_uncloudtask.py new file mode 100644 index 0000000..6d9b095 --- /dev/null +++ b/uncloud/migrations/0005_delete_uncloudtask.py @@ -0,0 +1,16 @@ +# Generated by Django 3.2.4 on 2021-07-07 15:11 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud', '0004_auto_20210101_1308'), + ] + + operations = [ + migrations.DeleteModel( + name='UncloudTask', + ), + ] diff --git a/uncloud/migrations/__init__.py b/uncloud/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/models.py b/uncloud/models.py new file mode 100644 index 0000000..c2b3cf9 --- /dev/null +++ b/uncloud/models.py @@ -0,0 +1,209 @@ +from django.db import models +from django.db.models import JSONField, Q +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ +from django.core.validators import MinValueValidator, MaxValueValidator +from django.core.exceptions import FieldError + +from uncloud import COUNTRIES +from .selectors import filter_for_when + +class UncloudModel(models.Model): + """ + This class extends the standard model with an + extra_data field that can be used to include public, + but internal information. + + For instance if you migrate from an existing virtualisation + framework to uncloud. + + The extra_data attribute should be considered a hack and whenever + data is necessary for running uncloud, it should **not** be stored + in there. + + """ + + extra_data = JSONField(editable=False, blank=True, null=True) + + class Meta: + abstract = True + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class UncloudStatus(models.TextChoices): + PENDING = 'PENDING', _('Pending') + AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment') + BEING_CREATED = 'BEING_CREATED', _('Being created') + SCHEDULED = 'SCHEDULED', _('Scheduled') # resource selected, waiting for dispatching + ACTIVE = 'ACTIVE', _('Active') + MODIFYING = 'MODIFYING', _('Modifying') # Resource is being changed + DELETED = 'DELETED', _('Deleted') # Resource has been deleted + DISABLED = 'DISABLED', _('Disabled') # Is usable, but cannot be used for new things + UNUSABLE = 'UNUSABLE', _('Unusable'), # Has some kind of error + + + +### +# General address handling +class CountryField(models.CharField): + def __init__(self, *args, **kwargs): + kwargs.setdefault('choices', COUNTRIES) + kwargs.setdefault('default', 'CH') + kwargs.setdefault('max_length', 2) + + super().__init__(*args, **kwargs) + + def get_internal_type(self): + return "CharField" + + +class UncloudAddress(models.Model): + full_name = models.CharField(max_length=256, null=False) + organization = models.CharField(max_length=256, blank=True, null=True) + street = models.CharField(max_length=256, null=False) + city = models.CharField(max_length=256, null=False) + postal_code = models.CharField(max_length=64) + country = CountryField(blank=False, null=False) + + class Meta: + abstract = True + + +class UncloudValidTimeFrame(models.Model): + """ + A model that allows to limit validity of something to a certain + time frame. Used for versioning basically. + + Logic: + + """ + + class Meta: + abstract = True + + constraints = [ + models.UniqueConstraint(fields=['owner'], + condition=models.Q(active=True), + name='one_active_card_per_user') + ] + + + valid_from = models.DateTimeField(default=timezone.now, null=True, blank=True) + valid_to = models.DateTimeField(null=True, blank=True) + + @classmethod + def get_current(cls, *args, **kwargs): + now = timezone.now() + + # With both given + cls.objects.filter(valid_from__lte=now, + valid_to__gte=now) + + # With to missing + cls.objects.filter(valid_from__lte=now, + valid_to__isnull=true) + + # With from missing + cls.objects.filter(valid_from__isnull=true, + valid_to__gte=now) + + # Both missing + cls.objects.filter(valid_from__isnull=true, + valid_to__gte=now) + + + + + +### +# UncloudNetworks are used as identifiers - such they are a base of uncloud + +class UncloudNetwork(models.Model): + """ + Storing IP networks + """ + + network_address = models.GenericIPAddressField(null=False, unique=True) + network_mask = models.IntegerField(null=False, + validators=[MinValueValidator(0), + MaxValueValidator(128)] + ) + + description = models.CharField(max_length=256) + + @classmethod + def populate_db_defaults(cls): + for net, desc in [ + ( "2a0a:e5c0:11::", "uncloud Billing" ), + ( "2a0a:e5c0:11:1::", "uncloud Referral" ), + ( "2a0a:e5c0:11:2::", "uncloud Coupon" ) + ]: + obj, created = cls.objects.get_or_create(network_address=net, + defaults= { + 'network_mask': 64, + 'description': desc + } + ) + + + def save(self, *args, **kwargs): + if not ':' in self.network_address and self.network_mask > 32: + raise FieldError("Mask cannot exceed 32 for IPv4") + + super().save(*args, **kwargs) + + + def __str__(self): + return f"{self.network_address}/{self.network_mask} {self.description}" + +### +# Who is running / providing this instance of uncloud? + +class UncloudProvider(UncloudAddress): + """ + A class resembling who is running this uncloud instance. + This might change over time so we allow starting/ending dates + + This also defines the taxation rules. + + starting/ending date define from when to when this is valid. This way + we can model address changes and have it correct in the bills. + """ + + # Meta: + # FIXMe: only allow non overlapping time frames -- how to define this as a constraint? + starting_date = models.DateField() + ending_date = models.DateField(blank=True, null=True) + + billing_network = models.ForeignKey(UncloudNetwork, related_name="uncloudproviderbill", on_delete=models.CASCADE) + referral_network = models.ForeignKey(UncloudNetwork, related_name="uncloudproviderreferral", on_delete=models.CASCADE) + coupon_network = models.ForeignKey(UncloudNetwork, related_name="uncloudprovidercoupon", on_delete=models.CASCADE) + + + @classmethod + def get_provider(cls, when=None): + """ + Find active provide at a certain time - if there was any + """ + + + return cls.objects.get(Q(starting_date__gte=when, ending_date__lte=when) | + Q(starting_date__gte=when, ending_date__isnull=True)) + + + @classmethod + def populate_db_defaults(cls): + obj, created = cls.objects.get_or_create(full_name="ungleich glarus ag", + street="Bahnhofstrasse 1", + postal_code="8783", + city="Linthal", + country="CH", + starting_date=timezone.now(), + billing_network=UncloudNetwork.objects.get(description="uncloud Billing"), + referral_network=UncloudNetwork.objects.get(description="uncloud Referral"), + coupon_network=UncloudNetwork.objects.get(description="uncloud Coupon") + ) + + + def __str__(self): + return f"{self.full_name} {self.country}" + diff --git a/uncloud/scheduler/main.py b/uncloud/scheduler/main.py deleted file mode 100755 index 5a4014f..0000000 --- a/uncloud/scheduler/main.py +++ /dev/null @@ -1,75 +0,0 @@ -# TODO -# 1. send an email to an email address defined by env['admin-email'] -# if resources are finished -# 2. Introduce a status endpoint of the scheduler - -# maybe expose a prometheus compatible output - -from uncloud.common.request import RequestEntry, RequestType -from uncloud.shared import shared -from uncloud.settings import settings -from .helper import ( - dead_host_mitigation, - dead_host_detection, - assign_host, - NoSuitableHostFound, -) -from . import logger - - -def main(): - for request_iterator in [ - shared.etcd_client.get_prefix( - settings["etcd"]["request_prefix"], value_in_json=True - ), - shared.etcd_client.watch_prefix( - settings["etcd"]["request_prefix"], - timeout=5, - value_in_json=True, - ), - ]: - for request_event in request_iterator: - request_entry = RequestEntry(request_event) - # Never Run time critical mechanism inside timeout - # mechanism because timeout mechanism only comes - # when no other event is happening. It means under - # heavy load there would not be a timeout event. - if request_entry.type == "TIMEOUT": - - # Detect hosts that are dead and set their status - # to "DEAD", and their VMs' status to "KILLED" - dead_hosts = dead_host_detection() - if dead_hosts: - logger.debug("Dead hosts: %s", dead_hosts) - dead_host_mitigation(dead_hosts) - - elif request_entry.type == RequestType.ScheduleVM: - print(request_event.value) - logger.debug( - "%s, %s", request_entry.key, request_entry.value - ) - - vm_entry = shared.vm_pool.get(request_entry.uuid) - if vm_entry is None: - logger.info( - "Trying to act on {} but it is deleted".format( - request_entry.uuid - ) - ) - continue - shared.etcd_client.client.delete( - request_entry.key - ) # consume Request - - try: - assign_host(vm_entry) - except NoSuitableHostFound: - vm_entry.add_log( - "Can't schedule VM. No Resource Left." - ) - shared.vm_pool.put(vm_entry) - - logger.info("No Resource Left. Emailing admin....") - - -if __name__ == "__main__": - main() diff --git a/uncloud/selectors.py b/uncloud/selectors.py new file mode 100644 index 0000000..52b8548 --- /dev/null +++ b/uncloud/selectors.py @@ -0,0 +1,23 @@ +from django.db.models import Q +from django.utils import timezone + +def filter_for_when(queryset, when=None): + """ + Return a filtered queryset which is valid for the given date + + Logic: + + Look for entries that have a starting date before when + and either + - No ending date + - Ending date after "when" + + Returns a queryset, you'll neet to apply .first() or similar on it + + """ + + if not when: + when = timezone.now() + + return queryset.filter(starting_date__lte=when).filter(Q(ending_date__gte=when) | + Q(ending_date__isnull=True)) diff --git a/uncloud/settings.py b/uncloud/settings.py new file mode 100644 index 0000000..be6cc11 --- /dev/null +++ b/uncloud/settings.py @@ -0,0 +1,265 @@ +""" +Django settings for uncloud project. + +Generated by 'django-admin startproject' using Django 3.0.3. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os +import re +import ldap +import sys + +from django.core.management.utils import get_random_secret_key +from django_auth_ldap.config import LDAPSearch, LDAPSearchUnion + + +LOGGING = {} + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +sys.modules['fontawesome_free'] = __import__('fontawesome-free') + +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +SITE_ID = 1 + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.sites', + 'allauth', + 'allauth.account', + 'allauth.socialaccount', + 'django.contrib.staticfiles', + 'django_extensions', + 'rest_framework', + 'bootstrap5', + 'django_q', + 'fontawesome_free', + 'uncloud', + 'uncloud_pay', + 'uncloud_auth', + 'uncloud_net', + 'uncloud_storage', + 'uncloud_vm', + 'uncloud_service', + 'opennebula', + 'matrixhosting', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'uncloud.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'uncloud.wsgi.application' + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] +############################################################################### +# Authall Settings +ACCOUNT_AUTHENTICATION_METHOD = "username" +ACCOUNT_EMAIL_CONFIRMATION_EXPIRE_DAYS = 1 +ACCOUNT_EMAIL_REQUIRED = False +ACCOUNT_EMAIL_VERIFICATION = "optional" +ACCOUNT_UNIQUE_EMAIL = False +################################################################################ +# AUTH/LDAP + +AUTH_LDAP_SERVER_URI = "" +AUTH_LDAP_BIND_DN = "" +AUTH_LDAP_BIND_PASSWORD = "" +AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=example,dc=com", + ldap.SCOPE_SUBTREE, + "(uid=%(user)s)") + +AUTH_LDAP_USER_ATTR_MAP = { + "first_name": "givenName", + "last_name": "sn", + "email": "mail" +} + +################################################################################ +# AUTH/Django +AUTHENTICATION_BACKENDS = [ + "django_auth_ldap.backend.LDAPBackend", + "django.contrib.auth.backends.ModelBackend", + 'allauth.account.auth_backends.AuthenticationBackend', +] + +AUTH_USER_MODEL = 'uncloud_auth.User' + + +################################################################################ +# AUTH/REST +REST_FRAMEWORK = { + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.BasicAuthentication', + 'rest_framework.authentication.SessionAuthentication', + ] +} + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ +STATIC_URL = '/static/' +STATICFILES_DIRS = [ os.path.join(BASE_DIR, "static") ] +STATICFILES_FINDERS = [ + 'django.contrib.staticfiles.finders.FileSystemFinder', + 'django.contrib.staticfiles.finders.AppDirectoriesFinder', +] + +#VM Deployment TEMPLATE +GITLAB_SERVER = 'https://code.ungleich.ch' +GITLAB_OAUTH_TOKEN = '' +GITLAB_PROJECT_ID = 388 +GITLAB_AUTHOR_EMAIL = '' +GITLAB_AUTHOR_NAME = '' +GITLAB_YAML_DIR = '' + +# XML-RPC interface of opennebula +OPENNEBULA_URL = 'https://opennebula.example.com:2634/RPC2' + +# user:pass for accessing opennebula +OPENNEBULA_USER_PASS = 'user:password' + +# Stripe (Credit Card payments) +STRIPE_KEY="" +STRIPE_PUBLIC_KEY="" +BILL_PAYMENT_DELAY = 0 +# The django secret key +SECRET_KEY=get_random_secret_key() + +ALLOWED_HOSTS = [] + +# required for hardcopy / pdf rendering: https://github.com/loftylabs/django-hardcopy +CHROME_PATH = '/usr/bin/chromium-browser' + +# Username that is created by default and owns the configuration objects +UNCLOUD_ADMIN_NAME = "uncloud-admin" + +LOGIN_REDIRECT_URL = '/' +LOGOUT_REDIRECT_URL = '/' + +# replace these in local_settings.py +AUTH_LDAP_SERVER_URI = "ldaps://ldap1.example.com,ldaps://ldap2.example.com" +AUTH_LDAP_BIND_DN="uid=django,ou=system,dc=example,dc=com" +AUTH_LDAP_BIND_PASSWORD="a very secure ldap password" +AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=example,dc=com", + ldap.SCOPE_SUBTREE, + "(uid=%(user)s)") + +# where to create customers +LDAP_CUSTOMER_DN="ou=customer,dc=example,dc=com" + +EMAIL_USE_TLS = True +EMAIL_HOST = '' +EMAIL_PORT = 465 +EMAIL_HOST_USER = DEFAULT_FROM_EMAIL = '' +EMAIL_HOST_PASSWORD = '' +DEFAULT_FROM_EMAIL = '' +RENEWAL_FROM_EMAIL = 'test@example.com' +# Should be removed in production +EMAIL_BACKEND = 'django.core.mail.backends.console.EmailBackend' + +############## +# Jobs +Q_CLUSTER = { + 'name': 'matrixhosting', + 'workers': 1, + 'recycle': 500, + 'timeout': 60, + 'compress': True, + 'cpu_affinity': 1, + 'save_limit': 250, + 'queue_limit': 500, + 'label': 'Django Q', + 'redis': { + 'host': '127.0.0.1', + 'port': 6379, + 'db': 0, } +} + +# Overwrite settings with local settings, if existing +try: + from uncloud.local_settings import * +except (ModuleNotFoundError, ImportError): + pass diff --git a/uncloud/settings/__init__.py b/uncloud/settings/__init__.py deleted file mode 100644 index 629660e..0000000 --- a/uncloud/settings/__init__.py +++ /dev/null @@ -1,128 +0,0 @@ -import configparser -import logging -import sys -import os - -from uncloud.common.etcd_wrapper import Etcd3Wrapper - -logger = logging.getLogger(__name__) - - -class CustomConfigParser(configparser.RawConfigParser): - def __getitem__(self, key): - try: - result = super().__getitem__(key) - except KeyError as err: - raise KeyError( - "Key '{}' not found in configuration. Make sure you configure uncloud.".format( - key - ) - ) from err - else: - return result - - -class Settings(object): - def __init__(self, config_key="/uncloud/config/"): - conf_name = "uncloud.conf" - conf_dir = os.environ.get( - "UCLOUD_CONF_DIR", os.path.expanduser("~/uncloud/") - ) - self.config_file = os.path.join(conf_dir, conf_name) - - self.config_parser = CustomConfigParser(allow_no_value=True) - self.config_key = config_key - - self.read_internal_values() - try: - self.config_parser.read(self.config_file) - except Exception as err: - logger.error("%s", err) - - def get_etcd_client(self): - args = tuple() - try: - kwargs = { - "host": self.config_parser.get("etcd", "url"), - "port": self.config_parser.get("etcd", "port"), - "ca_cert": self.config_parser.get("etcd", "ca_cert"), - "cert_cert": self.config_parser.get( - "etcd", "cert_cert" - ), - "cert_key": self.config_parser.get("etcd", "cert_key"), - } - except configparser.Error as err: - raise configparser.Error( - "{} in config file {}".format( - err.message, self.config_file - ) - ) from err - else: - try: - wrapper = Etcd3Wrapper(*args, **kwargs) - except Exception as err: - logger.error( - "etcd connection not successfull. Please check your config file." - "\nDetails: %s\netcd connection parameters: %s", - err, - kwargs, - ) - sys.exit(1) - else: - return wrapper - - def read_internal_values(self): - self.config_parser.read_dict( - { - "etcd": { - "file_prefix": "/files/", - "host_prefix": "/hosts/", - "image_prefix": "/images/", - "image_store_prefix": "/imagestore/", - "network_prefix": "/networks/", - "request_prefix": "/requests/", - "user_prefix": "/users/", - "vm_prefix": "/vms/", - } - } - ) - - def read_config_file_values(self, config_file): - try: - # Trying to read configuration file - with open(config_file, "r") as config_file_handle: - self.config_parser.read_file(config_file_handle) - except FileNotFoundError: - sys.exit( - "Configuration file {} not found!".format(config_file) - ) - except Exception as err: - logger.exception(err) - sys.exit("Error occurred while reading configuration file") - - def read_values_from_etcd(self): - etcd_client = self.get_etcd_client() - config_from_etcd = etcd_client.get( - self.config_key, value_in_json=True - ) - if config_from_etcd: - self.config_parser.read_dict(config_from_etcd.value) - else: - raise KeyError( - "Key '{}' not found in etcd. Please configure uncloud.".format( - self.config_key - ) - ) - - def __getitem__(self, key): - # Allow failing to read from etcd if we have - # it locally - try: - self.read_values_from_etcd() - except KeyError as e: - pass - - return self.config_parser[key] - - -settings = Settings() diff --git a/uncloud/shared/__init__.py b/uncloud/shared/__init__.py deleted file mode 100644 index db2093f..0000000 --- a/uncloud/shared/__init__.py +++ /dev/null @@ -1,34 +0,0 @@ -from uncloud.settings import settings -from uncloud.common.vm import VmPool -from uncloud.common.host import HostPool -from uncloud.common.request import RequestPool -from uncloud.common.storage_handlers import get_storage_handler - - -class Shared: - @property - def etcd_client(self): - return settings.get_etcd_client() - - @property - def host_pool(self): - return HostPool( - self.etcd_client, settings["etcd"]["host_prefix"] - ) - - @property - def vm_pool(self): - return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) - - @property - def request_pool(self): - return RequestPool( - self.etcd_client, settings["etcd"]["request_prefix"] - ) - - @property - def storage_handler(self): - return get_storage_handler() - - -shared = Shared() diff --git a/uncloud/static/uncloud/uncloud.css b/uncloud/static/uncloud/uncloud.css new file mode 100644 index 0000000..51d93ef --- /dev/null +++ b/uncloud/static/uncloud/uncloud.css @@ -0,0 +1,4 @@ +#content { + width: 400px; + margin: auto; +} diff --git a/uncloud/templates/uncloud/base.html b/uncloud/templates/uncloud/base.html new file mode 100644 index 0000000..cbf0686 --- /dev/null +++ b/uncloud/templates/uncloud/base.html @@ -0,0 +1,29 @@ +{% extends 'bootstrap5/bootstrap5.html' %} +{% block bootstrap5_before_content %} + + +{% endblock %} diff --git a/uncloud/templates/uncloud/index.html b/uncloud/templates/uncloud/index.html new file mode 100644 index 0000000..b8b5828 --- /dev/null +++ b/uncloud/templates/uncloud/index.html @@ -0,0 +1,170 @@ +{% extends 'uncloud/base.html' %} +{% block title %}Welcome to uncloud [beta]{% endblock %} + +{% block bootstrap5_content %} +
+
+
+
+

Welcome to uncloud [beta]

+
+
+ +
+

About uncloud

+
+

+ Welcome to uncloud, the Open Source cloud management + system by ungleich. + It is an API driven system with + some convience views provided by + the Django Rest + Framework. You can + freely access + the source code of uncloud. + This is a BETA service. As such, some + functionality might not be very sophisticated. +

+
+
+
+

Getting started

+
+

uncloud is designed to be as easy as possible to use. However, + there are some "real world" requirements that need to be met to + start using uncloud: + +

    +
  • First you need + to register an + account. If you already have one, you can + login. +
  • If you have forgotten your password or other issues with + logging in, you can contact the ungleich support + via support at ungleich.ch. + +
  • Secondy you will need to + create a billing + address. This is required for determining the correct + tax. +
  • Next you will need to + register a credit card + from which payments can be made. Your credit card will not + be charged without your consent. +
+
+
+
+

Introduction to uncloud concepts

+
+

We plan to offer many services on uncloud ranging from + for free, for a small amount or regular charges. As transfer + fees are a major challenge for our business, we based uncloud + on the pre-paid account model. Which means + that you can charge your account and then use your balance to + pay for product usage.

+
+
+ +
+

Credit cards

+
+

+ Credit cards are registered with stripe. We only save a the + last 4 digits and the expiry date of the card to make + identification for you easier. +

+
    +
  • Register a credit card + (this is required to be done via Javascript so that we never see + your credit card, but it is sent directly to stripe) +
  • You can list your + credit cards + By default the first credit card is used for charging + ("active: true") and later added cards will not be + used. To change this, first disable the active flag and + then set it on another credit card. +
+
+
+

Billing Address, Payments and Balance

+
+

Billing addresses behave similar to credit cards: you can + have many of them, but only one can be active. The active + billing address is taken for creating new orders.

+ +

In uncloud we use the pre-paid model: you can add money to + your account via payments. You can always check your + balance. The products you use will automatically be charged from + your existing balance. +

+ +

In the future you will be able opt-in to automatically + recharging your account at a certain time frame or whenever it + is below a certain amount

+ + + +
+
+ +
+

Networking

+
+

+ With uncloud you can use a variety of network related + services. +

+ + +
+
+ +
+

Current limitations

+
+
    +
  • Payments are only possible in CHF. +
+
+
+ {% if user.is_authenticated %} +
+

Account Settings

+
+
    +
    + {% csrf_token %} +
    + Delete User Account +

    Are you sure you want to delete your account? This will permanently delete your + profile and any orders you have generated.

    + {{ delete_form }} +
    +
    + +
    +
    +
+
+
+ {% endif %} +
+
+{% endblock %} diff --git a/uncloud/urls.py b/uncloud/urls.py new file mode 100644 index 0000000..14e45fd --- /dev/null +++ b/uncloud/urls.py @@ -0,0 +1,65 @@ +"""uncloud URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +""" + +from django.contrib import admin +from django.urls import path, include +from django.conf import settings +from django.conf.urls.static import static + +from rest_framework import routers +from rest_framework.schemas import get_schema_view + +#from opennebula import views as oneviews +from uncloud import views as uncloudviews +from uncloud_auth import views as authviews +from uncloud_net import views as netviews +from uncloud_pay import views as payviews +from uncloud_vm import views as vmviews +from uncloud_service import views as serviceviews +from matrixhosting import views as matrixviews + +router = routers.DefaultRouter() + +# Beta endpoints +router.register(r'beta/vm', vmviews.NicoVMProductViewSet, basename='nicovmproduct') + +################################################################################ +# v2 + +# Net +router.register(r'v2/net/wireguardvpn', netviews.WireGuardVPNViewSet, basename='wireguardvpnnetwork') +router.register(r'v2/net/wireguardvpnsizes', netviews.WireGuardVPNSizes, basename='wireguardvpnnetworksizes') + +# Payment related for a user +router.register(r'v2/payment/credit-card', payviews.CreditCardViewSet, basename='stripecreditcard') +router.register(r'v2/payment/payment', payviews.PaymentViewSet, basename='payment') +router.register(r'v2/payment/balance', payviews.BalanceViewSet, basename='payment-balance') +router.register(r'v2/payment/address', payviews.BillingAddressViewSet, basename='billingaddress') +router.register(r'v2/orders', payviews.OrderViewSet, basename='orders') +router.register(r'v2/bill', payviews.BillViewSet, basename='bills') +router.register(r'v2/machines', matrixviews.MachineViewSet, basename='machines') + +# Generic helper views that are usually not needed +router.register(r'v2/generic/vat-rate', payviews.VATRateViewSet, basename='vatrate') + + +urlpatterns = [ + path(r'api/', include(router.urls), name='api'), + + path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), # for login to REST API + path('openapi', get_schema_view( + title="uncloud", + description="uncloud API", + version="2.0.0" + ), name='openapi-schema'), + + path('admin/', admin.site.urls), + + path('accounts/', include('allauth.urls')), + path('cc/reg/', payviews.RegisterCard.as_view(), name="cc_register"), + path('matrix/', include('matrixhosting.urls', namespace='matrix')), + path('', uncloudviews.UncloudIndex.as_view(), name="uncloudindex"), +] diff --git a/uncloud/views.py b/uncloud/views.py new file mode 100644 index 0000000..a4bf683 --- /dev/null +++ b/uncloud/views.py @@ -0,0 +1,23 @@ +from django.views.generic.base import TemplateView +from django.contrib import messages +from django.shortcuts import redirect + +from uncloud_pay.selectors import get_balance_for_user +from .forms import UserDeleteForm + +class UncloudIndex(TemplateView): + template_name = "uncloud/index.html" + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + if self.request.user.is_authenticated: + context['balance'] = get_balance_for_user(self.request.user) + context['delete_form'] = UserDeleteForm(instance=self.request.user) + return context + + def post(self, request, *args, **kwargs): + UserDeleteForm(request.POST, instance=request.user) + user = request.user + user.delete() + messages.info(request, 'Your account has been deleted.') + return redirect('uncloudindex') \ No newline at end of file diff --git a/uncloud/wsgi.py b/uncloud/wsgi.py new file mode 100644 index 0000000..c4a07b8 --- /dev/null +++ b/uncloud/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for uncloud project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + +application = get_wsgi_application() diff --git a/uncloud_auth/__init__.py b/uncloud_auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_auth/admin.py b/uncloud_auth/admin.py new file mode 100644 index 0000000..f91be8f --- /dev/null +++ b/uncloud_auth/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin +from .models import User + +admin.site.register(User, UserAdmin) diff --git a/uncloud_auth/apps.py b/uncloud_auth/apps.py new file mode 100644 index 0000000..c16bd7a --- /dev/null +++ b/uncloud_auth/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class AuthConfig(AppConfig): + name = 'uncloud_auth' diff --git a/uncloud_auth/management/commands/make-admin.py b/uncloud_auth/management/commands/make-admin.py new file mode 100644 index 0000000..9157439 --- /dev/null +++ b/uncloud_auth/management/commands/make-admin.py @@ -0,0 +1,21 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model +import sys + +class Command(BaseCommand): + help = 'Give Admin rights to existing user' + + def add_arguments(self, parser): + parser.add_argument('username', type=str) + parser.add_argument('--superuser', action='store_true') + + def handle(self, *args, **options): + user = get_user_model().objects.get(username=options['username']) + user.is_staff = True + + if options['superuser']: + user.is_superuser = True + + user.save() + + print(f"{user.username} is now admin (superuser={user.is_superuser})") diff --git a/uncloud_auth/migrations/0001_initial.py b/uncloud_auth/migrations/0001_initial.py new file mode 100644 index 0000000..b263dc6 --- /dev/null +++ b/uncloud_auth/migrations/0001_initial.py @@ -0,0 +1,46 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +import django.contrib.auth.models +import django.contrib.auth.validators +import django.core.validators +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0012_alter_user_first_name_max_length'), + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=150, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('maximum_credit', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/uncloud_auth/migrations/__init__.py b/uncloud_auth/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_auth/models.py b/uncloud_auth/models.py new file mode 100644 index 0000000..90463e1 --- /dev/null +++ b/uncloud_auth/models.py @@ -0,0 +1,17 @@ +from django.contrib.auth.models import AbstractUser +from django.db import models +from django.core.validators import MinValueValidator + +from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS + +class User(AbstractUser): + """ + We use the standard user and add a maximum credit that is allowed + to be accumulated. After that we need to have warnings, cancellation, etc. + """ + + maximum_credit = models.DecimalField( + default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) diff --git a/uncloud_auth/serializers.py b/uncloud_auth/serializers.py new file mode 100644 index 0000000..c3f6694 --- /dev/null +++ b/uncloud_auth/serializers.py @@ -0,0 +1,72 @@ +from django.contrib.auth import get_user_model +from django.db import transaction +from ldap3.core.exceptions import LDAPEntryAlreadyExistsResult +from rest_framework import serializers + +from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS +from uncloud_pay.models import BillingAddress + +from .ungleich_ldap import LdapManager + + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = get_user_model() + read_only_fields = [ 'username', 'balance', 'maximum_credit' ] + fields = read_only_fields + [ 'email' ] # , 'primary_billing_address' ] + + def validate(self, data): + """ + Ensure that the primary billing address belongs to the user + """ + # The following is raising exceptions probably, it is WIP somewhere + # if 'primary_billing_address' in data: + # if not data['primary_billing_address'].owner == self.instance: + # raise serializers.ValidationError('Invalid data') + + return data + + def update(self, instance, validated_data): + ldap_manager = LdapManager() + return_val, _ = ldap_manager.change_user_details( + instance.username, {'mail': validated_data.get('email')} + ) + if not return_val: + raise serializers.ValidationError('Couldn\'t update email') + instance.email = validated_data.get('email') + instance.save() + return instance + + +class UserRegistrationSerializer(serializers.ModelSerializer): + class Meta: + model = get_user_model() + fields = ['username', 'first_name', 'last_name', 'email', 'password'] + extra_kwargs = { + 'password': {'style': {'input_type': 'password'}}, + 'first_name': {'allow_blank': False, 'required': True}, + 'last_name': {'allow_blank': False, 'required': True}, + 'email': {'allow_blank': False, 'required': True}, + } + + def create(self, validated_data): + ldap_manager = LdapManager() + try: + data = { + 'user': validated_data['username'], + 'password': validated_data['password'], + 'email': validated_data['email'], + 'firstname': validated_data['first_name'], + 'lastname': validated_data['last_name'], + } + ldap_manager.create_user(**data) + except LDAPEntryAlreadyExistsResult: + raise serializers.ValidationError( + {'username': ['A user with that username already exists.']} + ) + else: + return get_user_model().objects.create_user(**validated_data) + + +class ImportUserSerializer(serializers.Serializer): + username = serializers.CharField() diff --git a/uncloud_auth/templates/uncloud_auth/login.html b/uncloud_auth/templates/uncloud_auth/login.html new file mode 100644 index 0000000..2a3bfd6 --- /dev/null +++ b/uncloud_auth/templates/uncloud_auth/login.html @@ -0,0 +1,21 @@ +{% extends 'uncloud/base.html' %} +{% load bootstrap5 %} + +{% block bootstrap5_content %} +
+
+
+ +

Login to uncloud

+
+ {% csrf_token %} + {% bootstrap_form form %} + {% buttons %} + + {% endbuttons %} +
+
+
+
+ +{% endblock %} diff --git a/uncloud_auth/uldap.py b/uncloud_auth/uldap.py new file mode 100644 index 0000000..aa90c77 --- /dev/null +++ b/uncloud_auth/uldap.py @@ -0,0 +1,42 @@ +import ldap +# from django.conf import settings + +AUTH_LDAP_SERVER_URI = "ldaps://ldap1.ungleich.ch,ldaps://ldap2.ungleich.ch" +AUTH_LDAP_BIND_DN="uid=django-create,ou=system,dc=ungleich,dc=ch" +AUTH_LDAP_BIND_PASSWORD="kS#e+v\zjKn]L!,RIu2}V+DUS" +# AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=ungleich,dc=ch", +# ldap.SCOPE_SUBTREE, +# "(uid=%(user)s)") + + + +ldap_object = ldap.initialize(AUTH_LDAP_SERVER_URI) +cancelid = ldap_object.bind(AUTH_LDAP_BIND_DN, AUTH_LDAP_BIND_PASSWORD) + +res = ldap_object.search_s("dc=ungleich,dc=ch", ldap.SCOPE_SUBTREE, "(uid=nico)") +print(res) + +# class LDAP(object): +# """ +# Managing users in LDAP + +# Requires the following settings? + +# LDAP_USER_DN: where to create users in the tree + +# LDAP_ADMIN_DN: which DN to use for managing users +# LDAP_ADMIN_PASSWORD: which password to used + +# This module will reuse information from djagno_auth_ldap, including: + +# AUTH_LDAP_SERVER_URI + +# """ +# def __init__(self): +# pass + +# def create_user(self): +# pass + +# def change_password(self): +# pass diff --git a/uncloud_auth/ungleich_ldap.py b/uncloud_auth/ungleich_ldap.py new file mode 100644 index 0000000..f22b423 --- /dev/null +++ b/uncloud_auth/ungleich_ldap.py @@ -0,0 +1,284 @@ +import base64 +import hashlib +import logging +import random + +import ldap3 +from django.conf import settings + +logger = logging.getLogger(__name__) + + +class LdapManager: + __instance = None + def __new__(cls): + if LdapManager.__instance is None: + LdapManager.__instance = object.__new__(cls) + return LdapManager.__instance + + def __init__(self): + """ + Initialize the LDAP subsystem. + """ + self.rng = random.SystemRandom() + self.server = ldap3.Server(settings.AUTH_LDAP_SERVER) + + + def get_admin_conn(self): + """ + Return a bound :class:`ldap3.Connection` instance which has write + permissions on the dn in which the user accounts reside. + """ + conn = self.get_conn(user=settings.LDAP_ADMIN_DN, + password=settings.LDAP_ADMIN_PASSWORD, + raise_exceptions=True) + conn.bind() + return conn + + + def get_conn(self, **kwargs): + """ + Return an unbound :class:`ldap3.Connection` which talks to the configured + LDAP server. + + The *kwargs* are passed to the constructor of :class:`ldap3.Connection` and + can be used to set *user*, *password* and other useful arguments. + """ + return ldap3.Connection(self.server, **kwargs) + + + def _ssha_password(self, password): + """ + Apply the SSHA password hashing scheme to the given *password*. + *password* must be a :class:`bytes` object, containing the utf-8 + encoded password. + + Return a :class:`bytes` object containing ``ascii``-compatible data + which can be used as LDAP value, e.g. after armoring it once more using + base64 or decoding it to unicode from ``ascii``. + """ + SALT_BYTES = 15 + + sha1 = hashlib.sha1() + salt = self.rng.getrandbits(SALT_BYTES * 8).to_bytes(SALT_BYTES, + "little") + sha1.update(password) + sha1.update(salt) + + digest = sha1.digest() + passwd = b"{SSHA}" + base64.b64encode(digest + salt) + return passwd + + + def create_user(self, user, password, firstname, lastname, email): + conn = self.get_admin_conn() + uidNumber = self._get_max_uid() + 1 + logger.debug("uidNumber={uidNumber}".format(uidNumber=uidNumber)) + user_exists = True + while user_exists: + user_exists, _ = self.check_user_exists( + "", + '(&(objectClass=inetOrgPerson)(objectClass=posixAccount)' + '(objectClass=top)(uidNumber={uidNumber}))'.format( + uidNumber=uidNumber + ) + ) + if user_exists: + logger.debug( + "{uid} exists. Trying next.".format(uid=uidNumber) + ) + uidNumber += 1 + logger.debug("{uid} does not exist. Using it".format(uid=uidNumber)) + self._set_max_uid(uidNumber) + try: + uid = user # user.encode("utf-8") + conn.add("uid={uid},{customer_dn}".format( + uid=uid, customer_dn=settings.LDAP_CUSTOMER_DN + ), + ["inetOrgPerson", "posixAccount", "ldapPublickey"], + { + "uid": [uid], + "sn": [lastname.encode("utf-8")], + "givenName": [firstname.encode("utf-8")], + "cn": [uid], + "displayName": ["{} {}".format(firstname, lastname).encode("utf-8")], + "uidNumber": [str(uidNumber)], + "gidNumber": [str(settings.LDAP_CUSTOMER_GROUP_ID)], + "loginShell": ["/bin/bash"], + "homeDirectory": ["/home/{}".format(user).encode("utf-8")], + "mail": email.encode("utf-8"), + "userPassword": [self._ssha_password( + password.encode("utf-8") + )] + } + ) + logger.debug('Created user %s %s' % (user.encode('utf-8'), + uidNumber)) + except Exception as ex: + logger.debug('Could not create user %s' % user.encode('utf-8')) + logger.error("Exception: " + str(ex)) + raise + finally: + conn.unbind() + + + def change_password(self, uid, new_password): + """ + Changes the password of the user identified by user_dn + + :param uid: str The uid that identifies the user + :param new_password: str The new password string + :return: True if password was changed successfully False otherwise + """ + conn = self.get_admin_conn() + + # Make sure the user exists first to change his/her details + user_exists, entries = self.check_user_exists( + uid=uid, + search_base=settings.ENTIRE_SEARCH_BASE + ) + return_val = False + if user_exists: + try: + return_val = conn.modify( + entries[0].entry_dn, + { + "userpassword": ( + ldap3.MODIFY_REPLACE, + [self._ssha_password(new_password.encode("utf-8"))] + ) + } + ) + except Exception as ex: + logger.error("Exception: " + str(ex)) + else: + logger.error("User {} not found".format(uid)) + + conn.unbind() + return return_val + + def change_user_details(self, uid, details): + """ + Updates the user details as per given values in kwargs of the user + identified by user_dn. + + Assumes that all attributes passed in kwargs are valid. + + :param uid: str The uid that identifies the user + :param details: dict A dictionary containing the new values + :return: True if user details were updated successfully False otherwise + """ + conn = self.get_admin_conn() + + # Make sure the user exists first to change his/her details + user_exists, entries = self.check_user_exists( + uid=uid, + search_base=settings.ENTIRE_SEARCH_BASE + ) + + return_val = False + if user_exists: + details_dict = {k: (ldap3.MODIFY_REPLACE, [v.encode("utf-8")]) for + k, v in details.items()} + try: + return_val = conn.modify(entries[0].entry_dn, details_dict) + msg = "success" + except Exception as ex: + msg = str(ex) + logger.error("Exception: " + msg) + finally: + conn.unbind() + else: + msg = "User {} not found".format(uid) + logger.error(msg) + conn.unbind() + return return_val, msg + + def check_user_exists(self, uid, search_filter="", attributes=None, + search_base=settings.LDAP_CUSTOMER_DN): + """ + Check if the user with the given uid exists in the customer group. + + :param uid: str representing the user + :param search_filter: str representing the filter condition to find + users. If its empty, the search finds the user with + the given uid. + :param attributes: list A list of str representing all the attributes + to be obtained in the result entries + :param search_base: str + :return: tuple (bool, [ldap3.abstract.entry.Entry ..]) + A bool indicating if the user exists + A list of all entries obtained in the search + """ + conn = self.get_admin_conn() + entries = [] + try: + result = conn.search( + search_base=search_base, + search_filter=search_filter if len(search_filter)> 0 else + '(uid={uid})'.format(uid=uid), + attributes=attributes + ) + entries = conn.entries + finally: + conn.unbind() + return result, entries + + def delete_user(self, uid): + """ + Deletes the user with the given uid from ldap + + :param uid: str representing the user + :return: True if the delete was successful False otherwise + """ + conn = self.get_admin_conn() + try: + return_val = conn.delete( + ("uid={uid}," + settings.LDAP_CUSTOMER_DN).format(uid=uid), + ) + msg = "success" + except Exception as ex: + msg = str(ex) + logger.error("Exception: " + msg) + return_val = False + finally: + conn.unbind() + return return_val, msg + + def _set_max_uid(self, max_uid): + """ + a utility function to save max_uid value to a file + + :param max_uid: an integer representing the max uid + :return: + """ + with open(settings.LDAP_MAX_UID_FILE_PATH, 'w+') as handler: + handler.write(str(max_uid)) + + def _get_max_uid(self): + """ + A utility function to read the max uid value that was previously set + + :return: An integer representing the max uid value that was previously + set + """ + try: + with open(settings.LDAP_MAX_UID_FILE_PATH, 'r+') as handler: + try: + return_value = int(handler.read()) + except ValueError as ve: + logger.error( + "Error reading int value from {}. {}" + "Returning default value {} instead".format( + settings.LDAP_MAX_UID_PATH, + str(ve), + settings.LDAP_DEFAULT_START_UID + ) + ) + return_value = settings.LDAP_DEFAULT_START_UID + return return_value + except FileNotFoundError as fnfe: + logger.error("File not found : " + str(fnfe)) + return_value = settings.LDAP_DEFAULT_START_UID + logger.error("So, returning UID={}".format(return_value)) + return return_value diff --git a/uncloud_auth/views.py b/uncloud_auth/views.py new file mode 100644 index 0000000..9310a4c --- /dev/null +++ b/uncloud_auth/views.py @@ -0,0 +1,77 @@ +from django.contrib.auth import views as auth_views +from django.contrib.auth import logout + +from django_auth_ldap.backend import LDAPBackend +from rest_framework import mixins, permissions, status, viewsets +from rest_framework.decorators import action +from rest_framework.response import Response + +from .serializers import * + + +class LoginView(auth_views.LoginView): + template_name = 'uncloud_auth/login.html' + +class LogoutView(auth_views.LogoutView): + pass +# template_name = 'uncloud_auth/logo.html' + + +class UserViewSet(viewsets.GenericViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = UserSerializer + + def get_queryset(self): + return self.request.user + + def list(self, request, format=None): + # This is a bit stupid: we have a user, we create a queryset by + # matching on the username. But I don't know a "nicer" way. + # Nico, 2020-03-18 + user = request.user + serializer = self.get_serializer(user, context = {'request': request}) + return Response(serializer.data) + + @action(detail=False, methods=['post']) + def change_email(self, request): + serializer = self.get_serializer( + request.user, data=request.data, context={'request': request} + ) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data) + + +class AccountManagementViewSet(mixins.CreateModelMixin, viewsets.GenericViewSet): + serializer_class = UserRegistrationSerializer + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + self.perform_create(serializer) + headers = self.get_success_headers(serializer.data) + return Response( + serializer.data, status=status.HTTP_201_CREATED, headers=headers + ) + + +class AdminUserViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAdminUser] + + def get_serializer_class(self): + if self.action == 'import_from_ldap': + return ImportUserSerializer + else: + return UserSerializer + + def get_queryset(self): + return get_user_model().objects.all() + + @action(detail=False, methods=['post'], url_path='import_from_ldap') + def import_from_ldap(self, request, pk=None): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + ldap_username = serializer.validated_data.pop("username") + user = LDAPBackend().populate_user(ldap_username) + + return Response(UserSerializer(user, context = {'request': request}).data) diff --git a/uncloud_net/__init__.py b/uncloud_net/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_net/admin.py b/uncloud_net/admin.py new file mode 100644 index 0000000..ca6aaa1 --- /dev/null +++ b/uncloud_net/admin.py @@ -0,0 +1,7 @@ +from django.contrib import admin + +from .models import * + + +for m in [ ReverseDNSEntry, WireGuardVPNPool, WireGuardVPN ]: + admin.site.register(m) diff --git a/uncloud_net/apps.py b/uncloud_net/apps.py new file mode 100644 index 0000000..489beb1 --- /dev/null +++ b/uncloud_net/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudNetConfig(AppConfig): + name = 'uncloud_net' diff --git a/uncloud_net/forms.py b/uncloud_net/forms.py new file mode 100644 index 0000000..ad4e013 --- /dev/null +++ b/uncloud_net/forms.py @@ -0,0 +1,11 @@ +from django import forms + +from .models import * +from .selectors import * + +class WireGuardVPNForm(forms.ModelForm): + network_size = forms.ChoiceField(choices=allowed_vpn_network_reservation_size) + + class Meta: + model = WireGuardVPN + fields = [ "wireguard_public_key" ] diff --git a/uncloud_net/management/commands/vpn.py b/uncloud_net/management/commands/vpn.py new file mode 100644 index 0000000..9fdc80d --- /dev/null +++ b/uncloud_net/management/commands/vpn.py @@ -0,0 +1,44 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand + +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster + +import logging +log = logging.getLogger(__name__) + + + +peer_template=""" +# {username} +[Peer] +PublicKey = {public_key} +AllowedIPs = {vpnnetwork} +""" + +class Command(BaseCommand): + help = 'General uncloud commands' + + def add_arguments(self, parser): + parser.add_argument('--hostname', + action='store_true', + help='Name of this VPN Host', + required=True) + + def handle(self, *args, **options): + if options['bootstrap']: + self.bootstrap() + + self.create_vpn_config(options['hostname']) + + def create_vpn_config(self, hostname): + configs = [] + + for pool in VPNPool.objects.filter(vpn_hostname=hostname): + configs.append(pool_config) + + print(configs) diff --git a/uncloud_net/migrations/0001_initial.py b/uncloud_net/migrations/0001_initial.py new file mode 100644 index 0000000..6794156 --- /dev/null +++ b/uncloud_net/migrations/0001_initial.py @@ -0,0 +1,62 @@ +# Generated by Django 3.1 on 2020-12-13 13:42 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='MACAdress', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + ), + migrations.CreateModel( + name='WireGuardVPNPool', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('network', models.GenericIPAddressField(unique=True)), + ('network_mask', models.IntegerField(validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(128)])), + ('subnetwork_mask', models.IntegerField(validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(128)])), + ('vpn_server_hostname', models.CharField(max_length=256)), + ('wireguard_private_key', models.CharField(max_length=48)), + ], + ), + migrations.CreateModel( + name='WireGuardVPNFreeLeases', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pool_index', models.IntegerField(unique=True)), + ('vpnpool', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_net.wireguardvpnpool')), + ], + ), + migrations.CreateModel( + name='WireGuardVPN', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('pool_index', models.IntegerField(unique=True)), + ('wireguard_public_key', models.CharField(max_length=48)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('vpnpool', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_net.wireguardvpnpool')), + ], + ), + migrations.CreateModel( + name='ReverseDNSEntry', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('ip_address', models.GenericIPAddressField(unique=True)), + ('name', models.CharField(max_length=253)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/uncloud_net/migrations/0002_wireguardvpnpool_wireguard_public_key.py b/uncloud_net/migrations/0002_wireguardvpnpool_wireguard_public_key.py new file mode 100644 index 0000000..479aba1 --- /dev/null +++ b/uncloud_net/migrations/0002_wireguardvpnpool_wireguard_public_key.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2020-12-13 17:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='wireguardvpnpool', + name='wireguard_public_key', + field=models.CharField(default='', max_length=48), + preserve_default=False, + ), + ] diff --git a/uncloud_net/migrations/0003_wireguardvpnpool_wg_name.py b/uncloud_net/migrations/0003_wireguardvpnpool_wg_name.py new file mode 100644 index 0000000..9ecf52c --- /dev/null +++ b/uncloud_net/migrations/0003_wireguardvpnpool_wg_name.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2020-12-13 17:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0002_wireguardvpnpool_wireguard_public_key'), + ] + + operations = [ + migrations.AddField( + model_name='wireguardvpnpool', + name='wg_name', + field=models.CharField(default='wg0', max_length=15), + preserve_default=False, + ), + ] diff --git a/uncloud_net/migrations/0004_auto_20201213_1734.py b/uncloud_net/migrations/0004_auto_20201213_1734.py new file mode 100644 index 0000000..24e46e7 --- /dev/null +++ b/uncloud_net/migrations/0004_auto_20201213_1734.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1 on 2020-12-13 17:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0003_wireguardvpnpool_wg_name'), + ] + + operations = [ + migrations.AddConstraint( + model_name='wireguardvpnpool', + constraint=models.UniqueConstraint(fields=('wg_name', 'vpn_server_hostname'), name='unique_interface_name_per_host'), + ), + ] diff --git a/uncloud_net/migrations/0005_auto_20201220_1837.py b/uncloud_net/migrations/0005_auto_20201220_1837.py new file mode 100644 index 0000000..1dbabe6 --- /dev/null +++ b/uncloud_net/migrations/0005_auto_20201220_1837.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1 on 2020-12-20 18:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0004_auto_20201213_1734'), + ] + + operations = [ + migrations.AlterField( + model_name='wireguardvpn', + name='wireguard_public_key', + field=models.CharField(max_length=48, unique=True), + ), + ] diff --git a/uncloud_net/migrations/0006_auto_20201224_1626.py b/uncloud_net/migrations/0006_auto_20201224_1626.py new file mode 100644 index 0000000..c0dd2ef --- /dev/null +++ b/uncloud_net/migrations/0006_auto_20201224_1626.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1 on 2020-12-24 16:26 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0005_auto_20201220_1837'), + ] + + operations = [ + migrations.AddConstraint( + model_name='wireguardvpn', + constraint=models.UniqueConstraint(fields=('vpnpool', 'wireguard_public_key'), name='wg_key_unique_per_pool'), + ), + ] diff --git a/uncloud_net/migrations/__init__.py b/uncloud_net/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_net/models.py b/uncloud_net/models.py new file mode 100644 index 0000000..9865a08 --- /dev/null +++ b/uncloud_net/models.py @@ -0,0 +1,208 @@ +import uuid +import ipaddress + +from django.db import models +from django.contrib.auth import get_user_model +from django.core.validators import MinValueValidator, MaxValueValidator +from django.core.exceptions import FieldError, ValidationError + +from uncloud_pay.models import Order, Product + +class WireGuardVPNPool(models.Model): + """ + Network address pools from which VPNs can be created + """ + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['wg_name', 'vpn_server_hostname' ], + name='unique_interface_name_per_host') + ] + + + # Linux interface naming is restricing to max 15 characters + wg_name = models.CharField(max_length=15) + + network = models.GenericIPAddressField(unique=True) + network_mask = models.IntegerField(validators=[MinValueValidator(0), + MaxValueValidator(128)]) + + subnetwork_mask = models.IntegerField(validators=[ + MinValueValidator(0), + MaxValueValidator(128) + ]) + + vpn_server_hostname = models.CharField(max_length=256) + wireguard_private_key = models.CharField(max_length=48) + wireguard_public_key = models.CharField(max_length=48) + + @property + def max_pool_index(self): + """ + Return the highest possible network / last network id + """ + + bits = self.subnetwork_mask - self.network_mask + + return (2**bits)-1 + + @property + def ip_network(self): + """ + Return the IP network based on our address and mask + """ + return ipaddress.ip_network(f"{self.network}/{self.network_mask}") + + def __str__(self): + return f"{self.ip_network} (subnets: /{self.subnetwork_mask})" + + @property + def wireguard_config(self): + wireguard_config = [ f"[Interface]\nListenPort = 51820\nPrivateKey = {self.wireguard_private_key}\n" ] + + peers = [] + + for vpn in self.wireguardvpn_set.all(): + public_key = vpn.wireguard_public_key + peer_network = f"{vpn.address}/{self.subnetwork_mask}" + owner = vpn.owner + + peers.append(f"# Owner: {owner}\n[Peer]\nPublicKey = {public_key}\nAllowedIPs = {peer_network}\n\n") + + wireguard_config.extend(peers) + + return "\n".join(wireguard_config) + + +class WireGuardVPN(models.Model): + """ + Created VPNNetworks + """ + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + vpnpool = models.ForeignKey(WireGuardVPNPool, + on_delete=models.CASCADE) + + pool_index = models.IntegerField(unique=True) + + wireguard_public_key = models.CharField(max_length=48, unique=True) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['vpnpool', 'wireguard_public_key'], + name='wg_key_unique_per_pool') + ] + + + @property + def network_mask(self): + return self.vpnpool.subnetwork_mask + + @property + def vpn_server(self): + return self.vpnpool.vpn_server_hostname + + @property + def vpn_server_public_key(self): + return self.vpnpool.wireguard_public_key + + @property + def address(self): + """ + Locate the correct subnet in the supernet + + First get the network itself + + """ + + net = self.vpnpool.ip_network + subnet = net[(2**(128-self.vpnpool.subnetwork_mask)) * self.pool_index] + + return str(subnet) + + def __str__(self): + return f"{self.address} ({self.pool_index})" + + def create_product(self): + """ + Ensure we have a product for the WireguardVPN + """ + + pass + + # Product.objects.get_or_create( + # name="WireGuardVPN", + # description="Wireguard VPN", + # currency=Currency.CHF, + # config= + + +class WireGuardVPNFreeLeases(models.Model): + """ + Previously used VPNNetworks + """ + vpnpool = models.ForeignKey(WireGuardVPNPool, + on_delete=models.CASCADE) + + pool_index = models.IntegerField(unique=True) + +################################################################################ + +class MACAdress(models.Model): + default_prefix = 0x420000000000 + + +class ReverseDNSEntry(models.Model): + """ + A reverse DNS entry + """ + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + ip_address = models.GenericIPAddressField(null=False, unique=True) + + name = models.CharField(max_length=253, null=False) + + @property + def reverse_pointer(self): + return ipaddress.ip_address(self.ip_address).reverse_pointer + + def implement(self): + """ + The implement function implements the change + """ + + # Get all DNS entries (?) / update this DNS entry + # convert to DNS name + # + pass + + + def save(self, *args, **kwargs): + # Product.objects.filter(config__parameters__contains='reverse_dns_network') + # FIXME: check if order is still active / not replaced + + allowed = False + product = None + + for order in Order.objects.filter(config__parameters__reverse_dns_network__isnull=False, + owner=self.owner): + network = order.config['parameters']['reverse_dns_network'] + + net = ipaddress.ip_network(network) + addr = ipaddress.ip_address(self.ip_address) + + if addr in net: + allowed = True + product = order.product + break + + + if not allowed: + raise ValidationError(f"User {self.owner} does not have the right to create reverse DNS entry for {self.ip_address}") + + super().save(*args, **kwargs) + + + def __str__(self): + return f"{self.ip_address} - {self.name}" diff --git a/uncloud_net/selectors.py b/uncloud_net/selectors.py new file mode 100644 index 0000000..6e12e8b --- /dev/null +++ b/uncloud_net/selectors.py @@ -0,0 +1,43 @@ +from django.db import transaction +from django.db.models import Count, F +from .models import * + +def get_suitable_pools(subnetwork_mask): + """ + Find suitable pools for a certain network size. + + First, filter for all pools that offer the requested subnetwork_size. + + Then find those pools that are not fully exhausted: + + The number of available networks in a pool is 2^(subnetwork_size-network_size. + + The number of available networks in a pool is given by the number of VPNNetworkreservations. + + """ + + return WireGuardVPNPool.objects.annotate( + num_reservations=Count('wireguardvpn'), + max_reservations=2**(F('subnetwork_mask')-F('network_mask'))).filter( + num_reservations__lt=F('max_reservations'), + subnetwork_mask=subnetwork_mask) + + +def allowed_vpn_network_reservation_size(): + """ + Find all possible sizes of subnetworks that are available. + + Select all pools with free networks. + + Get their subnetwork sizes, reduce to a set + + """ + + pools = WireGuardVPNPool.objects.annotate(num_reservations=Count('wireguardvpn'), + max_reservations=2**(F('subnetwork_mask')-F('network_mask'))).filter( + num_reservations__lt=F('max_reservations')) + + # Need to return set of tuples, see + # https://docs.djangoproject.com/en/3.1/ref/models/fields/#field-choices +# return set([ (pool.subnetwork_mask, pool.subnetwork_mask) for pool in pools ]) + return set([pool.subnetwork_mask for pool in pools ]) diff --git a/uncloud_net/serializers.py b/uncloud_net/serializers.py new file mode 100644 index 0000000..09baa59 --- /dev/null +++ b/uncloud_net/serializers.py @@ -0,0 +1,57 @@ +import base64 + +from django.contrib.auth import get_user_model +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from .models import * +from .services import * +from .selectors import * + + +class WireGuardVPNSerializer(serializers.ModelSerializer): + address = serializers.CharField(read_only=True) + vpn_server = serializers.CharField(read_only=True) + vpn_server_public_key = serializers.CharField(read_only=True) + network_mask = serializers.IntegerField() + + class Meta: + model = WireGuardVPN + fields = [ 'wireguard_public_key', 'address', 'network_mask', 'vpn_server', + 'vpn_server_public_key' ] + + extra_kwargs = { + 'network_mask': {'write_only': True } + } + + + def validate_network_mask(self, value): + msg = _(f"No pool for network size {value}") + sizes = allowed_vpn_network_reservation_size() + + if not value in sizes: + raise serializers.ValidationError(msg) + + return value + + def validate_wireguard_public_key(self, value): + msg = _("Supplied key is not a valid wireguard public key") + + """ + Verify wireguard key. + See https://lists.zx2c4.com/pipermail/wireguard/2020-December/006221.html + """ + + try: + decoded_key = base64.standard_b64decode(value) + except Exception as e: + raise serializers.ValidationError(msg) + + if not len(decoded_key) == 32: + raise serializers.ValidationError(msg) + + return value + + +class WireGuardVPNSizesSerializer(serializers.Serializer): + size = serializers.IntegerField(min_value=0, max_value=128) diff --git a/uncloud_net/services.py b/uncloud_net/services.py new file mode 100644 index 0000000..8449394 --- /dev/null +++ b/uncloud_net/services.py @@ -0,0 +1,65 @@ +from django.db import transaction + +from .models import * +from .selectors import * +from .tasks import * +from django_q.tasks import async_task, result + +@transaction.atomic +def create_wireguard_vpn(owner, public_key, network_mask): + # Check if the user has a membership. + #------------------------------------ + # If yes, user is eligible for API access and 2 VPNs + # If user already has 2 VPNs, we deduct from the credit + # If deduction is higher than the allowed credit, we fail + + + # + # Check if the user has suitable balance + # Create order + # + return create_wireguard_vpn_tech(owner, public_key, network_mask) + +@transaction.atomic +def create_wireguard_vpn_tech(owner, public_key, network_mask): + pool = get_suitable_pools(network_mask)[0] + count = pool.wireguardvpn_set.count() + + # Try re-using previously used networks first + try: + free_lease = WireGuardVPNFreeLeases.objects.get(vpnpool=pool) + + vpn = WireGuardVPN.objects.create(owner=owner, + vpnpool=pool, + pool_index=free_lease.pool_index, + wireguard_public_key=public_key) + + free_lease.delete() + + except WireGuardVPNFreeLeases.DoesNotExist: + # First object + if count == 0: + vpn = WireGuardVPN.objects.create(owner=owner, + vpnpool=pool, + pool_index=0, + wireguard_public_key=public_key) + + else: # Select last network and try +1 it + last_net = WireGuardVPN.objects.filter(vpnpool=pool).order_by('pool_index').last() + + next_index = last_net.pool_index + 1 + + if next_index <= pool.max_pool_index: + vpn = WireGuardVPN.objects.create(owner=owner, + vpnpool=pool, + pool_index=next_index, + wireguard_public_key=public_key) + + + config = pool.wireguard_config + server = pool.vpn_server_hostname + wg_name = pool.wg_name + + async_task(configure_wireguard_server_on_host, (wg_name, config), queue=server) + + return vpn diff --git a/uncloud_net/tasks.py b/uncloud_net/tasks.py new file mode 100644 index 0000000..5684871 --- /dev/null +++ b/uncloud_net/tasks.py @@ -0,0 +1,74 @@ +from .models import * + +import os +import subprocess +import logging +import uuid +from django_q.tasks import async_task, result + +log = logging.getLogger(__name__) + + +def configure_wireguard_server_on_host(wg_name, config): + """ + - Create wireguard config (DB query -> string) + - Submit config to cdist worker + - Change config locally on worker / commit / shared + """ + + # Write config + fname = f"/etc/wireguard/{wg_name}.conf" + with open(fname, "w") as fd: + fd.write(config) + + # Ensure the device exists + subprocess.run(f"ip link show {wg_name} >/dev/null || sudo ip link add {{wg_name}} type wireguard", + shell=True, check=True) + + # Ensure the config is correct + subprocess.run(f"sudo wg setconf {wg_name} {fname}", + shell=True, check=True) + + + +def configure_wireguard_server_via_cdist(wireguardvpnpool): + """ + - Create wireguard config (DB query -> string) + - Submit config to cdist worker + - Change config locally on worker / commit / shared + + """ + + config = wireguardvpnpool.wireguard_config + server = wireguardvpnpool.vpn_server_hostname + + log.info(f"Configuring VPN server {server} (async)") + + async_task(cdist_configure_wireguard_server,config, server).id + + +def cdist_configure_wireguard_server(config, server): + """ + Create config and configure server. + + To be executed on the cdist worker. + """ + + dirname= "/home/app/.cdist/type/__ungleich_wireguard/files/" + fname = os.path.join(dirname,server) + + log.info(f"Configuring VPN server {server} (on cdist host)") + with open(fname, "w") as fd: + fd.write(config) + + log.debug("git committing wireguard changes") + subprocess.run(f"cd {dirname} && git pull && git add {server} && git commit -m 'Updating config for {server}' && git push", + shell=True, check=True) + + log.debug(f"Configuring VPN server {server} with cdist") + subprocess.run(f"cdist config {server}", shell=True, check=True) + + # FIXME: + # ensure logs are on the server + # ensure exit codes are known + return True diff --git a/uncloud_net/templates/uncloud_net/wireguardvpn_form.html b/uncloud_net/templates/uncloud_net/wireguardvpn_form.html new file mode 100644 index 0000000..1463f41 --- /dev/null +++ b/uncloud_net/templates/uncloud_net/wireguardvpn_form.html @@ -0,0 +1,25 @@ +{% extends 'uncloud/base.html' %} + +{% block body %} +
+
+
+

+

Create a VPN Network

+

+ Create a new wireguard based VPN network. +

+ +
+ +
+
+ {% csrf_token %} + {{ form }} + +
+
+
+
+ +{% endblock %} diff --git a/uncloud_net/tests.py b/uncloud_net/tests.py new file mode 100644 index 0000000..75bdafa --- /dev/null +++ b/uncloud_net/tests.py @@ -0,0 +1,53 @@ +from django.test import TestCase +from rest_framework.test import APIRequestFactory, force_authenticate + +from rest_framework.reverse import reverse +from django.contrib.auth import get_user_model +from django.core.exceptions import ValidationError, FieldError + +from .views import * +from .models import * + +from uncloud_pay.models import BillingAddress, Order +from uncloud.models import UncloudNetwork + +class UncloudNetworkTests(TestCase): + def test_invalid_IPv4_network(self): + with self.assertRaises(FieldError): + UncloudNetwork.objects.create(network_address="192.168.1.0", + network_mask=33) + +class VPNTests(TestCase): + def setUp(self): + self.user = get_user_model().objects.create_user('django-test-user', 'noreply@ungleich.ch') + self.admin_user = get_user_model().objects.create_user('django-test-adminuser', + 'noreply-admin@ungleich.ch') + + + + self.admin_user.is_staff = True + self.admin_user.save() + + self.pool_network = '2001:db8::' + self.pool_network2 = '2001:db8:1::' + self.pool_network_size = '48' + self.pool_subnetwork_size = '64' + self.pool_vpn_hostname = 'vpn.example.org' + self.pool_wireguard_private_key = 'MOz8kk0m4jhNtAXlge0qzexZh1MipIhu4HJwtdvZ2EY=' + + self.vpn_wireguard_public_key = 'B2b78eWBIXPMM1x4DDjkCDZepS0qDgcLN3T3PjcgXkY=' + + self.vpnpool = WireGuardVPNPool.objects.get_or_create(network=self.pool_network, + network_size=self.pool_network_size, + subnetwork_size=self.pool_subnetwork_size, + vpn_hostname=self.pool_vpn_hostname, + wireguard_private_key=self.pool_wireguard_private_key + ) + + self.factory = APIRequestFactory() + + + + def tearDown(self): + self.user.delete() + self.admin_user.delete() diff --git a/uncloud_net/views.py b/uncloud_net/views.py new file mode 100644 index 0000000..8e7e81b --- /dev/null +++ b/uncloud_net/views.py @@ -0,0 +1,63 @@ +from django.views.generic.edit import CreateView +from django.contrib.auth.mixins import LoginRequiredMixin +from django.contrib.messages.views import SuccessMessageMixin +from rest_framework.response import Response + +from django.shortcuts import render + +from rest_framework import viewsets, permissions + +from .models import * +from .serializers import * +from .selectors import * +from .services import * +from .forms import * +from .tasks import * + +class WireGuardVPNViewSet(viewsets.ModelViewSet): + serializer_class = WireGuardVPNSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + if self.request.user.is_superuser: + obj = WireGuardVPN.objects.all() + else: + obj = WireGuardVPN.objects.filter(owner=self.request.user) + + return obj + + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + vpn = create_wireguard_vpn( + owner=self.request.user, + public_key=serializer.validated_data['wireguard_public_key'], + network_mask=serializer.validated_data['network_mask'] + ) + + return Response(WireGuardVPNSerializer(vpn).data) + + +class WireGuardVPNCreateView(LoginRequiredMixin, SuccessMessageMixin, CreateView): + model = WireGuardVPN + + login_url = '/login/' + success_url = '/' + success_message = "%(network) was created successfully" + + form_class = WireGuardVPNForm + + def get_success_message(self, cleaned_data): + return self.success_message % dict(cleaned_data, + the_prefix = self.object.prefix) + +class WireGuardVPNSizes(viewsets.ViewSet): + def list(self, request): + sizes = allowed_vpn_network_reservation_size() + print(sizes) + + sizes = [ { 'size': size } for size in sizes ] + print(sizes) + + return Response(WireGuardVPNSizesSerializer(sizes, many=True).data) diff --git a/uncloud_pay/__init__.py b/uncloud_pay/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/uncloud_pay/__init__.py @@ -0,0 +1 @@ + diff --git a/uncloud_pay/admin.py b/uncloud_pay/admin.py new file mode 100644 index 0000000..cb7b650 --- /dev/null +++ b/uncloud_pay/admin.py @@ -0,0 +1,103 @@ +from django.contrib import admin +from django.template.response import TemplateResponse +from django.urls import path +from django.shortcuts import render +from django.conf.urls import url + +from hardcopy import bytestring_to_pdf +from django.core.files.temp import NamedTemporaryFile +from django.http import FileResponse +from django.template.loader import render_to_string + +from uncloud_pay.models import * + +class BillRecordInline(admin.TabularInline): + model = BillRecord + +class RecurringPeriodInline(admin.TabularInline): + model = ProductToRecurringPeriod + +class ProductAdmin(admin.ModelAdmin): + inlines = [ RecurringPeriodInline ] + +class BillAdmin(admin.ModelAdmin): + inlines = [ BillRecordInline ] + + def get_urls(self): + """ + Create URLs for PDF view + """ + + info = "%s_%s" % (self.model._meta.app_label, self.model._meta.model_name) + pat = lambda regex, fn: url(regex, self.admin_site.admin_view(fn), name='%s_%s' % (info, fn.__name__)) + + url_patterns = [ + pat(r'^([0-9]+)/as_pdf/$', self.as_pdf), + pat(r'^([0-9]+)/as_html/$', self.as_html), + ] + super().get_urls() + + return url_patterns + + def as_pdf(self, request, object_id): + bill = self.get_object(request, object_id=object_id) + print(bill) + + if bill is None: + raise self._get_404_exception(object_id) + + output_file = NamedTemporaryFile() + bill_html = render_to_string( + "uncloud_pay/bill.html.j2", + { + 'bill': bill, + 'bill_records': bill.billrecord_set.all() + } + ) + + bytestring_to_pdf(bill_html.encode('utf-8'), output_file) + response = FileResponse(output_file, content_type="application/pdf") + response['Content-Disposition'] = f'filename="bill_{bill}.pdf"' + + return response + + def as_html(self, request, object_id): + bill = self.get_object(request, object_id=object_id) + + if bill is None: + raise self._get_404_exception(object_id) + + return render(request, 'uncloud_pay/bill.html.j2', + {'bill': bill, + 'bill_records': bill.billrecord_set.all() + }) + + + bill_html = render_to_string("bill.html.j2", {'bill': bill, + 'bill_records': bill.billrecord_set.all() + }) + + bytestring_to_pdf(bill_html.encode('utf-8'), output_file) + response = FileResponse(output_file, content_type="application/pdf") + + response['Content-Disposition'] = f'filename="bill_{bill}.pdf"' + + return HttpResponse(template.render(context, request)) + return response + + +admin.site.register(Bill, BillAdmin) +admin.site.register(Product, ProductAdmin) + +for m in [ + BillingAddress, + Order, + BillRecord, + Payment, + ProductToRecurringPeriod, + RecurringPeriod, + StripeCreditCard, + StripeCustomer, + PricingPlan, + VATRate +]: + admin.site.register(m) diff --git a/uncloud_pay/apps.py b/uncloud_pay/apps.py new file mode 100644 index 0000000..051ffb4 --- /dev/null +++ b/uncloud_pay/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudPayConfig(AppConfig): + name = 'uncloud_pay' diff --git a/uncloud_pay/helpers.py b/uncloud_pay/helpers.py new file mode 100644 index 0000000..f791564 --- /dev/null +++ b/uncloud_pay/helpers.py @@ -0,0 +1,26 @@ +from functools import reduce +from datetime import datetime +from rest_framework import mixins +from rest_framework.viewsets import GenericViewSet +from django.utils import timezone +from calendar import monthrange + +def beginning_of_month(year, month): + tz = timezone.get_current_timezone() + return datetime(year=year, month=month, day=1, tzinfo=tz) + +def end_of_month(year, month): + (_, days) = monthrange(year, month) + tz = timezone.get_current_timezone() + return datetime(year=year, month=month, day=days, + hour=23, minute=59, second=59, tzinfo=tz) + +class ProductViewSet(mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.ListModelMixin, + GenericViewSet): + """ + A customer-facing viewset that provides default `create()`, `retrieve()` + and `list()`. + """ + pass diff --git a/uncloud_pay/management/commands/.gitignore b/uncloud_pay/management/commands/.gitignore new file mode 100644 index 0000000..cf5c7fa --- /dev/null +++ b/uncloud_pay/management/commands/.gitignore @@ -0,0 +1,2 @@ +# Customer tests +customer-*.py diff --git a/uncloud_pay/management/commands/add-opennebula-vm-orders.py b/uncloud_pay/management/commands/add-opennebula-vm-orders.py new file mode 100644 index 0000000..1d66790 --- /dev/null +++ b/uncloud_pay/management/commands/add-opennebula-vm-orders.py @@ -0,0 +1,140 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model + +from django.utils import timezone +from datetime import datetime, timedelta + +from uncloud_pay.models import * +from uncloud_vm.models import * + +import sys + +def vm_price_2020(cpu=1, ram=2, v6only=False): + if v6only: + discount = 9 + else: + discount = 0 + + return cpu*3 + ram*4 - discount + +def disk_price_2020(size_in_gb, disk_type): + if disk_type == VMDiskType.CEPH_SSD: + price = 3.5/10 + elif disk_type == VMDiskType.CEPH_HDD: + price = 1.5/100 + else: + raise Exception("not yet defined price") + + return size_in_gb * price + +class Command(BaseCommand): + help = 'Adding VMs / creating orders for user' + + def add_arguments(self, parser): + parser.add_argument('--username', type=str, required=True) + + def handle(self, *args, **options): + user = get_user_model().objects.get(username=options['username']) + + addr, created = BillingAddress.objects.get_or_create( + owner=user, + active=True, + defaults={'organization': 'Undefined organisation', + 'name': 'Undefined name', + 'street': 'Undefined Street', + 'city': 'Undefined city', + 'postal_code': '8750', + 'country': 'CH', + 'active': True + } + ) + + # 25206 + SSD + vm25206 = VMProduct.objects.create(name="one-25206", cores=1, ram_in_gb=4, owner=user) + vm25206.create_order_at(timezone.make_aware(datetime.datetime(2020,3,3))) + + # vm25206_ssd = VMDiskProduct.objects.create(vm=vm25206, owner=user, size_in_gb=30) + # vm25206_ssd.create_order_at(timezone.make_aware(datetime.datetime(2020,3,3))) + + # change 1 + vm25206.cores = 2 + vm25206.ram_in_gb = 8 + vm25206.save() + vm25206.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,4,17))) + + sys.exit(0) + + # change 2 + # vm25206_ssd.size_in_gb = 50 + # vm25206_ssd.save() + # vm25206_ssd.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,8,5))) + + # 25206 done. + + # 25615 + vm25615 = VMProduct.objects.create(name="one-25615", cores=1, ram_in_gb=4, owner=user) + vm25615.create_order_at(timezone.make_aware(datetime.datetime(2020,3,3))) + + # Change 2020-04-17 + vm25615.cores = 2 + vm25615.ram_in_gb = 8 + vm25615.save() + vm25615.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,4,17))) + + # vm25615_ssd = VMDiskProduct(vm=vm25615, owner=user, size_in_gb=30) + # vm25615_ssd.create_order_at(timezone.make_aware(datetime.datetime(2020,3,3))) + # vm25615_ssd.save() + + vm25208 = VMProduct.objects.create(name="one-25208", cores=1, ram_in_gb=4, owner=user) + vm25208.create_order_at(timezone.make_aware(datetime.datetime(2020,3,5))) + + vm25208.cores = 2 + vm25208.ram_in_gb = 8 + vm25208.save() + vm25208.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,4,17))) + + Bill.create_next_bills_for_user(user, ending_date=end_of_month(timezone.make_aware(datetime.datetime(2020,7,31)))) + + sys.exit(0) + + + vm25615_ssd.size_in_gb = 50 + vm25615_ssd.save() + vm25615_ssd.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,8,5))) + + + + vm25208_ssd = VMDiskProduct.objects.create(vm=vm25208, + owner=user, + size_in_gb=30) + + + + vm25208_ssd.size_in_gb = 50 + vm25208_ssd.save() + vm25208_ssd.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,8,5))) + + + # 25207 + vm25207 = VMProduct.objects.create(name="OpenNebula 25207", + cores=1, + ram_in_gb=4, + owner=user) + + vm25207_ssd = VMDiskProduct.objects.create(vm=vm25207, + owner=user, + size_in_gb=30) + + vm25207_ssd.size_in_gb = 50 + vm25207_ssd.save() + vm25207_ssd.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,8,5))) + + + vm25207.cores = 2 + vm25207.ram_in_gb = 8 + vm25207.save() + vm25207.create_or_update_order(when_to_start=timezone.make_aware(datetime.datetime(2020,6,19))) + + + # FIXES: check starting times (they are slightly different) + # add vm 25236 diff --git a/uncloud_pay/management/commands/bootstrap-user.py b/uncloud_pay/management/commands/bootstrap-user.py new file mode 100644 index 0000000..b78e80c --- /dev/null +++ b/uncloud_pay/management/commands/bootstrap-user.py @@ -0,0 +1,40 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model +import datetime + +from uncloud_pay.models import * + +class Command(BaseCommand): + help = 'Bootstrap user (for testing)' + + def add_arguments(self, parser): + parser.add_argument('--username', type=str, required=True) + + def handle(self, *args, **options): + user = get_user_model().objects.get(username=options['username']) + + addr = BillingAddress.objects.get_or_create( + owner=user, + active=True, + defaults={'organization': 'ungleich', + 'name': 'Nico Schottelius', + 'street': 'Hauptstrasse 14', + 'city': 'Luchsingen', + 'postal_code': '8775', + 'country': 'CH' } + ) + + + bills = Bill.objects.filter(owner=user) + + # not even one bill? create! + if bills: + bill = bills[0] + else: + bill = Bill.objects.create(owner=user) + + # find any order that is associated to this bill + orders = Order.objects.filter(owner=user) + + print(f"Addr: {addr}") + print(f"Bill: {bill}") diff --git a/uncloud_pay/management/commands/charge-negative-balance.py b/uncloud_pay/management/commands/charge-negative-balance.py new file mode 100644 index 0000000..8405bd3 --- /dev/null +++ b/uncloud_pay/management/commands/charge-negative-balance.py @@ -0,0 +1,28 @@ +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Order, Bill, get_balance_for_user +import uncloud_pay.stripe as uncloud_stripe + +from datetime import timedelta +from django.utils import timezone + +class Command(BaseCommand): + help = 'Generate bills and charge customers if necessary.' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + users = User.objects.all() + print("Processing {} users.".format(users.count())) + for user in users: + balance = get_balance_for_user(user) + if balance < 0: + print("User {} has negative balance ({}), charging.".format(user.username, balance)) + amount_to_be_charged = abs(balance) + result = uncloud_stripe.charge_customer(user, amount_to_be_charged) + if result.status != 'succeeded': + print("ERR: charging {} with method {} failed" + .format(user.username, result) + ) + print("=> Done.") diff --git a/uncloud_pay/management/commands/generate-bills.py b/uncloud_pay/management/commands/generate-bills.py new file mode 100644 index 0000000..5bd4519 --- /dev/null +++ b/uncloud_pay/management/commands/generate-bills.py @@ -0,0 +1,35 @@ +import logging + +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Order, Bill +from django.core.exceptions import ObjectDoesNotExist + +from datetime import timedelta, date +from django.utils import timezone +from uncloud_pay.models import Bill + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = 'Generate bills and charge customers if necessary.' + + def add_arguments(self, parser): + pass + + # TODO: use logger.* + def handle(self, *args, **options): + # Iterate over all 'active' users. + # TODO: filter out inactive users. + users = User.objects.all() + print("Processing {} users.".format(users.count())) + + for user in users: + now = timezone.now() + Bill.generate_for( + year=now.year, + month=now.month, + user=user) + + # We're done for this round :-) + print("=> Done.") diff --git a/uncloud_pay/management/commands/handle-overdue-bills.py b/uncloud_pay/management/commands/handle-overdue-bills.py new file mode 100644 index 0000000..595fbc2 --- /dev/null +++ b/uncloud_pay/management/commands/handle-overdue-bills.py @@ -0,0 +1,23 @@ +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Bill + +from datetime import timedelta +from django.utils import timezone + +class Command(BaseCommand): + help = 'Take action on overdue bills.' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + users = User.objects.all() + print("Processing {} users.".format(users.count())) + for user in users: + for bill in Bill.get_overdue_for(user): + print("/!\ Overdue bill for {}, {} with amount {}" + .format(user.username, bill.uuid, bill.amount)) + # TODO: take action? + + print("=> Done.") diff --git a/uncloud_pay/management/commands/import-vat-rates.py b/uncloud_pay/management/commands/import-vat-rates.py new file mode 100644 index 0000000..a741740 --- /dev/null +++ b/uncloud_pay/management/commands/import-vat-rates.py @@ -0,0 +1,50 @@ +from django.core.management.base import BaseCommand +from uncloud_pay.models import VATRate + +import logging +import urllib +import csv +import sys +import io + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = '''Imports VAT Rates. Assume vat rates of format https://github.com/kdeldycke/vat-rates/blob/master/vat_rates.csv''' + vat_url = "https://raw.githubusercontent.com/ungleich/vat-rates/main/vat_rates.csv" + + + def add_arguments(self, parser): + parser.add_argument('--vat-url', default=self.vat_url) + + def handle(self, *args, **options): + vat_url = options['vat_url'] + url_open = urllib.request.urlopen(vat_url) + + # map to fileio using stringIO + csv_file = io.StringIO(url_open.read().decode('utf-8')) + reader = csv.DictReader(csv_file) + + for row in reader: + if row["territory_codes"] and len(row["territory_codes"].splitlines()) > 1: + for code in row["territory_codes"].splitlines(): + VATRate.objects.get_or_create( + starting_date=row["start_date"], + ending_date=row["stop_date"] if row["stop_date"] != "" else None, + territory_codes=code, + currency_code=row["currency_code"], + rate=row["rate"], + rate_type=row["rate_type"], + description=row["description"] + ) + else: + VATRate.objects.get_or_create( + starting_date=row["start_date"], + ending_date=row["stop_date"] if row["stop_date"] != "" else None, + territory_codes=row["territory_codes"], + currency_code=row["currency_code"], + rate=row["rate"], + rate_type=row["rate_type"], + description=row["description"] + ) + logger.info('All VAT Rates have been added!') diff --git a/uncloud_pay/migrations/0001_initial.py b/uncloud_pay/migrations/0001_initial.py new file mode 100644 index 0000000..e65f3dd --- /dev/null +++ b/uncloud_pay/migrations/0001_initial.py @@ -0,0 +1,202 @@ +# Generated by Django 3.1 on 2020-12-28 22:19 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import uncloud.models +import uncloud_pay.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_auth', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Bill', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField(default=uncloud_pay.models.start_of_this_month)), + ('ending_date', models.DateTimeField()), + ('due_date', models.DateField(default=uncloud_pay.models.default_payment_delay)), + ('is_final', models.BooleanField(default=False)), + ], + ), + migrations.CreateModel( + name='BillingAddress', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('full_name', models.CharField(max_length=256)), + ('organization', models.CharField(blank=True, max_length=256, null=True)), + ('street', models.CharField(max_length=256)), + ('city', models.CharField(max_length=256)), + ('postal_code', models.CharField(max_length=64)), + ('country', uncloud.models.CountryField(blank=True, choices=[('AD', 'Andorra'), ('AE', 'United Arab Emirates'), ('AF', 'Afghanistan'), ('AG', 'Antigua & Barbuda'), ('AI', 'Anguilla'), ('AL', 'Albania'), ('AM', 'Armenia'), ('AN', 'Netherlands Antilles'), ('AO', 'Angola'), ('AQ', 'Antarctica'), ('AR', 'Argentina'), ('AS', 'American Samoa'), ('AT', 'Austria'), ('AU', 'Australia'), ('AW', 'Aruba'), ('AZ', 'Azerbaijan'), ('BA', 'Bosnia and Herzegovina'), ('BB', 'Barbados'), ('BD', 'Bangladesh'), ('BE', 'Belgium'), ('BF', 'Burkina Faso'), ('BG', 'Bulgaria'), ('BH', 'Bahrain'), ('BI', 'Burundi'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BN', 'Brunei Darussalam'), ('BO', 'Bolivia'), ('BR', 'Brazil'), ('BS', 'Bahama'), ('BT', 'Bhutan'), ('BV', 'Bouvet Island'), ('BW', 'Botswana'), ('BY', 'Belarus'), ('BZ', 'Belize'), ('CA', 'Canada'), ('CC', 'Cocos (Keeling) Islands'), ('CF', 'Central African Republic'), ('CG', 'Congo'), ('CH', 'Switzerland'), ('CI', 'Ivory Coast'), ('CK', 'Cook Iislands'), ('CL', 'Chile'), ('CM', 'Cameroon'), ('CN', 'China'), ('CO', 'Colombia'), ('CR', 'Costa Rica'), ('CU', 'Cuba'), ('CV', 'Cape Verde'), ('CX', 'Christmas Island'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DE', 'Germany'), ('DJ', 'Djibouti'), ('DK', 'Denmark'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('DZ', 'Algeria'), ('EC', 'Ecuador'), ('EE', 'Estonia'), ('EG', 'Egypt'), ('EH', 'Western Sahara'), ('ER', 'Eritrea'), ('ES', 'Spain'), ('ET', 'Ethiopia'), ('FI', 'Finland'), ('FJ', 'Fiji'), ('FK', 'Falkland Islands (Malvinas)'), ('FM', 'Micronesia'), ('FO', 'Faroe Islands'), ('FR', 'France'), ('FX', 'France, Metropolitan'), ('GA', 'Gabon'), ('GB', 'United Kingdom (Great Britain)'), ('GD', 'Grenada'), ('GE', 'Georgia'), ('GF', 'French Guiana'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GL', 'Greenland'), ('GM', 'Gambia'), ('GN', 'Guinea'), ('GP', 'Guadeloupe'), ('GQ', 'Equatorial Guinea'), ('GR', 'Greece'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GT', 'Guatemala'), ('GU', 'Guam'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HK', 'Hong Kong'), ('HM', 'Heard & McDonald Islands'), ('HN', 'Honduras'), ('HR', 'Croatia'), ('HT', 'Haiti'), ('HU', 'Hungary'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('IL', 'Israel'), ('IN', 'India'), ('IO', 'British Indian Ocean Territory'), ('IQ', 'Iraq'), ('IR', 'Islamic Republic of Iran'), ('IS', 'Iceland'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JO', 'Jordan'), ('JP', 'Japan'), ('KE', 'Kenya'), ('KG', 'Kyrgyzstan'), ('KH', 'Cambodia'), ('KI', 'Kiribati'), ('KM', 'Comoros'), ('KN', 'St. Kitts and Nevis'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('KW', 'Kuwait'), ('KY', 'Cayman Islands'), ('KZ', 'Kazakhstan'), ('LA', "Lao People's Democratic Republic"), ('LB', 'Lebanon'), ('LC', 'Saint Lucia'), ('LI', 'Liechtenstein'), ('LK', 'Sri Lanka'), ('LR', 'Liberia'), ('LS', 'Lesotho'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('LV', 'Latvia'), ('LY', 'Libyan Arab Jamahiriya'), ('MA', 'Morocco'), ('MC', 'Monaco'), ('MD', 'Moldova, Republic of'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('ML', 'Mali'), ('MN', 'Mongolia'), ('MM', 'Myanmar'), ('MO', 'Macau'), ('MP', 'Northern Mariana Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MS', 'Monserrat'), ('MT', 'Malta'), ('MU', 'Mauritius'), ('MV', 'Maldives'), ('MW', 'Malawi'), ('MX', 'Mexico'), ('MY', 'Malaysia'), ('MZ', 'Mozambique'), ('NA', 'Namibia'), ('NC', 'New Caledonia'), ('NE', 'Niger'), ('NF', 'Norfolk Island'), ('NG', 'Nigeria'), ('NI', 'Nicaragua'), ('NL', 'Netherlands'), ('NO', 'Norway'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('NU', 'Niue'), ('NZ', 'New Zealand'), ('OM', 'Oman'), ('PA', 'Panama'), ('PE', 'Peru'), ('PF', 'French Polynesia'), ('PG', 'Papua New Guinea'), ('PH', 'Philippines'), ('PK', 'Pakistan'), ('PL', 'Poland'), ('PM', 'St. Pierre & Miquelon'), ('PN', 'Pitcairn'), ('PR', 'Puerto Rico'), ('PT', 'Portugal'), ('PW', 'Palau'), ('PY', 'Paraguay'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SA', 'Saudi Arabia'), ('SB', 'Solomon Islands'), ('SC', 'Seychelles'), ('SD', 'Sudan'), ('SE', 'Sweden'), ('SG', 'Singapore'), ('SH', 'St. Helena'), ('SI', 'Slovenia'), ('SJ', 'Svalbard & Jan Mayen Islands'), ('SK', 'Slovakia'), ('SL', 'Sierra Leone'), ('SM', 'San Marino'), ('SN', 'Senegal'), ('SO', 'Somalia'), ('SR', 'Suriname'), ('ST', 'Sao Tome & Principe'), ('SV', 'El Salvador'), ('SY', 'Syrian Arab Republic'), ('SZ', 'Swaziland'), ('TC', 'Turks & Caicos Islands'), ('TD', 'Chad'), ('TF', 'French Southern Territories'), ('TG', 'Togo'), ('TH', 'Thailand'), ('TJ', 'Tajikistan'), ('TK', 'Tokelau'), ('TM', 'Turkmenistan'), ('TN', 'Tunisia'), ('TO', 'Tonga'), ('TP', 'East Timor'), ('TR', 'Turkey'), ('TT', 'Trinidad & Tobago'), ('TV', 'Tuvalu'), ('TW', 'Taiwan, Province of China'), ('TZ', 'Tanzania, United Republic of'), ('UA', 'Ukraine'), ('UG', 'Uganda'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VA', 'Vatican City State (Holy See)'), ('VC', 'St. Vincent & the Grenadines'), ('VE', 'Venezuela'), ('VG', 'British Virgin Islands'), ('VI', 'United States Virgin Islands'), ('VN', 'Viet Nam'), ('VU', 'Vanuatu'), ('WF', 'Wallis & Futuna Islands'), ('WS', 'Samoa'), ('YE', 'Yemen'), ('YT', 'Mayotte'), ('YU', 'Yugoslavia'), ('ZA', 'South Africa'), ('ZM', 'Zambia'), ('ZR', 'Zaire'), ('ZW', 'Zimbabwe')], default='CH', max_length=2)), + ('vat_number', models.CharField(blank=True, default='', max_length=100)), + ('active', models.BooleanField(default=False)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Product', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=256, unique=True)), + ('description', models.CharField(max_length=1024)), + ('config', models.JSONField()), + ('currency', models.CharField(choices=[('CHF', 'Swiss Franc'), ('EUR', 'Euro'), ('USD', 'US Dollar')], default='CHF', max_length=32)), + ], + ), + migrations.CreateModel( + name='RecurringPeriod', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=100, unique=True)), + ('duration_seconds', models.IntegerField(unique=True)), + ], + ), + migrations.CreateModel( + name='StripeCustomer', + fields=[ + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to='uncloud_auth.user')), + ('stripe_id', models.CharField(max_length=32)), + ], + ), + migrations.CreateModel( + name='VATRate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('starting_date', models.DateField(blank=True, null=True)), + ('ending_date', models.DateField(blank=True, null=True)), + ('territory_codes', models.TextField(blank=True, default='')), + ('currency_code', models.CharField(max_length=10)), + ('rate', models.FloatField()), + ('rate_type', models.TextField(blank=True, default='')), + ('description', models.TextField(blank=True, default='')), + ], + ), + migrations.CreateModel( + name='StripeCreditCard', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('card_name', models.CharField(default='My credit card', max_length=128)), + ('card_id', models.CharField(max_length=32)), + ('last4', models.CharField(max_length=4)), + ('brand', models.CharField(max_length=64)), + ('expiry_date', models.DateField()), + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='ProductToRecurringPeriod', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('is_default', models.BooleanField(default=False)), + ('product', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.product')), + ('recurring_period', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.recurringperiod')), + ], + ), + migrations.AddField( + model_name='product', + name='recurring_periods', + field=models.ManyToManyField(through='uncloud_pay.ProductToRecurringPeriod', to='uncloud_pay.RecurringPeriod'), + ), + migrations.CreateModel( + name='PaymentMethod', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('source', models.CharField(choices=[('stripe', 'Stripe'), ('unknown', 'Unknown')], default='stripe', max_length=256)), + ('description', models.TextField()), + ('primary', models.BooleanField(default=False, editable=False)), + ('stripe_payment_method_id', models.CharField(blank=True, max_length=32, null=True)), + ('stripe_setup_intent_id', models.CharField(blank=True, max_length=32, null=True)), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Payment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('amount', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('source', models.CharField(choices=[('wire', 'Wire Transfer'), ('stripe', 'Stripe'), ('voucher', 'Voucher'), ('referral', 'Referral'), ('unknown', 'Unknown')], default='unknown', max_length=256)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Order', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('description', models.TextField()), + ('config', models.JSONField()), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField(default=django.utils.timezone.now)), + ('ending_date', models.DateTimeField(blank=True, null=True)), + ('one_time_price', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('recurring_price', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('currency', models.CharField(choices=[('CHF', 'Swiss Franc'), ('EUR', 'Euro'), ('USD', 'US Dollar')], default='CHF', max_length=32)), + ('should_be_billed', models.BooleanField(default=True)), + ('billing_address', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.billingaddress')), + ('depends_on', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='parent_of', to='uncloud_pay.order')), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('product', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.product')), + ('recurring_period', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.recurringperiod')), + ('replaces', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='replaced_by', to='uncloud_pay.order')), + ], + ), + migrations.CreateModel( + name='Membership', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('starting_date', models.DateField(blank=True, null=True)), + ('ending_date', models.DateField(blank=True, null=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='BillRecord', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField()), + ('ending_date', models.DateTimeField()), + ('is_recurring_record', models.BooleanField()), + ('bill', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.bill')), + ('order', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.order')), + ], + ), + migrations.AddField( + model_name='bill', + name='billing_address', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.billingaddress'), + ), + migrations.AddField( + model_name='bill', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + migrations.AddConstraint( + model_name='producttorecurringperiod', + constraint=models.UniqueConstraint(condition=models.Q(is_default=True), fields=('product',), name='one_default_recurring_period_per_product'), + ), + migrations.AddConstraint( + model_name='producttorecurringperiod', + constraint=models.UniqueConstraint(fields=('product', 'recurring_period'), name='recurring_period_once_per_product'), + ), + migrations.AddConstraint( + model_name='billingaddress', + constraint=models.UniqueConstraint(condition=models.Q(active=True), fields=('owner',), name='one_active_billing_address_per_user'), + ), + migrations.AddConstraint( + model_name='bill', + constraint=models.UniqueConstraint(fields=('owner', 'starting_date', 'ending_date'), name='one_bill_per_month_per_user'), + ), + ] diff --git a/uncloud_pay/migrations/0002_auto_20201228_2244.py b/uncloud_pay/migrations/0002_auto_20201228_2244.py new file mode 100644 index 0000000..4665553 --- /dev/null +++ b/uncloud_pay/migrations/0002_auto_20201228_2244.py @@ -0,0 +1,24 @@ +# Generated by Django 3.1 on 2020-12-28 22:44 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='payment', + name='currency', + field=models.CharField(choices=[('CHF', 'Swiss Franc'), ('EUR', 'Euro'), ('USD', 'US Dollar')], default='CHF', max_length=32), + ), + migrations.AlterField( + model_name='payment', + name='amount', + field=models.DecimalField(decimal_places=2, max_digits=10, validators=[django.core.validators.MinValueValidator(0)]), + ), + ] diff --git a/uncloud_pay/migrations/0003_auto_20201228_2256.py b/uncloud_pay/migrations/0003_auto_20201228_2256.py new file mode 100644 index 0000000..b516bd5 --- /dev/null +++ b/uncloud_pay/migrations/0003_auto_20201228_2256.py @@ -0,0 +1,28 @@ +# Generated by Django 3.1 on 2020-12-28 22:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0002_auto_20201228_2244'), + ] + + operations = [ + migrations.AlterField( + model_name='order', + name='currency', + field=models.CharField(choices=[('CHF', 'Swiss Franc')], default='CHF', max_length=32), + ), + migrations.AlterField( + model_name='payment', + name='currency', + field=models.CharField(choices=[('CHF', 'Swiss Franc')], default='CHF', max_length=32), + ), + migrations.AlterField( + model_name='product', + name='currency', + field=models.CharField(choices=[('CHF', 'Swiss Franc')], default='CHF', max_length=32), + ), + ] diff --git a/uncloud_pay/migrations/0004_stripecreditcard_active.py b/uncloud_pay/migrations/0004_stripecreditcard_active.py new file mode 100644 index 0000000..3fb8015 --- /dev/null +++ b/uncloud_pay/migrations/0004_stripecreditcard_active.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1 on 2020-12-28 23:34 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0003_auto_20201228_2256'), + ] + + operations = [ + migrations.AddField( + model_name='stripecreditcard', + name='active', + field=models.BooleanField(default=True), + ), + ] diff --git a/uncloud_pay/migrations/0005_auto_20201228_2335.py b/uncloud_pay/migrations/0005_auto_20201228_2335.py new file mode 100644 index 0000000..814752e --- /dev/null +++ b/uncloud_pay/migrations/0005_auto_20201228_2335.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1 on 2020-12-28 23:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0004_stripecreditcard_active'), + ] + + operations = [ + migrations.AlterField( + model_name='stripecreditcard', + name='active', + field=models.BooleanField(default=False), + ), + ] diff --git a/uncloud_pay/migrations/0006_auto_20201228_2337.py b/uncloud_pay/migrations/0006_auto_20201228_2337.py new file mode 100644 index 0000000..a164767 --- /dev/null +++ b/uncloud_pay/migrations/0006_auto_20201228_2337.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1 on 2020-12-28 23:37 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0005_auto_20201228_2335'), + ] + + operations = [ + migrations.AlterField( + model_name='stripecreditcard', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/uncloud_pay/migrations/0007_auto_20201228_2338.py b/uncloud_pay/migrations/0007_auto_20201228_2338.py new file mode 100644 index 0000000..315a74b --- /dev/null +++ b/uncloud_pay/migrations/0007_auto_20201228_2338.py @@ -0,0 +1,17 @@ +# Generated by Django 3.1 on 2020-12-28 23:38 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0006_auto_20201228_2337'), + ] + + operations = [ + migrations.AddConstraint( + model_name='stripecreditcard', + constraint=models.UniqueConstraint(condition=models.Q(active=True), fields=('owner',), name='one_active_card_per_user'), + ), + ] diff --git a/uncloud_pay/migrations/0008_payment_external_reference.py b/uncloud_pay/migrations/0008_payment_external_reference.py new file mode 100644 index 0000000..0de20b6 --- /dev/null +++ b/uncloud_pay/migrations/0008_payment_external_reference.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1 on 2020-12-29 00:04 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0007_auto_20201228_2338'), + ] + + operations = [ + migrations.AddField( + model_name='payment', + name='external_reference', + field=models.CharField(default='', max_length=256), + ), + ] diff --git a/uncloud_pay/migrations/0009_auto_20201229_0037.py b/uncloud_pay/migrations/0009_auto_20201229_0037.py new file mode 100644 index 0000000..fc195e4 --- /dev/null +++ b/uncloud_pay/migrations/0009_auto_20201229_0037.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1 on 2020-12-29 00:37 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0008_payment_external_reference'), + ] + + operations = [ + migrations.AlterField( + model_name='payment', + name='external_reference', + field=models.CharField(blank=True, default='', max_length=256, null=True), + ), + ] diff --git a/uncloud_pay/migrations/0010_auto_20201229_0042.py b/uncloud_pay/migrations/0010_auto_20201229_0042.py new file mode 100644 index 0000000..6dd6a60 --- /dev/null +++ b/uncloud_pay/migrations/0010_auto_20201229_0042.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2020-12-29 00:42 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0009_auto_20201229_0037'), + ] + + operations = [ + migrations.AlterField( + model_name='payment', + name='timestamp', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + ] diff --git a/uncloud_pay/migrations/0011_auto_20210101_1308.py b/uncloud_pay/migrations/0011_auto_20210101_1308.py new file mode 100644 index 0000000..942f430 --- /dev/null +++ b/uncloud_pay/migrations/0011_auto_20210101_1308.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1 on 2021-01-01 13:08 + +from django.db import migrations +import uncloud.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0010_auto_20201229_0042'), + ] + + operations = [ + migrations.AlterField( + model_name='billingaddress', + name='country', + field=uncloud.models.CountryField(choices=[('AD', 'Andorra'), ('AE', 'United Arab Emirates'), ('AF', 'Afghanistan'), ('AG', 'Antigua & Barbuda'), ('AI', 'Anguilla'), ('AL', 'Albania'), ('AM', 'Armenia'), ('AN', 'Netherlands Antilles'), ('AO', 'Angola'), ('AQ', 'Antarctica'), ('AR', 'Argentina'), ('AS', 'American Samoa'), ('AT', 'Austria'), ('AU', 'Australia'), ('AW', 'Aruba'), ('AZ', 'Azerbaijan'), ('BA', 'Bosnia and Herzegovina'), ('BB', 'Barbados'), ('BD', 'Bangladesh'), ('BE', 'Belgium'), ('BF', 'Burkina Faso'), ('BG', 'Bulgaria'), ('BH', 'Bahrain'), ('BI', 'Burundi'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BN', 'Brunei Darussalam'), ('BO', 'Bolivia'), ('BR', 'Brazil'), ('BS', 'Bahama'), ('BT', 'Bhutan'), ('BV', 'Bouvet Island'), ('BW', 'Botswana'), ('BY', 'Belarus'), ('BZ', 'Belize'), ('CA', 'Canada'), ('CC', 'Cocos (Keeling) Islands'), ('CF', 'Central African Republic'), ('CG', 'Congo'), ('CH', 'Switzerland'), ('CI', 'Ivory Coast'), ('CK', 'Cook Iislands'), ('CL', 'Chile'), ('CM', 'Cameroon'), ('CN', 'China'), ('CO', 'Colombia'), ('CR', 'Costa Rica'), ('CU', 'Cuba'), ('CV', 'Cape Verde'), ('CX', 'Christmas Island'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DE', 'Germany'), ('DJ', 'Djibouti'), ('DK', 'Denmark'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('DZ', 'Algeria'), ('EC', 'Ecuador'), ('EE', 'Estonia'), ('EG', 'Egypt'), ('EH', 'Western Sahara'), ('ER', 'Eritrea'), ('ES', 'Spain'), ('ET', 'Ethiopia'), ('FI', 'Finland'), ('FJ', 'Fiji'), ('FK', 'Falkland Islands (Malvinas)'), ('FM', 'Micronesia'), ('FO', 'Faroe Islands'), ('FR', 'France'), ('FX', 'France, Metropolitan'), ('GA', 'Gabon'), ('GB', 'United Kingdom (Great Britain)'), ('GD', 'Grenada'), ('GE', 'Georgia'), ('GF', 'French Guiana'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GL', 'Greenland'), ('GM', 'Gambia'), ('GN', 'Guinea'), ('GP', 'Guadeloupe'), ('GQ', 'Equatorial Guinea'), ('GR', 'Greece'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GT', 'Guatemala'), ('GU', 'Guam'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HK', 'Hong Kong'), ('HM', 'Heard & McDonald Islands'), ('HN', 'Honduras'), ('HR', 'Croatia'), ('HT', 'Haiti'), ('HU', 'Hungary'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('IL', 'Israel'), ('IN', 'India'), ('IO', 'British Indian Ocean Territory'), ('IQ', 'Iraq'), ('IR', 'Islamic Republic of Iran'), ('IS', 'Iceland'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JO', 'Jordan'), ('JP', 'Japan'), ('KE', 'Kenya'), ('KG', 'Kyrgyzstan'), ('KH', 'Cambodia'), ('KI', 'Kiribati'), ('KM', 'Comoros'), ('KN', 'St. Kitts and Nevis'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('KW', 'Kuwait'), ('KY', 'Cayman Islands'), ('KZ', 'Kazakhstan'), ('LA', "Lao People's Democratic Republic"), ('LB', 'Lebanon'), ('LC', 'Saint Lucia'), ('LI', 'Liechtenstein'), ('LK', 'Sri Lanka'), ('LR', 'Liberia'), ('LS', 'Lesotho'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('LV', 'Latvia'), ('LY', 'Libyan Arab Jamahiriya'), ('MA', 'Morocco'), ('MC', 'Monaco'), ('MD', 'Moldova, Republic of'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('ML', 'Mali'), ('MN', 'Mongolia'), ('MM', 'Myanmar'), ('MO', 'Macau'), ('MP', 'Northern Mariana Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MS', 'Monserrat'), ('MT', 'Malta'), ('MU', 'Mauritius'), ('MV', 'Maldives'), ('MW', 'Malawi'), ('MX', 'Mexico'), ('MY', 'Malaysia'), ('MZ', 'Mozambique'), ('NA', 'Namibia'), ('NC', 'New Caledonia'), ('NE', 'Niger'), ('NF', 'Norfolk Island'), ('NG', 'Nigeria'), ('NI', 'Nicaragua'), ('NL', 'Netherlands'), ('NO', 'Norway'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('NU', 'Niue'), ('NZ', 'New Zealand'), ('OM', 'Oman'), ('PA', 'Panama'), ('PE', 'Peru'), ('PF', 'French Polynesia'), ('PG', 'Papua New Guinea'), ('PH', 'Philippines'), ('PK', 'Pakistan'), ('PL', 'Poland'), ('PM', 'St. Pierre & Miquelon'), ('PN', 'Pitcairn'), ('PR', 'Puerto Rico'), ('PT', 'Portugal'), ('PW', 'Palau'), ('PY', 'Paraguay'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SA', 'Saudi Arabia'), ('SB', 'Solomon Islands'), ('SC', 'Seychelles'), ('SD', 'Sudan'), ('SE', 'Sweden'), ('SG', 'Singapore'), ('SH', 'St. Helena'), ('SI', 'Slovenia'), ('SJ', 'Svalbard & Jan Mayen Islands'), ('SK', 'Slovakia'), ('SL', 'Sierra Leone'), ('SM', 'San Marino'), ('SN', 'Senegal'), ('SO', 'Somalia'), ('SR', 'Suriname'), ('ST', 'Sao Tome & Principe'), ('SV', 'El Salvador'), ('SY', 'Syrian Arab Republic'), ('SZ', 'Swaziland'), ('TC', 'Turks & Caicos Islands'), ('TD', 'Chad'), ('TF', 'French Southern Territories'), ('TG', 'Togo'), ('TH', 'Thailand'), ('TJ', 'Tajikistan'), ('TK', 'Tokelau'), ('TM', 'Turkmenistan'), ('TN', 'Tunisia'), ('TO', 'Tonga'), ('TP', 'East Timor'), ('TR', 'Turkey'), ('TT', 'Trinidad & Tobago'), ('TV', 'Tuvalu'), ('TW', 'Taiwan, Province of China'), ('TZ', 'Tanzania, United Republic of'), ('UA', 'Ukraine'), ('UG', 'Uganda'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VA', 'Vatican City State (Holy See)'), ('VC', 'St. Vincent & the Grenadines'), ('VE', 'Venezuela'), ('VG', 'British Virgin Islands'), ('VI', 'United States Virgin Islands'), ('VN', 'Viet Nam'), ('VU', 'Vanuatu'), ('WF', 'Wallis & Futuna Islands'), ('WS', 'Samoa'), ('YE', 'Yemen'), ('YT', 'Mayotte'), ('YU', 'Yugoslavia'), ('ZA', 'South Africa'), ('ZM', 'Zambia'), ('ZR', 'Zaire'), ('ZW', 'Zimbabwe')], default='CH', max_length=2), + ), + ] diff --git a/uncloud_pay/migrations/0012_auto_20210630_0742.py b/uncloud_pay/migrations/0012_auto_20210630_0742.py new file mode 100644 index 0000000..45e3dfe --- /dev/null +++ b/uncloud_pay/migrations/0012_auto_20210630_0742.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.4 on 2021-06-30 07:42 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0011_auto_20210101_1308'), + ] + + operations = [ + migrations.AddField( + model_name='billingaddress', + name='vat_number_verified', + field=models.BooleanField(default=False), + ), + migrations.AlterField( + model_name='payment', + name='source', + field=models.CharField(choices=[('wire', 'Wire Transfer'), ('stripe', 'Stripe'), ('voucher', 'Voucher'), ('referral', 'Referral')], max_length=256), + ), + ] diff --git a/uncloud_pay/migrations/0013_alter_billingaddress_owner.py b/uncloud_pay/migrations/0013_alter_billingaddress_owner.py new file mode 100644 index 0000000..7597129 --- /dev/null +++ b/uncloud_pay/migrations/0013_alter_billingaddress_owner.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.4 on 2021-07-03 15:23 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0012_auto_20210630_0742'), + ] + + operations = [ + migrations.AlterField( + model_name='billingaddress', + name='owner', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='billing_addresses', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/uncloud_pay/migrations/0014_auto_20210703_1747.py b/uncloud_pay/migrations/0014_auto_20210703_1747.py new file mode 100644 index 0000000..1c004d0 --- /dev/null +++ b/uncloud_pay/migrations/0014_auto_20210703_1747.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.4 on 2021-07-03 17:47 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0013_alter_billingaddress_owner'), + ] + + operations = [ + migrations.AddField( + model_name='billingaddress', + name='stripe_tax_id', + field=models.CharField(blank=True, default='', max_length=100), + ), + migrations.AddField( + model_name='billingaddress', + name='vat_number_validated_on', + field=models.DateTimeField(blank=True, null=True), + ), + ] diff --git a/uncloud_pay/migrations/0015_auto_20210705_0849.py b/uncloud_pay/migrations/0015_auto_20210705_0849.py new file mode 100644 index 0000000..dfb6d80 --- /dev/null +++ b/uncloud_pay/migrations/0015_auto_20210705_0849.py @@ -0,0 +1,34 @@ +# Generated by Django 3.2.4 on 2021-07-05 08:49 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0014_auto_20210703_1747'), + ] + + operations = [ + migrations.AddField( + model_name='order', + name='customer', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.stripecustomer'), + ), + migrations.AddField( + model_name='order', + name='status', + field=models.CharField(choices=[('draft', 'Draft'), ('declined', 'Declined'), ('approved', 'Approved')], default='draft', max_length=100), + ), + migrations.AddField( + model_name='order', + name='stripe_charge_id', + field=models.CharField(max_length=100, null=True), + ), + migrations.AddField( + model_name='order', + name='vm_id', + field=models.IntegerField(default=0), + ), + ] diff --git a/uncloud_pay/migrations/0016_pricingplan.py b/uncloud_pay/migrations/0016_pricingplan.py new file mode 100644 index 0000000..505c141 --- /dev/null +++ b/uncloud_pay/migrations/0016_pricingplan.py @@ -0,0 +1,29 @@ +# Generated by Django 3.2.4 on 2021-07-06 13:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0015_auto_20210705_0849'), + ] + + operations = [ + migrations.CreateModel( + name='PricingPlan', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(max_length=255, unique=True)), + ('vat_inclusive', models.BooleanField(default=True)), + ('vat_percentage', models.DecimalField(blank=True, decimal_places=5, default=0, max_digits=7)), + ('set_up_fees', models.DecimalField(decimal_places=2, default=0, max_digits=7)), + ('cores_unit_price', models.DecimalField(decimal_places=2, default=0, max_digits=7)), + ('ram_unit_price', models.DecimalField(decimal_places=2, default=0, max_digits=7)), + ('storage_unit_price', models.DecimalField(decimal_places=2, default=0, max_digits=7)), + ('discount_name', models.CharField(blank=True, max_length=255, null=True)), + ('discount_amount', models.DecimalField(decimal_places=2, default=0, max_digits=6)), + ('stripe_coupon_id', models.CharField(blank=True, max_length=255, null=True)), + ], + ), + ] diff --git a/uncloud_pay/migrations/0017_auto_20210706_1728.py b/uncloud_pay/migrations/0017_auto_20210706_1728.py new file mode 100644 index 0000000..1571b10 --- /dev/null +++ b/uncloud_pay/migrations/0017_auto_20210706_1728.py @@ -0,0 +1,23 @@ +# Generated by Django 3.2.4 on 2021-07-06 17:28 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0016_pricingplan'), + ] + + operations = [ + migrations.RemoveField( + model_name='paymentmethod', + name='owner', + ), + migrations.DeleteModel( + name='Payment', + ), + migrations.DeleteModel( + name='PaymentMethod', + ), + ] diff --git a/uncloud_pay/migrations/0018_payment.py b/uncloud_pay/migrations/0018_payment.py new file mode 100644 index 0000000..47d6e3a --- /dev/null +++ b/uncloud_pay/migrations/0018_payment.py @@ -0,0 +1,30 @@ +# Generated by Django 3.2.4 on 2021-07-06 17:47 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0017_auto_20210706_1728'), + ] + + operations = [ + migrations.CreateModel( + name='Payment', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('amount', models.DecimalField(decimal_places=2, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('source', models.CharField(choices=[('wire', 'Wire Transfer'), ('stripe', 'Stripe'), ('voucher', 'Voucher'), ('referral', 'Referral')], max_length=256)), + ('timestamp', models.DateTimeField(default=django.utils.timezone.now)), + ('currency', models.CharField(choices=[('CHF', 'Swiss Franc')], default='CHF', max_length=32)), + ('external_reference', models.CharField(blank=True, default='', max_length=256, null=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/uncloud_pay/migrations/0019_order_pricing_plan.py b/uncloud_pay/migrations/0019_order_pricing_plan.py new file mode 100644 index 0000000..5392ce6 --- /dev/null +++ b/uncloud_pay/migrations/0019_order_pricing_plan.py @@ -0,0 +1,19 @@ +# Generated by Django 3.2.4 on 2021-07-06 19:18 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0018_payment'), + ] + + operations = [ + migrations.AddField( + model_name='order', + name='pricing_plan', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.pricingplan'), + ), + ] diff --git a/uncloud_pay/migrations/0020_rename_is_final_bill_is_closed.py b/uncloud_pay/migrations/0020_rename_is_final_bill_is_closed.py new file mode 100644 index 0000000..f3419eb --- /dev/null +++ b/uncloud_pay/migrations/0020_rename_is_final_bill_is_closed.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.4 on 2021-07-07 20:18 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0019_order_pricing_plan'), + ] + + operations = [ + migrations.RenameField( + model_name='bill', + old_name='is_final', + new_name='is_closed', + ), + ] diff --git a/uncloud_pay/migrations/0021_auto_20210709_0914.py b/uncloud_pay/migrations/0021_auto_20210709_0914.py new file mode 100644 index 0000000..66e3dcb --- /dev/null +++ b/uncloud_pay/migrations/0021_auto_20210709_0914.py @@ -0,0 +1,21 @@ +# Generated by Django 3.2.4 on 2021-07-09 09:14 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0020_rename_is_final_bill_is_closed'), + ] + + operations = [ + migrations.RemoveField( + model_name='order', + name='stripe_charge_id', + ), + migrations.RemoveField( + model_name='order', + name='vm_id', + ), + ] diff --git a/uncloud_pay/migrations/0022_remove_order_status.py b/uncloud_pay/migrations/0022_remove_order_status.py new file mode 100644 index 0000000..2b51be8 --- /dev/null +++ b/uncloud_pay/migrations/0022_remove_order_status.py @@ -0,0 +1,17 @@ +# Generated by Django 3.2.4 on 2021-07-11 08:32 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0021_auto_20210709_0914'), + ] + + operations = [ + migrations.RemoveField( + model_name='order', + name='status', + ), + ] diff --git a/uncloud_pay/migrations/__init__.py b/uncloud_pay/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_pay/models.py b/uncloud_pay/models.py new file mode 100644 index 0000000..a4cf007 --- /dev/null +++ b/uncloud_pay/models.py @@ -0,0 +1,1197 @@ +import logging +import datetime +import json + +from math import ceil +from calendar import monthrange +from decimal import Decimal + +from django.conf import settings +from django.contrib.auth import get_user_model +from django.core.validators import MinValueValidator +from django.db import models +from django.db.models import Q +from django.utils.translation import gettext_lazy as _ +from django.utils import timezone +from django_q.tasks import schedule +from django_q.models import Schedule +# Verify whether or not to use them here +from django.core.exceptions import ObjectDoesNotExist, ValidationError + +import uncloud_pay +from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS +from uncloud.models import UncloudAddress, UncloudProvider +from uncloud.selectors import filter_for_when +from .services import * + +# Used to generate bill due dates. +BILL_PAYMENT_DELAY=datetime.timedelta(days=settings.BILL_PAYMENT_DELAY) + +# Initialize logger. +logger = logging.getLogger(__name__) + +def default_payment_delay(): + return timezone.now() + BILL_PAYMENT_DELAY + +class Currency(models.TextChoices): + """ + Possible currencies to be billed + """ + CHF = 'CHF', _('Swiss Franc') +# EUR = 'EUR', _('Euro') +# USD = 'USD', _('US Dollar') + + +### +# Stripe + +class StripeCustomer(models.Model): + owner = models.OneToOneField( get_user_model(), + primary_key=True, + on_delete=models.CASCADE) + stripe_id = models.CharField(max_length=32) + + def __str__(self): + return self.owner.username + + +class StripeCreditCard(models.Model): + owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + + card_name = models.CharField(null=False, max_length=128, default="My credit card") + card_id = models.CharField(null=False, max_length=32) + last4 = models.CharField(null=False, max_length=4) + brand = models.CharField(null=False, max_length=64) + expiry_date = models.DateField(null=False) + active = models.BooleanField(default=False) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['owner'], + condition=models.Q(active=True), + name='one_active_card_per_user') + ] + + + def __str__(self): + return f"{self.card_name}: {self.brand} {self.last4} ({self.expiry_date})" + +class Payment(models.Model): + owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + + amount = models.DecimalField( + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + source = models.CharField(max_length=256, + choices = ( + ('wire', 'Wire Transfer'), + ('stripe', 'Stripe'), + ('voucher', 'Voucher'), + ('referral', 'Referral'), + )) + + timestamp = models.DateTimeField(default=timezone.now) + + currency = models.CharField(max_length=32, choices=Currency.choices, default=Currency.CHF) + + external_reference = models.CharField(max_length=256, default="", null=True, blank=True) + + def __str__(self): + return f"{self.amount}{self.currency} from {self.owner} via {self.source} on {self.timestamp}" + + def save(self, *args, **kwargs): + # Try to charge the user via the active card before saving otherwise throw payment Error + if self.source == 'stripe': + try: + result = uncloud_pay.stripe.charge_customer(self.owner, self.amount, self.currency,) + if not result.status or result.status != 'succeeded': + raise Exception("The payment has been failed, please try to activate another card") + super().save(*args, **kwargs) + except Exception as e: + raise e + + + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class RecurringPeriodDefaultChoices(models.IntegerChoices): + """ + This is an old class and being superseeded by the database model below + """ + PER_365D = 365*24*3600, _('Per 365 days') + PER_30D = 30*24*3600, _('Per 30 days') + PER_WEEK = 7*24*3600, _('Per Week') + PER_DAY = 24*3600, _('Per Day') + PER_HOUR = 3600, _('Per Hour') + PER_MINUTE = 60, _('Per Minute') + PER_SECOND = 1, _('Per Second') + ONE_TIME = 0, _('Onetime') + +# RecurringPeriods +class RecurringPeriod(models.Model): + """ + Available recurring periods. + By default seeded from RecurringPeriodChoices + """ + + name = models.CharField(max_length=100, unique=True) + duration_seconds = models.IntegerField(unique=True) + + @classmethod + def populate_db_defaults(cls): + for (seconds, name) in RecurringPeriodDefaultChoices.choices: + obj, created = cls.objects.get_or_create(name=name, + defaults={ 'duration_seconds': seconds }) + + @staticmethod + def secs_to_name(secs): + name = "" + days = 0 + hours = 0 + + if secs > 24*3600: + days = secs // (24*3600) + secs -= (days*24*3600) + + if secs > 3600: + hours = secs // 3600 + secs -= hours*3600 + + return f"{days} days {hours} hours {secs} seconds" + + def __str__(self): + duration = self.secs_to_name(self.duration_seconds) + + return f"{self.name} ({duration})" + + +### +# Bills. + +class BillingAddress(UncloudAddress): + owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE, related_name='billing_addresses') + vat_number = models.CharField(max_length=100, default="", blank=True) + vat_number_verified = models.BooleanField(default=False) + vat_number_validated_on = models.DateTimeField(blank=True, null=True) + stripe_tax_id = models.CharField(max_length=100, default="", blank=True) + active = models.BooleanField(default=False) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['owner'], + condition=models.Q(active=True), + name='one_active_billing_address_per_user') + ] + + @classmethod + def populate_db_defaults(cls): + """ + Ensure we have at least one billing address that is associated with the uncloud-admin. + + This way we are sure that an UncloudProvider can be created. + + Cannot use get_or_create as that looks for exactly one. + + """ + + owner = get_user_model().objects.get(username=settings.UNCLOUD_ADMIN_NAME) + billing_address = cls.objects.filter(owner=owner).first() + + if not billing_address: + billing_address = cls.objects.create(owner=owner, + organization="uncloud admins", + full_name="Uncloud Admin", + street="Uncloudstreet. 42", + city="Luchsingen", + postal_code="8775", + country="CH", + active=True) + + def __str__(self): + return "{} - {}, {}, {} {}, {}".format( + self.owner, + self.full_name, self.street, self.postal_code, self.city, + self.country) + + @staticmethod + def get_address_for(user): + return BillingAddress.objects.get(owner=user) + +### +# VAT + +class VATRate(models.Model): + starting_date = models.DateField(blank=True, null=True) + ending_date = models.DateField(blank=True, null=True) + territory_codes = models.TextField(blank=True, default='') + currency_code = models.CharField(max_length=10) + rate = models.FloatField() + rate_type = models.TextField(blank=True, default='') + description = models.TextField(blank=True, default='') + + @staticmethod + def get_for_country(country_code): + vat_rate = None + try: + vat_rate = VATRate.objects.get( + territory_codes=country_code, start_date__isnull=False, stop_date=None + ) + return vat_rate.rate + except VATRate.DoesNotExist as dne: + logger.debug(str(dne)) + logger.debug("Did not find VAT rate for %s, returning 0" % country_code) + return 0 + + @staticmethod + def get_vat_rate(billing_address, when=None): + """ + Returns the VAT rate for business to customer. + + B2B is always 0% with the exception of trading within the own country + """ + + country = billing_address.country + + # Need to have a provider country + providers = UncloudProvider.objects.all() + vatrate = filter_for_when(VATRate.objects.filter(territory_codes=country), when).first() + + if not providers and not vatrate: + return 0 + + uncloud_provider = filter_for_when(providers).get() + + # By default we charge VAT. This affects: + # - Same country sales (VAT applied) + # - B2C to EU (VAT applied) + rate = vatrate.rate if vatrate else 0 + + # Exception: if... + # - the billing_address is in EU, + # - the vat_number has been set + # - the vat_number has been verified + # Then we do not charge VAT + + if uncloud_provider.country != country and billing_address.vat_number and billing_address.vat_number_verified: + rate = 0 + return rate + + + def __str__(self): + return f"{self.territory_codes}: {self.starting_date} - {self.ending_date or ''}: {self.rate_type}" + +### +# Products + +class Product(models.Model): + """ + A product is something a user can order. To record the pricing, we + create order that define a state in time. + + A product can have *one* one_time_order and/or *one* + recurring_order. + + If either of them needs to be updated, a new order of the same + type will be created and links to the previous order. + + """ + + name = models.CharField(max_length=256, unique=True) + description = models.CharField(max_length=1024) + config = models.JSONField() + recurring_periods = models.ManyToManyField(RecurringPeriod, through='ProductToRecurringPeriod') + currency = models.CharField(max_length=32, choices=Currency.choices, default=Currency.CHF) + + @property + def default_recurring_period(self): + """ + Return the default recurring Period + """ + return self.recurring_periods.get(producttorecurringperiod__is_default=True) + + @classmethod + def populate_db_defaults(cls): + recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + obj, created = cls.objects.get_or_create(name="Dual Stack Virtual Machine v1", + description="A standard virtual machine", + currency=Currency.CHF, + config={ + 'features': { + 'cores': + { 'min': 1, + 'max': 48 + }, + 'ram_gb': + { 'min': 1, + 'max': 256 + }, + 'ssd_gb': + { 'min': 10 + }, + 'hdd_gb': + { 'min': 0, + }, + 'additional_ipv4_address': + { 'min': 0, + }, + } + } + ) + + obj.recurring_periods.add(recurring_period, through_defaults= { 'is_default': True }) + + obj, created = cls.objects.get_or_create(name="Dual Stack Virtual Machine v2", + description="A standard virtual machine", + currency=Currency.CHF, + config={ + 'features': { + 'base': + { 'min': 1, + 'max': 1, + }, + 'cores': + { 'min': 1, + 'max': 48, + }, + 'ram_gb': + { 'min': 1, + 'max': 256, + }, + 'ssd_gb': + { 'min': 10 + }, + 'hdd_gb': + { 'min': 0 + }, + 'additional_ipv4_address': + { 'min': 0,}, + } + } + ) + + obj.recurring_periods.add(recurring_period, through_defaults= { 'is_default': True }) + + obj, created = cls.objects.get_or_create(name="reverse DNS", + description="Reverse DNS network", + currency=Currency.CHF, + config={ + 'parameters': [ + 'network' + ] + }) + obj.recurring_periods.add(recurring_period, through_defaults= { 'is_default': True }) + + + def __str__(self): + return f"{self.name} - {self.description}" + + @property + def recurring_orders(self): + return self.orders.order_by('id').exclude(recurring_price=0) + + @property + def last_recurring_order(self): + return self.recurring_orders.last() + + @property + def one_time_orders(self): + return self.orders.order_by('id').filter(recurring_price=0) + + @property + def last_one_time_order(self): + return self.one_time_orders.last() + + # FIXME: this could/should be part of Order (?) + def create_or_update_recurring_order(self, when_to_start=None, recurring_period=None): + if not self.recurring_price: + return + + if not recurring_period: + recurring_period = self.default_recurring_period + + if not when_to_start: + when_to_start = timezone.now() + + if self.last_recurring_order: + if self.recurring_price < self.last_recurring_order.price: + + if when_to_start < self.last_recurring_order.next_cancel_or_downgrade_date: + when_to_start = start_after(self.last_recurring_order.next_cancel_or_downgrade_date) + + when_to_end = end_before(when_to_start) + + new_order = Order.objects.create(owner=self.owner, + billing_address=self.last_recurring_order.billing_address, + starting_date=when_to_start, + price=self.recurring_price, + recurring_period=recurring_period, + description=str(self), + replaces=self.last_recurring_order) + + self.last_recurring_order.replace_with(new_order) + self.orders.add(new_order) + else: + self.create_order(when_to_start, recurring_period) + + @property + def is_recurring(self): + return self.recurring_price > 0 + + @property + def billing_address(self): + return self.order.billing_address + + def discounted_price_by_period(self, requested_period): + """ + Each product has a standard recurring period for which + we define a pricing. I.e. VPN is usually year, VM is usually monthly. + + The user can opt-in to use a different period, which influences the price: + The longer a user commits, the higher the discount. + + Products can also be limited in the available periods. For instance + a VPN only makes sense to be bought for at least one day. + + Rules are as follows: + + given a standard recurring period of ..., changing to ... modifies price ... + + + # One month for free if buying / year, compared to a month: about 8.33% discount + per_year -> per_month -> /11 + per_month -> per_year -> *11 + + # Month has 30.42 days on average. About 7.9% discount to go monthly + per_month -> per_day -> /28 + per_day -> per_month -> *28 + + # Day has 24h, give one for free + per_day -> per_hour -> /23 + per_hour -> per_day -> /23 + + + Examples + + VPN @ 120CHF/y becomes + - 10.91 CHF/month (130.91 CHF/year) + - 0.39 CHF/day (142.21 CHF/year) + + VM @ 15 CHF/month becomes + - 165 CHF/month (13.75 CHF/month) + - 0.54 CHF/day (16.30 CHF/month) + + """ + + # FIXME: This logic needs to be phased out / replaced by product specific (?) + # proportions. Maybe using the RecurringPeriod table to link the possible discounts/add ups + + if self.default_recurring_period == RecurringPeriod.PER_365D: + if requested_period == RecurringPeriod.PER_365D: + return self.recurring_price + if requested_period == RecurringPeriod.PER_30D: + return self.recurring_price/11. + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price/11./28. + + elif self.default_recurring_period == RecurringPeriod.PER_30D: + if requested_period == RecurringPeriod.PER_365D: + return self.recurring_price*11 + if requested_period == RecurringPeriod.PER_30D: + return self.recurring_price + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price/28. + + elif self.default_recurring_period == RecurringPeriod.PER_DAY: + if requested_period == RecurringPeriod.PER_365D: + return self.recurring_price*11*28 + if requested_period == RecurringPeriod.PER_30D: + return self.recurring_price*28 + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price + else: + # FIXME: use the right type of exception here! + raise Exception("Did not implement the discounter for this case") + + + def save(self, *args, **kwargs): + # try: + # ba = BillingAddress.get_address_for(self.owner) + # except BillingAddress.DoesNotExist: + # raise ValidationError("User does not have a billing address") + + # if not ba.active: + # raise ValidationError("User does not have an active billing address") + + + # Verify the required JSON fields + + super().save(*args, **kwargs) + + +### +# Pricing +###### +import logging + +from django.db import models + +logger = logging.getLogger(__name__) + +class PricingPlan(models.Model): + name = models.CharField(max_length=255, unique=True) + vat_inclusive = models.BooleanField(default=True) + vat_percentage = models.DecimalField( + max_digits=7, decimal_places=5, blank=True, default=0 + ) + set_up_fees = models.DecimalField( + max_digits=7, decimal_places=2, default=0 + ) + cores_unit_price = models.DecimalField( + max_digits=7, decimal_places=2, default=0 + ) + ram_unit_price = models.DecimalField( + max_digits=7, decimal_places=2, default=0 + ) + storage_unit_price = models.DecimalField( + max_digits=7, decimal_places=2, default=0 + ) + discount_name = models.CharField(max_length=255, null=True, blank=True) + discount_amount = models.DecimalField( + max_digits=6, decimal_places=2, default=0 + ) + stripe_coupon_id = models.CharField(max_length=255, null=True, blank=True) + + def __str__(self): + display_str = self.name + ' => ' + ' - '.join([ + '{} Setup'.format(self.set_up_fees.normalize()), + '{}/Core'.format(self.cores_unit_price.normalize()), + '{}/GB RAM'.format(self.ram_unit_price.normalize()), + '{}/GB SSD'.format(self.storage_unit_price.normalize()), + '{}% VAT'.format(self.vat_percentage.normalize()) + if not self.vat_inclusive else 'VAT-Incl', + ]) + if self.discount_amount: + display_str = ' - '.join([ + display_str, + '{} {}'.format( + self.discount_amount, + self.discount_name if self.discount_name else 'Discount' + ) + ]) + return display_str + + @classmethod + def get_by_name(cls, name): + try: + pricing = PricingPlan.objects.get(name=name) + except Exception as e: + logger.error( + "Error getting VMPricing with name {name}. " + "Details: {details}. Attempting to return default" + "pricing.".format(name=name, details=str(e)) + ) + pricing = PricingPlan.get_default_pricing() + return pricing + + @classmethod + def get_default_pricing(cls): + """ Returns the default pricing or None """ + try: + default_pricing = PricingPlan.objects.get(name='default') + except Exception as e: + logger.error(str(e)) + default_pricing = None + return default_pricing + +### +# Orders. +class Order(models.Model): + """ + Order are assumed IMMUTABLE and used as SOURCE OF TRUST for generating + bills. Do **NOT** mutate then! + + An one time order is "closed" (does not need to be billed anymore) + if it has one bill record. Having more than one is a programming + error. + + A recurring order is closed if it has been replaced + (replaces__isnull=False) AND the ending_date is set AND it was + billed the last time it needed to be billed (how to check the last + item?) + + BOTH are closed, if they are ended/closed AND have been fully + charged. + + Fully charged == fully billed: sum_of_order_usage == sum_of_bill_records + + """ + + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE, + editable=True) + + billing_address = models.ForeignKey(BillingAddress, + on_delete=models.CASCADE) + + customer = models.ForeignKey(StripeCustomer, on_delete=models.CASCADE, null=True) + + description = models.TextField() + + product = models.ForeignKey(Product, blank=False, null=False, on_delete=models.CASCADE) + config = models.JSONField() + + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField(default=timezone.now) + ending_date = models.DateTimeField(blank=True, null=True) + + recurring_period = models.ForeignKey(RecurringPeriod, + on_delete=models.CASCADE, + editable=True) + + one_time_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + recurring_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + currency = models.CharField(max_length=32, choices=Currency.choices, default=Currency.CHF) + + replaces = models.ForeignKey('self', + related_name='replaced_by', + on_delete=models.CASCADE, + blank=True, + null=True) + + depends_on = models.ForeignKey('self', + related_name='parent_of', + on_delete=models.CASCADE, + blank=True, + null=True) + pricing_plan = models.ForeignKey(PricingPlan, blank=False, null=True, on_delete=models.CASCADE) + + should_be_billed = models.BooleanField(default=True) + + @property + def earliest_ending_date(self): + """ + Recurring orders cannot end before finishing at least one recurring period. + + One time orders have a recurring period of 0, so this work universally + """ + + return self.starting_date + datetime.timedelta(seconds=self.recurring_period.duration_seconds) + + + def next_cancel_or_downgrade_date(self, until_when=None): + """ + Return the next proper ending date after n times the + recurring_period, where n is an integer that applies for downgrading + or cancelling. + """ + + if not until_when: + until_when = timezone.now() + + if until_when < self.starting_date: + raise ValidationError("Cannot end before start of start of order") + + if self.recurring_period.duration_seconds > 0: + delta = until_when - self.starting_date + + num_times = ceil(delta.total_seconds() / self.recurring_period.duration_seconds) + + next_date = self.starting_date + datetime.timedelta(seconds=num_times * self.recurring_period.duration_seconds) + else: + next_date = self.starting_date + + return next_date + + def get_ending_date_for_bill(self, bill): + """ + Determine the ending date given a specific bill + """ + + # If the order is quit, charge the final amount / finish (????) + # Probably not a good idea -- FIXME :continue until usual + if self.ending_date: + this_ending_date = self.ending_date + else: + if self.next_cancel_or_downgrade_date(bill.ending_date) > bill.ending_date: + this_ending_date = self.next_cancel_or_downgrade_date(bill.ending_date) + else: + this_ending_date = bill.ending_date + + return this_ending_date + + + @property + def count_billed(self): + """ + How many times this order was billed so far. + This logic is mainly thought to be for recurring bills, but also works for one time bills + """ + + return sum([ br.quantity for br in self.bill_records.all() ]) + + def cancel(self): + self.ending_date = timezone.now() + self.should_be_billed = False + self.save() + if self.instance_id: + last_bill_record = BillRecord.objects.filter(order=self).order_by('id').last() + schedule('matrixhosting.tasks.delete_instance', + self.instance_id, + schedule_type=Schedule.ONCE, + next_run=last_bill_record.ending_date or (timezone.now() + datetime.timedelta(hours=1))) + + def count_used(self, when=None): + """ + How many times this order was billed so far. + This logic is mainly thought to be for recurring bills, but also works for one time bills + """ + + if self.is_one_time: + return 1 + + if not when: + when = timezone.now() + + # Cannot be used after it ended + if self.ending_date and when > self.ending_date: + when = self.ending_date + + return (when - self.starting_date) / self.default_recurring_period + + @property + def all_usage_billed(self, when=None): + """ + Returns true if this order does not need any further billing + ever. In other words: is this order "closed"? + """ + + if self.count_billed == self.count_used(when): + return True + else: + return False + + @property + def is_closed(self): + if self.all_usage_billed and self.ending_date: + return True + else: + return False + + @property + def is_recurring(self): + return self.recurring_price > 0 + + @property + def is_one_time(self): + return not self.is_recurring + + def replace_with(self, new_order): + new_order.replaces = self + self.ending_date = end_before(new_order.starting_date) + self.save() + + def update_order(self, config, starting_date=None): + """ + Updating an order means creating a new order and reference the previous order + """ + + if not starting_date: + starting_date = timezone.now() + + new_order = self.__class__(owner=self.owner, + billing_address=self.billing_address, + description=self.description, + product=self.product, + config=config, + pricing_plan=self.pricing_plan, + starting_date=starting_date, + currency=self.currency + ) + + new_order.recurring_price = new_order.calculate_recurring_price() + new_order.replaces = self + new_order.save() + + self.ending_date = end_before(new_order.starting_date) + self.save() + + return new_order + + + def create_bill_record(self, bill): + br = None + + if self.recurring_price != 0: + records = BillRecord.objects.filter(order=self).all() + if not records: + if self.one_time_price: + br = BillRecord.objects.create(bill=bill, + order=self, + starting_date=self.starting_date, + ending_date=bill.ending_date, + is_recurring_record=False) + else: + br = self.create_new_bill_record_for_recurring_order(bill) + else: + opened_recurring_record = BillRecord.objects.filter(bill=bill, order=self, is_recurring_record=True).first() + if opened_recurring_record: + br = opened_recurring_record + self.update_bill_record_for_recurring_order(br, bill) + else: + br = self.create_new_bill_record_for_recurring_order(bill) + return br + + def update_bill_record_for_recurring_order(self, + bill_record, + bill): + """ + Possibly update a bill record according to the information in the bill + """ + + # If the order has an ending date set, we might need to adjust the bill_record + if self.ending_date: + if bill_record.ending_date != self.ending_date: + bill_record.ending_date = self.ending_date + + else: + # recurring, not terminated, should go until at least end of bill + if bill_record.ending_date < bill.ending_date: + bill_record.ending_date = bill.ending_date + + bill_record.save() + + def create_new_bill_record_for_recurring_order(self, bill): + """ + Create a new bill record + """ + last_bill_record = BillRecord.objects.filter(order=self).order_by('id').last() + + starting_date=self.starting_date + + if last_bill_record: + # We already charged beyond the end of this bill's period + if last_bill_record.ending_date >= bill.ending_date: + return + + # This order is terminated or replaced + if self.ending_date: + # And the last bill record already covered us -> nothing to be done anymore + if last_bill_record.ending_date == self.ending_date: + return + + starting_date = start_after(last_bill_record.ending_date) + ending_date = self.get_ending_date_for_bill(bill) + + return BillRecord.objects.create(bill=bill, + order=self, + starting_date=starting_date, + ending_date=ending_date, + is_recurring_record=True) + + def calculate_recurring_price(self): + try: + config = json.loads(self.config) + recurring_price = 0 + if 'cores' in config: + recurring_price += self.pricing_plan.cores_unit_price * int(config['cores']) + if 'memory' in config: + recurring_price += self.pricing_plan.ram_unit_price * int(config['memory']) + if 'storage' in config: + recurring_price += self.pricing_plan.storage_unit_price * int(config['storage']) + + vat_rate = VATRate.get_vat_rate(self.billing_address) + vat_validation_status = "verified" if self.billing_address.vat_number_validated_on and self.billing_address.vat_number_verified else False + subtotal, subtotal_after_discount, price_after_discount_with_vat, vat, vat_percent, discount = uncloud_pay.utils.apply_vat_discount( + recurring_price, self.pricing_plan, + vat_rate=vat_rate * 100, vat_validation_status = vat_validation_status + ) + return price_after_discount_with_vat + except Exception as e: + logger.error("An error occurred while parsing the config obj", e) + return 0 + + def check_parameters(self): + if 'parameters' in self.product.config: + for parameter in self.product.config['parameters']: + if not parameter in self.config['parameters']: + raise ValidationError(f"Required parameter '{parameter}' is missing.") + + + def save(self, *args, **kwargs): + # Calculate the price of the order when we create it + # IMMUTABLE fields -- need to create new order to modify them + # However this is not enforced here... + if self._state.adding: + self.recurring_price = self.calculate_recurring_price() + + if self.recurring_period_id is None: + self.recurring_period = self.product.default_recurring_period + + try: + prod_period = self.product.recurring_periods.get(producttorecurringperiod__recurring_period=self.recurring_period) + except ObjectDoesNotExist: + raise ValidationError(f"Recurring Period {self.recurring_period} not allowed for product {self.product}") + + self.check_parameters() + + if self.ending_date and self.ending_date < self.starting_date: + raise ValidationError("End date cannot be before starting date") + + + super().save(*args, **kwargs) + + + def __str__(self): + return f"Order {self.id}: {self.description}" + +class Bill(models.Model): + """ + A bill is a representation of usage at a specific time + """ + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField(default=start_of_this_month) + ending_date = models.DateTimeField() + due_date = models.DateField(default=default_payment_delay) + + + billing_address = models.ForeignKey(BillingAddress, + on_delete=models.CASCADE, + editable=True, + null=False) + + # FIXME: editable=True -> is in the admin, but also editable in DRF + # Maybe filter fields in the serializer? + + is_closed = models.BooleanField(default=False) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['owner', + 'starting_date', + 'ending_date' ], + name='one_bill_per_month_per_user') + ] + + def close(self): + """ + Close/finish a bill + """ + self.is_closed = True + if not self.ending_date: + self.ending_date = timezone.now() + self.save() + + @property + def sum(self): + bill_records = BillRecord.objects.filter(bill=self) + return sum([ br.sum for br in bill_records ]) + + @property + def vat_rate(self): + return VATRate.get_vat_rate(self.billing_address, when=self.ending_date) + + + @classmethod + def create_bills_for_all_users(cls): + """ + Create next bill for each user + """ + + for owner in get_user_model().objects.all(): + cls.create_next_bills_for_user(owner) + + @classmethod + def create_next_bills_for_user(cls, owner, ending_date=None): + """ + Create one bill per billing address, as the VAT rates might be different + for each address + """ + + bills = [] + for billing_address in BillingAddress.objects.filter(owner=owner): + bill = cls.create_next_bill_for_user_address(billing_address, ending_date) + if bill: + bills.append(bill) + + return bills + + @classmethod + def create_next_bill_for_user_address(cls, billing_address, ending_date=None): + """ + Create the next bill for a specific billing address of a user + """ + + owner = billing_address.owner + + all_orders = Order.objects.filter(Q(owner__id=owner.id), Q(should_be_billed=True), + Q(billing_address__id=billing_address.id) + ).order_by('id') + + if len(all_orders) > 0: + bill = cls.get_or_create_bill(billing_address, ending_date=ending_date) + for order in all_orders: + order.create_bill_record(bill) + return bill + else: + # This Customer Hasn't any active orders + return False + + + @classmethod + def get_or_create_bill(cls, billing_address, ending_date=None): + """ + Get / reuse last bill if it is not yet closed + + Create bill, if there is no bill or if bill is closed. + """ + + last_bill = cls.objects.filter(billing_address=billing_address).order_by('id').last() + + all_orders = Order.objects.filter(billing_address=billing_address).order_by('id') + first_order = all_orders.first() + + bill = None + + # Get date & bill from previous bill, if it exists + if last_bill: + if not last_bill.is_closed: + bill = last_bill + starting_date = last_bill.starting_date + ending_date = bill.ending_date + else: + starting_date = last_bill.ending_date + datetime.timedelta(seconds=1) + else: + # Might be an idea to make this the start of the month, too + if first_order: + starting_date = first_order.starting_date + else: + starting_date = timezone.now() + + if not ending_date: + ending_date = end_of_month(starting_date) + + if not bill: + bill = cls.objects.create( + owner=billing_address.owner, + starting_date=starting_date, + ending_date=ending_date, + billing_address=billing_address) + + + return bill + + def __str__(self): + return f"{self.owner}-{self.id}" + + +class BillRecord(models.Model): + """ + Entry of a bill, dynamically generated from an order. + """ + + bill = models.ForeignKey(Bill, on_delete=models.CASCADE) + order = models.ForeignKey(Order, on_delete=models.CASCADE) + + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField() + ending_date = models.DateTimeField() + + is_recurring_record = models.BooleanField(blank=False, null=False) + + @property + def quantity(self): + """ Determine the quantity by the duration""" + if not self.is_recurring_record: + return 1 + + record_delta = self.ending_date.date() - self.starting_date.date() + if self.order.recurring_period and self.order.recurring_period.duration_seconds > 0: + return int(record_delta.total_seconds() / self.order.recurring_period.duration_seconds) + else: + return 1 + + @property + def sum(self): + if self.is_recurring_record: + return self.order.recurring_price * Decimal(self.quantity) + else: + return self.order.one_time_price + + @property + def price(self): + if self.is_recurring_record: + return self.order.recurring_price + else: + return self.order.one_time_price + + def __str__(self): + if self.is_recurring_record: + bill_line = f"{self.starting_date} - {self.ending_date}: {self.quantity} x {self.order}" + else: + bill_line = f"{self.starting_date}: {self.order}" + + return bill_line + + def save(self, *args, **kwargs): + if self.ending_date < self.starting_date: + raise ValidationError("End date cannot be before starting date") + + super().save(*args, **kwargs) + + +class ProductToRecurringPeriod(models.Model): + """ + Intermediate manytomany mapping class that allows storing the default recurring period + for a product + """ + + recurring_period = models.ForeignKey(RecurringPeriod, on_delete=models.CASCADE) + product = models.ForeignKey(Product, on_delete=models.CASCADE) + + is_default = models.BooleanField(default=False) + + class Meta: + constraints = [ + models.UniqueConstraint(fields=['product'], + condition=models.Q(is_default=True), + name='one_default_recurring_period_per_product'), + models.UniqueConstraint(fields=['product', 'recurring_period'], + name='recurring_period_once_per_product') + ] + + def __str__(self): + return f"{self.product} - {self.recurring_period} (default: {self.is_default})" + + +class Membership(models.Model): + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + starting_date = models.DateField(blank=True, null=True) + ending_date = models.DateField(blank=True, null=True) + + + @classmethod + def user_has_membership(user, when): + """ + Return true if user has membership at a point of time, + return false if that is not the case + """ + + pass + + # cls.objects.filter(owner=user, + # starting_date) diff --git a/uncloud_pay/selectors.py b/uncloud_pay/selectors.py new file mode 100644 index 0000000..a634e30 --- /dev/null +++ b/uncloud_pay/selectors.py @@ -0,0 +1,31 @@ +from django.utils import timezone +from django.db import transaction +from .models import * + +def get_payments_for_user(user): + payments = [ payment.amount for payment in Payment.objects.filter(owner=user) ] + + return sum(payments) + +def get_spendings_for_user(user): + bills = Bill.objects.filter(owner=user) + + amount = 0 + for bill in bills: + amount += bill.sum + + return amount + +@transaction.atomic +def get_balance_for_user(user): + return get_payments_for_user(user) - get_spendings_for_user(user) + +@transaction.atomic +def has_enough_balance(user, due_amount): + balance = get_balance_for_user(user) + if balance >= due_amount: + return True + return False + +def get_billing_address_for_user(user): + return BillingAddress.objects.filter(owner=user, active=True).first() diff --git a/uncloud_pay/serializers.py b/uncloud_pay/serializers.py new file mode 100644 index 0000000..4ea4104 --- /dev/null +++ b/uncloud_pay/serializers.py @@ -0,0 +1,127 @@ +from django.contrib.auth import get_user_model +from rest_framework import serializers +from uncloud_auth.serializers import UserSerializer +from django.utils.translation import gettext_lazy as _ + +from .models import * +import uncloud_pay.stripe as uncloud_stripe +from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS + +### +# 2020-12 Checked code + +class StripeCreditCardSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = StripeCreditCard + exclude = [ "card_id", "owner" ] + read_only_fields = [ "last4", "brand", "expiry_date" ] + +class PaymentSerializer(serializers.ModelSerializer): + owner = serializers.HiddenField(default=serializers.CurrentUserDefault()) + + class Meta: + model = Payment + fields = '__all__' + read_only_fields = [ "external_reference", "source", "timestamp" ] + + def validate(self, data): + payment_intent = uncloud_stripe.charge_customer(data['owner'], + data['amount']) + + data["external_reference"] = payment_intent["id"] + data["source"] = "stripe" + + return data + +class BalanceSerializer(serializers.Serializer): + balance = serializers.DecimalField(max_digits=AMOUNT_MAX_DIGITS, decimal_places=AMOUNT_DECIMALS) + +class BillingAddressSerializer(serializers.ModelSerializer): + class Meta: + model = BillingAddress + exclude = [ "owner" ] + + +class VATRateSerializer(serializers.ModelSerializer): + + class Meta: + model = VATRate + fields = '__all__' + + +################################################################################ +# Unchecked code + + +### +# Orders & Products. + +class OrderSerializer(serializers.ModelSerializer): + owner = serializers.PrimaryKeyRelatedField(queryset=get_user_model().objects.all()) + + def __init__(self, *args, **kwargs): + # Don't pass the 'fields' arg up to the superclass + admin = kwargs.pop('admin', None) + + # Instantiate the superclass normally + super(OrderSerializer, self).__init__(*args, **kwargs) + + # Only allows owner in admin mode. + if not admin: + self.fields.pop('owner') + + def create(self, validated_data): + billing_address = BillingAddress.get_preferred_address_for(validated_data["owner"]) + instance = Order(billing_address=billing_address, **validated_data) + instance.save() + + return instance + + def validate_owner(self, value): + if BillingAddress.get_preferred_address_for(value) == None: + raise serializers.ValidationError("Owner does not have a valid billing address.") + + return value + + class Meta: + model = Order + read_only_fields = ['replaced_by', 'depends_on'] + fields = ['owner', 'description', 'creation_date', 'starting_date', 'ending_date', + 'recurring_period', 'recurring_price', 'one_time_price', + 'config', 'pricing_plan', 'should_be_billed'] + read_only_fields + + +### +# Bills + +# TODO: remove magic numbers for decimal fields +class BillRecordSerializer(serializers.Serializer): + order = serializers.HyperlinkedRelatedField( + view_name='order-detail', + read_only=True) + description = serializers.CharField() + one_time_price = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + recurring_price = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) +# recurring_period = serializers.ChoiceField() + recurring_count = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + vat_rate = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + vat_amount = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + amount = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + total = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + +class BillSerializer(serializers.ModelSerializer): + billing_address = BillingAddressSerializer(read_only=True) + records = BillRecordSerializer(many=True, read_only=True) + + class Meta: + model = Bill + fields = ['owner', 'sum', 'vat_rate', + 'due_date', 'creation_date', 'starting_date', 'ending_date', + 'records', 'is_closed', 'billing_address'] + +# We do not want users to mutate the country / VAT number of an address, as it +# will change VAT on existing bills. +class UpdateBillingAddressSerializer(serializers.ModelSerializer): + class Meta: + model = BillingAddress + fields = ['street', 'city', 'postal_code'] diff --git a/uncloud_pay/services.py b/uncloud_pay/services.py new file mode 100644 index 0000000..84a7c8d --- /dev/null +++ b/uncloud_pay/services.py @@ -0,0 +1,34 @@ +import datetime +from calendar import monthrange +from django.utils import timezone + + +def start_of_month(a_day): + """ Returns first of the month of a given datetime object""" + return a_day.replace(day=1,hour=0,minute=0,second=0, microsecond=0) + +def end_of_month(a_day): + """ Returns first of the month of a given datetime object""" + + _, last_day = monthrange(a_day.year, a_day.month) + return a_day.replace(day=last_day,hour=23,minute=59,second=59, microsecond=0) + +def start_of_this_month(): + """ Returns first of this month""" + a_day = timezone.now() + return a_day.replace(day=1,hour=0,minute=0,second=0, microsecond=0) + +def end_of_this_month(): + """ Returns first of this month""" + a_day = timezone.now() + + _, last_day = monthrange(a_day.year, a_day.month) + return a_day.replace(day=last_day,hour=23,minute=59,second=59, microsecond=0) + +def end_before(a_date): + """ Return suitable datetimefield for ending just before a_date """ + return a_date - datetime.timedelta(seconds=1) + +def start_after(a_date): + """ Return suitable datetimefield for starting just after a_date """ + return a_date + datetime.timedelta(seconds=1) diff --git a/uncloud_pay/stripe.py b/uncloud_pay/stripe.py new file mode 100644 index 0000000..a59456e --- /dev/null +++ b/uncloud_pay/stripe.py @@ -0,0 +1,249 @@ +import stripe +import stripe.error +import logging +import datetime + +from django.core.exceptions import ObjectDoesNotExist, ValidationError +from django.conf import settings +from django.contrib.auth import get_user_model + +from .models import StripeCustomer, StripeCreditCard + +logger = logging.getLogger(__name__) + +CURRENCY = 'chf' + +stripe.api_key = settings.STRIPE_KEY + +def handle_stripe_error(f): + def handle_problems(*args, **kwargs): + response = { + 'paid': False, + 'response_object': None, + 'error': None + } + + common_message = "Currently it is not possible to make payments. Please try agin later." + try: + response_object = f(*args, **kwargs) + return response_object + except stripe.error.CardError as e: + # Since it's a decline, stripe.error.CardError will be caught + body = e.json_body + logging.error(str(e)) + + raise e # For error handling. + except stripe.error.RateLimitError: + logging.error("Too many requests made to the API too quickly.") + raise Exception(common_message) + except stripe.error.InvalidRequestError as e: + logging.error(str(e)) + raise Exception('Invalid parameters.') + except stripe.error.AuthenticationError as e: + # Authentication with Stripe's API failed + # (maybe you changed API keys recently) + logging.error(str(e)) + raise Exception(common_message) + except stripe.error.APIConnectionError as e: + logging.error(str(e)) + raise Exception(common_message) + except stripe.error.StripeError as e: + # XXX: maybe send email + logging.error(str(e)) + raise Exception(common_message) + + return handle_problems + +def public_api_key(): + return settings.STRIPE_PUBLIC_KEY + +def get_customer_id_for(user): + try: + # .get() raise if there is no matching entry. + return StripeCustomer.objects.get(owner=user).stripe_id + except ObjectDoesNotExist: + # No entry yet - making a new one. + try: + customer = create_customer(user.username, user.email) + uncloud_stripe_mapping = StripeCustomer.objects.create( + owner=user, stripe_id=customer.id) + return uncloud_stripe_mapping.stripe_id + except Exception as e: + return None + +@handle_stripe_error +def create_setup_intent(customer_id): + return stripe.SetupIntent.create(customer=customer_id) + +@handle_stripe_error +def get_setup_intent(setup_intent_id): + return stripe.SetupIntent.retrieve(setup_intent_id) + +@handle_stripe_error +def get_payment_method(payment_method_id): + return stripe.PaymentMethod.retrieve(payment_method_id) + +@handle_stripe_error +def get_card_from_payment(user, payment_method_id): + payment_method = stripe.PaymentMethod.retrieve(payment_method_id) + if payment_method: + if 'card' in payment_method: + sync_cards_for_user(user) + return payment_method['card'] + return False + + +@handle_stripe_error +def attach_payment_method(payment_method_id, customer_id): + return stripe.PaymentMethod.attach(payment_method_id, customer=customer_id) + +@handle_stripe_error +def create_customer(name, email): + return stripe.Customer.create(name=name, email=email) + +@handle_stripe_error +def get_customer(customer_id): + return stripe.Customer.retrieve(customer_id) + +@handle_stripe_error +def get_customer_cards(customer_id): + print(f"getting cards for: {customer_id}") + + cards = [] + stripe_cards = stripe.PaymentMethod.list( + customer=customer_id, + type="card", + ) + + for stripe_card in stripe_cards["data"]: + card = {} + card['brand'] = stripe_card["card"]["brand"] + card['last4'] = stripe_card["card"]["last4"] + card['month'] = stripe_card["card"]["exp_month"] + card['year'] = stripe_card["card"]["exp_year"] + card['id'] = stripe_card["id"] + + cards.append(card) + + return cards + +def sync_cards_for_user(user): + customer_id = get_customer_id_for(user) + cards = get_customer_cards(customer_id) + + active_cards = StripeCreditCard.objects.filter(owner=user, + active=True) + + if len(active_cards) > 0: + has_active_card = True + else: + has_active_card = False + + for card in cards: + active = False + + if not has_active_card: + active = True + has_active_card = True + + StripeCreditCard.objects.get_or_create(card_id=card['id'], + owner = user, + defaults = { + 'last4': card['last4'], + 'brand': card['brand'], + 'expiry_date': datetime.date(card['year'], + card['month'], + 1), + 'active': active + } + ) + +@handle_stripe_error +def charge_customer(user, amount, currency='CHF', card=False): + # Amount is in CHF but stripes requires smallest possible unit. + # https://stripe.com/docs/api/payment_intents/create#create_payment_intent-amount + # FIXME: might need to be adjusted for other currencies + + if currency == 'CHF': + adjusted_amount = int(amount * 100) + else: + return Exception("Programming error: unsupported currency") + + try: + card = card or StripeCreditCard.objects.get(owner=user, + active=True) + + except StripeCreditCard.DoesNotExist: + raise ValidationError("No active credit card - cannot create payment") + + customer_id = get_customer_id_for(user) + + return stripe.PaymentIntent.create( + amount=adjusted_amount, + currency=currency, + customer=customer_id, + payment_method=card.card_id, + off_session=True, + confirm=True, + ) + +@handle_stripe_error +def get_payment_intent(user, amount, currency='CHF', card=False): + # Amount is in CHF but stripes requires smallest possible unit. + # https://stripe.com/docs/api/payment_intents/create#create_payment_intent-amount + # FIXME: might need to be adjusted for other currencies + + if currency == 'CHF': + adjusted_amount = int(amount * 100) + else: + return Exception("Programming error: unsupported currency") + + try: + card = card or StripeCreditCard.objects.get(owner=user, + active=True) + + except StripeCreditCard.DoesNotExist: + raise ValidationError("No active credit card - cannot create payment") + + customer_id = get_customer_id_for(user) + + return stripe.PaymentIntent.create( + amount=adjusted_amount, + currency=currency, + customer=customer_id, + payment_method=card.card_id, + setup_future_usage='off_session', + confirm=False, + ) + +@handle_stripe_error +def get_or_create_tax_id_for_user(stripe_customer_id, vat_number, + type="eu_vat", country=""): + def compare_vat_numbers(vat1, vat2): + _vat1 = vat1.replace(" ", "").replace(".", "").replace("-","") + _vat2 = vat2.replace(" ", "").replace(".", "").replace("-","") + return True if _vat1 == _vat2 else False + + tax_ids_list = stripe.Customer.list_tax_ids( + stripe_customer_id, + limit=100, + ) + for tax_id_obj in tax_ids_list.data: + if compare_vat_numbers(tax_id_obj.value, vat_number): + return tax_id_obj + else: + logger.debug( + "{val1} is not equal to {val2} or {con1} not same as " + "{con2}".format(val1=tax_id_obj.value, val2=vat_number, + con1=tax_id_obj.country.lower(), + con2=country.lower().strip())) + logger.debug( + "tax id obj does not exist for {val}. Creating a new one".format( + val=vat_number + )) + tax_id_obj = stripe.Customer.create_tax_id( + stripe_customer_id, + type=type, + value=vat_number, + ) + return tax_id_obj diff --git a/uncloud_pay/templates/uncloud_pay/bill.html.j2 b/uncloud_pay/templates/uncloud_pay/bill.html.j2 new file mode 100644 index 0000000..7cf10f8 --- /dev/null +++ b/uncloud_pay/templates/uncloud_pay/bill.html.j2 @@ -0,0 +1,1059 @@ +{% load static %} + + + + + + + + + {{ bill }} + + + + + + +
+ +
+
+ ungleich glarus ag +
Bahnhofstrasse 1 +
8783 Linthal +
Switzerland +
+
+
+ {{ bill.billing_address.organization }}
+ {{ bill.billing_address.name }}
+ {{ bill.owner.email }}
+ {{ bill.billing_address.street }}
+ {{ bill.billing_address.country }} {{ bill.billing_address.postal_code }} {{ bill.billing_address.city }}
+ +
+
+
+ Bill id: {{ bill }} +
{{ bill.starting_date|date:"Ymd" }} - + {{ bill.ending_date|date:"Ymd" }} +
+
+
+
+

Invoice

+
+ + + + + + + + + + + {% for record in bill_records %} + + + + + + + {% endfor %} + +
DetailPrice/UnitUnitsTotal price
{{ record.starting_date|date:"Ymd-H:i:s" }} + - {{ record.ending_date|date:"Ymd-H:i:s" }} + {{ record.order }} + {{ record.price|floatformat:2 }}{{ record.quantity|floatformat:2 }}{{ record.sum|floatformat:2 }}
+
+

+ Total (excl. VAT) + {{ bill.amount }} +

+

+ VAT 7.7% + {{ bill.vat_amount|floatformat:2 }} +

+
+
+

+ Total amount to be paid + {{ bill.sum|floatformat:2 }} +

+
+
+
+

+ +41 (0) 55 505-62-66

+ +

+ buchhaltung-ag@ungleich.ch

+ +

+ https://ungleich.ch

+

+ @ungleich

+
+
+

Glarner Kantonalbank

+

+ IBAN: CH 4300 7730 0055 5931 177 +

+

+ BIC: GLKBCH22 +

+

Mwst-Nummer: CHE-156.970.649 MWST

+
+
+ + + diff --git a/uncloud_pay/templates/uncloud_pay/register_stripe.html b/uncloud_pay/templates/uncloud_pay/register_stripe.html new file mode 100644 index 0000000..0eed76a --- /dev/null +++ b/uncloud_pay/templates/uncloud_pay/register_stripe.html @@ -0,0 +1,78 @@ +{% extends 'uncloud/base.html' %} +{% block bootstrap5_extra_head %} + + +{% endblock %} +{% block bootstrap5_content %} +
+ {% csrf_token %} +
+

Register Credit Card with Stripe

+

+ By submitting I authorise to send instructions to + the financial institution that issued my card to take + payments from my card account in accordance with the + terms of my agreement with you. +

+ + + + + +
+
+ + +
+
The card will be registered with stripe.
+ + +
+ + + + +{% endblock %} diff --git a/uncloud_pay/tests.py b/uncloud_pay/tests.py new file mode 100644 index 0000000..0fde03d --- /dev/null +++ b/uncloud_pay/tests.py @@ -0,0 +1,532 @@ +from django.test import TestCase +from django.contrib.auth import get_user_model +from datetime import datetime, date, timedelta +from django.utils import timezone + +from .models import * +from uncloud_service.models import GenericServiceProduct +from uncloud.models import UncloudProvider, UncloudNetwork + +import json + +vm_product_config = { + 'features': { + 'cores': + { 'min': 1, + 'max': 48 + }, + 'ram_gb': + { 'min': 1, + 'max': 256 + }, + }, +} + +vm_order_config = json.dumps({ + 'cores': 1, + 'memory': 2, + 'storage': 100 +}) + +vm_order_downgrade_config = { + 'features': { + 'cores': 1, + 'ram_gb': 1 + } +} + +vm_order_upgrade_config = { + 'features': { + 'cores': 4, + 'ram_gb': 4 + } +} + + +class ProductTestCase(TestCase): + """ + Test products and products <-> order interaction + """ + + def setUp(self): + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + + self.ba = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="somewhere else", + active=True) + + RecurringPeriod.populate_db_defaults() + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + def test_create_product(self): + """ + Create a sample product + """ + p = Product.objects.create(name="Testproduct", + description="Only for testing", + config=vm_product_config) + + p.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + +class OrderTestCase(TestCase): + """ + The heart of ordering products + """ + + def setUp(self): + self.pricing_plan = PricingPlan.objects.create(name="PricingSample", set_up_fees=35, cores_unit_price=3, + ram_unit_price=4, storage_unit_price=0.02) + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + + self.ba = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="somewhere else", + active=True) + + self.product = Product.objects.create(name="Testproduct", + description="Only for testing", + config=vm_product_config) + + RecurringPeriod.populate_db_defaults() + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + self.product.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + + def test_order_invalid_recurring_period(self): + """ + Order a products with a recurringperiod that is not added to the product + """ + + order_config = json.dumps({ + 'cores': 1, + 'memory':2, + 'storage': 100 + }) + o = Order.objects.create(owner=self.user, + billing_address=self.ba, + pricing_plan = self.pricing_plan, + product=self.product, + config=order_config) + + + def test_order_product(self): + """ + Order a product, ensure the order has correct price setup + """ + order_config = json.dumps({ + 'cores': 1, + 'memory':2, + 'storage': 100 + }) + o = Order.objects.create(owner=self.user, + billing_address=self.ba, + pricing_plan = self.pricing_plan, + product=self.product, + config=order_config) + + self.assertEqual(o.one_time_price, 0) + self.assertEqual(o.recurring_price, 13.0) + + def test_change_order(self): + """ + Change an order and ensure that + - a new order is created + - the price is correct in the new order + """ + order_config = json.dumps({ + 'cores': 2, + 'memory':4, + 'storage': 200 + }) + order1 = Order.objects.create(owner=self.user, + billing_address=self.ba, + pricing_plan = self.pricing_plan, + product=self.product, + config=order_config) + + self.assertEqual(order1.one_time_price, 0) + self.assertEqual(order1.recurring_price, 26.0) + + +class ModifyOrderTestCase(TestCase): + """ + Test typical order flows like + - cancelling + - downgrading + - upgrading + """ + + def setUp(self): + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + self.pricing_plan = PricingPlan.objects.create(name="PricingSample", set_up_fees=35, cores_unit_price=3, + ram_unit_price=4, storage_unit_price=0.02) + self.order1_config = json.dumps({ + 'cores': 2, + 'memory':4, + 'storage': 200 + }) + self.order2_config = json.dumps({ + 'cores': 1, + 'memory':2, + 'storage': 100 + }) + self.ba = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="somewhere else", + active=True) + + self.product = Product.objects.create(name="Testproduct", + description="Only for testing", + config=vm_product_config) + + RecurringPeriod.populate_db_defaults() + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + self.product.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + + def test_change_order(self): + """ + Test changing an order + + Expected result: + + - Old order should be closed before new order starts + - New order should start at starting data + """ + + user = self.user + + starting_price = 16 + downgrade_price = 8 + + starting_date = timezone.make_aware(datetime.datetime(2019,3,3)) + ending1_date = starting_date + datetime.timedelta(days=15) + change1_date = start_after(ending1_date) + + bill_ending_date = change1_date + datetime.timedelta(days=1) + + + order1 = Order.objects.create(owner=self.user, + billing_address=BillingAddress.get_address_for(self.user), + product=self.product, + config=self.order1_config, + pricing_plan=self.pricing_plan, + starting_date=starting_date) + + order1.update_order(self.order2_config, starting_date=change1_date) + + bills = Bill.create_next_bills_for_user(user, ending_date=bill_ending_date) + + bill = bills[0] + bill_records = BillRecord.objects.filter(bill=bill) + + self.assertEqual(len(bill_records), 2) + + self.assertEqual(bill_records[0].starting_date, starting_date) + self.assertEqual(bill_records[0].ending_date, ending1_date) + + self.assertEqual(bill_records[1].starting_date, change1_date) + + + + def test_downgrade_product(self): + """ + Test downgrading behaviour: + + We create a recurring product (recurring time: 30 days) and downgrade after 15 days. + + We create the bill right AFTER the end of the first order. + + Expected result: + + - First bill record for 30 days + - Second bill record starting after 30 days + - Bill contains two bill records + + """ + + user = self.user + + starting_price = 16 + downgrade_price = 8 + + starting_date = timezone.make_aware(datetime.datetime(2019,3,3)) + first_order_should_end_at = starting_date + datetime.timedelta(days=30) + change1_date = start_after(starting_date + datetime.timedelta(days=15)) + bill_ending_date = change1_date + datetime.timedelta(days=1) + order1 = Order.objects.create(owner=self.user, + billing_address=BillingAddress.get_address_for(self.user), + product=self.product, + pricing_plan=self.pricing_plan, + config=self.order1_config, + starting_date=starting_date) + + bills = Bill.create_next_bills_for_user(user, ending_date=bill_ending_date) + + bill = bills[0] + bill_records = BillRecord.objects.filter(bill=bill) + + self.assertEqual(len(bill_records), 1) + self.assertEqual(bill_records[0].starting_date, starting_date) + + order1.update_order(self.order2_config, starting_date=change1_date) + bills = Bill.create_next_bills_for_user(user, ending_date=bill_ending_date) + bill_records = BillRecord.objects.filter(bill=bill) + self.assertEqual(len(bill_records), 2) + self.assertEqual(bill_records[0].order.ending_date.date(), change1_date.date()) + + +class BillTestCase(TestCase): + """ + Test aspects of billing / creating a bill + """ + + def setUp(self): + RecurringPeriod.populate_db_defaults() + + self.pricing_plan = PricingPlan.objects.create(name="PricingSample", set_up_fees=35, cores_unit_price=3, + ram_unit_price=4, storage_unit_price=0.02) + + self.user_without_address = get_user_model().objects.create( + username='no_home_person', + email='far.away@domain.tld') + + self.user = get_user_model().objects.create( + username='jdoe', + email='john.doe@domain.tld') + + self.recurring_user = get_user_model().objects.create( + username='recurrent_product_user', + email='jane.doe@domain.tld') + + self.user_addr = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="unknown", + active=True) + + self.recurring_user_addr = BillingAddress.objects.create( + owner=self.recurring_user, + organization = 'Test org', + street="Somewhere", + city="Else", + postal_code="unknown", + active=True) + + self.order_meta = {} + self.order_meta[1] = { + 'starting_date': timezone.make_aware(datetime.datetime(2020,3,3)), + 'ending_date': timezone.make_aware(datetime.datetime(2020,4,17)), + 'price': 15, + 'description': '' + } + + self.product = Product.objects.create(name="Product Sample", + description="Not only for testing, but for joy", + config=vm_product_config) + + + self.vm = Product.objects.create(name="Super Fast VM", + description="Zooooom", + config=vm_product_config) + + + RecurringPeriod.populate_db_defaults() + self.default_recurring_period = RecurringPeriod.objects.get(name="Per 30 days") + + self.onetime_recurring_period = RecurringPeriod.objects.get(name="Onetime") + + self.product.recurring_periods.add(self.onetime_recurring_period, + through_defaults= { 'is_default': True }) + + self.vm.recurring_periods.add(self.default_recurring_period, + through_defaults= { 'is_default': True }) + + + # used for generating multiple bills + self.bill_dates = [ + timezone.make_aware(datetime.datetime(2020,3,31)), + timezone.make_aware(datetime.datetime(2020,4,30)), + timezone.make_aware(datetime.datetime(2020,5,31)), + ] + + + def order_product(self): + return Order.objects.create( + owner=self.user, + recurring_period=RecurringPeriod.objects.get(name="Onetime"), + product=self.product, + billing_address=BillingAddress.get_address_for(self.user), + starting_date=self.order_meta[1]['starting_date'], + ending_date=self.order_meta[1]['ending_date'], + pricing_plan=self.pricing_plan, + config=vm_order_config) + + def order_vm(self, owner=None): + + if not owner: + owner = self.recurring_user + + return Order.objects.create( + owner=owner, + product=self.vm, + config=vm_order_config, + pricing_plan=self.pricing_plan, + billing_address=BillingAddress.get_address_for(self.recurring_user), + starting_date=timezone.make_aware(datetime.datetime(2020,3,3)), + ) + + def test_bill_one_time_with_recurring(self): + """ + Validate that if the order contains one_time_price and recurring_pricing + One Bill records should be created + """ + + order = Order.objects.create( + owner=self.user, + product=self.vm, + config=vm_order_config, + pricing_plan=self.pricing_plan, + one_time_price = 35, + billing_address=BillingAddress.get_address_for(self.user), + starting_date=timezone.make_aware(datetime.datetime(2020,3,3)), + ) + + bill = Bill.create_next_bill_for_user_address(self.user_addr) + + self.assertEqual(order.billrecord_set.count(), 1) + record = order.billrecord_set.first() + self.assertEqual(record.is_recurring_record, False) + self.assertEqual(record.price, 35) + self.assertEqual(record.quantity, 1) + self.assertEqual(record.sum, 35) + #close the bill as it has been paid + bill.close() + bill2 = Bill.create_next_bill_for_user_address(self.user_addr) + self.assertNotEqual(bill.id, bill2.id) + self.assertEqual(order.billrecord_set.count(), 2) + record = BillRecord.objects.filter(bill=bill2, order=order).first() + self.assertEqual(record.is_recurring_record, True) + self.assertEqual(record.price, 13) + self.assertEqual(record.quantity, 1) + self.assertEqual(record.sum, 13) + + def test_bill_one_time_one_bill_record(self): + """ + Ensure there is only 1 bill record per order + """ + + order = self.order_product() + + bill = Bill.create_next_bill_for_user_address(self.user_addr) + + self.assertEqual(order.billrecord_set.count(), 1) + + def test_bill_sum_onetime(self): + """ + Check the bill sum for a single one time order + """ + + order = self.order_product() + self.assertEqual(order.recurring_price, 13.0) + bill = Bill.create_next_bill_for_user_address(self.user_addr) + self.assertEqual(order.billrecord_set.count(), 1) + record = order.billrecord_set.first() + self.assertEqual(record.price, 13) + self.assertEqual(record.quantity, 1) + self.assertEqual(bill.sum, 13) + + + def test_bill_creates_record_for_recurring_order(self): + """ + Ensure there is only 1 bill record per order + """ + + order = self.order_vm() + bill = Bill.create_next_bill_for_user_address(self.recurring_user_addr) + + self.assertEqual(order.billrecord_set.count(), 1) + self.assertEqual(bill.billrecord_set.count(), 1) + + + def test_new_bill_after_closing(self): + """ + After closing a bill and the user has a recurring product, + the next bill run should create e new bill + """ + + order = self.order_vm() + + for ending_date in self.bill_dates: + b = Bill.create_next_bill_for_user_address(self.recurring_user_addr, ending_date) + b.close() + + bill_count = Bill.objects.filter(owner=self.recurring_user).count() + + self.assertEqual(len(self.bill_dates), bill_count) + + + +class BillingAddressTestCase(TestCase): + def setUp(self): + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + + + def test_user_no_address(self): + """ + Raise an error, when there is no address + """ + + self.assertRaises(BillingAddress.DoesNotExist, + BillingAddress.get_address_for, + self.user) + +class VATRatesTestCase(TestCase): + def setUp(self): + self.user = get_user_model().objects.create( + username='random_user', + email='jane.random@domain.tld') + + self.user_addr = BillingAddress.objects.create( + owner=self.user, + organization = 'Test org', + street="unknown", + city="unknown", + postal_code="unknown", + active=True) + + UncloudNetwork.populate_db_defaults() + UncloudProvider.populate_db_defaults() + + + + def test_get_rate_for_user(self): + """ + Raise an error, when there is no address + """ diff --git a/uncloud_pay/utils.py b/uncloud_pay/utils.py new file mode 100644 index 0000000..c80ae3a --- /dev/null +++ b/uncloud_pay/utils.py @@ -0,0 +1,155 @@ +import logging +import decimal +import datetime + +from . import stripe as uncloud_stripe +import stripe +from .models import PricingPlan, BillingAddress + +logger = logging.getLogger(__name__) + +eu_countries = ['at', 'be', 'bg', 'ch', 'cy', 'cz', 'hr', 'dk', + 'ee', 'fi', 'fr', 'mc', 'de', 'gr', 'hu', 'ie', 'it', + 'lv', 'lu', 'mt', 'nl', 'po', 'pt', 'ro','sk', 'si', 'es', + 'se', 'gb'] + +def validate_vat_number(stripe_customer_id, billing_address_id): + try: + billing_address = BillingAddress.objects.get(id=billing_address_id) + except BillingAddress.DoesNotExist as dne: + billing_address = None + except BillingAddress.MultipleObjectsReturned as mor: + billing_address = BillingAddress.objects.filter(id=billing_address_id).order_by('-id').first() + if billing_address is not None: + logger.debug("BillingAddress found: %s %s" % ( + billing_address_id, str(billing_address))) + if billing_address.country.lower().strip() not in eu_countries: + return { + "validated_on": "", + "status": "not_needed" + } + if billing_address.vat_number_validated_on and billing_address.vat_number_verified: + return { + "validated_on": billing_address.vat_number_validated_on, + "status": "verified" + } + else: + if billing_address.stripe_tax_id: + logger.debug("We have a tax id %s" % billing_address.stripe_tax_id) + tax_id_obj = stripe.Customer.retrieve_tax_id( + stripe_customer_id, + billing_address.stripe_tax_id, + ) + if tax_id_obj.verification.status == "verified": + logger.debug("Latest status on Stripe=%s. Updating" % + tax_id_obj.verification.status) + # update billing address + billing_address.vat_number_validated_on = datetime.datetime.now() + billing_address.vat_number_verified = True + billing_address.save() + return { + "status": "verified", + "validated_on": billing_address.vat_number_validated_on + } + else: + billing_address.vat_number_validated_on = datetime.datetime.now() + billing_address.vat_number_verified = False + billing_address.save() + else: + logger.debug("Creating a tax id") + tax_id_obj = create_tax_id( + stripe_customer_id, billing_address_id, + "ch_vat" if billing_address.country.lower() == "ch" else "eu_vat", + ) + else: + logger.debug("invalid billing address") + return { + "status": "invalid billing address", + "validated_on": "" + } + return { + "status": tax_id_obj.verification.status if 'verification' in tax_id_obj else "unknown", + "validated_on": datetime.datetime.now() if tax_id_obj.verification.status == "verified" else "" + } + +def create_tax_id(stripe_customer_id, billing_address_id, type): + try: + billing_address = BillingAddress.objects.get(id=billing_address_id) + except BillingAddress.DoesNotExist as dne: + billing_address = None + logger.debug("BillingAddress does not exist for %s" % billing_address_id) + except BillingAddress.MultipleObjectsReturned as mor: + logger.debug("Multiple BillingAddress exist for %s" % billing_address_id) + billing_address = BillingAddress.objects.filter(billing_address_id).order_by('-id').first() + + tax_id_obj = None + if billing_address: + try: + tax_id_obj = uncloud_stripe.get_or_create_tax_id_for_user( + stripe_customer_id, + vat_number=billing_address.vat_number, + type=type, + country=billing_address.country + ) + billing_address.stripe_tax_id = tax_id_obj.id + billing_address.vat_number_verified = True if tax_id_obj.verification.status == "verified" else False + billing_address.save() + return tax_id_obj + except Exception as e: + logger.debug("Received none in tax_id_obj") + return { + 'verification': None, + 'error': str(e) + } + +def apply_vat_discount(subtotal, pricing_plan, vat_rate=False, vat_validation_status=False): + vat_percent = vat_rate or pricing_plan.vat_percentage + if pricing_plan.vat_inclusive or (vat_validation_status and vat_validation_status in ["verified", "not_needed"]): + vat_percent = decimal.Decimal(0) + vat = decimal.Decimal(0) + else: + vat = subtotal * decimal.Decimal(vat_rate) * decimal.Decimal(0.01) + discount_amount = 0 + discount_amount_with_vat = 0 + if pricing_plan.discount_amount: + discount_amount = round(float(pricing_plan.discount_amount), 2) + discount_amount_with_vat = decimal.Decimal(discount_amount) * (1 + decimal.Decimal(vat_rate) * decimal.Decimal(0.01)) + discount_amount_with_vat = discount_amount_with_vat + + subtotal = round(float(subtotal), 2) + vat_percent = round(float(vat_percent), 2) + discount = { + 'name': pricing_plan.discount_name, + 'amount': discount_amount, + 'amount_with_vat': round(float(discount_amount_with_vat), 2) + } + subtotal_after_discount = subtotal - discount["amount"] + price_after_discount_with_vat = round((subtotal - discount['amount']) * (1 + vat_percent * 0.01), 2) + + return (subtotal, round(float(subtotal_after_discount), 2), price_after_discount_with_vat, + round(float(vat), 2), vat_percent, discount) + + +def get_order_total_with_vat(cores, memory, storage, + pricing_name='default', vat_rate=False, vat_validation_status=False): + try: + pricing = PricingPlan.objects.get(name=pricing_name) + except Exception as ex: + logger.error( + "Error getting PricingPlan object for {pricing_name}." + "Details: {details}".format( + pricing_name=pricing_name, details=str(ex) + ) + ) + return None + + subtotal = ( + pricing.set_up_fees + + (decimal.Decimal(cores) * pricing.cores_unit_price) + + (decimal.Decimal(memory) * pricing.ram_unit_price) + + (decimal.Decimal(storage) * (pricing.storage_unit_price)) + ) + return apply_vat_discount(subtotal, pricing, vat_rate, vat_validation_status) + + + diff --git a/uncloud_pay/views.py b/uncloud_pay/views.py new file mode 100644 index 0000000..e90dda1 --- /dev/null +++ b/uncloud_pay/views.py @@ -0,0 +1,310 @@ +from django.contrib.auth.mixins import LoginRequiredMixin +from django.views.generic.base import TemplateView +from django.shortcuts import render +from django.db import transaction +from django.contrib.auth import get_user_model +from rest_framework import viewsets, mixins, permissions, status, views +from rest_framework.renderers import TemplateHTMLRenderer +from rest_framework.response import Response +from rest_framework.decorators import action +from rest_framework.reverse import reverse +from rest_framework.decorators import renderer_classes +from vat_validator import validate_vat, vies +from vat_validator.countries import EU_COUNTRY_CODES +from hardcopy import bytestring_to_pdf +from django.core.files.temp import NamedTemporaryFile +from django.http import FileResponse +from django.template.loader import render_to_string +from copy import deepcopy + +import json +import logging + +from .models import * +from .serializers import * +from .selectors import * + +from datetime import datetime +from vat_validator import sanitize_vat +import uncloud_pay.stripe as uncloud_stripe +from django.contrib.auth.decorators import login_required +from django.utils.decorators import method_decorator +from django.http import JsonResponse +import stripe + +logger = logging.getLogger(__name__) + +### +# 2020-12 checked code + +class RegisterCard(TemplateView): + template_name = "uncloud_pay/register_stripe.html" + + @method_decorator(login_required) + def dispatch(self, *args, **kwargs): + return super().dispatch(*args, **kwargs) + + + def get_context_data(self, **kwargs): + + customer_id = uncloud_stripe.get_customer_id_for(self.request.user) + setup_intent = uncloud_stripe.create_setup_intent(customer_id) + context = super().get_context_data(**kwargs) + context['client_secret'] = setup_intent.client_secret + context['username'] = self.request.user.username + context['stripe_pk'] = uncloud_stripe.public_api_key + return context + + +class CreditCardViewSet(mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.DestroyModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet): + + serializer_class = StripeCreditCardSerializer + permission_classes = [permissions.IsAuthenticated] + + def list(self, request): + uncloud_stripe.sync_cards_for_user(self.request.user) + return super().list(request) + + def get_queryset(self): + return StripeCreditCard.objects.filter(owner=self.request.user) + +class PaymentViewSet(viewsets.ModelViewSet): + serializer_class = PaymentSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Payment.objects.filter(owner=self.request.user) + +class BalanceViewSet(viewsets.ViewSet): + permission_classes = [permissions.IsAuthenticated] + + def list(self, request): + serializer = BalanceSerializer(data={ + 'balance': get_balance_for_user(self.request.user) + }) + serializer.is_valid() + return Response(serializer.data) + + +class ListCards(TemplateView): + template_name = "uncloud_pay/list_stripe.html" + + @method_decorator(login_required) + def dispatch(self, *args, **kwargs): + return super().dispatch(*args, **kwargs) + + def get_context_data(self, **kwargs): + customer_id = uncloud_stripe.get_customer_id_for(self.request.user) + cards = uncloud_stripe.get_customer_cards(customer_id) + + context = super().get_context_data(**kwargs) + context['cards'] = cards + context['username'] = self.request.user + + return context + +### +# Bills and Orders. + +class BillViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = BillSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Bill.objects.filter(owner=self.request.user) + + + @action(detail=False, methods=['get']) + def unpaid(self, request): + serializer = self.get_serializer( + Bill.get_unpaid_for(self.request.user), + many=True) + return Response(serializer.data) + + @action(detail=True, methods=['get']) + def download(self, *args, **kwargs): + """ + Allow to download + """ + bill = self.get_object() + provider = UncloudProvider.get_provider() + output_file = NamedTemporaryFile() + bill_html = render_to_string("bill.html.j2", {'bill': bill}) + + bytestring_to_pdf(bill_html.encode('utf-8'), output_file) + response = FileResponse(output_file, content_type="application/pdf") + response['Content-Disposition'] = 'filename="{}_{}.pdf"'.format( + bill.reference, bill.uuid + ) + + return response + + +class OrderViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = OrderSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Order.objects.filter(owner=self.request.user) + +class VATRateViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = VATRateSerializer + permission_classes = [permissions.IsAuthenticated] + queryset = VATRate.objects.all() + +class BillingAddressViewSet(mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_serializer_class(self): + if self.action == 'update': + return UpdateBillingAddressSerializer + else: + return BillingAddressSerializer + + def get_queryset(self): + return self.request.user.billing_addresses.all() + + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Validate VAT numbers. + country = serializer.validated_data["country"] + + # We ignore empty VAT numbers. + if 'vat_number' in serializer.validated_data and serializer.validated_data["vat_number"] != "": + vat_number = serializer.validated_data["vat_number"] + + if not validate_vat(country, vat_number): + return Response( + {'error': 'Malformed VAT number.'}, + status=status.HTTP_400_BAD_REQUEST) + elif country in EU_COUNTRY_CODES: + # XXX: make a synchroneous call to a third patry API here might not be a good idea.. + try: + vies_state = vies.check_vat(country, vat_number) + if not vies_state.valid: + return Response( + {'error': 'European VAT number does not exist in VIES.'}, + status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + logger.warning(e) + return Response( + {'error': 'Could not validate EU VAT number against VIES. Try again later..'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + + serializer.save(owner=request.user) + return Response(serializer.data) + +### +# Admin stuff. + +class AdminPaymentViewSet(viewsets.ModelViewSet): + serializer_class = PaymentSerializer + permission_classes = [permissions.IsAdminUser] + + def get_queryset(self): + return Payment.objects.all() + + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save(timestamp=datetime.now()) + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + +# Bills are generated from orders and should not be created or updated by hand. +class AdminBillViewSet(BillViewSet): + serializer_class = BillSerializer + permission_classes = [permissions.IsAdminUser] + + def get_queryset(self): + return Bill.objects.all() + + @action(detail=False, methods=['get']) + def unpaid(self, request): + unpaid_bills = [] + # XXX: works but we can do better than number of users + 1 SQL requests... + for user in get_user_model().objects.all(): + unpaid_bills = unpaid_bills + Bill.get_unpaid_for(self.request.user) + + serializer = self.get_serializer(unpaid_bills, many=True) + return Response(serializer.data) + + @action(detail=False, methods=['post']) + def generate(self, request): + users = get_user_model().objects.all() + + generated_bills = [] + for user in users: + now = timezone.now() + generated_bills = generated_bills + Bill.generate_for( + year=now.year, + month=now.month, + user=user) + + return Response( + map(lambda b: b.reference, generated_bills), + status=status.HTTP_200_OK) + +class AdminOrderViewSet(mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.UpdateModelMixin, + viewsets.GenericViewSet): + serializer_class = OrderSerializer + permission_classes = [permissions.IsAdminUser] + + def get_serializer(self, *args, **kwargs): + return self.serializer_class(*args, **kwargs, admin=True) + + def get_queryset(self): + return Order.objects.all() + + # Updates create a new order and terminate the 'old' one. + @transaction.atomic + def update(self, request, *args, **kwargs): + order = self.get_object() + partial = kwargs.pop('partial', False) + serializer = self.get_serializer(order, data=request.data, partial=partial) + serializer.is_valid(raise_exception=True) + + # Clone existing order for replacement. + replacing_order = deepcopy(order) + + # Yes, that's how you make a new entry in DB: + # https://docs.djangoproject.com/en/3.0/topics/db/queries/#copying-model-instances + replacing_order.pk = None + + for attr, value in serializer.validated_data.items(): + setattr(replacing_order, attr, value) + + # Save replacing order and terminate 'previous' one. + replacing_order.save() + order.replaced_by = replacing_order + order.save() + order.terminate() + + return Response(replacing_order) + + @action(detail=True, methods=['post']) + def terminate(self, request, pk): + order = self.get_object() + if order.is_terminated: + return Response( + {'error': 'Order is already terminated.'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + else: + order.terminate() + return Response({}, status=status.HTTP_200_OK) diff --git a/uncloud_service/__init__.py b/uncloud_service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_service/admin.py b/uncloud_service/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_service/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_service/apps.py b/uncloud_service/apps.py new file mode 100644 index 0000000..190bd35 --- /dev/null +++ b/uncloud_service/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UngleichServiceConfig(AppConfig): + name = 'uncloud_service' diff --git a/uncloud_service/models.py b/uncloud_service/models.py new file mode 100644 index 0000000..a37e42b --- /dev/null +++ b/uncloud_service/models.py @@ -0,0 +1,63 @@ +from django.db import models +from uncloud_pay.models import Product, RecurringPeriod, AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS +from uncloud_vm.models import VMProduct, VMDiskImageProduct +from django.core.validators import MinValueValidator + +class MatrixServiceProduct(models.Model): + monthly_managment_fee = 20 + + description = "Managed Matrix HomeServer" + + # Specific to Matrix-as-a-Service + vm = models.ForeignKey( + VMProduct, on_delete=models.CASCADE + ) + domain = models.CharField(max_length=255, default='domain.tld') + + # Default recurring price is PER_MONT, see Product class. + # def recurring_price(self, recurring_period=RecurringPeriod.PER_30D): + # return self.monthly_managment_fee + + @staticmethod + def base_image(): + # TODO: find a way to safely reference debian 10 image. +#e return VMDiskImageProduct.objects.get(uuid="93e564c5-adb3-4741-941f-718f76075f02") + return False + + # @staticmethod + # def allowed_recurring_periods(): + # return list(filter( + # lambda pair: pair[0] in [RecurringPeriod.PER_30D], + # RecurringPeriod.choices)) + + @property + def one_time_price(self): + return 30 + +class GenericServiceProduct(models.Model): + custom_description = models.TextField() + custom_recurring_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + custom_one_time_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + @property + def recurring_price(self): + # FIXME: handle recurring_period somehow. + return self.custom_recurring_price + + @property + def description(self): + return self.custom_description + + @property + def one_time_price(self): + return self.custom_one_time_price + + @staticmethod + def allowed_recurring_periods(): + return RecurringPeriod.choices diff --git a/uncloud_service/serializers.py b/uncloud_service/serializers.py new file mode 100644 index 0000000..bc6d753 --- /dev/null +++ b/uncloud_service/serializers.py @@ -0,0 +1,60 @@ +from rest_framework import serializers +from .models import * +from uncloud_vm.serializers import ManagedVMProductSerializer +from uncloud_vm.models import VMProduct +from uncloud_pay.models import RecurringPeriod, BillingAddress + +# XXX: the OrderSomethingSomthingProductSerializer classes add a lot of +# boilerplate: can we reduce it somehow? + +class MatrixServiceProductSerializer(serializers.ModelSerializer): + vm = ManagedVMProductSerializer() + + class Meta: + model = MatrixServiceProduct + fields = ['order', 'owner', 'status', 'vm', 'domain', + 'recurring_period'] + read_only_fields = ['order', 'owner', 'status'] + +class OrderMatrixServiceProductSerializer(MatrixServiceProductSerializer): + # recurring_period = serializers.ChoiceField( + # choices=MatrixServiceProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(OrderMatrixServiceProductSerializer, self).__init__(*args, **kwargs) + self.fields['billing_address'] = serializers.ChoiceField( + choices=BillingAddress.get_addresses_for( + self.context['request'].user) + ) + + class Meta: + model = MatrixServiceProductSerializer.Meta.model + fields = MatrixServiceProductSerializer.Meta.fields + [ + 'recurring_period', 'billing_address' + ] + read_only_fields = MatrixServiceProductSerializer.Meta.read_only_fields + +class GenericServiceProductSerializer(serializers.ModelSerializer): + class Meta: + model = GenericServiceProduct + fields = ['order', 'owner', 'status', 'custom_recurring_price', + 'custom_description', 'custom_one_time_price'] + read_only_fields = [ 'owner', 'status'] + +class OrderGenericServiceProductSerializer(GenericServiceProductSerializer): + # recurring_period = serializers.ChoiceField( + # choices=GenericServiceProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(OrderGenericServiceProductSerializer, self).__init__(*args, **kwargs) + self.fields['billing_address'] = serializers.ChoiceField( + choices=BillingAddress.get_addresses_for( + self.context['request'].user) + ) + + class Meta: + model = GenericServiceProductSerializer.Meta.model + fields = GenericServiceProductSerializer.Meta.fields + [ + 'recurring_period', 'billing_address' + ] + read_only_fields = GenericServiceProductSerializer.Meta.read_only_fields diff --git a/uncloud_service/tests.py b/uncloud_service/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/uncloud_service/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/uncloud_service/views.py b/uncloud_service/views.py new file mode 100644 index 0000000..abd4a05 --- /dev/null +++ b/uncloud_service/views.py @@ -0,0 +1,128 @@ +from rest_framework import viewsets, permissions +from rest_framework.response import Response +from django.db import transaction +from django.utils import timezone + +from .models import * +from .serializers import * + +from uncloud_pay.helpers import ProductViewSet +from uncloud_pay.models import Order +from uncloud_vm.models import VMProduct, VMDiskProduct + +def create_managed_vm(cores, ram, disk_size, image, order): + # Create VM + disk = VMDiskProduct( + owner=order.owner, + order=order, + size_in_gb=disk_size, + image=image) + vm = VMProduct( + name="Managed Service Host", + owner=order.owner, + cores=cores, + ram_in_gb=ram, + primary_disk=disk) + disk.vm = vm + + vm.save() + disk.save() + + return vm + + +class MatrixServiceProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = MatrixServiceProductSerializer + + def get_queryset(self): + return MatrixServiceProduct.objects.filter(owner=self.request.user) + + def get_serializer_class(self): + if self.action == 'create': + return OrderMatrixServiceProductSerializer + else: + return MatrixServiceProductSerializer + + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + order_billing_address = serializer.validated_data.pop("billing_address") + + # Create base order.) + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user, + billing_address=order_billing_address, + starting_date=timezone.now() + ) + order.save() + + # Create unerderlying VM. + data = serializer.validated_data.pop('vm') + vm = create_managed_vm( + order=order, + cores=data['cores'], + ram=data['ram_in_gb'], + disk_size=data['primary_disk']['size_in_gb'], + image=MatrixServiceProduct.base_image()) + + # Create service. + service = serializer.save( + order=order, + owner=request.user, + vm=vm) + + return Response(serializer.data) + +class GenericServiceProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return GenericServiceProduct.objects.filter(owner=self.request.user) + + def get_serializer_class(self): + if self.action == 'create': + return OrderGenericServiceProductSerializer + else: + return GenericServiceProductSerializer + + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + order_billing_address = serializer.validated_data.pop("billing_address") + + # Create base order. + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user, + billing_address=order_billing_address, + starting_date=timezone.now() + ) + order.save() + + # Create service. + print(serializer.validated_data) + service = serializer.save(order=order, owner=request.user) + + # XXX: Move this to some kind of on_create hook in parent + # Product class? + order.add_record( + service.one_time_price, + service.recurring_price, + service.description) + + # XXX: Move this to some kind of on_create hook in parent + # Product class? + order.add_record( + service.one_time_price, + service.recurring_price, + service.description) + + return Response(serializer.data) diff --git a/uncloud_storage/__init__.py b/uncloud_storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_storage/admin.py b/uncloud_storage/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_storage/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_storage/apps.py b/uncloud_storage/apps.py new file mode 100644 index 0000000..38b2301 --- /dev/null +++ b/uncloud_storage/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudStorageConfig(AppConfig): + name = 'uncloud_storage' diff --git a/uncloud_storage/models.py b/uncloud_storage/models.py new file mode 100644 index 0000000..0dac5c2 --- /dev/null +++ b/uncloud_storage/models.py @@ -0,0 +1,7 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class StorageClass(models.TextChoices): + HDD = 'HDD', _('HDD') + SSD = 'SSD', _('SSD') diff --git a/uncloud_storage/tests.py b/uncloud_storage/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/uncloud_storage/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/uncloud_storage/views.py b/uncloud_storage/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/uncloud_storage/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/uncloud_vm/__init__.py b/uncloud_vm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_vm/admin.py b/uncloud_vm/admin.py new file mode 100644 index 0000000..6f3bc50 --- /dev/null +++ b/uncloud_vm/admin.py @@ -0,0 +1,19 @@ +from django.contrib import admin + +# Register your models here. +from uncloud_vm.models import * +from uncloud_pay.models import Order + +class VMDiskInline(admin.TabularInline): + model = VMDiskProduct + +class OrderInline(admin.TabularInline): + model = Order + +class VMProductAdmin(admin.ModelAdmin): + inlines = [ + VMDiskInline + ] + +admin.site.register(VMProduct, VMProductAdmin) +admin.site.register(VMDiskProduct) diff --git a/uncloud_vm/apps.py b/uncloud_vm/apps.py new file mode 100644 index 0000000..c5e94a5 --- /dev/null +++ b/uncloud_vm/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudVmConfig(AppConfig): + name = 'uncloud_vm' diff --git a/uncloud_vm/management/commands/vm.py b/uncloud_vm/management/commands/vm.py new file mode 100644 index 0000000..667c5ad --- /dev/null +++ b/uncloud_vm/management/commands/vm.py @@ -0,0 +1,119 @@ +import json + +import uncloud.secrets as secrets + +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model + +from uncloud_vm.models import VMSnapshotProduct, VMProduct, VMHost +from datetime import datetime + +class Command(BaseCommand): + help = 'Select VM Host for VMs' + + def add_arguments(self, parser): + parser.add_argument('--this-hostname', required=True) + parser.add_argument('--this-cluster', required=True) + + parser.add_argument('--create-vm-snapshots', action='store_true') + parser.add_argument('--schedule-vms', action='store_true') + parser.add_argument('--start-vms', action='store_true') + + + def handle(self, *args, **options): + for cmd in [ 'create_vm_snapshots', 'schedule_vms', 'start_vms' ]: + if options[cmd]: + f = getattr(self, cmd) + f(args, options) + + def schedule_vms(self, *args, **options): + for pending_vm in VMProduct.objects.filter(status='PENDING'): + cores_needed = pending_vm.cores + ram_needed = pending_vm.ram_in_gb + + # Database filtering + possible_vmhosts = VMHost.objects.filter(physical_cores__gte=cores_needed) + + # Logical filtering + possible_vmhosts = [ vmhost for vmhost in possible_vmhosts + if vmhost.available_cores >=cores_needed + and vmhost.available_ram_in_gb >= ram_needed ] + + if not possible_vmhosts: + log.error("No suitable Host found - cannot schedule VM {}".format(pending_vm)) + continue + + vmhost = possible_vmhosts[0] + pending_vm.vmhost = vmhost + pending_vm.status = 'SCHEDULED' + pending_vm.save() + + print("Scheduled VM {} on VMHOST {}".format(pending_vm, pending_vm.vmhost)) + + print(self) + + def start_vms(self, *args, **options): + vmhost = VMHost.objects.get(hostname=options['this_hostname']) + + if not vmhost: + raise Exception("No vmhost {} exists".format(options['vmhostname'])) + + # not active? done here + if not vmhost.status = 'ACTIVE': + return + + vms_to_start = VMProduct.objects.filter(vmhost=vmhost, + status='SCHEDULED') + for vm in vms_to_start: + """ run qemu: + check if VM is not already active / qemu running + prepare / create the Qemu arguments + """ + print("Starting VM {}".format(VM)) + + def check_vms(self, *args, **options): + """ + Check if all VMs that are supposed to run are running + """ + + def modify_vms(self, *args, **options): + """ + Check all VMs that are requested to be modified and restart them + """ + + def create_vm_snapshots(self, *args, **options): + this_cluster = VMCluster(option['this_cluster']) + + for snapshot in VMSnapshotProduct.objects.filter(status='PENDING', + cluster=this_cluster): + if not snapshot.extra_data: + snapshot.extra_data = {} + + # TODO: implement locking here + if 'creating_hostname' in snapshot.extra_data: + pass + + snapshot.extra_data['creating_hostname'] = options['this_hostname'] + snapshot.extra_data['creating_start'] = str(datetime.now()) + snapshot.save() + + # something on the line of: + # for disk im vm.disks: + # rbd snap create pool/image-name@snapshot name + # snapshot.extra_data['snapshots'] + # register the snapshot names in extra_data (?) + + print(snapshot) + + def check_health(self, *args, **options): + pending_vms = VMProduct.objects.filter(status='PENDING') + vmhosts = VMHost.objects.filter(status='active') + + # 1. Check that all active hosts reported back N seconds ago + # 2. Check that no VM is running on a dead host + # 3. Migrate VMs if necessary + # 4. Check that no VMs have been pending for longer than Y seconds + + # If VM snapshots exist without a VM -> notify user (?) + + print("Nothing is good, you should implement me") diff --git a/uncloud_vm/migrations/0001_initial.py b/uncloud_vm/migrations/0001_initial.py new file mode 100644 index 0000000..4ec089a --- /dev/null +++ b/uncloud_vm/migrations/0001_initial.py @@ -0,0 +1,111 @@ +# Generated by Django 3.1 on 2020-12-13 10:38 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='VMCluster', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('extra_data', models.JSONField(blank=True, editable=False, null=True)), + ('name', models.CharField(max_length=128, unique=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VMDiskImageProduct', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('extra_data', models.JSONField(blank=True, editable=False, null=True)), + ('name', models.CharField(max_length=256)), + ('is_os_image', models.BooleanField(default=False)), + ('is_public', models.BooleanField(default=False, editable=False)), + ('size_in_gb', models.FloatField(blank=True, null=True)), + ('import_url', models.URLField(blank=True, null=True)), + ('image_source', models.CharField(max_length=128, null=True)), + ('image_source_type', models.CharField(max_length=128, null=True)), + ('storage_class', models.CharField(choices=[('HDD', 'HDD'), ('SSD', 'SSD')], default='SSD', max_length=32)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32)), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VMHost', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('extra_data', models.JSONField(blank=True, editable=False, null=True)), + ('hostname', models.CharField(max_length=253, unique=True)), + ('physical_cores', models.IntegerField(default=0)), + ('usable_cores', models.IntegerField(default=0)), + ('usable_ram_in_gb', models.FloatField(default=0)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32)), + ('vmcluster', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmcluster')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VMProduct', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('name', models.CharField(blank=True, max_length=32, null=True)), + ('cores', models.IntegerField()), + ('ram_in_gb', models.FloatField()), + ('vmcluster', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmcluster')), + ('vmhost', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmhost')), + ], + ), + migrations.CreateModel( + name='VMSnapshotProduct', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('gb_ssd', models.FloatField(editable=False)), + ('gb_hdd', models.FloatField(editable=False)), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='snapshots', to='uncloud_vm.vmproduct')), + ], + ), + migrations.CreateModel( + name='VMNetworkCard', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('mac_address', models.BigIntegerField()), + ('ip_address', models.GenericIPAddressField(blank=True, null=True)), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmproduct')), + ], + ), + migrations.CreateModel( + name='VMDiskProduct', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('size_in_gb', models.FloatField(blank=True)), + ('disk_type', models.CharField(choices=[('ceph/ssd', 'Ceph Ssd'), ('ceph/hdd', 'Ceph Hdd'), ('local/ssd', 'Local Ssd'), ('local/hdd', 'Local Hdd')], default='ceph/ssd', max_length=20)), + ('image', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmdiskimageproduct')), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmproduct')), + ], + ), + migrations.CreateModel( + name='VMWithOSProduct', + fields=[ + ('vmproduct_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='uncloud_vm.vmproduct')), + ('primary_disk', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.vmdiskproduct')), + ], + bases=('uncloud_vm.vmproduct',), + ), + ] diff --git a/uncloud_vm/migrations/__init__.py b/uncloud_vm/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_vm/models.py b/uncloud_vm/models.py new file mode 100644 index 0000000..c605779 --- /dev/null +++ b/uncloud_vm/models.py @@ -0,0 +1,189 @@ +from django.db import models +from django.contrib.auth import get_user_model + +from uncloud_pay.models import Product, RecurringPeriod +from uncloud.models import UncloudModel, UncloudStatus + +import uncloud_pay.models as pay_models +import uncloud_storage.models + +class VMCluster(UncloudModel): + name = models.CharField(max_length=128, unique=True) + +class VMHost(UncloudModel): + # 253 is the maximum DNS name length + hostname = models.CharField(max_length=253, unique=True) + + vmcluster = models.ForeignKey( + VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + # indirectly gives a maximum number of cores / VM - f.i. 32 + physical_cores = models.IntegerField(default=0) + + # determines the maximum usable cores - f.i. 320 if you overbook by a factor of 10 + usable_cores = models.IntegerField(default=0) + + # ram that can be used of the server + usable_ram_in_gb = models.FloatField(default=0) + + status = models.CharField( + max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING + ) + + @property + def vms(self): + return VMProduct.objects.filter(vmhost=self) + + @property + def used_ram_in_gb(self): + return sum([vm.ram_in_gb for vm in VMProduct.objects.filter(vmhost=self)]) + + @property + def available_ram_in_gb(self): + return self.usable_ram_in_gb - self.used_ram_in_gb + + @property + def available_cores(self): + return self.usable_cores - sum([vm.cores for vm in self.vms ]) + + + +class VMProduct(models.Model): + vmhost = models.ForeignKey( + VMHost, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + vmcluster = models.ForeignKey( + VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + name = models.CharField(max_length=32, blank=True, null=True) + cores = models.IntegerField() + ram_in_gb = models.FloatField() + + @property + def recurring_price(self): + return self.cores * 3 + self.ram_in_gb * 4 + + @property + def description(self): + return "Virtual machine '{}': {} core(s), {}GB memory".format( + self.name, self.cores, self.ram_in_gb) + + # @staticmethod + # def allowed_recurring_periods(): + # return list(filter( + # lambda pair: pair[0] in [RecurringPeriod.PER_365D, + # RecurringPeriod.PER_30D, RecurringPeriod.PER_HOUR], + # RecurringPeriod.choices)) + + + def __str__(self): + return f"VM id={self.id},name={self.name},cores={self.cores},ram_in_gb={self.ram_in_gb}" + + +class VMWithOSProduct(VMProduct): + primary_disk = models.ForeignKey('VMDiskProduct', on_delete=models.CASCADE, null=True) + + +class VMDiskImageProduct(UncloudModel): + """ + Images are used for cloning/linking. + + They are the base for images. + + """ + + owner = models.ForeignKey( + get_user_model(), on_delete=models.CASCADE, editable=False + ) + + name = models.CharField(max_length=256) + is_os_image = models.BooleanField(default=False) + is_public = models.BooleanField(default=False, editable=False) # only allow admins to set this + + size_in_gb = models.FloatField(null=True, blank=True) + import_url = models.URLField(null=True, blank=True) + image_source = models.CharField(max_length=128, null=True) + image_source_type = models.CharField(max_length=128, null=True) + + storage_class = models.CharField(max_length=32, + choices = uncloud_storage.models.StorageClass.choices, + default = uncloud_storage.models.StorageClass.SSD) + + status = models.CharField( + max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING + ) + + def __str__(self): + return "VMDiskImage {} ({}): {} gb".format(self.id, + self.name, + self.size_in_gb) + + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class VMDiskType(models.TextChoices): + """ + Types of disks that can be attached to VMs + """ + CEPH_SSD = 'ceph/ssd' + CEPH_HDD = 'ceph/hdd' + LOCAL_SSD = 'local/ssd' + LOCAL_HDD = 'local/hdd' + + +class VMDiskProduct(models.Model): + """ + The VMDiskProduct is attached to a VM. + + It is based on a VMDiskImageProduct that will be used as a basis. + + It can be enlarged, but not shrinked compared to the VMDiskImageProduct. + """ + + vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE) + + image = models.ForeignKey(VMDiskImageProduct, on_delete=models.CASCADE, + blank=True, null=True) + + size_in_gb = models.FloatField(blank=True) + + disk_type = models.CharField( + max_length=20, + choices=VMDiskType.choices, + default=VMDiskType.CEPH_SSD) + + def __str__(self): + return f"Disk {self.size_in_gb}GB ({self.disk_type}) for {self.vm}" + + @property + def recurring_price(self): + if self.disk_type == VMDiskType.CEPH_SSD: + price_per_gb = 3.5/10 + elif self.disk_type == VMDiskType.CEPH_HDD: + price_per_gb = 1.5/100 + elif self.disk_type == VMDiskType.LOCAL_SSD: + price_per_gb = 3.5/10 + elif self.disk_type == VMDiskType.CEPH_HDD: + price_per_gb = 1.5/100 + + return self.size_in_gb * price_per_gb + + +class VMNetworkCard(models.Model): + vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE) + + mac_address = models.BigIntegerField() + + ip_address = models.GenericIPAddressField(blank=True, + null=True) + + +class VMSnapshotProduct(models.Model): + gb_ssd = models.FloatField(editable=False) + gb_hdd = models.FloatField(editable=False) + + vm = models.ForeignKey(VMProduct, + related_name='snapshots', + on_delete=models.CASCADE) diff --git a/uncloud_vm/serializers.py b/uncloud_vm/serializers.py new file mode 100644 index 0000000..a60d10b --- /dev/null +++ b/uncloud_vm/serializers.py @@ -0,0 +1,143 @@ +from django.contrib.auth import get_user_model + +from rest_framework import serializers + +from .models import * +from uncloud_pay.models import RecurringPeriod, BillingAddress + +# XXX: does not seem to be used? + +GB_SSD_PER_DAY=0.012 +GB_HDD_PER_DAY=0.0006 + +GB_SSD_PER_DAY=0.012 +GB_HDD_PER_DAY=0.0006 + +### +# Admin views. + +class VMHostSerializer(serializers.HyperlinkedModelSerializer): + vms = serializers.PrimaryKeyRelatedField(many=True, read_only=True) + + class Meta: + model = VMHost + fields = '__all__' + read_only_fields = [ 'vms' ] + +class VMClusterSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = VMCluster + fields = '__all__' + + +### +# Disks. + +class VMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = '__all__' + +class CreateVMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = ['size_in_gb', 'image'] + +class CreateManagedVMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = ['size_in_gb'] + +class VMDiskImageProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskImageProduct + fields = '__all__' + +class VMSnapshotProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMSnapshotProduct + fields = '__all__' + + + # verify that vm.owner == user.request + def validate_vm(self, value): + if not value.owner == self.context['request'].user: + raise serializers.ValidationError("VM {} not found for owner {}.".format(value, + self.context['request'].user)) + disks = VMDiskProduct.objects.filter(vm=value) + + if len(disks) == 0: + raise serializers.ValidationError("VM {} does not have any disks, cannot snapshot".format(value.id)) + + return value + + pricing = {} + pricing['per_gb_ssd'] = 0.012 + pricing['per_gb_hdd'] = 0.0006 + pricing['recurring_period'] = 'per_day' + +### +# VMs + +# Helper used in uncloud_service for services allocating VM. +class ManagedVMProductSerializer(serializers.ModelSerializer): + """ + Managed VM serializer used in ungleich_service app. + """ + primary_disk = CreateManagedVMDiskProductSerializer() + class Meta: + model = VMWithOSProduct + fields = [ 'cores', 'ram_in_gb', 'primary_disk'] + +class VMProductSerializer(serializers.ModelSerializer): + primary_disk = CreateVMDiskProductSerializer() + snapshots = VMSnapshotProductSerializer(many=True, read_only=True) + disks = VMDiskProductSerializer(many=True, read_only=True) + + class Meta: + model = VMWithOSProduct + fields = ['order', 'owner', 'status', 'name', 'cores', + 'ram_in_gb', 'primary_disk', 'snapshots', 'disks', 'extra_data'] + read_only_fields = ['order', 'owner', 'status'] + +class OrderVMProductSerializer(VMProductSerializer): + # recurring_period = serializers.ChoiceField( + # choices=VMWithOSProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(VMProductSerializer, self).__init__(*args, **kwargs) + + class Meta: + model = VMProductSerializer.Meta.model + fields = VMProductSerializer.Meta.fields + [ 'recurring_period' ] + read_only_fields = VMProductSerializer.Meta.read_only_fields + +# Nico's playground. +class NicoVMProductSerializer(serializers.ModelSerializer): + snapshots = VMSnapshotProductSerializer(many=True, read_only=True) + order = serializers.StringRelatedField() + + class Meta: + model = VMProduct + read_only_fields = ['order', 'owner', 'status', + 'vmhost', 'vmcluster', 'snapshots', + 'extra_data' ] + fields = read_only_fields + [ 'name', + 'cores', + 'ram_in_gb' + ] + +class DCLVMProductSerializer(serializers.HyperlinkedModelSerializer): + """ + Create an interface similar to standard DCL + """ + + # Custom field used at creation (= ordering) only. + # recurring_period = serializers.ChoiceField( + # choices=VMProduct.allowed_recurring_periods()) + + os_disk_uuid = serializers.UUIDField() + # os_disk_size = + + class Meta: + model = VMProduct diff --git a/uncloud_vm/tests.py b/uncloud_vm/tests.py new file mode 100644 index 0000000..e5d403f --- /dev/null +++ b/uncloud_vm/tests.py @@ -0,0 +1,98 @@ +import datetime + +import parsedatetime + +from django.test import TestCase +from django.contrib.auth import get_user_model +from django.utils import timezone +from django.core.exceptions import ValidationError + +from uncloud_vm.models import VMDiskImageProduct, VMDiskProduct, VMProduct, VMHost +from uncloud_pay.models import Order, RecurringPeriod + +User = get_user_model() +cal = parsedatetime.Calendar() + + +# If you want to check the test database using some GUI/cli tool +# then use the following connecting parameters + +# host: localhost +# database: test_uncloud +# user: root +# password: +# port: 5432 + +class VMTestCase(TestCase): + @classmethod + def setUpClass(cls): + # Setup vm host + cls.vm_host, created = VMHost.objects.get_or_create( + hostname='serverx.placey.ungleich.ch', physical_cores=32, usable_cores=320, + usable_ram_in_gb=512.0, status='active' + ) + super().setUpClass() + + def setUp(self) -> None: + # Setup two users as it is common to test with different user + self.user = User.objects.create_user( + username='testuser', email='test@test.com', first_name='Test', last_name='User' + ) + self.user2 = User.objects.create_user( + username='Meow', email='meow123@test.com', first_name='Meow', last_name='Cat' + ) + super().setUp() + + def create_sample_vm(self, owner): + one_month_later, parse_status = cal.parse("1 month later") + return VMProduct.objects.create( + vmhost=self.vm_host, cores=2, ram_in_gb=4, owner=owner, + order=Order.objects.create( + owner=owner, + creation_date=datetime.datetime.now(tz=timezone.utc), + starting_date=datetime.datetime.now(tz=timezone.utc), + ending_date=datetime.datetime(*one_month_later[:6], tzinfo=timezone.utc), + recurring_period=RecurringPeriod.PER_MONTH + ) + ) + +# TODO: the logic tested by this test is not implemented yet. +# def test_disk_product(self): +# """Ensures that a VMDiskProduct can only be created from a VMDiskImageProduct +# that is in status 'active'""" +# +# vm = self.create_sample_vm(owner=self.user) +# +# pending_disk_image = VMDiskImageProduct.objects.create( +# owner=self.user, name='pending_disk_image', is_os_image=True, is_public=True, size_in_gb=10, +# status='pending' +# ) +# try: +# vm_disk_product = VMDiskProduct.objects.create( +# owner=self.user, vm=vm, image=pending_disk_image, size_in_gb=10 +# ) +# except ValidationError: +# vm_disk_product = None +# +# self.assertIsNone( +# vm_disk_product, +# msg='VMDiskProduct created with disk image whose status is not active.' +# ) + +# TODO: the logic tested by this test is not implemented yet. +# def test_vm_disk_product_creation_for_someone_else(self): +# """Ensure that a user can only create a VMDiskProduct for his/her own VM""" +# +# # Create a VM which is ownership of self.user2 +# someone_else_vm = self.create_sample_vm(owner=self.user2) +# +# # 'self.user' would try to create a VMDiskProduct for 'user2's VM +# with self.assertRaises(ValidationError, msg='User created a VMDiskProduct for someone else VM.'): +# vm_disk_product = VMDiskProduct.objects.create( +# owner=self.user, vm=someone_else_vm, +# size_in_gb=10, +# image=VMDiskImageProduct.objects.create( +# owner=self.user, name='disk_image', is_os_image=True, is_public=True, size_in_gb=10, +# status='active' +# ) +# ) diff --git a/uncloud_vm/views.py b/uncloud_vm/views.py new file mode 100644 index 0000000..67f8656 --- /dev/null +++ b/uncloud_vm/views.py @@ -0,0 +1,261 @@ +from django.db import transaction +from django.shortcuts import render +from django.utils import timezone + +from django.contrib.auth.models import User +from django.shortcuts import get_object_or_404 + +from rest_framework import viewsets, permissions +from rest_framework.response import Response +from rest_framework.exceptions import ValidationError + +from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster +from uncloud_pay.models import Order, BillingAddress + +from .serializers import * +from uncloud_pay.helpers import ProductViewSet + +import datetime + +### +# Generic disk image views. Do not require orders / billing. + +class VMDiskImageProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskImageProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMDiskImageProduct.objects.all() + else: + obj = VMDiskImageProduct.objects.filter(owner=self.request.user) | VMDiskImageProduct.objects.filter(is_public=True) + + return obj + + + def create(self, request): + serializer = VMDiskImageProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + + # did not specify size NOR import url? + if not serializer.validated_data['size_in_gb']: + if not serializer.validated_data['import_url']: + raise ValidationError(detail={ 'error_mesage': 'Specify either import_url or size_in_gb' }) + + serializer.save(owner=request.user) + return Response(serializer.data) + +class VMDiskImageProductPublicViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskImageProductSerializer + + def get_queryset(self): + return VMDiskImageProduct.objects.filter(is_public=True) + +### +# User VM disk and snapshots. + +class VMDiskProductViewSet(viewsets.ModelViewSet): + """ + Let a user modify their own VMDisks + """ + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMDiskProduct.objects.all() + else: + obj = VMDiskProduct.objects.filter(owner=self.request.user) + + return obj + + def create(self, request): + serializer = VMDiskProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + + # get disk size from image, if not specified + if not 'size_in_gb' in serializer.validated_data: + size_in_gb = serializer.validated_data['image'].size_in_gb + else: + size_in_gb = serializer.validated_data['size_in_gb'] + + if size_in_gb < serializer.validated_data['image'].size_in_gb: + raise ValidationError(detail={ 'error_mesage': 'Size is smaller than original image' }) + + serializer.save(owner=request.user, size_in_gb=size_in_gb) + return Response(serializer.data) + +class VMSnapshotProductViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMSnapshotProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMSnapshotProduct.objects.all() + else: + obj = VMSnapshotProduct.objects.filter(owner=self.request.user) + + return obj + + def create(self, request): + serializer = VMSnapshotProductSerializer(data=request.data, context={'request': request}) + + # This verifies that the VM belongs to the request user + serializer.is_valid(raise_exception=True) + + vm = vm=serializer.validated_data['vm'] + disks = VMDiskProduct.objects.filter(vm=vm) + ssds_size = sum([d.size_in_gb for d in disks if d.image.storage_class == 'ssd']) + hdds_size = sum([d.size_in_gb for d in disks if d.image.storage_class == 'hdd']) + + recurring_price = serializer.pricing['per_gb_ssd'] * ssds_size + serializer.pricing['per_gb_hdd'] * hdds_size + recurring_period = serializer.pricing['recurring_period'] + + # Create order + now = datetime.datetime.now() + order = Order(owner=request.user, + recurring_period=recurring_period) + order.save() + order.add_record(one_time_price=0, + recurring_price=recurring_price, + description="Snapshot of VM {} from {}".format(vm, now)) + + serializer.save(owner=request.user, + order=order, + gb_ssd=ssds_size, + gb_hdd=hdds_size) + + return Response(serializer.data) + +### +# User VMs. + +class VMProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMWithOSProduct.objects.all() + else: + obj = VMWithOSProduct.objects.filter(owner=self.request.user) + + return obj + + def get_serializer_class(self): + if self.action == 'create': + return OrderVMProductSerializer + else: + return VMProductSerializer + + # Use a database transaction so that we do not get half-created structure + # if something goes wrong. + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + + # Create disk image. + disk = VMDiskProduct(owner=request.user, + **serializer.validated_data.pop("primary_disk")) + vm = VMWithOSProduct(owner=request.user, primary_disk=disk, + **serializer.validated_data) + disk.vm = vm # XXX: Is this really needed? + + # Create VM and Disk orders. + vm_order = Order.from_product( + vm, + recurring_period=order_recurring_period, + starting_date=timezone.now() + ) + + disk_order = Order.from_product( + disk, + recurring_period=order_recurring_period, + starting_date=timezone.now(), + depends_on=vm_order + ) + + + # Commit to DB. + vm.order = vm_order + vm.save() + vm_order.save() + + disk.order = disk_order + disk_order.save() + disk.save() + + return Response(VMProductSerializer(vm, context={'request': request}).data) + +class NicoVMProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = NicoVMProductSerializer + + def get_queryset(self): + obj = VMProduct.objects.filter(owner=self.request.user) + return obj + + def create(self, request): + serializer = self.serializer_class(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + vm = serializer.save(owner=request.user) + + return Response(serializer.data) + + +### +# Admin stuff. + +class VMHostViewSet(viewsets.ModelViewSet): + serializer_class = VMHostSerializer + queryset = VMHost.objects.all() + permission_classes = [permissions.IsAdminUser] + +class VMClusterViewSet(viewsets.ModelViewSet): + serializer_class = VMClusterSerializer + queryset = VMCluster.objects.all() + permission_classes = [permissions.IsAdminUser] + +## +# Nico's playground. + +# Also create: +# - /dcl/available_os +# Basically a view of public and my disk images +# - +class DCLCreateVMProductViewSet(ProductViewSet): + """ + This view resembles the way how DCL VMs are created by default. + + The user chooses an OS, os disk size, ram, cpu and whether or not to have a mapped IPv4 address + """ + + permission_classes = [permissions.IsAuthenticated] + serializer_class = DCLVMProductSerializer + + def get_queryset(self): + return VMProduct.objects.filter(owner=self.request.user) + + # Use a database transaction so that we do not get half-created structure + # if something goes wrong. + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = VMProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + + # Create base order. + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user + ) + order.save() + + # Create VM. + vm = serializer.save(owner=request.user, order=order) + + return Response(serializer.data)