diff --git a/.gitignore b/.gitignore index 5c55899..ab6a151 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,12 @@ -.idea -.vscode +.idea/ +.vscode/ +__pycache__/ +pay.conf +log.txt +test.py +STRIPE +venv/ uncloud/docs/build logs.txt @@ -16,3 +22,6 @@ uncloud/version.py build/ venv/ dist/ + +*.iso +*.sqlite3 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..e468591 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,18 @@ +stages: + - lint + - test + +run-tests: + stage: test + image: code.ungleich.ch:5050/uncloud/uncloud/uncloud-ci:latest + services: + - postgres:latest + variables: + DATABASE_HOST: postgres + DATABASE_USER: postgres + POSTGRES_HOST_AUTH_METHOD: trust + coverage: /^TOTAL.+?(\d+\%)$/ + script: + - pip install -r requirements.txt + - coverage run --source='.' ./manage.py test + - coverage report diff --git a/README.md b/README.md index 0e32f57..8c53654 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,62 @@ -# ucloud +# Uncloud -Checkout https://ungleich.ch/ucloud/ for the documentation of ucloud. +Cloud management platform, the ungleich way. + + +[![pipeline status](https://code.ungleich.ch/uncloud/uncloud/badges/master/pipeline.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master) +[![coverage report](https://code.ungleich.ch/uncloud/uncloud/badges/master/coverage.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master) + +## Useful commands + +* `./manage.py import-vat-rates path/to/csv` +* `./manage.py make-admin username` + +## Development setup + +Install system dependencies: + +* On Fedora, you will need the following packages: `python3-virtualenv python3-devel openldap-devel gcc chromium` + +NOTE: you will need to configure a LDAP server and credentials for authentication. See `uncloud/settings.py`. + +``` +# Initialize virtualenv. +» virtualenv .venv +Using base prefix '/usr' +New python executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python3 +Also creating executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python +Installing setuptools, pip, wheel... +done. + +# Enter virtualenv. +» source .venv/bin/activate + +# Install dependencies. +» pip install -r requirements.txt +[...] + +# Run migrations. +» ./manage.py migrate +Operations to perform: + Apply all migrations: admin, auth, contenttypes, opennebula, sessions, uncloud_auth, uncloud_net, uncloud_pay, uncloud_service, uncloud_vm +Running migrations: + [...] + +# Run webserver. +» ./manage.py runserver +Watching for file changes with StatReloader +Performing system checks... + +System check identified no issues (0 silenced). +May 07, 2020 - 10:17:08 +Django version 3.0.6, using settings 'uncloud.settings' +Starting development server at http://127.0.0.1:8000/ +Quit the server with CONTROL-C. +``` + +### Note on PGSQL + +If you want to use Postgres: + +* Install on configure PGSQL on your base system. +* OR use a container! `podman run --rm -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -it postgres:latest` diff --git a/archive/issues.org b/archive/issues.org new file mode 100644 index 0000000..840ec3c --- /dev/null +++ b/archive/issues.org @@ -0,0 +1,6 @@ +* Intro + This file lists issues that should be handled, are small and likely + not yet high prio. +* Issues +** TODO Register prefered address in User model +** TODO Allow to specify different recurring periods diff --git a/archive/uncloud_django_based/hacks/abk-hacks.py b/archive/uncloud_django_based/hacks/abk-hacks.py new file mode 100644 index 0000000..abc63d3 --- /dev/null +++ b/archive/uncloud_django_based/hacks/abk-hacks.py @@ -0,0 +1,55 @@ +""" +investigate into a simple python function that maps an ldap user to a vat percentage. Basically you need to +lookup the customer address, check if she is a business/registered tax number and if not apply the local +vat +""" + +import iso3166 +import datetime + +from csv import DictReader + + +def get_vat(street_address, city, postal_code, country, vat_number=None): + vat = { + 'Austria': [ + {'period': '1984-01-01/', 'rate': 0.2}, + {'period': '1976-01-01/1984-01-01', 'rate': 0.18}, + {'period': '1973-01-01/1976-01-01', 'rate': 0.16}, + ] + } + return iso3166.countries.get(country) + + # return iso3166.countries_by_name[country] + + +def main(): + # vat = get_vat( + # street_address='82 Nasheman-e-Iqbal near Wapda Town', + # city='Lahore', + # postal_code=53700, + # country='Pakistan', + # ) + # print(vat) + vat_rates = {} + with open('vat_rates.csv', newline='') as csvfile: + reader = DictReader(csvfile) + for row in reader: + territory_codes = row['territory_codes'].split('\n') + for code in territory_codes: + if code not in vat_rates: + vat_rates[code] = {} + + start_date = row['start_date'] + stop_data = row['stop_date'] + time_period = f'{start_date}|{stop_data}' + r = row.copy() + del r['start_date'] + del r['stop_date'] + del r['territory_codes'] + vat_rates[code][time_period] = r + print(vat_rates) + + +if __name__ == '__main__': + main() diff --git a/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py b/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py new file mode 100644 index 0000000..c0bbaf8 --- /dev/null +++ b/archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py @@ -0,0 +1,46 @@ +import importlib +import sys +import os + +from os.path import join as join_path +from xmlrpc.client import ServerProxy as RPCClient + +root = os.path.dirname(os.getcwd()) +sys.path.append(join_path(root, 'uncloud')) +secrets = importlib.import_module('uncloud.secrets') + + +class OpenNebula: + def __init__(self, url, session_string): + self.session_string = session_string + self.client = RPCClient(secrets.OPENNEBULA_URL) + + def create_user(self, username, password, authentication_driver='', group_id=None): + # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-user-allocate + + if group_id is None: + group_id = [] + + return self.client.one.user.allocate( + self.session_string, + username, + password, + authentication_driver, + group_id + ) + + def chmod(self, vm_id, user_id=-1, group_id=-1): + # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-vm-chown + + return self.client.one.vm.chown(self.session_string, vm_id, user_id, group_id) + + +one = OpenNebula(secrets.OPENNEBULA_URL, secrets.OPENNEBULA_USER_PASS) + +# Create User in OpenNebula +# success, response, *_ = one.create_user(username='meow12345', password='hello_world') +# print(success, response) + +# Change owner of a VM +# success, response, *_ = one.chmod(vm_id=25589, user_id=706) +# print(success, response) diff --git a/archive/uncloud_django_based/hacks/command-wrapper.sh b/archive/uncloud_django_based/hacks/command-wrapper.sh new file mode 100644 index 0000000..d6ddd13 --- /dev/null +++ b/archive/uncloud_django_based/hacks/command-wrapper.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +dbhost=$1; shift + +ssh -L5432:localhost:5432 "$dbhost" & + +python manage.py "$@" + + + +# command only needs to be active while manage command is running + +# -T no pseudo terminal + + +# alternatively: commands output shell code + +# ssh uncloud@dbhost "python manage.py --hostname xxx ..." diff --git a/archive/uncloud_django_based/meow-payv1/README.md b/archive/uncloud_django_based/meow-payv1/README.md new file mode 100644 index 0000000..fe6a2a3 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/README.md @@ -0,0 +1,51 @@ +# uncloud-pay + +The generic product/payment system. + +## Installation + +```shell script +pip3 install -r requirements.txt +``` + +## Getting Started + +```shell script +python ucloud_pay.py +``` + +## Usage + +#### 1. Adding of products +```shell script +http --json http://[::]:5000/product/add username=your_username_here password=your_password_here specs:=@ipv6-only-vm.json +``` + +#### 2. Listing of products +```shell script +http --json http://[::]:5000/product/list +``` + +#### 3. Registering user's payment method (credit card for now using Stripe) + +```shell script +http --json http://[::]:5000/user/register_payment card_number=4111111111111111 cvc=123 expiry_year=2020 expiry_month=8 card_holder_name="The test user" username=your_username_here password=your_password_here line1="your_billing_address" city="your_city" country="your_country" +``` + +#### 4. Ordering products + +First of all, user have to buy the membership first. + +```shell script +http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=membership pay=True +``` + +```shell script +http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=ipv6-only-vm cpu=1 ram=1 os-disk-space=10 os=alpine pay=True +``` + +#### 5. Listing users orders + +```shell script +http --json POST http://[::]:5000/order/list username=your_username_here password=your_password_here +``` diff --git a/archive/uncloud_django_based/meow-payv1/config.py b/archive/uncloud_django_based/meow-payv1/config.py new file mode 100644 index 0000000..16804af --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/config.py @@ -0,0 +1,21 @@ +import os + +from ungleich_common.ldap.ldap_manager import LdapManager +from ungleich_common.std.configparser import StrictConfigParser +from ungleich_common.etcd.etcd_wrapper import EtcdWrapper + +config_file = os.environ.get('meow-pay-config-file', default='pay.conf') + +config = StrictConfigParser(allow_no_value=True) +config.read(config_file) + +etcd_client = EtcdWrapper( + host=config.get('etcd', 'host'), port=config.get('etcd', 'port'), + ca_cert=config.get('etcd', 'ca_cert'), cert_key=config.get('etcd', 'cert_key'), + cert_cert=config.get('etcd', 'cert_cert') +) + +ldap_manager = LdapManager( + server=config.get('ldap', 'server'), admin_dn=config.get('ldap', 'admin_dn'), + admin_password=config.get('ldap', 'admin_password') +) diff --git a/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py b/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py new file mode 100644 index 0000000..e6bfb43 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/hack-a-vpn.py @@ -0,0 +1,213 @@ +from flask import Flask, request +from flask_restful import Resource, Api +import etcd3 +import json +import logging +from functools import wraps + +from ldaptest import is_valid_ldap_user + +def authenticate(func): + @wraps(func) + def wrapper(*args, **kwargs): + if not getattr(func, 'authenticated', True): + return func(*args, **kwargs) + + # pass in username/password ! + acct = basic_authentication() # custom account lookup function + + if acct: + return func(*args, **kwargs) + + flask_restful.abort(401) + return wrapper + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + + +class DB(object): + def __init__(self, config, prefix="/"): + self.config = config + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + + self.connect() + + @readable_errors + def connect(self): + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) + + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) + + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: + value = json.dumps(value) + + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + +class Membership(Resource): + def __init__(self, config): + self.config = config + + def get(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + return {'message': 'Order successful' }, 200 + + def post(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + return {'message': 'Order 2x successful' }, 200 + + +class Order(Resource): + def __init__(self, config): + self.config = config + + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + +class Product(Resource): + def __init__(self, config): + self.config = config + + self.products = [] + self.products.append( + { "name": "membership-free", + "description": """ +This membership gives you access to the API and includes a VPN +with 1 IPv6 address. +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "a3883466-0012-4d01-80ff-cbf7469957af", + "recurring": True, + "recurring_time_frame": "per_year", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 0 + } + ] + } + ) + self.products.append( + { "name": "membership-standard", + "description": """ +This membership gives you access to the API and includes an IPv6-VPN with +one IPv6 address ("Road warrior") +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "1d85296b-0863-4dd6-a543-a6d5a4fbe4a6", + "recurring": True, + "recurring_time_frame": "per_month", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 5 + } + + ] + } + ) + self.products.append( + { "name": "membership-premium", + "description": """ +This membership gives you access to the API and includes an +IPv6-VPN with a /48 IPv6 network. +See https://redmine.ungleich.ch/issues/7747? +""", + "uuid": "bfd63fd2-d227-436f-a8b8-600de74dd6ce", + "recurring": True, + "recurring_time_frame": "per_month", + "features": [ + { "name": "membership", + "price_one_time": 0, + "price_recurring": 5 + } + + ] + } + ) + self.products.append( + { "name": "ipv6-vpn-with-/48", + "description": """ +An IPv6 VPN with a /48 network included. +""", + "uuid": "fe5753f8-6fe1-4dc4-9b73-7b803de4c597", + "recurring": True, + "recurring_time_frame": "per_year", + "features": [ + { "name": "vpn", + "price_one_time": 0, + "price_recurring": 120 + } + ] + } + ) + + + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + def get(self): + data = request.get_json(silent=True) or {} + print("{} {}".format(data, config)) + + return self.products + + + + + +if __name__ == '__main__': + app = Flask(__name__) + + config = {} + + config['etcd_url']="https://etcd1.ungleich.ch" + config['ldap_url']="ldaps://ldap1.ungleich.ch" + + api = Api(app) + api.add_resource(Order, '/orders', resource_class_args=( config, )) + api.add_resource(Product, '/products', resource_class_args=( config, )) + api.add_resource(Membership, '/membership', resource_class_args=( config, )) + + app.run(host='::', port=5000, debug=True) diff --git a/archive/uncloud_django_based/meow-payv1/helper.py b/archive/uncloud_django_based/meow-payv1/helper.py new file mode 100644 index 0000000..65a5155 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/helper.py @@ -0,0 +1,87 @@ +import logging + +import parsedatetime + +from datetime import datetime +from stripe_utils import StripeUtils + + +def get_plan_id_from_product(product): + plan_id = 'ucloud-v1-' + plan_id += product['name'].strip().replace(' ', '-') + return plan_id + + +def get_pricing(price_in_chf_cents, product_type, recurring_period): + if product_type == 'recurring': + return 'CHF {}/{}'.format(price_in_chf_cents/100, recurring_period) + elif product_type == 'one-time': + return 'CHF {} (One time charge)'.format(price_in_chf_cents/100) + + +def get_user_friendly_product(product_dict): + uf_product = { + 'name': product_dict['name'], + 'description': product_dict['description'], + 'product_id': product_dict['usable-id'], + 'pricing': get_pricing( + product_dict['price'], product_dict['type'], product_dict['recurring_period'] + ) + } + if product_dict['type'] == 'recurring': + uf_product['minimum_subscription_period'] = product_dict['minimum_subscription_period'] + return uf_product + + +def get_token(card_number, cvc, exp_month, exp_year): + stripe_utils = StripeUtils() + token_response = stripe_utils.get_token_from_card( + card_number, cvc, exp_month, exp_year + ) + if token_response['response_object']: + return token_response['response_object'].id + else: + return None + + +def resolve_product(usable_id, etcd_client): + products = etcd_client.get_prefix('/v1/products/', value_in_json=True) + for p in products: + if p.value['usable-id'] == usable_id: + return p.value + return None + + +def calculate_charges(specification, data): + logging.debug('Calculating charges for specs:{} and data:{}'.format(specification, data)) + one_time_charge = 0 + recurring_charge = 0 + for feature_name, feature_detail in specification['features'].items(): + if feature_detail['constant']: + data[feature_name] = 1 + + if feature_detail['unit']['type'] != 'str': + one_time_charge += feature_detail['one_time_fee'] + recurring_charge += ( + feature_detail['price_per_unit_per_period'] * data[feature_name] / + feature_detail['unit']['value'] + ) + return one_time_charge, recurring_charge + + +def is_order_valid(order_timestamp, renewal_period): + """ + Sample Code Usage + + >> current_datetime, status = cal.parse('Now') + >> current_datetime = datetime(*current_datetime[:6]) + + >> print('Is order valid: ', is_order_valid(current_datetime, '1 month')) + >> True + """ + cal = parsedatetime.Calendar() + + renewal_datetime, status = cal.parse(renewal_period) + renewal_datetime = datetime(*renewal_datetime[:6]) + + return order_timestamp <= renewal_datetime diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json new file mode 100644 index 0000000..110027a --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json @@ -0,0 +1,28 @@ +{ + "usable-id": "ipv6-only-django-hosting", + "active": true, + "name": "IPv6 Only Django Hosting", + "description": "Host your Django application on our shiny IPv6 Only VM", + "recurring_period": "month", + "quantity": "inf", + "features": { + "cpu": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 3, + "one_time_fee": 0, + "constant": false + }, + "ram": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 4, + "one_time_fee": 0, + "constant": false + }, + "os-disk-space": { + "unit": {"value": 10, "type":"int"}, + "one_time_fee": 0, + "price_per_unit_per_period": 3.5, + "constant": false + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json new file mode 100644 index 0000000..d07ad6c --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json @@ -0,0 +1,34 @@ +{ + "usable-id": "ipv6-only-vm", + "active": true, + "name": "IPv6 Only VM", + "description": "IPv6 Only VM are accessible to only those having IPv6 for themselves", + "recurring_period": "month", + "quantity": "inf", + "features": { + "cpu": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 3, + "one_time_fee": 0, + "constant": false + }, + "ram": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 4, + "one_time_fee": 0, + "constant": false + }, + "os-disk-space": { + "unit": {"value": 10, "type":"int"}, + "one_time_fee": 0, + "price_per_unit_per_period": 4, + "constant": false + }, + "os": { + "unit": {"value": 1, "type":"str"}, + "one_time_fee": 0, + "price_per_unit_per_period": 0, + "constant": false + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json new file mode 100644 index 0000000..38c6201 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json @@ -0,0 +1,16 @@ +{ + "usable-id": "ipv6-only-vpn", + "active": true, + "name": "IPv6 Only VPN", + "description": "IPv6 VPN enable you to access IPv6 only websites and more", + "recurring_period": "month", + "quantity": "inf", + "features": { + "vpn": { + "unit": {"value": 1, "type": "int"}, + "price_per_unit_per_period": 10, + "one_time_fee": 0, + "constant": true + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/ipv6box.json b/archive/uncloud_django_based/meow-payv1/products/ipv6box.json new file mode 100644 index 0000000..eca11f0 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/ipv6box.json @@ -0,0 +1,16 @@ +{ + "usable-id": "ipv6-box", + "active": true, + "name": "IPv6 Box", + "description": "A ready-to-go IPv6 Box: it creates a VPN to ungleich and distributes IPv6 addresses to all your computers.", + "recurring_period": "eternity", + "quantity": 4, + "features": { + "ipv6-box": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 0, + "one_time_fee": 250, + "constant": true + } + } +} diff --git a/archive/uncloud_django_based/meow-payv1/products/membership.json b/archive/uncloud_django_based/meow-payv1/products/membership.json new file mode 100644 index 0000000..4003330 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/products/membership.json @@ -0,0 +1,17 @@ +{ + "usable-id": "membership", + "active": true, + "name": "Membership", + "description": "Membership to use uncloud-pay", + "recurring_period": "month", + "quantity": "inf", + "features": { + "membership": { + "unit": {"value": 1, "type":"int"}, + "price_per_unit_per_period": 5, + "one_time_fee": 0, + "constant": true + } + }, + "max_per_user": "1" +} diff --git a/archive/uncloud_django_based/meow-payv1/requirements.txt b/archive/uncloud_django_based/meow-payv1/requirements.txt new file mode 100644 index 0000000..0b758ca --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/requirements.txt @@ -0,0 +1,7 @@ +stripe +flask +Flask-RESTful +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-etcd&subdirectory=etcd +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-std&subdirectory=std +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-schemas&subdirectory=schemas diff --git a/archive/uncloud_django_based/meow-payv1/sample-pay.conf b/archive/uncloud_django_based/meow-payv1/sample-pay.conf new file mode 100644 index 0000000..5d1fe61 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/sample-pay.conf @@ -0,0 +1,17 @@ +[etcd] +host = 127.0.0.1 +port = 2379 +ca_cert +cert_cert +cert_key + +[stripe] +private_key=stripe_private_key + +[app] +port = 5000 + +[ldap] +server = ldap_server_url +admin_dn = ldap_admin_dn +admin_password = ldap_admin_password diff --git a/archive/uncloud_django_based/meow-payv1/schemas.py b/archive/uncloud_django_based/meow-payv1/schemas.py new file mode 100644 index 0000000..2e3aef7 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/schemas.py @@ -0,0 +1,136 @@ +import logging +import config +import json +import math + +from config import ldap_manager, etcd_client +from helper import resolve_product +from ungleich_common.schemas.schemas import BaseSchema, Field, ValidationException + + +class AddProductSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.add_schema(UserCredentialSchema, data) + self.specs = Field('specs', dict, **self.get(data, 'specs')) + self.update = Field('update', bool, **self.get(data, 'update', return_default=True, default=False)) + + def validation(self): + user = self.objects['user'] + user = json.loads(user.entry_to_json()) + uid, ou, *dc = user['dn'].replace('ou=', '').replace('dc=', '').replace('uid=', '').split(',') + if ou != config.config.get('ldap', 'internal_user_ou', fallback='users'): + raise ValidationException('You do not have access to create product.') + + product = resolve_product(self.specs.value['usable-id'], etcd_client) + if product: + self.objects['product'] = product + + +class AddressSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.line1 = Field('line1', str, **self.get(data, 'line1')) + self.line2 = Field('line2', str, **self.get(data, 'line2', return_default=True)) + self.city = Field('city', str, **self.get(data, 'city')) + self.country = Field('country', str, **self.get(data, 'country')) + self.state = Field('state', str, **self.get(data, 'state', return_default=True)) + self.postal_code = Field('postal_code', str, **self.get(data, 'postal_code', return_default=True)) + + +class UserRegisterPaymentSchema(BaseSchema): + def __init__(self, data): + super().__init__() + + self.add_schema(UserCredentialSchema, data) + self.add_schema(AddressSchema, data, under_field_name='address') + + self.card_number = Field('card_number', str, **self.get(data, 'card_number')) + self.cvc = Field('cvc', str, **self.get(data, 'cvc')) + self.expiry_year = Field('expiry_year', int, **self.get(data, 'expiry_year')) + self.expiry_month = Field('expiry_month', int, **self.get(data, 'expiry_month')) + self.card_holder_name = Field('card_holder_name', str, **self.get(data, 'card_holder_name')) + + +class UserCredentialSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.username = Field('username', str, **self.get(data, 'username')) + self.password = Field('password', str, **self.get(data, 'password')) + + def validation(self): + try: + entry = ldap_manager.is_password_valid(self.username.value, self.password.value, query_key='uid') + except ValueError: + raise ValidationException('No user with \'{}\' username found. You can create account at ' + 'https://account.ungleich.ch'.format(self.username.value)) + except Exception: + raise ValidationException('Invalid username/password.') + else: + self.objects['user'] = entry + + +class ProductOrderSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.product_id = Field( + 'product_id', str, **self.get(data, 'product_id'), validators=[self.product_id_validation] + ) + self.pay_consent = Field('pay', bool, **self.get(data, 'pay', return_default=True, default=False)) + self.add_schema(UserCredentialSchema, data) + + def product_id_validation(self): + product = resolve_product(self.product_id.value, etcd_client) + if product: + product['quantity'] = float(product['quantity']) + self.product_id.value = product['uuid'] + self.objects['product'] = product + logging.debug('Got product {}'.format(product)) + + if not product['active']: + raise ValidationException('Product is not active at the moment.') + + if product['quantity'] <= 0: + raise ValidationException('Out of stock.') + else: + raise ValidationException('No such product exists.') + + def validation(self): + username = self.objects['user'].uid + customer_previous_orders = etcd_client.get_prefix('/v1/user/{}'.format(username), value_in_json=True) + customer_previous_orders = [o.value for o in customer_previous_orders] + membership = next(filter(lambda o: o['product'] == 'membership', customer_previous_orders), None) + if membership is None and self.objects['product']['usable-id'] != 'membership': + raise ValidationException('Please buy membership first to use this facility') + max_quantity_user_can_order = float(self.objects['product'].get('max_per_user', math.inf)) + previous_order_of_same_product = [ + o for o in customer_previous_orders if o['product'] == self.objects['product']['usable-id'] + ] + if len(previous_order_of_same_product) >= max_quantity_user_can_order: + raise ValidationException( + 'You cannot buy {} more than {} times'.format( + self.objects['product']['name'], int(max_quantity_user_can_order) + ) + ) + + +class OrderListSchema(BaseSchema): + def __init__(self, data): + super().__init__() + self.add_schema(UserCredentialSchema, data) + + +def make_return_message(err, status_code=200): + logging.debug('message: {}'.format(str(err))) + return {'message': str(err)}, status_code + + +def create_schema(specification, data): + fields = {} + for feature_name, feature_detail in specification['features'].items(): + if not feature_detail['constant']: + fields[feature_name] = Field( + feature_name, eval(feature_detail['unit']['type']), **BaseSchema.get(data, feature_name) + ) + + return type('{}Schema'.format(specification['name']), (BaseSchema,), fields) diff --git a/archive/uncloud_django_based/meow-payv1/stripe_hack.py b/archive/uncloud_django_based/meow-payv1/stripe_hack.py new file mode 100644 index 0000000..f436c62 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/stripe_hack.py @@ -0,0 +1,7 @@ +import stripe_utils +import os + + +if __name__ == '__main__': + s = stripe_utils.StripeUtils(os.environ['STRIPE_PRIVATE_KEY']) + print(s.get_stripe_customer_from_email('coder.purple+2002@gmail.com')) diff --git a/archive/uncloud_django_based/meow-payv1/stripe_utils.py b/archive/uncloud_django_based/meow-payv1/stripe_utils.py new file mode 100644 index 0000000..6a2cd29 --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/stripe_utils.py @@ -0,0 +1,491 @@ +import re +import stripe +import stripe.error +import logging + +from config import etcd_client as client, config as config + +stripe.api_key = config.get('stripe', 'private_key') + + +def handle_stripe_error(f): + def handle_problems(*args, **kwargs): + response = { + 'paid': False, + 'response_object': None, + 'error': None + } + + common_message = "Currently it's not possible to make payments." + try: + response_object = f(*args, **kwargs) + response = { + 'response_object': response_object, + 'error': None + } + return response + except stripe.error.CardError as e: + # Since it's a decline, stripe.error.CardError will be caught + body = e.json_body + err = body['error'] + response.update({'error': err['message']}) + logging.error(str(e)) + return response + except stripe.error.RateLimitError: + response.update( + {'error': "Too many requests made to the API too quickly"}) + return response + except stripe.error.InvalidRequestError as e: + logging.error(str(e)) + response.update({'error': "Invalid parameters"}) + return response + except stripe.error.AuthenticationError as e: + # Authentication with Stripe's API failed + # (maybe you changed API keys recently) + logging.error(str(e)) + response.update({'error': common_message}) + return response + except stripe.error.APIConnectionError as e: + logging.error(str(e)) + response.update({'error': common_message}) + return response + except stripe.error.StripeError as e: + # maybe send email + logging.error(str(e)) + response.update({'error': common_message}) + return response + except Exception as e: + # maybe send email + logging.error(str(e)) + response.update({'error': common_message}) + return response + + return handle_problems + + +class StripeUtils(object): + CURRENCY = 'chf' + INTERVAL = 'month' + SUCCEEDED_STATUS = 'succeeded' + STRIPE_PLAN_ALREADY_EXISTS = 'Plan already exists' + STRIPE_NO_SUCH_PLAN = 'No such plan' + PLAN_EXISTS_ERROR_MSG = 'Plan {} exists already.\nCreating a local StripePlan now.' + PLAN_DOES_NOT_EXIST_ERROR_MSG = 'Plan {} does not exist.' + + def __init__(self, private_key): + self.stripe = stripe + stripe.api_key = private_key + + @handle_stripe_error + def card_exists(self, customer, cc_number, exp_month, exp_year, cvc): + token_obj = stripe.Token.create( + card={ + 'number': cc_number, + 'exp_month': exp_month, + 'exp_year': exp_year, + 'cvc': cvc, + }, + ) + cards = stripe.Customer.list_sources( + customer, + limit=20, + object='card' + ) + + for card in cards.data: + if (card.fingerprint == token_obj.card.fingerprint and + int(card.exp_month) == int(exp_month) and int(card.exp_year) == int(exp_year)): + return True + return False + + @staticmethod + def get_stripe_customer_from_email(email): + customer = stripe.Customer.list(limit=1, email=email) + return customer.data[0] if len(customer.data) == 1 else None + + @staticmethod + def update_customer_token(customer, token): + customer.source = token + customer.save() + + @handle_stripe_error + def get_token_from_card(self, cc_number, cvc, expiry_month, expiry_year): + token_obj = stripe.Token.create( + card={ + 'number': cc_number, + 'exp_month': expiry_month, + 'exp_year': expiry_year, + 'cvc': cvc, + }, + ) + return token_obj + + @handle_stripe_error + def associate_customer_card(self, stripe_customer_id, token, + set_as_default=False): + customer = stripe.Customer.retrieve(stripe_customer_id) + card = customer.sources.create(source=token) + if set_as_default: + customer.default_source = card.id + customer.save() + return True + + @handle_stripe_error + def dissociate_customer_card(self, stripe_customer_id, card_id): + customer = stripe.Customer.retrieve(stripe_customer_id) + card = customer.sources.retrieve(card_id) + card.delete() + + @handle_stripe_error + def update_customer_card(self, customer_id, token): + customer = stripe.Customer.retrieve(customer_id) + current_card_token = customer.default_source + customer.sources.retrieve(current_card_token).delete() + customer.source = token + customer.save() + credit_card_raw_data = customer.sources.data.pop() + new_card_data = { + 'last4': credit_card_raw_data.last4, + 'brand': credit_card_raw_data.brand + } + return new_card_data + + @handle_stripe_error + def get_card_details(self, customer_id): + customer = stripe.Customer.retrieve(customer_id) + credit_card_raw_data = customer.sources.data.pop() + card_details = { + 'last4': credit_card_raw_data.last4, + 'brand': credit_card_raw_data.brand, + 'exp_month': credit_card_raw_data.exp_month, + 'exp_year': credit_card_raw_data.exp_year, + 'fingerprint': credit_card_raw_data.fingerprint, + 'card_id': credit_card_raw_data.id + } + return card_details + + @handle_stripe_error + def get_all_invoices(self, customer_id, created_gt): + return_list = [] + has_more_invoices = True + starting_after = False + while has_more_invoices: + if starting_after: + invoices = stripe.Invoice.list( + limit=10, customer=customer_id, created={'gt': created_gt}, + starting_after=starting_after + ) + else: + invoices = stripe.Invoice.list( + limit=10, customer=customer_id, created={'gt': created_gt} + ) + has_more_invoices = invoices.has_more + for invoice in invoices.data: + sub_ids = [] + for line in invoice.lines.data: + if line.type == 'subscription': + sub_ids.append(line.id) + elif line.type == 'invoiceitem': + sub_ids.append(line.subscription) + else: + sub_ids.append('') + invoice_details = { + 'created': invoice.created, + 'receipt_number': invoice.receipt_number, + 'invoice_number': invoice.number, + 'paid_at': invoice.status_transitions.paid_at if invoice.paid else 0, + 'period_start': invoice.period_start, + 'period_end': invoice.period_end, + 'billing_reason': invoice.billing_reason, + 'discount': invoice.discount.coupon.amount_off if invoice.discount else 0, + 'total': invoice.total, + # to see how many line items we have in this invoice and + # then later check if we have more than 1 + 'lines_data_count': len(invoice.lines.data) if invoice.lines.data is not None else 0, + 'invoice_id': invoice.id, + 'lines_meta_data_csv': ','.join( + [line.metadata.VM_ID if hasattr(line.metadata, 'VM_ID') else '' for line in invoice.lines.data] + ), + 'subscription_ids_csv': ','.join(sub_ids), + 'line_items': invoice.lines.data + } + starting_after = invoice.id + return_list.append(invoice_details) + return return_list + + @handle_stripe_error + def get_cards_details_from_token(self, token): + stripe_token = stripe.Token.retrieve(token) + card_details = { + 'last4': stripe_token.card.last4, + 'brand': stripe_token.card.brand, + 'exp_month': stripe_token.card.exp_month, + 'exp_year': stripe_token.card.exp_year, + 'fingerprint': stripe_token.card.fingerprint, + 'card_id': stripe_token.card.id + } + return card_details + + def check_customer(self, stripe_cus_api_id, user, token): + try: + customer = stripe.Customer.retrieve(stripe_cus_api_id) + except stripe.error.InvalidRequestError: + customer = self.create_customer(token, user.email, user.name) + user.stripecustomer.stripe_id = customer.get( + 'response_object').get('id') + user.stripecustomer.save() + if type(customer) is dict: + customer = customer['response_object'] + return customer + + @handle_stripe_error + def get_customer(self, stripe_api_cus_id): + customer = stripe.Customer.retrieve(stripe_api_cus_id) + # data = customer.get('response_object') + return customer + + @handle_stripe_error + def create_customer(self, token, email, name=None, address=None): + if name is None or name.strip() == "": + name = email + customer = self.stripe.Customer.create( + source=token, + description=name, + email=email, + address=address + ) + return customer + + @handle_stripe_error + def make_charge(self, amount=None, customer=None): + _amount = float(amount) + amount = int(_amount * 100) # stripe amount unit, in cents + charge = self.stripe.Charge.create( + amount=amount, # in cents + currency=self.CURRENCY, + customer=customer + ) + return charge + + @staticmethod + def _get_all_stripe_plans(): + all_stripe_plans = client.get("/v1/stripe_plans") + all_stripe_plans_set = set() + if all_stripe_plans: + all_stripe_plans_obj = all_stripe_plans.value + if all_stripe_plans_obj and len(all_stripe_plans_obj['plans']) > 0: + all_stripe_plans_set = set(all_stripe_plans_obj["plans"]) + return all_stripe_plans_set + + @staticmethod + def _save_all_stripe_plans(stripe_plans): + client.put("/v1/stripe_plans", {"plans": list(stripe_plans)}) + + @handle_stripe_error + def get_or_create_stripe_plan(self, product_name, amount, stripe_plan_id, + interval=INTERVAL): + """ + This function checks if a StripePlan with the given + stripe_plan_id already exists. If it exists then the function + returns this object otherwise it creates a new StripePlan and + returns the new object. + + :param amount: The amount in CHF cents + :param product_name: The name of the Stripe plan (product) to be created. + :param stripe_plan_id: The id of the Stripe plan to be + created. Use get_stripe_plan_id_string function to + obtain the name of the plan to be created + :param interval: The interval for subscription {month, year}. Defaults + to month if not provided + :return: The StripePlan object if it exists else creates a + Plan object in Stripe and a local StripePlan and + returns it. Returns None in case of Stripe error + """ + _amount = float(amount) + amount = int(_amount * 100) # stripe amount unit, in cents + all_stripe_plans = self._get_all_stripe_plans() + if stripe_plan_id in all_stripe_plans: + logging.debug("{} plan exists in db.".format(stripe_plan_id)) + else: + logging.debug(("{} plan DOES NOT exist in db. " + "Creating").format(stripe_plan_id)) + try: + plan_obj = self.stripe.Plan.retrieve(id=stripe_plan_id) + logging.debug("{} plan exists in Stripe".format(stripe_plan_id)) + all_stripe_plans.add(stripe_plan_id) + except stripe.error.InvalidRequestError as e: + if "No such plan" in str(e): + logging.debug("Plan {} does not exist in Stripe, Creating") + plan_obj = self.stripe.Plan.create( + amount=amount, + product={'name': product_name}, + interval=interval, + currency=self.CURRENCY, + id=stripe_plan_id) + logging.debug(self.PLAN_EXISTS_ERROR_MSG.format(stripe_plan_id)) + all_stripe_plans.add(stripe_plan_id) + self._save_all_stripe_plans(all_stripe_plans) + return stripe_plan_id + + @handle_stripe_error + def delete_stripe_plan(self, stripe_plan_id): + """ + Deletes the Plan in Stripe and also deletes the local db copy + of the plan if it exists + + :param stripe_plan_id: The stripe plan id that needs to be + deleted + :return: True if the plan was deleted successfully from + Stripe, False otherwise. + """ + return_value = False + try: + plan = self.stripe.Plan.retrieve(stripe_plan_id) + plan.delete() + return_value = True + all_stripe_plans = self._get_all_stripe_plans() + all_stripe_plans.remove(stripe_plan_id) + self._save_all_stripe_plans(all_stripe_plans) + except stripe.error.InvalidRequestError as e: + if self.STRIPE_NO_SUCH_PLAN in str(e): + logging.debug( + self.PLAN_DOES_NOT_EXIST_ERROR_MSG.format(stripe_plan_id)) + return return_value + + @handle_stripe_error + def subscribe_customer_to_plan(self, customer, plans, trial_end=None): + """ + Subscribes the given customer to the list of given plans + + :param customer: The stripe customer identifier + :param plans: A list of stripe plans. + :param trial_end: An integer representing when the Stripe subscription + is supposed to end + Ref: https://stripe.com/docs/api/python#create_subscription-items + e.g. + plans = [ + { + "plan": "dcl-v1-cpu-2-ram-5gb-ssd-10gb", + }, + ] + :return: The subscription StripeObject + """ + + subscription_result = self.stripe.Subscription.create( + customer=customer, items=plans, trial_end=trial_end + ) + return subscription_result + + @handle_stripe_error + def set_subscription_metadata(self, subscription_id, metadata): + subscription = stripe.Subscription.retrieve(subscription_id) + subscription.metadata = metadata + subscription.save() + + @handle_stripe_error + def unsubscribe_customer(self, subscription_id): + """ + Cancels a given subscription + + :param subscription_id: The Stripe subscription id string + :return: + """ + sub = stripe.Subscription.retrieve(subscription_id) + return sub.delete() + + @handle_stripe_error + def make_payment(self, customer, amount, token): + charge = self.stripe.Charge.create( + amount=amount, # in cents + currency=self.CURRENCY, + customer=customer + ) + return charge + + @staticmethod + def get_stripe_plan_id(cpu, ram, ssd, version, app='dcl', hdd=None, + price=None): + """ + Returns the Stripe plan id string of the form + `dcl-v1-cpu-2-ram-5gb-ssd-10gb` based on the input parameters + + :param cpu: The number of cores + :param ram: The size of the RAM in GB + :param ssd: The size of ssd storage in GB + :param hdd: The size of hdd storage in GB + :param version: The version of the Stripe plans + :param app: The application to which the stripe plan belongs + to. By default it is 'dcl' + :param price: The price for this plan + :return: A string of the form `dcl-v1-cpu-2-ram-5gb-ssd-10gb` + """ + dcl_plan_string = 'cpu-{cpu}-ram-{ram}gb-ssd-{ssd}gb'.format(cpu=cpu, + ram=ram, + ssd=ssd) + if hdd is not None: + dcl_plan_string = '{dcl_plan_string}-hdd-{hdd}gb'.format( + dcl_plan_string=dcl_plan_string, hdd=hdd) + stripe_plan_id_string = '{app}-v{version}-{plan}'.format( + app=app, + version=version, + plan=dcl_plan_string + ) + if price is not None: + stripe_plan_id_string_with_price = '{}-{}chf'.format( + stripe_plan_id_string, + round(price, 2) + ) + return stripe_plan_id_string_with_price + else: + return stripe_plan_id_string + + @staticmethod + def get_vm_config_from_stripe_id(stripe_id): + """ + Given a string like "dcl-v1-cpu-2-ram-5gb-ssd-10gb" return different + configuration params as a dict + + :param stripe_id|str + :return: dict + """ + pattern = re.compile(r'^dcl-v(\d+)-cpu-(\d+)-ram-(\d+\.?\d*)gb-ssd-(\d+)gb-?(\d*\.?\d*)(chf)?$') + match_res = pattern.match(stripe_id) + if match_res is not None: + price = None + try: + price = match_res.group(5) + except IndexError: + logging.debug("Did not find price in {}".format(stripe_id)) + return { + 'version': match_res.group(1), + 'cores': match_res.group(2), + 'ram': match_res.group(3), + 'ssd': match_res.group(4), + 'price': price + } + + @staticmethod + def get_stripe_plan_name(cpu, memory, disk_size, price): + """ + Returns the Stripe plan name + :return: + """ + return "{cpu} Cores, {memory} GB RAM, {disk_size} GB SSD, " \ + "{price} CHF".format( + cpu=cpu, + memory=memory, + disk_size=disk_size, + price=round(price, 2) + ) + + @handle_stripe_error + def set_subscription_meta_data(self, subscription_id, meta_data): + """ + Adds VM metadata to a subscription + :param subscription_id: Stripe identifier for the subscription + :param meta_data: A dict of meta data to be added + :return: + """ + subscription = stripe.Subscription.retrieve(subscription_id) + subscription.metadata = meta_data + subscription.save() diff --git a/archive/uncloud_django_based/meow-payv1/ucloud_pay.py b/archive/uncloud_django_based/meow-payv1/ucloud_pay.py new file mode 100644 index 0000000..dbc0d2c --- /dev/null +++ b/archive/uncloud_django_based/meow-payv1/ucloud_pay.py @@ -0,0 +1,338 @@ +import logging + +from datetime import datetime +from uuid import uuid4 + +from flask import Flask, request +from flask_restful import Resource, Api +from werkzeug.exceptions import HTTPException +from config import etcd_client as client, config as config +from stripe_utils import StripeUtils +from schemas import ( + make_return_message, ValidationException, UserRegisterPaymentSchema, + AddProductSchema, ProductOrderSchema, OrderListSchema, create_schema +) +from helper import get_plan_id_from_product, calculate_charges + + +class ListProducts(Resource): + @staticmethod + def get(): + products = client.get_prefix('/v1/products/') + products = [ + product + for product in [p.value for p in products] + if product['active'] + ] + prod_dict = {} + for p in products: + prod_dict[p['usable-id']] = { + 'name': p['name'], + 'description': p['description'], + } + logger.debug('Products = {}'.format(prod_dict)) + return prod_dict, 200 + +class AddProduct(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + logger.debug('Got data: {}'.format(str(data))) + validator = AddProductSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + previous_product = cleaned_values.get('product', None) + if previous_product: + if not cleaned_values['update']: + return make_return_message('Product already exists. Pass --update to update the product.') + else: + product_uuid = previous_product.pop('uuid') + else: + product_uuid = uuid4().hex + + product_value = cleaned_values['specs'] + + product_key = '/v1/products/{}'.format(product_uuid) + product_value['uuid'] = product_uuid + + logger.debug('Adding product data: {}'.format(str(product_value))) + client.put(product_key, product_value) + if not previous_product: + return make_return_message('Product created.') + else: + return make_return_message('Product updated.') + +################################################################################ +# Nico-ok-marker + + +class UserRegisterPayment(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + logger.debug('Got data: {}'.format(str(data))) + validator = UserRegisterPaymentSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + last4 = data['card_number'].strip()[-4:] + + stripe_utils = StripeUtils() + + # Does customer already exist ? + stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail']) + + # Does customer already exist ? + if stripe_customer is not None: + logger.debug('Customer {}-{} exists already'.format( + cleaned_values['username'], cleaned_values['user']['mail']) + ) + + # Check if the card already exists + ce_response = stripe_utils.card_exists( + stripe_customer.id, cc_number=data['card_number'], + exp_month=int(data['expiry_month']), + exp_year=int(data['expiry_year']), + cvc=data['cvc']) + + if ce_response['response_object']: + message = 'The given card ending in {} exists already.'.format(last4) + return make_return_message(message, 400) + + elif ce_response['response_object'] is False: + # Associate card with user + logger.debug('Adding card ending in {}'.format(last4)) + token_response = stripe_utils.get_token_from_card( + data['card_number'], data['cvc'], data['expiry_month'], + data['expiry_year'] + ) + if token_response['response_object']: + logger.debug('Token {}'.format(token_response['response_object'].id)) + resp = stripe_utils.associate_customer_card( + stripe_customer.id, token_response['response_object'].id + ) + if resp['response_object']: + return make_return_message( + 'Card ending in {} registered as your payment source'.format(last4) + ) + else: + return make_return_message('Error with payment gateway. Contact support', 400) + else: + return make_return_message('Error: {}'.format(ce_response['error']), 400) + else: + # Stripe customer does not exist, create a new one + logger.debug( + 'Customer {} does not exist, creating new'.format(cleaned_values['user']['mail']) + ) + token_response = stripe_utils.get_token_from_card( + cleaned_values['card_number'], cleaned_values['cvc'], + cleaned_values['expiry_month'], cleaned_values['expiry_year'] + ) + if token_response['response_object']: + logger.debug('Token {}'.format(token_response['response_object'].id)) + + # Create stripe customer + stripe_customer_resp = stripe_utils.create_customer( + name=cleaned_values['card_holder_name'], + token=token_response['response_object'].id, + email=cleaned_values['user']['mail'], + address=cleaned_values['address'] + ) + stripe_customer = stripe_customer_resp['response_object'] + + if stripe_customer: + logger.debug('Created stripe customer {}'.format(stripe_customer.id)) + return make_return_message( + 'Card ending in {} registered as your payment source'.format(last4) + ) + else: + return make_return_message('Error with card. Contact support', 400) + else: + return make_return_message('Error with payment gateway. Contact support', 400) + + +class ProductOrder(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + validator = ProductOrderSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + stripe_utils = StripeUtils() + + product = cleaned_values['product'] + + # Check the user has a payment source added + stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail']) + + if not stripe_customer or len(stripe_customer.sources) == 0: + return make_return_message('Please register your payment method first.', 400) + + try: + product_schema = create_schema(product, data) + product_schema = product_schema() + product_schema.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + transformed_data = product_schema.get_cleaned_values() + logger.debug('Tranformed data: {}'.format(transformed_data)) + one_time_charge, recurring_charge = calculate_charges(product, transformed_data) + recurring_charge = int(recurring_charge) + + if not cleaned_values['pay']: + return make_return_message( + 'You would be charged {} CHF one time and {} CHF every {}. ' + 'Add --pay to command to order.'.format( + one_time_charge, recurring_charge, product['recurring_period'] + ) + ) + + with client.client.lock('product-order') as _: + # Initiate a one-time/subscription based on product type + if recurring_charge > 0: + logger.debug('Product {} is recurring payment'.format(product['name'])) + plan_id = get_plan_id_from_product(product) + res = stripe_utils.get_or_create_stripe_plan( + product_name=product['name'], + stripe_plan_id=plan_id, amount=recurring_charge, + interval=product['recurring_period'], + ) + if res['response_object']: + logger.debug('Obtained plan {}'.format(plan_id)) + subscription_res = stripe_utils.subscribe_customer_to_plan( + stripe_customer.id, + [{'plan': plan_id}] + ) + subscription_obj = subscription_res['response_object'] + if subscription_obj is None or subscription_obj.status != 'active': + return make_return_message( + 'Error subscribing to plan. Detail: {}'.format(subscription_res['error']), 400 + ) + else: + order_obj = { + 'order-id': uuid4().hex, + 'ordered-at': datetime.now().isoformat(), + 'product': product['usable-id'], + 'one-time-price': one_time_charge, + 'recurring-price': recurring_charge, + 'recurring-period': product['recurring_period'] + } + client.put( + '/v1/user/{}/orders/{}'.format( + cleaned_values['username'], order_obj['order-id'] + ), order_obj + ) + product['quantity'] -= 1 + client.put('/v1/products/{}'.format(product['uuid']), product) + + return { + 'message': 'Order Successful.', + **order_obj + } + else: + logger.error('Could not create plan {}'.format(plan_id)) + return make_return_message('Something wrong happened. Contact administrator', 400) + + elif recurring_charge == 0 and one_time_charge > 0: + logger.debug('Product {} is one-time payment'.format(product['name'])) + charge_response = stripe_utils.make_charge( + amount=one_time_charge, + customer=stripe_customer.id + ) + stripe_onetime_charge = charge_response.get('response_object') + + # Check if the payment was approved + if not stripe_onetime_charge: + msg = charge_response.get('error') + return make_return_message('Error subscribing to plan. Details: {}'.format(msg), 400) + + order_obj = { + 'order-id': uuid4().hex, + 'ordered-at': datetime.now().isoformat(), + 'product': product['usable-id'], + 'one-time-price': one_time_charge, + } + client.put( + '/v1/user/{}/orders/{}'.format(cleaned_values['username'], order_obj['order-id']), + order_obj + ) + product['quantity'] -= 1 + client.put('/v1/products/{}'.format(product['uuid']), product) + + return {'message': 'Order successful', **order_obj}, 200 + + +class OrderList(Resource): + @staticmethod + def post(): + data = request.get_json(silent=True) or {} + + try: + validator = OrderListSchema(data) + validator.is_valid() + except ValidationException as err: + return make_return_message(err, 400) + else: + cleaned_values = validator.get_cleaned_values() + orders = client.get_prefix('/v1/user/{}/orders'.format(cleaned_values['username'])) + orders_dict = { + order.value['order-id']: { + **order.value + } + for order in orders + } + logger.debug('Orders = {}'.format(orders_dict)) + return {'orders': orders_dict}, 200 + + +if __name__ == '__main__': + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + log_formater = logging.Formatter('[%(filename)s:%(lineno)d] %(message)s') + + stream_logger = logging.StreamHandler() + stream_logger.setFormatter(log_formater) + + # file_logger = logging.FileHandler('log.txt') + # file_logger.setLevel(logging.DEBUG) + # file_logger.setFormatter(log_formater) + + logger.addHandler(stream_logger) + # logger.addHandler(file_logger) + + app = Flask(__name__) + + api = Api(app) + api.add_resource(ListProducts, '/product/list') + api.add_resource(AddProduct, '/product/add') + api.add_resource(ProductOrder, '/product/order') + api.add_resource(UserRegisterPayment, '/user/register_payment') + api.add_resource(OrderList, '/order/list') + + app.run(host='::', port=config.get('app', 'port', fallback=5000), debug=True) + + + @app.errorhandler(Exception) + def handle_exception(e): + app.logger.error(e) + # pass through HTTP errors + if isinstance(e, HTTPException): + return e + + # now you're handling non-HTTP exceptions only + return {'message': 'Server Error'}, 500 diff --git a/archive/uncloud_django_based/notes-abk.md b/archive/uncloud_django_based/notes-abk.md new file mode 100644 index 0000000..6d5c223 --- /dev/null +++ b/archive/uncloud_django_based/notes-abk.md @@ -0,0 +1,11 @@ +## TODO 2020-02-22 + +* ~~move the current rest api to /opennebula~~ +* ~~make the /opennebula api only accessible by an admin account~~ +* ~~create a new filtered api on /vm/list that~~ + * ~~a) requires authentication~~ + * ~~b) only shows the VMs of the current user~~ +* ~~the new api should not contain all details, but: cpus (as read by the vcpu field), ram, ips, disks~~ +* ~~also make a (random) uuid the primary key for VMs - everything in this uncloud hack will use uuids as the id~~ +* ~~still expose the opennebula id as opennebula_id~~ +* ~~note put all secrets/configs into uncloud.secrets - I added a sample file into the repo~~ diff --git a/archive/uncloud_django_based/notes-nico.org b/archive/uncloud_django_based/notes-nico.org new file mode 100644 index 0000000..811fbff --- /dev/null +++ b/archive/uncloud_django_based/notes-nico.org @@ -0,0 +1,102 @@ +* snapshot feature +** product: vm-snapshot +** flow +*** list all my VMs +**** get the uuid of the VM I want to take a snapshot of +*** request a snapshot +``` +vmuuid=$(http nicocustomer +http -a nicocustomer:xxx http://uncloud.ch/vm/create_snapshot uuid= +password=... +``` +** backend realisation +*** list snapshots + - have them in the DB + - create an entry on create +*** creating snapshots + - vm sync / fsync? + - rbd snapshot + - host/cluster mapping? + - need image(s) + +* steps +** DONE authenticate via ldap + CLOSED: [2020-02-20 Thu 19:05] +** DONE Make classes / views require authentication + CLOSED: [2020-02-20 Thu 19:05] +** TODO register credit card +*** TODO find out what saving with us +*** Info +**** should not be fully saved in the DB +**** model needs to be a bit different +* Decide where to save sensitive data +** stripe access key, etc. +* python requirements (nicohack202002) + django djangorestframework django-auth-ldap stripe +* os package requirements (alpine) + openldap-dev +* VPN case +** put on /orders with uuid +** register cc +* CC +** TODO check whether we can register or not at stripe +* membership +** required for "smaller" / "shorter" products + +* TODO Membership missing +* Flows to be implemented - see https://redmine.ungleich.ch/issues/7609 +** Membership +*** 5 CHF +** Django Hosting +*** One time payment 35 CHF +*** Monthly payment depends on VM size +*** Parameters: same as IPv6 only VM +** IPv6 VPN +*** Parameters: none +*** Is for free if the customer has an active VM +** IPv6 only VM +*** Parameters: cores, ram, os_disk_size, OS +* Django rest framework +** viewset: .list and .create +** view: .get .post +* TODO register CC +* DONE list products + CLOSED: [2020-02-24 Mon 20:15] +* An ungleich account - can be registered for free on + https://account.ungleich.ch +* httpie installed (provides the http command) + +## Get a membership + + +## Registering a payment method + +To be able to pay for the membership, you will need to register a +credit card or apply for payment on bill (TO BE IMPLEMENTED). + +### Register credit card + +``` +http POST https://api.ungleich.ch/membership \ + username=nico password=yourpassword \ + cc_number=.. \ + cc_ + +``` + + + +### Request payment via bill + + + + +## Create the membership + + +``` +http POST https://api.ungleich.ch/membership username=nico password=yourpassword + +``` + +## List available products diff --git a/archive/uncloud_django_based/plan.org b/archive/uncloud_django_based/plan.org new file mode 100644 index 0000000..9f172c2 --- /dev/null +++ b/archive/uncloud_django_based/plan.org @@ -0,0 +1,6 @@ +* TODO register CC +* TODO list products +* ahmed +** schemas +*** field: is_valid? - used by schemas +*** definition of a "schema" diff --git a/archive/uncloud_django_based/uncloud/.gitignore b/archive/uncloud_django_based/uncloud/.gitignore new file mode 100644 index 0000000..71202e1 --- /dev/null +++ b/archive/uncloud_django_based/uncloud/.gitignore @@ -0,0 +1,4 @@ +db.sqlite3 +uncloud/secrets.py +debug.log +uncloud/local_settings.py \ No newline at end of file diff --git a/bin/gen-version b/archive/uncloud_etcd_based/bin/gen-version similarity index 63% rename from bin/gen-version rename to archive/uncloud_etcd_based/bin/gen-version index a2e2882..06c3e22 100755 --- a/bin/gen-version +++ b/archive/uncloud_etcd_based/bin/gen-version @@ -1,22 +1,22 @@ #!/bin/sh # -*- coding: utf-8 -*- # -# 2019 Nico Schottelius (nico-ucloud at schottelius.org) +# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org) # -# This file is part of ucloud. +# This file is part of uncloud. # -# ucloud is free software: you can redistribute it and/or modify +# uncloud is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation, either version 3 of the License, or # (at your option) any later version. # -# ucloud is distributed in the hope that it will be useful, +# uncloud is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License -# along with ucloud. If not, see . +# along with uncloud. If not, see . # # @@ -26,4 +26,4 @@ dir=${0%/*} # Ensure version is present - the bundled/shipped version contains a static version, # the git version contains a dynamic version -printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py +printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py diff --git a/bin/uncloud b/archive/uncloud_etcd_based/bin/uncloud similarity index 100% rename from bin/uncloud rename to archive/uncloud_etcd_based/bin/uncloud diff --git a/bin/uncloud-run-reinstall b/archive/uncloud_etcd_based/bin/uncloud-run-reinstall similarity index 92% rename from bin/uncloud-run-reinstall rename to archive/uncloud_etcd_based/bin/uncloud-run-reinstall index 18e95c0..b211613 100755 --- a/bin/uncloud-run-reinstall +++ b/archive/uncloud_etcd_based/bin/uncloud-run-reinstall @@ -24,6 +24,6 @@ dir=${0%/*} ${dir}/gen-version; -pip uninstall -y uncloud -python setup.py install +pip uninstall -y uncloud >/dev/null +python setup.py install >/dev/null ${dir}/uncloud "$@" diff --git a/conf/uncloud.conf b/archive/uncloud_etcd_based/conf/uncloud.conf similarity index 100% rename from conf/uncloud.conf rename to archive/uncloud_etcd_based/conf/uncloud.conf diff --git a/uncloud/docs/Makefile b/archive/uncloud_etcd_based/docs/Makefile similarity index 93% rename from uncloud/docs/Makefile rename to archive/uncloud_etcd_based/docs/Makefile index 5e7ea85..246b56c 100644 --- a/uncloud/docs/Makefile +++ b/archive/uncloud_etcd_based/docs/Makefile @@ -7,7 +7,7 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = source/ BUILDDIR = build/ -DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/ +DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/ .PHONY: all build clean diff --git a/uncloud/docs/README.md b/archive/uncloud_etcd_based/docs/README.md similarity index 100% rename from uncloud/docs/README.md rename to archive/uncloud_etcd_based/docs/README.md diff --git a/uncloud/cli/__init__.py b/archive/uncloud_etcd_based/docs/__init__.py similarity index 100% rename from uncloud/cli/__init__.py rename to archive/uncloud_etcd_based/docs/__init__.py diff --git a/uncloud/configure/__init__.py b/archive/uncloud_etcd_based/docs/source/__init__.py similarity index 100% rename from uncloud/configure/__init__.py rename to archive/uncloud_etcd_based/docs/source/__init__.py diff --git a/uncloud/docs/source/admin-guide b/archive/uncloud_etcd_based/docs/source/admin-guide.rst similarity index 72% rename from uncloud/docs/source/admin-guide rename to archive/uncloud_etcd_based/docs/source/admin-guide.rst index ec6597d..b62808d 100644 --- a/uncloud/docs/source/admin-guide +++ b/archive/uncloud_etcd_based/docs/source/admin-guide.rst @@ -56,40 +56,13 @@ To start host we created earlier, execute the following command ucloud host ungleich.ch -Create OS Image ---------------- +File & image scanners +-------------------------- -Create ucloud-init ready OS image (Optional) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -This step is optional if you just want to test ucloud. However, sooner or later -you want to create OS images with ucloud-init to properly -contexualize VMs. - -1. Start a VM with OS image on which you want to install ucloud-init -2. Execute the following command on the started VM - - .. code-block:: sh - - apk add git - git clone https://code.ungleich.ch/ucloud/ucloud-init.git - cd ucloud-init - sh ./install.sh -3. Congratulations. Your image is now ucloud-init ready. - - -Upload Sample OS Image -~~~~~~~~~~~~~~~~~~~~~~ -Execute the following to get the sample OS image file. - -.. code-block:: sh - - mkdir /var/www/admin - (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download) - -Run File Scanner and Image Scanner -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make -images from tracked files. So, we need to track the file by running File Scanner +Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our +uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by +ucloud. We can only make images from tracked files. So, we need to track the +file by running File Scanner .. code-block:: sh diff --git a/uncloud/docs/source/conf.py b/archive/uncloud_etcd_based/docs/source/conf.py similarity index 100% rename from uncloud/docs/source/conf.py rename to archive/uncloud_etcd_based/docs/source/conf.py diff --git a/uncloud/docs/source/diagram-code/ucloud b/archive/uncloud_etcd_based/docs/source/diagram-code/ucloud similarity index 100% rename from uncloud/docs/source/diagram-code/ucloud rename to archive/uncloud_etcd_based/docs/source/diagram-code/ucloud diff --git a/archive/uncloud_etcd_based/docs/source/hacking.rst b/archive/uncloud_etcd_based/docs/source/hacking.rst new file mode 100644 index 0000000..1c750d6 --- /dev/null +++ b/archive/uncloud_etcd_based/docs/source/hacking.rst @@ -0,0 +1,36 @@ +Hacking +======= +Using uncloud in hacking (aka development) mode. + + +Get the code +------------ +.. code-block:: sh + :linenos: + + git clone https://code.ungleich.ch/uncloud/uncloud.git + + + +Install python requirements +--------------------------- +You need to have python3 installed. + +.. code-block:: sh + :linenos: + + cd uncloud! + python -m venv venv + . ./venv/bin/activate + ./bin/uncloud-run-reinstall + + + +Install os requirements +----------------------- +Install the following software packages: **dnsmasq**. + +If you already have a working IPv6 SLAAC and DNS setup, +this step can be skipped. + +Note that you need at least one /64 IPv6 network to run uncloud. diff --git a/uncloud/docs/source/images/ucloud.svg b/archive/uncloud_etcd_based/docs/source/images/ucloud.svg similarity index 100% rename from uncloud/docs/source/images/ucloud.svg rename to archive/uncloud_etcd_based/docs/source/images/ucloud.svg diff --git a/uncloud/docs/source/index.rst b/archive/uncloud_etcd_based/docs/source/index.rst similarity index 90% rename from uncloud/docs/source/index.rst rename to archive/uncloud_etcd_based/docs/source/index.rst index b31cff3..fad1f88 100644 --- a/uncloud/docs/source/index.rst +++ b/archive/uncloud_etcd_based/docs/source/index.rst @@ -11,14 +11,13 @@ Welcome to ucloud's documentation! :caption: Contents: introduction - user-guide setup-install + vm-images + user-guide admin-guide - user-guide/how-to-create-an-os-image-for-ucloud troubleshooting hacking - Indices and tables ================== diff --git a/uncloud/docs/source/introduction.rst b/archive/uncloud_etcd_based/docs/source/introduction.rst similarity index 100% rename from uncloud/docs/source/introduction.rst rename to archive/uncloud_etcd_based/docs/source/introduction.rst diff --git a/uncloud/docs/source/misc/todo.rst b/archive/uncloud_etcd_based/docs/source/misc/todo.rst similarity index 100% rename from uncloud/docs/source/misc/todo.rst rename to archive/uncloud_etcd_based/docs/source/misc/todo.rst diff --git a/uncloud/docs/source/setup-install.rst b/archive/uncloud_etcd_based/docs/source/setup-install.rst similarity index 100% rename from uncloud/docs/source/setup-install.rst rename to archive/uncloud_etcd_based/docs/source/setup-install.rst diff --git a/uncloud/docs/source/theory/summary.rst b/archive/uncloud_etcd_based/docs/source/theory/summary.rst similarity index 100% rename from uncloud/docs/source/theory/summary.rst rename to archive/uncloud_etcd_based/docs/source/theory/summary.rst diff --git a/uncloud/docs/source/troubleshooting.rst b/archive/uncloud_etcd_based/docs/source/troubleshooting.rst similarity index 100% rename from uncloud/docs/source/troubleshooting.rst rename to archive/uncloud_etcd_based/docs/source/troubleshooting.rst diff --git a/uncloud/docs/source/user-guide.rst b/archive/uncloud_etcd_based/docs/source/user-guide.rst similarity index 100% rename from uncloud/docs/source/user-guide.rst rename to archive/uncloud_etcd_based/docs/source/user-guide.rst diff --git a/uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst b/archive/uncloud_etcd_based/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst similarity index 100% rename from uncloud/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst rename to archive/uncloud_etcd_based/docs/source/user-guide/how-to-create-an-os-image-for-ucloud.rst diff --git a/archive/uncloud_etcd_based/docs/source/vm-images.rst b/archive/uncloud_etcd_based/docs/source/vm-images.rst new file mode 100644 index 0000000..4b2758a --- /dev/null +++ b/archive/uncloud_etcd_based/docs/source/vm-images.rst @@ -0,0 +1,66 @@ +VM images +================================== + +Overview +--------- + +ucloud tries to be least invasise towards VMs and only require +strictly necessary changes for running in a virtualised +environment. This includes configurations for: + +* Configuring the network +* Managing access via ssh keys +* Resizing the attached disk(s) + +Upstream images +--------------- + +The 'official' uncloud images are defined in the `uncloud/images +`_ repository. + +How to make you own Uncloud images +---------------------------------- + +.. note:: + It is fairly easy to create your own images for uncloud, as the common + operations (which are detailed below) can be automatically handled by the + `uncloud/uncloud-init `_ tool. + +Network configuration +~~~~~~~~~~~~~~~~~~~~~ +All VMs in ucloud are required to support IPv6. The primary network +configuration is always done using SLAAC. A VM thus needs only to be +configured to + +* accept router advertisements on all network interfaces +* use the router advertisements to configure the network interfaces +* accept the DNS entries from the router advertisements + + +Configuring SSH keys +~~~~~~~~~~~~~~~~~~~~ + +To be able to access the VM, ucloud support provisioning SSH keys. + +To accept ssh keys in your VM, request the URL +*http://metadata/ssh_keys*. Add the content to the appropriate user's +**authorized_keys** file. Below you find sample code to accomplish +this task: + +.. code-block:: sh + + tmp=$(mktemp) + curl -s http://metadata/ssk_keys > "$tmp" + touch ~/.ssh/authorized_keys # ensure it exists + cat ~/.ssh/authorized_keys >> "$tmp" + sort "$tmp" | uniq > ~/.ssh/authorized_keys + + +Disk resize +~~~~~~~~~~~ +In virtualised environments, the disk sizes might grow. The operating +system should detect disks that are bigger than the existing partition +table and resize accordingly. This task is os specific. + +ucloud does not support shrinking disks due to the complexity and +intra OS dependencies. diff --git a/archive/uncloud_etcd_based/scripts/uncloud b/archive/uncloud_etcd_based/scripts/uncloud new file mode 100755 index 0000000..9517b01 --- /dev/null +++ b/archive/uncloud_etcd_based/scripts/uncloud @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +import logging +import sys +import importlib +import argparse +import os + +from etcd3.exceptions import ConnectionFailedError + +from uncloud.common import settings +from uncloud import UncloudException +from uncloud.common.cli import resolve_otp_credentials + +# Components that use etcd +ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', + 'imagescanner', 'metadata', 'configure', 'hack'] + +ALL_COMPONENTS = ETCD_COMPONENTS.copy() +ALL_COMPONENTS.append('oneshot') +#ALL_COMPONENTS.append('cli') + + +if __name__ == '__main__': + arg_parser = argparse.ArgumentParser() + subparsers = arg_parser.add_subparsers(dest='command') + + parent_parser = argparse.ArgumentParser(add_help=False) + parent_parser.add_argument('--debug', '-d', action='store_true', default=False, + help='More verbose logging') + parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory', + default=os.path.expanduser('~/uncloud')) + + etcd_parser = argparse.ArgumentParser(add_help=False) + etcd_parser.add_argument('--etcd-host') + etcd_parser.add_argument('--etcd-port') + etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') + etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') + etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') + + for component in ALL_COMPONENTS: + mod = importlib.import_module('uncloud.{}.main'.format(component)) + parser = getattr(mod, 'arg_parser') + + if component in ETCD_COMPONENTS: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) + else: + subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) + + arguments = vars(arg_parser.parse_args()) + etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value] + etcd_arguments = { + 'etcd': { + key.replace('etcd_', ''): arguments[key] + for key in etcd_arguments + } + } + if not arguments['command']: + arg_parser.print_help() + else: + # Initializing Settings and resolving otp_credentials + # It is neccessary to resolve_otp_credentials after argument parsing is done because + # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and + # providing the default values for --name, --realm and --seed arguments from the values + # we read from file. But, now we are asking user about where the config file lives. So, + # to providing default value is not possible before parsing arguments. So, we are doing + # it after.. +# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments) +# resolve_otp_credentials(arguments) + + name = arguments.pop('command') + mod = importlib.import_module('uncloud.{}.main'.format(name)) + main = getattr(mod, 'main') + + if arguments['debug']: + logging.basicConfig(level=logging.DEBUG) + else: + logging.basicConfig(level=logging.INFO) + + log = logging.getLogger() + + try: + main(arguments) + except UncloudException as err: + log.error(err) + sys.exit(1) +# except ConnectionFailedError as err: +# log.error('Cannot connect to etcd: {}'.format(err)) + except Exception as err: + log.exception(err) diff --git a/setup.py b/archive/uncloud_etcd_based/setup.py similarity index 96% rename from setup.py rename to archive/uncloud_etcd_based/setup.py index 12da6b8..f5e0718 100644 --- a/setup.py +++ b/archive/uncloud_etcd_based/setup.py @@ -40,7 +40,8 @@ setup( "pynetbox", "colorama", "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3", - "marshmallow" + "marshmallow", + "ldap3" ], scripts=["scripts/uncloud"], data_files=[ diff --git a/uncloud/docs/__init__.py b/archive/uncloud_etcd_based/test/__init__.py similarity index 100% rename from uncloud/docs/__init__.py rename to archive/uncloud_etcd_based/test/__init__.py diff --git a/archive/uncloud_etcd_based/test/test_mac_local.py b/archive/uncloud_etcd_based/test/test_mac_local.py new file mode 100644 index 0000000..3a4ac3a --- /dev/null +++ b/archive/uncloud_etcd_based/test/test_mac_local.py @@ -0,0 +1,37 @@ +import unittest +from unittest.mock import Mock + +from uncloud.hack.mac import MAC +from uncloud import UncloudException + +class TestMacLocal(unittest.TestCase): + def setUp(self): + self.config = Mock() + self.config.arguments = {"no_db":True} + self.mac = MAC(self.config) + self.mac.create() + + def testMacInt(self): + self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong first MAC index") + + def testMacRepr(self): + self.assertEqual(self.mac.__repr__(), '420000000001', "wrong first MAC index") + + def testMacStr(self): + self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong first MAC index") + + def testValidationRaise(self): + with self.assertRaises(UncloudException): + self.mac.validate_mac("2") + + def testValidation(self): + self.assertTrue(self.mac.validate_mac("42:00:00:00:00:01"), "Validation of a given MAC not working properly") + + def testNextMAC(self): + self.mac.create() + self.assertEqual(self.mac.__repr__(), '420000000001', "wrong second MAC index") + self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong second MAC index") + self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong second MAC index") + +if __name__ == '__main__': + unittest.main() diff --git a/archive/uncloud_etcd_based/uncloud/__init__.py b/archive/uncloud_etcd_based/uncloud/__init__.py new file mode 100644 index 0000000..2920f47 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/__init__.py @@ -0,0 +1,2 @@ +class UncloudException(Exception): + pass diff --git a/uncloud/api/README.md b/archive/uncloud_etcd_based/uncloud/api/README.md similarity index 100% rename from uncloud/api/README.md rename to archive/uncloud_etcd_based/uncloud/api/README.md diff --git a/uncloud/api/__init__.py b/archive/uncloud_etcd_based/uncloud/api/__init__.py similarity index 100% rename from uncloud/api/__init__.py rename to archive/uncloud_etcd_based/uncloud/api/__init__.py diff --git a/uncloud/api/common_fields.py b/archive/uncloud_etcd_based/uncloud/api/common_fields.py similarity index 85% rename from uncloud/api/common_fields.py rename to archive/uncloud_etcd_based/uncloud/api/common_fields.py index d1fcb64..ba9fb37 100755 --- a/uncloud/api/common_fields.py +++ b/archive/uncloud_etcd_based/uncloud/api/common_fields.py @@ -1,7 +1,6 @@ import os from uncloud.common.shared import shared -from uncloud.common.settings import settings class Optional: @@ -54,9 +53,7 @@ class VmUUIDField(Field): def vm_uuid_validation(self): r = shared.etcd_client.get( - os.path.join(settings["etcd"]["vm_prefix"], self.uuid) + os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid) ) if not r: - self.add_error( - "VM with uuid {} does not exists".format(self.uuid) - ) + self.add_error("VM with uuid {} does not exists".format(self.uuid)) diff --git a/uncloud/api/create_image_store.py b/archive/uncloud_etcd_based/uncloud/api/create_image_store.py similarity index 73% rename from uncloud/api/create_image_store.py rename to archive/uncloud_etcd_based/uncloud/api/create_image_store.py index 1040e97..90e0f92 100755 --- a/uncloud/api/create_image_store.py +++ b/archive/uncloud_etcd_based/uncloud/api/create_image_store.py @@ -4,7 +4,6 @@ import os from uuid import uuid4 from uncloud.common.shared import shared -from uncloud.common.settings import settings data = { 'is_public': True, @@ -15,6 +14,6 @@ data = { } shared.etcd_client.put( - os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex), + os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex), json.dumps(data), ) diff --git a/uncloud/api/helper.py b/archive/uncloud_etcd_based/uncloud/api/helper.py similarity index 88% rename from uncloud/api/helper.py rename to archive/uncloud_etcd_based/uncloud/api/helper.py index 0805280..8ceb3a6 100755 --- a/uncloud/api/helper.py +++ b/archive/uncloud_etcd_based/uncloud/api/helper.py @@ -7,7 +7,6 @@ import requests from pyotp import TOTP from uncloud.common.shared import shared -from uncloud.common.settings import settings logger = logging.getLogger(__name__) @@ -15,9 +14,9 @@ logger = logging.getLogger(__name__) def check_otp(name, realm, token): try: data = { - "auth_name": settings["otp"]["auth_name"], - "auth_token": TOTP(settings["otp"]["auth_seed"]).now(), - "auth_realm": settings["otp"]["auth_realm"], + "auth_name": shared.settings["otp"]["auth_name"], + "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(), + "auth_realm": shared.settings["otp"]["auth_realm"], "name": name, "realm": realm, "token": token, @@ -25,13 +24,13 @@ def check_otp(name, realm, token): except binascii.Error as err: logger.error( "Cannot compute OTP for seed: {}".format( - settings["otp"]["auth_seed"] + shared.settings["otp"]["auth_seed"] ) ) return 400 response = requests.post( - settings["otp"]["verification_controller_url"], json=data + shared.settings["otp"]["verification_controller_url"], json=data ) return response.status_code @@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client): ) images = etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) # Try to find image with name == image_name and store_name == store_name @@ -111,9 +110,7 @@ def random_bytes(num=6): return [random.randrange(256) for _ in range(num)] -def generate_mac( - uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x" -): +def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"): mac = random_bytes() if oui: if type(oui) == str: @@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix): lower_part = ipaddress.IPv6Address(":".join(ipv6_parts)) prefix = ipaddress.IPv6Address(prefix) return str(prefix + int(lower_part)) + diff --git a/uncloud/api/main.py b/archive/uncloud_etcd_based/uncloud/api/main.py similarity index 89% rename from uncloud/api/main.py rename to archive/uncloud_etcd_based/uncloud/api/main.py index 34e1dd1..73e8e21 100644 --- a/uncloud/api/main.py +++ b/archive/uncloud_etcd_based/uncloud/api/main.py @@ -15,9 +15,8 @@ from uncloud.common.shared import shared from uncloud.common import counters from uncloud.common.vm import VMStatus from uncloud.common.request import RequestEntry, RequestType -from uncloud.common.settings import settings -from . import schemas -from .helper import generate_mac, mac2ipv6 +from uncloud.api import schemas +from uncloud.api.helper import generate_mac, mac2ipv6 from uncloud import UncloudException logger = logging.getLogger(__name__) @@ -50,7 +49,7 @@ class CreateVM(Resource): validator = schemas.CreateVMSchema(data) if validator.is_valid(): vm_uuid = uuid4().hex - vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid) + vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) specs = { 'cpu': validator.specs['cpu'], 'ram': validator.specs['ram'], @@ -60,7 +59,7 @@ class CreateVM(Resource): macs = [generate_mac() for _ in range(len(data['network']))] tap_ids = [ counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['tap_counter'] + shared.etcd_client, shared.settings['etcd']['tap_counter'] ) for _ in range(len(data['network'])) ] @@ -84,7 +83,7 @@ class CreateVM(Resource): r = RequestEntry.from_scratch( type=RequestType.ScheduleVM, uuid=vm_uuid, - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) @@ -99,7 +98,7 @@ class VmStatus(Resource): validator = schemas.VMStatusSchema(data) if validator.is_valid(): vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) vm_value = vm.value.copy() vm_value['ip'] = [] @@ -107,7 +106,7 @@ class VmStatus(Resource): network_name, mac, tap = network_mac_and_tap network = shared.etcd_client.get( join_path( - settings['etcd']['network_prefix'], + shared.settings['etcd']['network_prefix'], data['name'], network_name, ), @@ -130,7 +129,7 @@ class CreateImage(Resource): validator = schemas.CreateImageSchema(data) if validator.is_valid(): file_entry = shared.etcd_client.get( - join_path(settings['etcd']['file_prefix'], data['uuid']) + join_path(shared.settings['etcd']['file_prefix'], data['uuid']) ) file_entry_value = json.loads(file_entry.value) @@ -144,7 +143,7 @@ class CreateImage(Resource): } shared.etcd_client.put( join_path( - settings['etcd']['image_prefix'], data['uuid'] + shared.settings['etcd']['image_prefix'], data['uuid'] ), json.dumps(image_entry_json), ) @@ -157,7 +156,7 @@ class ListPublicImages(Resource): @staticmethod def get(): images = shared.etcd_client.get_prefix( - settings['etcd']['image_prefix'], value_in_json=True + shared.settings['etcd']['image_prefix'], value_in_json=True ) r = {'images': []} for image in images: @@ -178,7 +177,7 @@ class VMAction(Resource): if validator.is_valid(): vm_entry = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], data['uuid']) + join_path(shared.settings['etcd']['vm_prefix'], data['uuid']) ) action = data['action'] @@ -208,7 +207,7 @@ class VMAction(Resource): type='{}VM'.format(action.title()), uuid=data['uuid'], hostname=vm_entry.hostname, - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) return ( @@ -231,10 +230,10 @@ class VMMigration(Resource): type=RequestType.InitVMMigration, uuid=vm.uuid, hostname=join_path( - settings['etcd']['host_prefix'], + shared.settings['etcd']['host_prefix'], validator.destination.value, ), - request_prefix=settings['etcd']['request_prefix'], + request_prefix=shared.settings['etcd']['request_prefix'], ) shared.request_pool.put(r) @@ -254,7 +253,7 @@ class ListUserVM(Resource): if validator.is_valid(): vms = shared.etcd_client.get_prefix( - settings['etcd']['vm_prefix'], value_in_json=True + shared.settings['etcd']['vm_prefix'], value_in_json=True ) return_vms = [] user_vms = filter( @@ -287,7 +286,7 @@ class ListUserFiles(Resource): if validator.is_valid(): files = shared.etcd_client.get_prefix( - settings['etcd']['file_prefix'], value_in_json=True + shared.settings['etcd']['file_prefix'], value_in_json=True ) return_files = [] user_files = [f for f in files if f.value['owner'] == data['name']] @@ -312,7 +311,7 @@ class CreateHost(Resource): validator = schemas.CreateHostSchema(data) if validator.is_valid(): host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': data['specs'], @@ -354,7 +353,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/ etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -372,7 +371,7 @@ class GetSSHKeys(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -405,7 +404,7 @@ class AddSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -439,7 +438,7 @@ class RemoveSSHKey(Resource): # {user_prefix}/{realm}/{name}/key/{key_name} etcd_key = join_path( - settings['etcd']['user_prefix'], + shared.settings['etcd']['user_prefix'], data['realm'], data['name'], 'key', @@ -471,23 +470,23 @@ class CreateNetwork(Resource): network_entry = { 'id': counters.increment_etcd_counter( - shared.etcd_client, settings['etcd']['vxlan_counter'] + shared.etcd_client, shared.settings['etcd']['vxlan_counter'] ), 'type': data['type'], } if validator.user.value: try: nb = pynetbox.api( - url=settings['netbox']['url'], - token=settings['netbox']['token'], + url=shared.settings['netbox']['url'], + token=shared.settings['netbox']['token'], ) nb_prefix = nb.ipam.prefixes.get( - prefix=settings['network']['prefix'] + prefix=shared.settings['network']['prefix'] ) prefix = nb_prefix.available_prefixes.create( data={ 'prefix_length': int( - settings['network']['prefix_length'] + shared.settings['network']['prefix_length'] ), 'description': '{}\'s network "{}"'.format( data['name'], data['network_name'] @@ -506,7 +505,7 @@ class CreateNetwork(Resource): network_entry['ipv6'] = 'fd00::/64' network_key = join_path( - settings['etcd']['network_prefix'], + shared.settings['etcd']['network_prefix'], data['name'], data['network_name'], ) @@ -526,7 +525,7 @@ class ListUserNetwork(Resource): if validator.is_valid(): prefix = join_path( - settings['etcd']['network_prefix'], data['name'] + shared.settings['etcd']['network_prefix'], data['name'] ) networks = shared.etcd_client.get_prefix( prefix, value_in_json=True @@ -570,7 +569,7 @@ def main(arguments): try: image_stores = list( shared.etcd_client.get_prefix( - settings['etcd']['image_store_prefix'], value_in_json=True + shared.settings['etcd']['image_store_prefix'], value_in_json=True ) ) except KeyError: @@ -590,7 +589,7 @@ def main(arguments): # shared.etcd_client.put( # join_path( - # settings['etcd']['image_store_prefix'], uuid4().hex + # shared.settings['etcd']['image_store_prefix'], uuid4().hex # ), # json.dumps(data), # ) diff --git a/uncloud/api/schemas.py b/archive/uncloud_etcd_based/uncloud/api/schemas.py similarity index 96% rename from uncloud/api/schemas.py rename to archive/uncloud_etcd_based/uncloud/api/schemas.py index e4de9a8..87f20c9 100755 --- a/uncloud/api/schemas.py +++ b/archive/uncloud_etcd_based/uncloud/api/schemas.py @@ -22,7 +22,6 @@ import bitmath from uncloud.common.host import HostStatus from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from uncloud.common.settings import settings from . import helper, logger from .common_fields import Field, VmUUIDField from .helper import check_otp, resolve_vm_name @@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema): def file_uuid_validation(self): file_entry = shared.etcd_client.get( os.path.join( - settings["etcd"]["file_prefix"], self.uuid.value + shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value ) ) if file_entry is None: @@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema): def image_store_name_validation(self): image_stores = list( shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"] + shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"] ) ) @@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema): for net in _network: network = shared.etcd_client.get( os.path.join( - settings["etcd"]["network_prefix"], + shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, net, ), @@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema): self.add_error("Can't migrate non-running VM") if vm.hostname == os.path.join( - settings["etcd"]["host_prefix"], self.destination.value + shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value ): self.add_error( "Destination host couldn't be same as Source Host" @@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema): super().__init__(data, fields=fields) def network_name_validation(self): - print(self.name.value, self.network_name.value) - key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) - print(key) + key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value) network = shared.etcd_client.get(key, value_in_json=True) if network: self.add_error( diff --git a/uncloud/docs/source/__init__.py b/archive/uncloud_etcd_based/uncloud/cli/__init__.py similarity index 100% rename from uncloud/docs/source/__init__.py rename to archive/uncloud_etcd_based/uncloud/cli/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/cli/helper.py b/archive/uncloud_etcd_based/uncloud/cli/helper.py new file mode 100644 index 0000000..51a4355 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/cli/helper.py @@ -0,0 +1,46 @@ +import requests +import json +import argparse +import binascii + +from pyotp import TOTP +from os.path import join as join_path +from uncloud.common.shared import shared + + +def get_otp_parser(): + otp_parser = argparse.ArgumentParser('otp') + otp_parser.add_argument('--name') + otp_parser.add_argument('--realm') + otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED') + + return otp_parser + + +def load_dump_pretty(content): + if isinstance(content, bytes): + content = content.decode('utf-8') + parsed = json.loads(content) + return json.dumps(parsed, indent=4, sort_keys=True) + + +def make_request(*args, data=None, request_method=requests.post): + try: + r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data) + except requests.exceptions.RequestException: + print('Error occurred while connecting to API server.') + else: + try: + print(load_dump_pretty(r.content)) + except Exception: + print('Error occurred while getting output from api server.') + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except binascii.Error: + raise argparse.ArgumentTypeError('Invalid seed') + else: + return token diff --git a/uncloud/cli/host.py b/archive/uncloud_etcd_based/uncloud/cli/host.py similarity index 100% rename from uncloud/cli/host.py rename to archive/uncloud_etcd_based/uncloud/cli/host.py diff --git a/uncloud/cli/image.py b/archive/uncloud_etcd_based/uncloud/cli/image.py similarity index 100% rename from uncloud/cli/image.py rename to archive/uncloud_etcd_based/uncloud/cli/image.py diff --git a/uncloud/cli/main.py b/archive/uncloud_etcd_based/uncloud/cli/main.py similarity index 100% rename from uncloud/cli/main.py rename to archive/uncloud_etcd_based/uncloud/cli/main.py diff --git a/uncloud/cli/network.py b/archive/uncloud_etcd_based/uncloud/cli/network.py similarity index 100% rename from uncloud/cli/network.py rename to archive/uncloud_etcd_based/uncloud/cli/network.py diff --git a/uncloud/cli/user.py b/archive/uncloud_etcd_based/uncloud/cli/user.py similarity index 100% rename from uncloud/cli/user.py rename to archive/uncloud_etcd_based/uncloud/cli/user.py diff --git a/uncloud/cli/vm.py b/archive/uncloud_etcd_based/uncloud/cli/vm.py similarity index 100% rename from uncloud/cli/vm.py rename to archive/uncloud_etcd_based/uncloud/cli/vm.py diff --git a/uncloud/network/__init__.py b/archive/uncloud_etcd_based/uncloud/client/__init__.py similarity index 100% rename from uncloud/network/__init__.py rename to archive/uncloud_etcd_based/uncloud/client/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/client/main.py b/archive/uncloud_etcd_based/uncloud/client/main.py new file mode 100644 index 0000000..062308c --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/client/main.py @@ -0,0 +1,23 @@ +import argparse +import etcd3 +from uncloud.common.etcd_wrapper import Etcd3Wrapper + +arg_parser = argparse.ArgumentParser('client', add_help=False) +arg_parser.add_argument('--dump-etcd-contents-prefix', help="Dump contents below the given prefix") + +def dump_etcd_contents(prefix): + etcd = Etcd3Wrapper() + for k,v in etcd.get_prefix_raw(prefix): + k = k.decode('utf-8') + v = v.decode('utf-8') + print("{} = {}".format(k,v)) +# print("{} = {}".format(k,v)) + +# for k,v in etcd.get_prefix(prefix): +# + print("done") + + +def main(arguments): + if 'dump_etcd_contents_prefix' in arguments: + dump_etcd_contents(prefix=arguments['dump_etcd_contents_prefix']) diff --git a/uncloud/common/__init__.py b/archive/uncloud_etcd_based/uncloud/common/__init__.py similarity index 100% rename from uncloud/common/__init__.py rename to archive/uncloud_etcd_based/uncloud/common/__init__.py diff --git a/uncloud/common/classes.py b/archive/uncloud_etcd_based/uncloud/common/classes.py similarity index 100% rename from uncloud/common/classes.py rename to archive/uncloud_etcd_based/uncloud/common/classes.py diff --git a/archive/uncloud_etcd_based/uncloud/common/cli.py b/archive/uncloud_etcd_based/uncloud/common/cli.py new file mode 100644 index 0000000..3d3c248 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/cli.py @@ -0,0 +1,26 @@ +from uncloud.common.shared import shared +from pyotp import TOTP + + +def get_token(seed): + if seed is not None: + try: + token = TOTP(seed).now() + except Exception: + raise Exception('Invalid seed') + else: + return token + + +def resolve_otp_credentials(kwargs): + d = { + 'name': shared.settings['client']['name'], + 'realm': shared.settings['client']['realm'], + 'token': get_token(shared.settings['client']['seed']) + } + + for k, v in d.items(): + if k in kwargs and kwargs[k] is None: + kwargs.update({k: v}) + + return d diff --git a/uncloud/common/counters.py b/archive/uncloud_etcd_based/uncloud/common/counters.py similarity index 100% rename from uncloud/common/counters.py rename to archive/uncloud_etcd_based/uncloud/common/counters.py diff --git a/uncloud/common/etcd_wrapper.py b/archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py similarity index 100% rename from uncloud/common/etcd_wrapper.py rename to archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py diff --git a/uncloud/common/host.py b/archive/uncloud_etcd_based/uncloud/common/host.py similarity index 100% rename from uncloud/common/host.py rename to archive/uncloud_etcd_based/uncloud/common/host.py diff --git a/uncloud/common/network.py b/archive/uncloud_etcd_based/uncloud/common/network.py similarity index 100% rename from uncloud/common/network.py rename to archive/uncloud_etcd_based/uncloud/common/network.py diff --git a/uncloud/common/parser.py b/archive/uncloud_etcd_based/uncloud/common/parser.py similarity index 100% rename from uncloud/common/parser.py rename to archive/uncloud_etcd_based/uncloud/common/parser.py diff --git a/uncloud/common/request.py b/archive/uncloud_etcd_based/uncloud/common/request.py similarity index 100% rename from uncloud/common/request.py rename to archive/uncloud_etcd_based/uncloud/common/request.py diff --git a/uncloud/common/schemas.py b/archive/uncloud_etcd_based/uncloud/common/schemas.py similarity index 100% rename from uncloud/common/schemas.py rename to archive/uncloud_etcd_based/uncloud/common/schemas.py diff --git a/uncloud/common/settings.py b/archive/uncloud_etcd_based/uncloud/common/settings.py similarity index 92% rename from uncloud/common/settings.py rename to archive/uncloud_etcd_based/uncloud/common/settings.py index 0d524a7..8503f42 100644 --- a/uncloud/common/settings.py +++ b/archive/uncloud_etcd_based/uncloud/common/settings.py @@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper from os.path import join as join_path logger = logging.getLogger(__name__) +settings = None class CustomConfigParser(configparser.RawConfigParser): @@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser): class Settings(object): - def __init__(self): + def __init__(self, conf_dir, seed_value=None): conf_name = 'uncloud.conf' - conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/')) self.config_file = join_path(conf_dir, conf_name) # this is used to cache config from etcd for 1 minutes. Without this we @@ -38,15 +38,19 @@ class Settings(object): self.config_parser.add_section('etcd') self.config_parser.set('etcd', 'base_prefix', '/') - try: + if os.access(self.config_file, os.R_OK): self.config_parser.read(self.config_file) - except Exception as err: - logger.error('%s', err) - + else: + raise FileNotFoundError('Config file %s not found!', self.config_file) self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/') self.read_internal_values() + if seed_value is None: + seed_value = dict() + + self.config_parser.read_dict(seed_value) + def get_etcd_client(self): args = tuple() try: @@ -128,4 +132,5 @@ class Settings(object): return self.config_parser[key] -settings = Settings() +def get_settings(): + return settings diff --git a/archive/uncloud_etcd_based/uncloud/common/shared.py b/archive/uncloud_etcd_based/uncloud/common/shared.py new file mode 100644 index 0000000..aea7cbc --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/common/shared.py @@ -0,0 +1,34 @@ +from uncloud.common.settings import get_settings +from uncloud.common.vm import VmPool +from uncloud.common.host import HostPool +from uncloud.common.request import RequestPool +import uncloud.common.storage_handlers as storage_handlers + + +class Shared: + @property + def settings(self): + return get_settings() + + @property + def etcd_client(self): + return self.settings.get_etcd_client() + + @property + def host_pool(self): + return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"]) + + @property + def vm_pool(self): + return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"]) + + @property + def request_pool(self): + return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"]) + + @property + def storage_handler(self): + return storage_handlers.get_storage_handler() + + +shared = Shared() diff --git a/uncloud/common/storage_handlers.py b/archive/uncloud_etcd_based/uncloud/common/storage_handlers.py similarity index 92% rename from uncloud/common/storage_handlers.py rename to archive/uncloud_etcd_based/uncloud/common/storage_handlers.py index 6f9b29e..58c2dc2 100644 --- a/uncloud/common/storage_handlers.py +++ b/archive/uncloud_etcd_based/uncloud/common/storage_handlers.py @@ -6,8 +6,7 @@ import stat from abc import ABC from . import logger from os.path import join as join_path - -from uncloud.common.settings import settings as config +import uncloud.common.shared as shared class ImageStorageHandler(ABC): @@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler): def get_storage_handler(): - __storage_backend = config["storage"]["storage_backend"] + __storage_backend = shared.shared.settings["storage"]["storage_backend"] if __storage_backend == "filesystem": return FileSystemBasedImageStorageHandler( - vm_base=config["storage"]["vm_dir"], - image_base=config["storage"]["image_dir"], + vm_base=shared.shared.settings["storage"]["vm_dir"], + image_base=shared.shared.settings["storage"]["image_dir"], ) elif __storage_backend == "ceph": return CEPHBasedImageStorageHandler( - vm_base=config["storage"]["ceph_vm_pool"], - image_base=config["storage"]["ceph_image_pool"], + vm_base=shared.shared.settings["storage"]["ceph_vm_pool"], + image_base=shared.shared.settings["storage"]["ceph_image_pool"], ) else: - raise Exception("Unknown Image Storage Handler") + raise Exception("Unknown Image Storage Handler") \ No newline at end of file diff --git a/uncloud/common/vm.py b/archive/uncloud_etcd_based/uncloud/common/vm.py similarity index 100% rename from uncloud/common/vm.py rename to archive/uncloud_etcd_based/uncloud/common/vm.py diff --git a/uncloud/scheduler/tests/__init__.py b/archive/uncloud_etcd_based/uncloud/configure/__init__.py similarity index 100% rename from uncloud/scheduler/tests/__init__.py rename to archive/uncloud_etcd_based/uncloud/configure/__init__.py diff --git a/uncloud/configure/main.py b/archive/uncloud_etcd_based/uncloud/configure/main.py similarity index 86% rename from uncloud/configure/main.py rename to archive/uncloud_etcd_based/uncloud/configure/main.py index e190460..87f5752 100644 --- a/uncloud/configure/main.py +++ b/archive/uncloud_etcd_based/uncloud/configure/main.py @@ -1,7 +1,6 @@ import os import argparse -from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('configure', add_help=False) @@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True) def update_config(section, kwargs): - uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True) + uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True) if not uncloud_config: uncloud_config = {} else: uncloud_config = uncloud_config.value uncloud_config[section] = kwargs - shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True) + shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True) -def main(**kwargs): - subcommand = kwargs.pop('subcommand') +def main(arguments): + subcommand = arguments['subcommand'] if not subcommand: arg_parser.print_help() else: - update_config(subcommand, kwargs) + update_config(subcommand, arguments) diff --git a/uncloud/filescanner/__init__.py b/archive/uncloud_etcd_based/uncloud/filescanner/__init__.py similarity index 100% rename from uncloud/filescanner/__init__.py rename to archive/uncloud_etcd_based/uncloud/filescanner/__init__.py diff --git a/uncloud/filescanner/main.py b/archive/uncloud_etcd_based/uncloud/filescanner/main.py similarity index 89% rename from uncloud/filescanner/main.py rename to archive/uncloud_etcd_based/uncloud/filescanner/main.py index c5660dd..046f915 100755 --- a/uncloud/filescanner/main.py +++ b/archive/uncloud_etcd_based/uncloud/filescanner/main.py @@ -9,7 +9,6 @@ import bitmath from uuid import uuid4 from . import logger -from uncloud.common.settings import settings from uncloud.common.shared import shared arg_parser = argparse.ArgumentParser('filescanner', add_help=False) @@ -53,7 +52,7 @@ def track_file(file, base_dir, host): file_path = file_path.relative_to(owner) creation_date = time.ctime(os.stat(file_str).st_ctime) - entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4())) + entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4())) entry_value = { 'filename': str(file_path), 'owner': owner, @@ -70,7 +69,7 @@ def track_file(file, base_dir, host): def main(arguments): hostname = arguments['hostname'] - base_dir = settings['storage']['file_dir'] + base_dir = shared.settings['storage']['file_dir'] # Recursively Get All Files and Folder below BASE_DIR files = glob.glob('{}/**'.format(base_dir), recursive=True) files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()] @@ -78,7 +77,7 @@ def main(arguments): # Files that are already tracked tracked_files = [ pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename'])) - for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True) + for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True) if f.value['host'] == hostname ] untracked_files = set(files) - set(tracked_files) diff --git a/uncloud/hack/README.org b/archive/uncloud_etcd_based/uncloud/hack/README.org similarity index 100% rename from uncloud/hack/README.org rename to archive/uncloud_etcd_based/uncloud/hack/README.org diff --git a/archive/uncloud_etcd_based/uncloud/hack/__init__.py b/archive/uncloud_etcd_based/uncloud/hack/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/__init__.py @@ -0,0 +1 @@ + diff --git a/uncloud/hack/conf.d/ucloud-host b/archive/uncloud_etcd_based/uncloud/hack/conf.d/ucloud-host similarity index 100% rename from uncloud/hack/conf.d/ucloud-host rename to archive/uncloud_etcd_based/uncloud/hack/conf.d/ucloud-host diff --git a/archive/uncloud_etcd_based/uncloud/hack/config.py b/archive/uncloud_etcd_based/uncloud/hack/config.py new file mode 100644 index 0000000..7e2655d --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/config.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +class Config(object): + def __init__(self, arguments): + """ read arguments dicts as a base """ + + self.arguments = arguments + + # Split them so *etcd_args can be used and we can + # iterate over etcd_hosts + self.etcd_hosts = [ arguments['etcd_host'] ] + self.etcd_args = { + 'ca_cert': arguments['etcd_ca_cert'], + 'cert_cert': arguments['etcd_cert_cert'], + 'cert_key': arguments['etcd_cert_key'], +# 'user': None, +# 'password': None + } + self.etcd_prefix = '/nicohack/' diff --git a/archive/uncloud_etcd_based/uncloud/hack/db.py b/archive/uncloud_etcd_based/uncloud/hack/db.py new file mode 100644 index 0000000..3d5582e --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/db.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . +# +# + +import etcd3 +import json +import logging +import datetime +import re + +from functools import wraps +from uncloud import UncloudException + +log = logging.getLogger(__name__) + +def db_logentry(message): + timestamp = datetime.datetime.now() + return { + "timestamp": str(timestamp), + "message": message + } + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError as e: + raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e)) + except etcd3.exceptions.ConnectionTimeoutError as e: + raise UncloudException('etcd connection timeout. {}'.format(e)) + + return wrapper + + +class DB(object): + def __init__(self, config, prefix="/"): + self.config = config + + # Root for everything + self.base_prefix= '/nicohack' + + # Can be set from outside + self.prefix = prefix + + try: + self.connect() + except FileNotFoundError as e: + raise UncloudException("Is the path to the etcd certs correct? {}".format(e)) + + @readable_errors + def connect(self): + self._db_clients = [] + for endpoint in self.config.etcd_hosts: + client = etcd3.client(host=endpoint, **self.config.etcd_args) + self._db_clients.append(client) + + def realkey(self, key): + return "{}{}/{}".format(self.base_prefix, + self.prefix, + key) + + @readable_errors + def get(self, key, as_json=False, **kwargs): + value, _ = self._db_clients[0].get(self.realkey(key), **kwargs) + + if as_json: + value = json.loads(value) + + return value + + @readable_errors + def get_prefix(self, key, as_json=False, **kwargs): + for value, meta in self._db_clients[0].get_prefix(self.realkey(key), **kwargs): + k = meta.key.decode("utf-8") + value = value.decode("utf-8") + if as_json: + value = json.loads(value) + + yield (k, value) + + + @readable_errors + def set(self, key, value, as_json=False, **kwargs): + if as_json: + value = json.dumps(value) + + log.debug("Setting {} = {}".format(self.realkey(key), value)) + # FIXME: iterate over clients in case of failure ? + return self._db_clients[0].put(self.realkey(key), value, **kwargs) + + + @readable_errors + def list_and_filter(self, key, filter_key=None, filter_regexp=None): + for k,v in self.get_prefix(key, as_json=True): + + if filter_key and filter_regexp: + if filter_key in v: + if re.match(filter_regexp, v[filter_key]): + yield v + else: + yield v + + + @readable_errors + def increment(self, key, **kwargs): + print(self.realkey(key)) + + + print("prelock") + lock = self._db_clients[0].lock('/nicohack/foo') + print("prelockacq") + lock.acquire() + print("prelockrelease") + lock.release() + + with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock: + print("in lock") + pass + +# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs)) +# self.set(self.realkey(key), str(value + 1), **kwargs) + + +if __name__ == '__main__': + endpoints = [ "https://etcd1.ungleich.ch:2379", + "https://etcd2.ungleich.ch:2379", + "https://etcd3.ungleich.ch:2379" ] + + db = DB(url=endpoints) diff --git a/uncloud/hack/hackcloud/.gitignore b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore similarity index 100% rename from uncloud/hack/hackcloud/.gitignore rename to archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py @@ -0,0 +1 @@ + diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh new file mode 100644 index 0000000..ab102a5 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \ + --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \ + --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \ + --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@" diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh new file mode 100755 index 0000000..5753099 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo $@ diff --git a/uncloud/hack/hackcloud/ifup.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh similarity index 100% rename from uncloud/hack/hackcloud/ifup.sh rename to archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last new file mode 100644 index 0000000..8c5f254 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last @@ -0,0 +1 @@ +000000000252 diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix new file mode 100644 index 0000000..5084a2f --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix @@ -0,0 +1 @@ +02:00 diff --git a/uncloud/hack/hackcloud/net.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/net.sh similarity index 100% rename from uncloud/hack/hackcloud/net.sh rename to archive/uncloud_etcd_based/uncloud/hack/hackcloud/net.sh diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules new file mode 100644 index 0000000..636c63d --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules @@ -0,0 +1,31 @@ +flush ruleset + +table bridge filter { + chain prerouting { + type filter hook prerouting priority 0; + policy accept; + + ibrname br100 jump br100 + } + + chain br100 { + # Allow all incoming traffic from outside + iifname vxlan100 accept + + # Default blocks: router advertisements, dhcpv6, dhcpv4 + icmpv6 type nd-router-advert drop + ip6 version 6 udp sport 547 drop + ip version 4 udp sport 67 drop + + jump br100_vmlist + drop + } + chain br100_vmlist { + # VM1 + iifname tap1 ether saddr 02:00:f0:a9:c4:4e ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44e accept + + # VM2 + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:888:0:f0ff:fea9:c44f accept + iifname v343a-0 ether saddr 02:00:f0:a9:c4:4f ip6 saddr 2a0a:e5c1:111:1234::/64 accept + } +} diff --git a/uncloud/hack/hackcloud/nftrules b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules-v2 similarity index 100% rename from uncloud/hack/hackcloud/nftrules rename to archive/uncloud_etcd_based/uncloud/hack/hackcloud/nftrules-v2 diff --git a/uncloud/hack/hackcloud/radvd.conf b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.conf similarity index 100% rename from uncloud/hack/hackcloud/radvd.conf rename to archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.conf diff --git a/uncloud/hack/hackcloud/radvd.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.sh similarity index 100% rename from uncloud/hack/hackcloud/radvd.sh rename to archive/uncloud_etcd_based/uncloud/hack/hackcloud/radvd.sh diff --git a/uncloud/hack/hackcloud/vm.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm-2.sh similarity index 100% rename from uncloud/hack/hackcloud/vm.sh rename to archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm-2.sh diff --git a/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh new file mode 100755 index 0000000..dd9be84 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/hackcloud/vm.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# if [ $# -ne 1 ]; then +# echo "$0: owner" +# exit 1 +# fi + +qemu=/usr/bin/qemu-system-x86_64 + +accel=kvm +#accel=tcg + +memory=1024 +cores=2 +uuid=$(uuidgen) +mac=$(./mac-gen.py) +owner=nico + +export bridge=br100 + +set -x +$qemu -name "uncloud-${uuid}" \ + -machine pc,accel=${accel} \ + -m ${memory} \ + -smp ${cores} \ + -uuid ${uuid} \ + -drive file=alpine-virt-3.11.2-x86_64.iso,media=cdrom \ + -netdev tap,id=netmain,script=./ifup.sh,downscript=./ifdown.sh \ + -device virtio-net-pci,netdev=netmain,id=net0,mac=${mac} diff --git a/archive/uncloud_etcd_based/uncloud/hack/host.py b/archive/uncloud_etcd_based/uncloud/hack/host.py new file mode 100644 index 0000000..06ccf98 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/host.py @@ -0,0 +1,75 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +import uuid + +from uncloud.hack.db import DB +from uncloud import UncloudException + +class Host(object): + def __init__(self, config, db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/hosts") + + if db_entry: + self.db_entry = db_entry + + + def list_hosts(self, filter_key=None, filter_regexp=None): + """ Return list of all hosts """ + for entry in self.db.list_and_filter("", filter_key, filter_regexp): + yield self.__class__(self.config, db_entry=entry) + + def cmdline_add_host(self): + """ FIXME: make this a bit smarter and less redundant """ + + for required_arg in [ + 'add_vm_host', + 'max_cores_per_vm', + 'max_cores_total', + 'max_memory_in_gb' ]: + if not required_arg in self.config.arguments: + raise UncloudException("Missing argument: {}".format(required_arg)) + + return self.add_host( + self.config.arguments['add_vm_host'], + self.config.arguments['max_cores_per_vm'], + self.config.arguments['max_cores_total'], + self.config.arguments['max_memory_in_gb']) + + + def add_host(self, + hostname, + max_cores_per_vm, + max_cores_total, + max_memory_in_gb): + + db_entry = {} + db_entry['uuid'] = str(uuid.uuid4()) + db_entry['hostname'] = hostname + db_entry['max_cores_per_vm'] = max_cores_per_vm + db_entry['max_cores_total'] = max_cores_total + db_entry['max_memory_in_gb'] = max_memory_in_gb + db_entry["db_version"] = 1 + db_entry["log"] = [] + + self.db.set(db_entry['uuid'], db_entry, as_json=True) + + return self.__class__(self.config, db_entry) diff --git a/archive/uncloud_etcd_based/uncloud/hack/mac.py b/archive/uncloud_etcd_based/uncloud/hack/mac.py new file mode 100755 index 0000000..e35cd9f --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/mac.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2012 Nico Schottelius (nico-cinv at schottelius.org) +# +# This file is part of cinv. +# +# cinv is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cinv is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cinv. If not, see . +# +# + +import argparse +import logging +import os.path +import os +import re +import json + +from uncloud import UncloudException +from uncloud.hack.db import DB + +log = logging.getLogger(__name__) + + +class MAC(object): + def __init__(self, config): + self.config = config + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(config, prefix="/mac") + + self.prefix = 0x420000000000 + self._number = 0 # Not set by default + + @staticmethod + def validate_mac(mac): + if not re.match(r'([0-9A-F]{2}[-:]){5}[0-9A-F]{2}$', mac, re.I): + raise UncloudException("Not a valid mac address: %s" % mac) + else: + return True + + def last_used_index(self): + if not self.no_db: + value = self.db.get("last_used_index") + if not value: + self.db.set("last_used_index", "0") + value = self.db.get("last_used_index") + + else: + value = "0" + + return int(value) + + def last_used_mac(self): + return self.int_to_mac(self.prefix + self.last_used_index()) + + def to_colon_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ':'.join(format(s, '02x') for s in b) + + def to_str_format(self): + b = self._number.to_bytes(6, byteorder="big") + return ''.join(format(s, '02x') for s in b) + + def create(self): + last_number = self.last_used_index() + + if last_number == int('0xffffffff', 16): + raise UncloudException("Exhausted all possible mac addresses - try to free some") + + next_number = last_number + 1 + self._number = self.prefix + next_number + + #next_number_string = "{:012x}".format(next_number) + #next_mac = self.int_to_mac(next_mac_number) + # db_entry = {} + # db_entry['vm_uuid'] = vmuuid + # db_entry['index'] = next_number + # db_entry['mac_address'] = next_mac + + # should be one transaction + # self.db.increment("last_used_index") + # self.db.set("used/{}".format(next_mac), + # db_entry, as_json=True) + + def __int__(self): + return self._number + + def __repr__(self): + return self.to_str_format() + + def __str__(self): + return self.to_colon_format() diff --git a/archive/uncloud_etcd_based/uncloud/hack/main.py b/archive/uncloud_etcd_based/uncloud/hack/main.py new file mode 100644 index 0000000..0ddd8fb --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/main.py @@ -0,0 +1,186 @@ +import argparse +import logging +import re + +import ldap3 + + +from uncloud.hack.vm import VM +from uncloud.hack.host import Host +from uncloud.hack.config import Config +from uncloud.hack.mac import MAC +from uncloud.hack.net import VXLANBridge, DNSRA + +from uncloud import UncloudException +from uncloud.hack.product import ProductOrder + +arg_parser = argparse.ArgumentParser('hack', add_help=False) + #description="Commands that are unfinished - use at own risk") +arg_parser.add_argument('--last-used-mac', action='store_true') +arg_parser.add_argument('--get-new-mac', action='store_true') + +arg_parser.add_argument('--init-network', help="Initialise networking", action='store_true') +arg_parser.add_argument('--create-vxlan', help="Initialise networking", action='store_true') +arg_parser.add_argument('--network', help="/64 IPv6 network") +arg_parser.add_argument('--vxlan-uplink-device', help="The VXLAN underlay device, i.e. eth0") +arg_parser.add_argument('--vni', help="VXLAN ID (decimal)", type=int) +arg_parser.add_argument('--run-dns-ra', action='store_true', + help="Provide router advertisements and DNS resolution via dnsmasq") +arg_parser.add_argument('--use-sudo', help="Use sudo for command requiring root!", action='store_true') + +arg_parser.add_argument('--create-vm', action='store_true') +arg_parser.add_argument('--destroy-vm', action='store_true') +arg_parser.add_argument('--get-vm-status', action='store_true') +arg_parser.add_argument('--get-vm-vnc', action='store_true') +arg_parser.add_argument('--list-vms', action='store_true') +arg_parser.add_argument('--memory', help="Size of memory (GB)", type=int, default=2) +arg_parser.add_argument('--cores', help="Amount of CPU cores", type=int, default=1) +arg_parser.add_argument('--image', help="Path (under hackprefix) to OS image") + +arg_parser.add_argument('--image-format', help="Image format: qcow2 or raw", choices=['raw', 'qcow2']) +arg_parser.add_argument('--uuid', help="VM UUID") + +arg_parser.add_argument('--no-db', help="Disable connection to etcd. For local testing only!", action='store_true') +arg_parser.add_argument('--hackprefix', help="hackprefix, if you need it you know it (it's where the iso is located and ifup/down.sh") + +# order based commands => later to be shifted below "order" +arg_parser.add_argument('--order', action='store_true') +arg_parser.add_argument('--list-orders', help="List all orders", action='store_true') +arg_parser.add_argument('--filter-order-key', help="Which key to filter on") +arg_parser.add_argument('--filter-order-regexp', help="Which regexp the value should match") + +arg_parser.add_argument('--process-orders', help="Process all (pending) orders", action='store_true') + +arg_parser.add_argument('--product', choices=["dualstack-vm"]) +arg_parser.add_argument('--os-image-name', help="Name of OS image (successor to --image)") +arg_parser.add_argument('--os-image-size', help="Size of OS image in GB", type=int, default=10) + +arg_parser.add_argument('--username') +arg_parser.add_argument('--password') + +arg_parser.add_argument('--api', help="Run the API") +arg_parser.add_argument('--mode', + choices=["direct", "api", "client"], + default="client", + help="Directly manipulate etcd, spawn the API server or behave as a client") + + +arg_parser.add_argument('--add-vm-host', help="Add a host that can run VMs") +arg_parser.add_argument('--list-vm-hosts', action='store_true') + +arg_parser.add_argument('--max-cores-per-vm') +arg_parser.add_argument('--max-cores-total') +arg_parser.add_argument('--max-memory-in-gb') + + +log = logging.getLogger(__name__) + +def authenticate(username, password, totp_token=None): + server = ldap3.Server("ldaps://ldap1.ungleich.ch") + dn = "uid={},ou=customer,dc=ungleich,dc=ch".format(username) + + log.debug("LDAP: connecting to {} as {}".format(server, dn)) + + try: + conn = ldap3.Connection(server, dn, password, auto_bind=True) + except ldap3.core.exceptions.LDAPBindError as e: + raise UncloudException("Credentials not verified by LDAP server: {}".format(e)) + + + +def order(config): + for required_arg in [ 'product', 'username', 'password' ]: + if not config.arguments[required_arg]: + raise UncloudException("Missing required argument: {}".format(required_arg)) + + if config.arguments['product'] == 'dualstack-vm': + for required_arg in [ 'cores', 'memory', 'os_image_name', 'os_image_size' ]: + if not config.arguments[required_arg]: + raise UncloudException("Missing required argument: {}".format(required_arg)) + + log.debug(config.arguments) + authenticate(config.arguments['username'], config.arguments['password']) + + # create DB entry for VM + vm = VM(config) + return vm.product.place_order(owner=config.arguments['username']) + + + + + +def main(arguments): + config = Config(arguments) + + if arguments['add_vm_host']: + h = Host(config) + h.cmdline_add_host() + + if arguments['list_vm_hosts']: + h = Host(config) + + for host in h.list_hosts(filter_key=arguments['filter_order_key'], + filter_regexp=arguments['filter_order_regexp']): + print("Host {}: {}".format(host.db_entry['uuid'], host.db_entry)) + + if arguments['order']: + print("Created order: {}".format(order(config))) + + if arguments['list_orders']: + p = ProductOrder(config) + for product_order in p.list_orders(filter_key=arguments['filter_order_key'], + filter_regexp=arguments['filter_order_regexp']): + print("Order {}: {}".format(product_order.db_entry['uuid'], product_order.db_entry)) + + if arguments['process_orders']: + p = ProductOrder(config) + p.process_orders() + + if arguments['create_vm']: + vm = VM(config) + vm.create() + + if arguments['destroy_vm']: + vm = VM(config) + vm.stop() + + if arguments['get_vm_status']: + vm = VM(config) + vm.status() + + if arguments['get_vm_vnc']: + vm = VM(config) + vm.vnc_addr() + + if arguments['list_vms']: + vm = VM(config) + vm.list() + + if arguments['last_used_mac']: + m = MAC(config) + print(m.last_used_mac()) + + if arguments['get_new_mac']: + print(MAC(config).get_next()) + + #if arguments['init_network']: + if arguments['create_vxlan']: + if not arguments['network'] or not arguments['vni'] or not arguments['vxlan_uplink_device']: + raise UncloudException("Initialising the network requires an IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + vb = VXLANBridge(vni=arguments['vni'], + route=arguments['network'], + uplinkdev=arguments['vxlan_uplink_device'], + use_sudo=arguments['use_sudo']) + vb._setup_vxlan() + vb._setup_bridge() + vb._add_vxlan_to_bridge() + vb._route_network() + + if arguments['run_dns_ra']: + if not arguments['network'] or not arguments['vni']: + raise UncloudException("Providing DNS/RAs requires a /64 IPv6 network and a VNI. You can use fd00::/64 and vni=1 for testing (non production!)") + + dnsra = DNSRA(route=arguments['network'], + vni=arguments['vni'], + use_sudo=arguments['use_sudo']) + dnsra._setup_dnsmasq() diff --git a/archive/uncloud_etcd_based/uncloud/hack/net.py b/archive/uncloud_etcd_based/uncloud/hack/net.py new file mode 100644 index 0000000..4887e04 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/net.py @@ -0,0 +1,116 @@ +import subprocess +import ipaddress +import logging + + +from uncloud import UncloudException + +log = logging.getLogger(__name__) + + +class VXLANBridge(object): + cmd_create_vxlan = "{sudo}ip -6 link add {vxlandev} type vxlan id {vni_dec} dstport 4789 group {multicast_address} dev {uplinkdev} ttl 5" + cmd_up_dev = "{sudo}ip link set {dev} up" + cmd_create_bridge="{sudo}ip link add {bridgedev} type bridge" + cmd_add_to_bridge="{sudo}ip link set {vxlandev} master {bridgedev} up" + cmd_add_addr="{sudo}ip addr add {ip} dev {bridgedev}" + cmd_add_route_dev="{sudo}ip route add {route} dev {bridgedev}" + + # VXLAN ids are at maximum 24 bit - use a /104 + multicast_network = ipaddress.IPv6Network("ff05::/104") + max_vni = (2**24)-1 + + def __init__(self, + vni, + uplinkdev, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' + + self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + self.config['multicast_address'] = self.multicast_network[vni] + + self.config['route_network'] = ipaddress.IPv6Network(route) + self.config['route'] = route + + self.config['uplinkdev'] = uplinkdev + self.config['vxlandev'] = "vx{}".format(self.config['vni_hex']) + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + + def setup_networking(self): + pass + + def _setup_vxlan(self): + self._execute_cmd(self.cmd_create_vxlan) + self._execute_cmd(self.cmd_up_dev, dev=self.config['vxlandev']) + + def _setup_bridge(self): + self._execute_cmd(self.cmd_create_bridge) + self._execute_cmd(self.cmd_up_dev, dev=self.config['bridgedev']) + + def _route_network(self): + self._execute_cmd(self.cmd_add_route_dev) + + def _add_vxlan_to_bridge(self): + self._execute_cmd(self.cmd_add_to_bridge) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + +class ManagementBridge(VXLANBridge): + pass + + +class DNSRA(object): + # VXLAN ids are at maximum 24 bit + max_vni = (2**24)-1 + + + # Command to start dnsmasq + cmd_start_dnsmasq="{sudo}dnsmasq --interface={bridgedev} --bind-interfaces --dhcp-range={route},ra-only,infinite --enable-ra --no-daemon" + + def __init__(self, + vni, + route=None, + use_sudo=False): + self.config = {} + + if vni > self.max_vni: + raise UncloudException("VNI must be in the range of 0 .. {}".format(self.max_vni)) + + if use_sudo: + self.config['sudo'] = 'sudo ' + else: + self.config['sudo'] = '' + + #TODO: remove if not needed + #self.config['vni_dec'] = vni + self.config['vni_hex'] = "{:x}".format(vni) + + # dnsmasq only wants the network without the prefix, therefore, cut it off + self.config['route'] = ipaddress.IPv6Network(route).network_address + self.config['bridgedev'] = "br{}".format(self.config['vni_hex']) + + def _setup_dnsmasq(self): + self._execute_cmd(self.cmd_start_dnsmasq) + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.config, **kwargs) + log.info("Executing: {}".format(cmd)) + print("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + +class Firewall(object): + pass diff --git a/uncloud/hack/nftables.conf b/archive/uncloud_etcd_based/uncloud/hack/nftables.conf similarity index 100% rename from uncloud/hack/nftables.conf rename to archive/uncloud_etcd_based/uncloud/hack/nftables.conf diff --git a/archive/uncloud_etcd_based/uncloud/hack/product.py b/archive/uncloud_etcd_based/uncloud/hack/product.py new file mode 100755 index 0000000..f979268 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/product.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +import json +import uuid +import logging +import re +import importlib + +from uncloud import UncloudException +from uncloud.hack.db import DB, db_logentry + +log = logging.getLogger(__name__) + +class ProductOrder(object): + def __init__(self, config, product_entry=None, db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/orders") + self.db_entry = {} + self.db_entry["product"] = product_entry + + # Overwrite if we are loading an existing product order + if db_entry: + self.db_entry = db_entry + + # FIXME: this should return a list of our class! + def list_orders(self, filter_key=None, filter_regexp=None): + for entry in self.db.list_and_filter("", filter_key, filter_regexp): + yield self.__class__(self.config, db_entry=entry) + + + def set_required_values(self): + """Set values that are required to make the db entry valid""" + if not "uuid" in self.db_entry: + self.db_entry["uuid"] = str(uuid.uuid4()) + if not "status" in self.db_entry: + self.db_entry["status"] = "NEW" + if not "owner" in self.db_entry: + self.db_entry["owner"] = "UNKNOWN" + if not "log" in self.db_entry: + self.db_entry["log"] = [] + if not "db_version" in self.db_entry: + self.db_entry["db_version"] = 1 + + def validate_status(self): + if "status" in self.db_entry: + if self.db_entry["status"] in [ "NEW", + "SCHEDULED", + "CREATED_ACTIVE", + "CANCELLED", + "REJECTED" ]: + return False + return True + + def order(self): + self.set_required_values() + if not self.db_entry["status"] == "NEW": + raise UncloudException("Cannot re-order same order. Status: {}".format(self.db_entry["status"])) + self.db.set(self.db_entry["uuid"], self.db_entry, as_json=True) + + return self.db_entry["uuid"] + + def process_orders(self): + """processing orders can be done stand alone on server side""" + for order in self.list_orders(): + if order.db_entry["status"] == "NEW": + log.info("Handling new order: {}".format(order)) + + # FIXME: these all should be a transactions! -> fix concurrent access! ! + if not "log" in order.db_entry: + order.db_entry['log'] = [] + + is_valid = True + # Verify the order entry + for must_attribute in [ "owner", "product" ]: + if not must_attribute in order.db_entry: + message = "Missing {} entry in order, rejecting order".format(must_attribute) + log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message)) + + order.db_entry['log'].append(db_logentry(message)) + order.db_entry['status'] = "REJECTED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + is_valid = False + + # Rejected the order + if not is_valid: + continue + + # Verify the product entry + for must_attribute in [ "python_product_class", "python_product_module" ]: + if not must_attribute in order.db_entry['product']: + message = "Missing {} entry in product of order, rejecting order".format(must_attribute) + log.info("Rejecting order {}: {}".format(order.db_entry["uuid"], message)) + + order.db_entry['log'].append(db_logentry(message)) + order.db_entry['status'] = "REJECTED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + is_valid = False + + # Rejected the order + if not is_valid: + continue + + print(order.db_entry["product"]["python_product_class"]) + + # Create the product + m = importlib.import_module(order.db_entry["product"]["python_product_module"]) + c = getattr(m, order.db_entry["product"]["python_product_class"]) + + product = c(config, db_entry=order.db_entry["product"]) + + # STOPPED + product.create_product() + + order.db_entry['status'] = "SCHEDULED" + self.db.set(order.db_entry['uuid'], order.db_entry, as_json=True) + + + + def __str__(self): + return str(self.db_entry) + +class Product(object): + def __init__(self, + config, + product_name, + product_class, + db_entry=None): + self.config = config + self.db = DB(self.config, prefix="/orders") + + self.db_entry = {} + self.db_entry["product_name"] = product_name + self.db_entry["python_product_class"] = product_class.__qualname__ + self.db_entry["python_product_module"] = product_class.__module__ + self.db_entry["db_version"] = 1 + self.db_entry["log"] = [] + self.db_entry["features"] = {} + + # Existing product? Read in db_entry + if db_entry: + self.db_entry = db_entry + + self.valid_periods = [ "per_year", "per_month", "per_week", + "per_day", "per_hour", + "per_minute", "per_second" ] + + def define_feature(self, + name, + one_time_price, + recurring_price, + recurring_period, + minimum_period): + + self.db_entry['features'][name] = {} + self.db_entry['features'][name]['one_time_price'] = one_time_price + self.db_entry['features'][name]['recurring_price'] = recurring_price + + if not recurring_period in self.valid_periods: + raise UncloudException("Invalid recurring period: {}".format(recurring_period)) + + self.db_entry['features'][name]['recurring_period'] = recurring_period + + if not minimum_period in self.valid_periods: + raise UncloudException("Invalid recurring period: {}".format(recurring_period)) + + recurring_index = self.valid_periods.index(recurring_period) + minimum_index = self.valid_periods.index(minimum_period) + + if minimum_index < recurring_index: + raise UncloudException("Minimum period for product '{}' feature '{}' must be shorter or equal than/as recurring period: {} > {}".format(self.db_entry['product_name'], name, minimum_period, recurring_period)) + + self.db_entry['features'][name]['minimum_period'] = minimum_period + + + def validate_product(self): + for feature in self.db_entry['features']: + pass + + def place_order(self, owner): + """ Schedule creating the product in etcd """ + order = ProductOrder(self.config, product_entry=self.db_entry) + order.db_entry["owner"] = owner + return order.order() + + def __str__(self): + return json.dumps(self.db_entry) diff --git a/uncloud/hack/rc-scripts/ucloud-api b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-api similarity index 100% rename from uncloud/hack/rc-scripts/ucloud-api rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-api diff --git a/uncloud/hack/rc-scripts/ucloud-host b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-host similarity index 100% rename from uncloud/hack/rc-scripts/ucloud-host rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-host diff --git a/uncloud/hack/rc-scripts/ucloud-metadata b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-metadata similarity index 100% rename from uncloud/hack/rc-scripts/ucloud-metadata rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-metadata diff --git a/uncloud/hack/rc-scripts/ucloud-scheduler b/archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-scheduler similarity index 100% rename from uncloud/hack/rc-scripts/ucloud-scheduler rename to archive/uncloud_etcd_based/uncloud/hack/rc-scripts/ucloud-scheduler diff --git a/uncloud/hack/uncloud-hack-init-host b/archive/uncloud_etcd_based/uncloud/hack/uncloud-hack-init-host similarity index 100% rename from uncloud/hack/uncloud-hack-init-host rename to archive/uncloud_etcd_based/uncloud/hack/uncloud-hack-init-host diff --git a/uncloud/hack/uncloud-run-vm b/archive/uncloud_etcd_based/uncloud/hack/uncloud-run-vm similarity index 100% rename from uncloud/hack/uncloud-run-vm rename to archive/uncloud_etcd_based/uncloud/hack/uncloud-run-vm diff --git a/archive/uncloud_etcd_based/uncloud/hack/vm.py b/archive/uncloud_etcd_based/uncloud/hack/vm.py new file mode 100755 index 0000000..4b0ca14 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/hack/vm.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2020 Nico Schottelius (nico.schottelius at ungleich.ch) +# +# This file is part of uncloud. +# +# uncloud is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# uncloud is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with uncloud. If not, see . + +# This module is directly called from the hack module, and can be used as follow: +# +# Create a new VM with default CPU/Memory. The path of the image file is relative to $hackprefix. +# `uncloud hack --hackprefix /tmp/hackcloud --create-vm --image mysuperimage.qcow2` +# +# List running VMs (returns a list of UUIDs). +# `uncloud hack --hackprefix /tmp/hackcloud --list-vms +# +# Get VM status: +# `uncloud hack --hackprefix /tmp/hackcloud --get-vm-status --uuid my-vm-uuid` +# +# Stop a VM: +# `uncloud hack --hackprefix /tmp/hackcloud --destroy-vm --uuid my-vm-uuid` +# `` + +import subprocess +import uuid +import os +import logging + +from uncloud.hack.db import DB +from uncloud.hack.mac import MAC +from uncloud.vmm import VMM +from uncloud.hack.product import Product + +log = logging.getLogger(__name__) +log.setLevel(logging.DEBUG) + +class VM(object): + def __init__(self, config, db_entry=None): + self.config = config + + #TODO: Enable etcd lookup + self.no_db = self.config.arguments['no_db'] + if not self.no_db: + self.db = DB(self.config, prefix="/vm") + + if db_entry: + self.db_entry = db_entry + + # General CLI arguments. + self.hackprefix = self.config.arguments['hackprefix'] + self.uuid = self.config.arguments['uuid'] + self.memory = self.config.arguments['memory'] or '1024M' + self.cores = self.config.arguments['cores'] or 1 + + if self.config.arguments['image']: + self.image = os.path.join(self.hackprefix, self.config.arguments['image']) + else: + self.image = None + + if self.config.arguments['image_format']: + self.image_format=self.config.arguments['image_format'] + else: + self.image_format='qcow2' + + # External components. + + # This one is broken: + # TypeError: expected str, bytes or os.PathLike object, not NoneType + # Fix before re-enabling + # self.vmm = VMM(vmm_backend=self.hackprefix) + self.mac = MAC(self.config) + + # Harcoded & generated values. + self.owner = 'uncloud' + self.accel = 'kvm' + self.threads = 1 + self.ifup = os.path.join(self.hackprefix, "ifup.sh") + self.ifdown = os.path.join(self.hackprefix, "ifdown.sh") + self.ifname = "uc{}".format(self.mac.to_str_format()) + + self.vm = {} + + self.product = Product(config, product_name="dualstack-vm", + product_class=self.__class__) + self.product.define_feature(name="base", + one_time_price=0, + recurring_price=9, + recurring_period="per_month", + minimum_period="per_hour") + + + self.features = [] + + + def get_qemu_args(self): + command = ( + "-name {owner}-{name}" + " -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname}" + " -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" + ).format( + owner=self.owner, name=self.uuid, + accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + ifup=self.ifup, ifdown=self.ifdown, ifname=self.ifname, + mac=self.mac + ) + + return command.split(" ") + + def create_product(self): + """Find a VM host and schedule on it""" + pass + + def create(self): + # New VM: new UUID, new MAC. + self.uuid = str(uuid.uuid4()) + self.mac=MAC(self.config) + self.mac.create() + + qemu_args = self.get_qemu_args() + log.debug("QEMU args passed to VMM: {}".format(qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + + self.mac.create() + self.vm['mac'] = self.mac + self.vm['ifname'] = "uc{}".format(self.mac.__repr__()) + + # FIXME: TODO: turn this into a string and THEN + # .split() it later -- easier for using .format() + #self.vm['commandline'] = [ "{}".format(self.sudo), + self.vm['commandline'] = "{sudo}{qemu} -name uncloud-{uuid} -machine pc,accel={accel} -m {memory} -smp {cores} -uuid {uuid} -drive file={os_image},media=cdrom -netdev tap,id=netmain,script={ifup},downscript={ifdown},ifname={ifname} -device virtio-net-pci,netdev=netmain,id=net0,mac={mac}" +# self.vm['commandline'] = [ "{}".format(self.sudo), +# "{}".format(self.qemu), +# "-name", "uncloud-{}".format(self.vm['uuid']), +# "-machine", "pc,accel={}".format(self.accel), +# "-m", "{}".format(self.vm['memory']), +# "-smp", "{}".format(self.vm['cores']), +# "-uuid", "{}".format(self.vm['uuid']), +# "-drive", "file={},media=cdrom".format(self.vm['os_image']), +# "-netdev", "tap,id=netmain,script={},downscript={},ifname={}".format(self.ifup, self.ifdown, self.vm['ifname']), +# "-device", "virtio-net-pci,netdev=netmain,id=net0,mac={}".format(self.vm['mac']) +# ] + + def _execute_cmd(self, cmd_string, **kwargs): + cmd = cmd_string.format(**self.vm, **kwargs) + log.info("Executing: {}".format(cmd)) + subprocess.run(cmd.split()) + + def stop(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + self.vmm.stop(self.uuid) + + def status(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_status(self.uuid)) + + def vnc_addr(self): + if not self.uuid: + print("Please specific an UUID with the --uuid flag.") + exit(1) + + print(self.vmm.get_vnc(self.uuid)) + + def list(self): + print(self.vmm.discover()) diff --git a/uncloud/host/__init__.py b/archive/uncloud_etcd_based/uncloud/host/__init__.py similarity index 100% rename from uncloud/host/__init__.py rename to archive/uncloud_etcd_based/uncloud/host/__init__.py diff --git a/uncloud/host/main.py b/archive/uncloud_etcd_based/uncloud/host/main.py similarity index 90% rename from uncloud/host/main.py rename to archive/uncloud_etcd_based/uncloud/host/main.py index ccffd77..f680991 100755 --- a/uncloud/host/main.py +++ b/archive/uncloud_etcd_based/uncloud/host/main.py @@ -6,7 +6,6 @@ from uuid import uuid4 from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared -from uncloud.common.settings import settings from uncloud.common.vm import VMStatus from uncloud.vmm import VMM from os.path import join as join_path @@ -36,7 +35,7 @@ def maintenance(host): if vmm.is_running(vm_uuid) and vmm.get_status(vm_uuid) == 'running': logger.debug('VM {} is running on {}'.format(vm_uuid, host)) vm = shared.vm_pool.get( - join_path(settings['etcd']['vm_prefix'], vm_uuid) + join_path(shared.settings['etcd']['vm_prefix'], vm_uuid) ) vm.status = VMStatus.running vm.vnc_socket = vmm.get_vnc(vm_uuid) @@ -52,7 +51,7 @@ def main(arguments): # Does not yet exist, create it if not host: host_key = join_path( - settings['etcd']['host_prefix'], uuid4().hex + shared.settings['etcd']['host_prefix'], uuid4().hex ) host_entry = { 'specs': '', @@ -80,9 +79,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for events_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False) ]: for request_event in events_iterator: @@ -95,7 +94,7 @@ def main(arguments): shared.request_pool.client.client.delete(request_event.key) vm_entry = shared.etcd_client.get( - join_path(settings['etcd']['vm_prefix'], request_event.uuid) + join_path(shared.settings['etcd']['vm_prefix'], request_event.uuid) ) logger.debug('VM hostname: {}'.format(vm_entry.value)) diff --git a/uncloud/host/virtualmachine.py b/archive/uncloud_etcd_based/uncloud/host/virtualmachine.py similarity index 96% rename from uncloud/host/virtualmachine.py rename to archive/uncloud_etcd_based/uncloud/host/virtualmachine.py index 2f6a5e3..a592efc 100755 --- a/uncloud/host/virtualmachine.py +++ b/archive/uncloud_etcd_based/uncloud/host/virtualmachine.py @@ -17,7 +17,6 @@ from uncloud.common.network import create_dev, delete_network_interface from uncloud.common.schemas import VMSchema, NetworkSchema from uncloud.host import logger from uncloud.common.shared import shared -from uncloud.common.settings import settings from uncloud.vmm import VMM from marshmallow import ValidationError @@ -91,7 +90,7 @@ class VM: self.vmm.socket_dir, self.uuid ), destination_host_key=destination_host_key, # Where source host transfer VM - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) else: @@ -119,7 +118,7 @@ class VM: network_name, mac, tap = network_mac_and_tap _key = os.path.join( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], self.vm["owner"], network_name, ) @@ -133,13 +132,13 @@ class VM: if network["type"] == "vxlan": tap = create_vxlan_br_tap( _id=network["id"], - _dev=settings["network"]["vxlan_phy_dev"], + _dev=shared.settings["network"]["vxlan_phy_dev"], tap_id=tap, ip=network["ipv6"], ) all_networks = shared.etcd_client.get_prefix( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], value_in_json=True, ) @@ -229,7 +228,7 @@ class VM: def resolve_network(network_name, network_owner): network = shared.etcd_client.get( join_path( - settings["etcd"]["network_prefix"], + shared.settings["etcd"]["network_prefix"], network_owner, network_name, ), diff --git a/uncloud/imagescanner/__init__.py b/archive/uncloud_etcd_based/uncloud/imagescanner/__init__.py similarity index 100% rename from uncloud/imagescanner/__init__.py rename to archive/uncloud_etcd_based/uncloud/imagescanner/__init__.py diff --git a/uncloud/imagescanner/main.py b/archive/uncloud_etcd_based/uncloud/imagescanner/main.py similarity index 94% rename from uncloud/imagescanner/main.py rename to archive/uncloud_etcd_based/uncloud/imagescanner/main.py index 1803213..ee9da2e 100755 --- a/uncloud/imagescanner/main.py +++ b/archive/uncloud_etcd_based/uncloud/imagescanner/main.py @@ -4,7 +4,6 @@ import argparse import subprocess as sp from os.path import join as join_path -from uncloud.common.settings import settings from uncloud.common.shared import shared from uncloud.imagescanner import logger @@ -33,7 +32,7 @@ def qemu_img_type(path): def main(arguments): # We want to get images entries that requests images to be created images = shared.etcd_client.get_prefix( - settings["etcd"]["image_prefix"], value_in_json=True + shared.settings["etcd"]["image_prefix"], value_in_json=True ) images_to_be_created = list( filter(lambda im: im.value["status"] == "TO_BE_CREATED", images) @@ -46,13 +45,13 @@ def main(arguments): image_filename = image.value["filename"] image_store_name = image.value["store_name"] image_full_path = join_path( - settings["storage"]["file_dir"], + shared.settings["storage"]["file_dir"], image_owner, image_filename, ) image_stores = shared.etcd_client.get_prefix( - settings["etcd"]["image_store_prefix"], + shared.settings["etcd"]["image_store_prefix"], value_in_json=True, ) user_image_store = next( diff --git a/uncloud/metadata/__init__.py b/archive/uncloud_etcd_based/uncloud/metadata/__init__.py similarity index 100% rename from uncloud/metadata/__init__.py rename to archive/uncloud_etcd_based/uncloud/metadata/__init__.py diff --git a/uncloud/metadata/main.py b/archive/uncloud_etcd_based/uncloud/metadata/main.py similarity index 92% rename from uncloud/metadata/main.py rename to archive/uncloud_etcd_based/uncloud/metadata/main.py index ccda60e..374260e 100644 --- a/uncloud/metadata/main.py +++ b/archive/uncloud_etcd_based/uncloud/metadata/main.py @@ -5,7 +5,6 @@ from flask import Flask, request from flask_restful import Resource, Api from werkzeug.exceptions import HTTPException -from uncloud.common.settings import settings from uncloud.common.shared import shared app = Flask(__name__) @@ -13,8 +12,10 @@ api = Api(app) app.logger.handlers.clear() +DEFAULT_PORT=1234 + arg_parser = argparse.ArgumentParser('metadata', add_help=False) -arg_parser.add_argument('--port', '-p', default=80, help='By default bind to port 80') +arg_parser.add_argument('--port', '-p', default=DEFAULT_PORT, help='By default bind to port {}'.format(DEFAULT_PORT)) @app.errorhandler(Exception) @@ -72,7 +73,7 @@ class Root(Resource): ) else: etcd_key = os.path.join( - settings["etcd"]["user_prefix"], + shared.settings["etcd"]["user_prefix"], data.value["owner_realm"], data.value["owner"], "key", diff --git a/uncloud/network/README b/archive/uncloud_etcd_based/uncloud/network/README similarity index 100% rename from uncloud/network/README rename to archive/uncloud_etcd_based/uncloud/network/README diff --git a/archive/uncloud_etcd_based/uncloud/network/__init__.py b/archive/uncloud_etcd_based/uncloud/network/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/network/create-bridge.sh b/archive/uncloud_etcd_based/uncloud/network/create-bridge.sh similarity index 100% rename from uncloud/network/create-bridge.sh rename to archive/uncloud_etcd_based/uncloud/network/create-bridge.sh diff --git a/uncloud/network/create-tap.sh b/archive/uncloud_etcd_based/uncloud/network/create-tap.sh similarity index 100% rename from uncloud/network/create-tap.sh rename to archive/uncloud_etcd_based/uncloud/network/create-tap.sh diff --git a/uncloud/network/create-vxlan.sh b/archive/uncloud_etcd_based/uncloud/network/create-vxlan.sh similarity index 100% rename from uncloud/network/create-vxlan.sh rename to archive/uncloud_etcd_based/uncloud/network/create-vxlan.sh diff --git a/uncloud/network/radvd-template.conf b/archive/uncloud_etcd_based/uncloud/network/radvd-template.conf similarity index 100% rename from uncloud/network/radvd-template.conf rename to archive/uncloud_etcd_based/uncloud/network/radvd-template.conf diff --git a/uncloud/scheduler/__init__.py b/archive/uncloud_etcd_based/uncloud/oneshot/__init__.py similarity index 100% rename from uncloud/scheduler/__init__.py rename to archive/uncloud_etcd_based/uncloud/oneshot/__init__.py diff --git a/archive/uncloud_etcd_based/uncloud/oneshot/main.py b/archive/uncloud_etcd_based/uncloud/oneshot/main.py new file mode 100644 index 0000000..dbb3b32 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/oneshot/main.py @@ -0,0 +1,123 @@ +import argparse +import os + + +from pathlib import Path +from uncloud.vmm import VMM +from uncloud.host.virtualmachine import update_radvd_conf, create_vxlan_br_tap + +from . import virtualmachine, logger + +### +# Argument parser loaded by scripts/uncloud. +arg_parser = argparse.ArgumentParser('oneshot', add_help=False) + +# Actions. +arg_parser.add_argument('--list', action='store_true', + help='list UUID and name of running VMs') +arg_parser.add_argument('--start', nargs=4, + metavar=('NAME', 'IMAGE', 'UPSTREAM_INTERFACE', 'NETWORK'), + help='start a VM using the OS IMAGE (full path), configuring networking on NETWORK IPv6 prefix') +arg_parser.add_argument('--stop', metavar='UUID', + help='stop a VM') +arg_parser.add_argument('--get-status', metavar='UUID', + help='return the status of the VM') +arg_parser.add_argument('--get-vnc', metavar='UUID', + help='return the path of the VNC socket of the VM') +arg_parser.add_argument('--reconfigure-radvd', metavar='NETWORK', + help='regenerate and reload RADVD configuration for NETWORK IPv6 prefix') + +# Arguments. +arg_parser.add_argument('--workdir', default=Path.home(), + help='Working directory, defaulting to $HOME') +arg_parser.add_argument('--mac', + help='MAC address of the VM to create (--start)') +arg_parser.add_argument('--memory', type=int, + help='Memory (MB) to allocate (--start)') +arg_parser.add_argument('--cores', type=int, + help='Number of cores to allocate (--start)') +arg_parser.add_argument('--threads', type=int, + help='Number of threads to allocate (--start)') +arg_parser.add_argument('--image-format', choices=['raw', 'qcow2'], + help='Format of OS image (--start)') +arg_parser.add_argument('--accel', choices=['kvm', 'tcg'], default='kvm', + help='QEMU acceleration to use (--start)') +arg_parser.add_argument('--upstream-interface', default='eth0', + help='Name of upstream interface (--start)') + +### +# Helpers. + +# XXX: check if it is possible to use the type returned by ETCD queries. +class UncloudEntryWrapper: + def __init__(self, value): + self.value = value + + def value(self): + return self.value + +def status_line(vm): + return "VM: {} {} {}".format(vm.get_uuid(), vm.get_name(), vm.get_status()) + +### +# Entrypoint. + +def main(arguments): + # Initialize VMM. + workdir = arguments['workdir'] + vmm = VMM(vmm_backend=workdir) + + # Harcoded debug values. + net_id = 0 + + # Build VM configuration. + vm_config = {} + vm_options = [ + 'mac', 'memory', 'cores', 'threads', 'image', 'image_format', + '--upstream_interface', 'upstream_interface', 'network', 'accel' + ] + for option in vm_options: + if arguments.get(option): + vm_config[option] = arguments[option] + + vm_config['net_id'] = net_id + + # Execute requested VM action. + if arguments['reconfigure_radvd']: + # TODO: check that RADVD is available. + prefix = arguments['reconfigure_radvd'] + network = UncloudEntryWrapper({ + 'id': net_id, + 'ipv6': prefix + }) + + # Make use of uncloud.host.virtualmachine for network configuration. + update_radvd_conf([network]) + elif arguments['start']: + # Extract from --start positional arguments. Quite fragile. + vm_config['name'] = arguments['start'][0] + vm_config['image'] = arguments['start'][1] + vm_config['network'] = arguments['start'][2] + vm_config['upstream_interface'] = arguments['start'][3] + + vm_config['tap_interface'] = "uc{}".format(len(vmm.discover())) + vm = virtualmachine.VM(vmm, vm_config) + vm.start() + elif arguments['stop']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['stop']}) + vm.stop() + elif arguments['get_status']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_status']}) + print(status_line(vm)) + elif arguments['get_vnc']: + vm = virtualmachine.VM(vmm, {'uuid': arguments['get_vnc']}) + print(vm.get_vnc_addr()) + elif arguments['list']: + vms = vmm.discover() + print("Found {} VMs.".format(len(vms))) + for uuid in vms: + vm = virtualmachine.VM(vmm, {'uuid': uuid}) + print(status_line(vm)) + else: + print('Please specify an action: --start, --stop, --list,\ +--get-status, --get-vnc, --reconfigure-radvd') diff --git a/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py b/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py new file mode 100644 index 0000000..5749bee --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/oneshot/virtualmachine.py @@ -0,0 +1,81 @@ +import uuid +import os + +from uncloud.host.virtualmachine import create_vxlan_br_tap +from uncloud.oneshot import logger + +class VM(object): + def __init__(self, vmm, config): + self.config = config + self.vmm = vmm + + # Extract VM specs/metadata from configuration. + self.name = config.get('name', 'no-name') + self.memory = config.get('memory', 1024) + self.cores = config.get('cores', 1) + self.threads = config.get('threads', 1) + self.image_format = config.get('image_format', 'qcow2') + self.image = config.get('image') + self.uuid = config.get('uuid', str(uuid.uuid4())) + self.mac = config.get('mac') + self.accel = config.get('accel', 'kvm') + + self.net_id = config.get('net_id', 0) + self.upstream_interface = config.get('upstream_interface', 'eth0') + self.tap_interface = config.get('tap_interface', 'uc0') + self.network = config.get('network') + + def get_qemu_args(self): + command = ( + "-uuid {uuid} -name {name} -machine pc,accel={accel}" + " -drive file={image},format={image_format},if=virtio" + " -device virtio-rng-pci" + " -m {memory} -smp cores={cores},threads={threads}" + " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no" + " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}" + ).format( + uuid=self.uuid, name=self.name, accel=self.accel, + image=self.image, image_format=self.image_format, + memory=self.memory, cores=self.cores, threads=self.threads, + net_id=self.net_id, tap=self.tap_interface, mac=self.mac + ) + + return command.split(" ") + + def start(self): + # Check that VM image is available. + if not os.path.isfile(self.image): + logger.error("Image {} does not exist. Aborting.".format(self.image)) + + # Create Bridge, VXLAN and tap interface for VM. + create_vxlan_br_tap( + self.net_id, self.upstream_interface, self.tap_interface, self.network + ) + + # Generate config for and run QEMU. + qemu_args = self.get_qemu_args() + logger.debug("QEMU args for VM {}: {}".format(self.uuid, qemu_args)) + self.vmm.start( + uuid=self.uuid, + migration=False, + *qemu_args + ) + + def stop(self): + self.vmm.stop(self.uuid) + + def get_status(self): + return self.vmm.get_status(self.uuid) + + def get_vnc_addr(self): + return self.vmm.get_vnc(self.uuid) + + def get_uuid(self): + return self.uuid + + def get_name(self): + success, json = self.vmm.execute_command(self.uuid, 'query-name') + if success: + return json['return']['name'] + + return None diff --git a/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py b/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py new file mode 100644 index 0000000..eea436a --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/scheduler/__init__.py @@ -0,0 +1,3 @@ +import logging + +logger = logging.getLogger(__name__) diff --git a/uncloud/scheduler/helper.py b/archive/uncloud_etcd_based/uncloud/scheduler/helper.py similarity index 97% rename from uncloud/scheduler/helper.py rename to archive/uncloud_etcd_based/uncloud/scheduler/helper.py index 108d126..79db322 100755 --- a/uncloud/scheduler/helper.py +++ b/archive/uncloud_etcd_based/uncloud/scheduler/helper.py @@ -7,7 +7,6 @@ from uncloud.common.host import HostStatus from uncloud.common.request import RequestEntry, RequestType from uncloud.common.vm import VMStatus from uncloud.common.shared import shared -from uncloud.common.settings import settings def accumulated_specs(vms_specs): @@ -130,7 +129,7 @@ def assign_host(vm): type=RequestType.StartVM, uuid=vm.uuid, hostname=vm.hostname, - request_prefix=settings["etcd"]["request_prefix"], + request_prefix=shared.settings["etcd"]["request_prefix"], ) shared.request_pool.put(r) diff --git a/uncloud/scheduler/main.py b/archive/uncloud_etcd_based/uncloud/scheduler/main.py similarity index 90% rename from uncloud/scheduler/main.py rename to archive/uncloud_etcd_based/uncloud/scheduler/main.py index c25700b..38c07bf 100755 --- a/uncloud/scheduler/main.py +++ b/archive/uncloud_etcd_based/uncloud/scheduler/main.py @@ -6,7 +6,6 @@ import argparse -from uncloud.common.settings import settings from uncloud.common.request import RequestEntry, RequestType from uncloud.common.shared import shared from uncloud.scheduler import logger @@ -24,9 +23,9 @@ def main(arguments): # get prefix until either success or deamon death comes. while True: for request_iterator in [ - shared.etcd_client.get_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.get_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), - shared.etcd_client.watch_prefix(settings['etcd']['request_prefix'], value_in_json=True, + shared.etcd_client.watch_prefix(shared.settings['etcd']['request_prefix'], value_in_json=True, raise_exception=False), ]: for request_event in request_iterator: diff --git a/archive/uncloud_etcd_based/uncloud/scheduler/tests/__init__.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud/scheduler/tests/test_basics.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/test_basics.py similarity index 100% rename from uncloud/scheduler/tests/test_basics.py rename to archive/uncloud_etcd_based/uncloud/scheduler/tests/test_basics.py diff --git a/uncloud/scheduler/tests/test_dead_host_mechanism.py b/archive/uncloud_etcd_based/uncloud/scheduler/tests/test_dead_host_mechanism.py similarity index 100% rename from uncloud/scheduler/tests/test_dead_host_mechanism.py rename to archive/uncloud_etcd_based/uncloud/scheduler/tests/test_dead_host_mechanism.py diff --git a/archive/uncloud_etcd_based/uncloud/version.py b/archive/uncloud_etcd_based/uncloud/version.py new file mode 100644 index 0000000..ccf3980 --- /dev/null +++ b/archive/uncloud_etcd_based/uncloud/version.py @@ -0,0 +1 @@ +VERSION = "0.0.5-30-ge91fd9e" diff --git a/uncloud/vmm/__init__.py b/archive/uncloud_etcd_based/uncloud/vmm/__init__.py similarity index 95% rename from uncloud/vmm/__init__.py rename to archive/uncloud_etcd_based/uncloud/vmm/__init__.py index 4c893f6..6db61eb 100644 --- a/uncloud/vmm/__init__.py +++ b/archive/uncloud_etcd_based/uncloud/vmm/__init__.py @@ -100,9 +100,9 @@ class TransferVM(Process): class VMM: # Virtual Machine Manager def __init__( - self, - qemu_path="/usr/bin/qemu-system-x86_64", - vmm_backend=os.path.expanduser("~/uncloud/vmm/"), + self, + qemu_path="/usr/bin/qemu-system-x86_64", + vmm_backend=os.path.expanduser("~/uncloud/vmm/"), ): self.qemu_path = qemu_path self.vmm_backend = vmm_backend @@ -125,7 +125,7 @@ class VMM: os.makedirs(self.socket_dir, exist_ok=True) def is_running(self, uuid): - sock_path = os.path.join(self.vmm_backend, uuid) + sock_path = os.path.join(self.socket_dir, uuid) try: sock = socket.socket(socket.AF_UNIX) sock.connect(sock_path) @@ -163,7 +163,7 @@ class VMM: qmp_arg = ( "-qmp", "unix:{},server,nowait".format( - join_path(self.vmm_backend, uuid) + join_path(self.socket_dir, uuid) ), ) vnc_arg = ( @@ -212,7 +212,7 @@ class VMM: def execute_command(self, uuid, command, **kwargs): # execute_command -> sucess?, output try: - with VMQMPHandles(os.path.join(self.vmm_backend, uuid)) as ( + with VMQMPHandles(os.path.join(self.socket_dir, uuid)) as ( sock_handle, file_handle, ): @@ -255,8 +255,8 @@ class VMM: def discover(self): vms = [ uuid - for uuid in os.listdir(self.vmm_backend) - if not isdir(join_path(self.vmm_backend, uuid)) + for uuid in os.listdir(self.socket_dir) + if not isdir(join_path(self.socket_dir, uuid)) ] return vms diff --git a/doc/README-how-to-configure-remote-uncloud-clients.org b/doc/README-how-to-configure-remote-uncloud-clients.org new file mode 100644 index 0000000..b48886b --- /dev/null +++ b/doc/README-how-to-configure-remote-uncloud-clients.org @@ -0,0 +1,28 @@ +* What is a remote uncloud client? +** Systems that configure themselves for the use with uncloud +** Examples are VMHosts, VPN Servers, cdist control server, etc. +* Which access do these clients need? +** They need read / write access to the database +* Possible methods +** Overview +| | pros | cons | +| SSL based | Once setup, can access all django parts natively, locally | X.509 infrastructure | +| SSH -L tunnel | All nodes can use [::1]:5432 | SSH setup can be fragile | +| ssh djangohost manage.py | All DB ops locally | Code is only executed on django host | +| https + token | Rest alike / consistent access | Code is only executed on django host | +| from_django | Everything is on the django host | main host can become bottleneck | +** remote vs. local Django code execution + - If manage.py is executed locally (= on the client), it can + check/modify local configs + - However local execution requires a pyvenv + packages + db access + - Local execution also *could* make use of postgresql notify for + triggering actions (which is quite neat) + - Remote execution (= on the primary django host) can acess the db + via unix socket + - However remote execution cannot check local state +** from_django + - might reuse existing methods like celery + - reduces the amount of things to be installed on the client to + almost zero + - follows the opennebula model + - has a single point of failurebin diff --git a/doc/README-identifiers.org b/doc/README-identifiers.org new file mode 100644 index 0000000..3dbb4b5 --- /dev/null +++ b/doc/README-identifiers.org @@ -0,0 +1,29 @@ +* Identifiers +** Problem description + Identifiers can be integers, strings or other objects. They should + be unique. +** Approach 1: integers + Integers are somewhat easy to remember, but also include + predictable growth, which might allow access to guessed hacking + (obivously proper permissions should prevent this). +** Approach 2: random uuids + UUIDs are 128 bit integers. Python supports uuid.uuid4() for random + uuids. +** Approach 3: IPv6 addresses + uncloud heavily depends on IPv6 in the first place. uncloud could + use a /48 to identify all objects. Objects that have IPv6 addresses + on their own, don't need to draw from the system /48. +*** Possible Subnetworks + Assuming uncloud uses a /48 to represent all resources. + + | Network | Name | Description | + |-----------------+-----------------+----------------------------------------------| + | 2001:db8::/48 | uncloud network | All identifiers drawn from here | + | 2001:db8:1::/64 | VM network | Every VM has an IPv6 address in this network | + | 2001:db8:2::/64 | Bill network | Every bill has an IPv6 address | + | 2001:db8:3::/64 | Order network | Every order has an IPv6 address | + | 2001:db8:5::/64 | Product network | Every product (?) has an IPv6 address | + | 2001:db8:4::/64 | Disk network | Every disk is identified | + +*** Tests + [15:47:37] black3.place6:~# rbd create -s 10G ssd/2a0a:e5c0:1::8 diff --git a/doc/README-object-relations.md b/doc/README-object-relations.md new file mode 100644 index 0000000..58f2413 --- /dev/null +++ b/doc/README-object-relations.md @@ -0,0 +1,82 @@ +## Introduction + +This article describes how models relate to each other and what the +design ideas are. It is meant to prevent us from double implementing +something or changing something that is already solved. + + +## Products + +A product is something someone can order. We might have "low level" +products that need to be composed (= higher degree of flexibility, but +more amount of details necessary) and "composed products" that present +some defaults or select other products automatically (f.i. a "dual +stack VM" can be a VM + a disk + an IPv4 address). + + +## Bills + +Bills represent active orders of a month. Bills can be shown during a +month but only become definitive at the end of the month. + +## Orders + +When customer X order a (set) of product, it generates an order for billing +purposes. The ordered products point to that order and register an Order Record +at creation. + +Orders and Order Records are assumed immutable => they are used to generate +bills and should not be mutated. If a product is updated (e.g. adding RAM to +VM), a new order should be generated. + +The order MUST NOT be deleted when a product is deleted, as it is used for +billing (including past bills). + +### Order record + +Used to store billing details of a product at creation: will stay there even if +the product change (e.g. new pricing, updated) and act as some kind of archive. +Used to generate bills. + +## Payment Methods + +Users/customers can register payment methods. + +## Sample flows / products + +### A VM snapshot + +A VM snapshot creates a snapshot of all disks attached to a VM to be +able to rollback the VM to a previous state. + +Creating a VM snapshot (-product) creates a related order. Deleting a +VMSnapshotproduct sets the order to deleted. + +### Object Storage + +(tbd by Balazs) + +### A "raw" VM + +(tbd by Ahmed) + +### An IPv6 only VM + +(tbd by Ahmed) + +### A dual stack VM + +(tbd by Ahmed) + +### A managed service (e.g. Matrix-as-a-Service) + +Customer orders service with: + * Service-specific configuration: e.g. domain name for matrix + * VM configuration: + - CPU + - Memory + - Disk (soon) + +It creates a new Order with two products/records: + * Service itself (= management) + * Underlying VM diff --git a/doc/README-postgresql.org b/doc/README-postgresql.org new file mode 100644 index 0000000..9e5cc10 --- /dev/null +++ b/doc/README-postgresql.org @@ -0,0 +1,8 @@ +* uncloud clients access the data base from a variety of outside hosts +* So the postgresql data base needs to be remotely accessible +* Instead of exposing the tcp socket, we make postgresql bind to localhost via IPv6 +** ::1, port 5432 +* Then we remotely connect to the database server with ssh tunneling +** ssh -L5432:localhost:5432 uncloud-database-host +* Configuring your database for SSH based remote access +** host all all ::1/128 trust diff --git a/doc/README-products.md b/doc/README-products.md new file mode 100644 index 0000000..1b1190d --- /dev/null +++ b/doc/README-products.md @@ -0,0 +1,34 @@ +## Introduction + +This document describes how to create, modify or +delete a product and use it. + +A product (like a VMSnapshotproduct) creates an order when ordered. +The "order" is used to combine products together. + +Sub-products or related products link to the same order. +Each product has one (?) orderrecord + + +## How to delete a product (logic 1) + +If a user want so delete (=cancel) a product, the following steps +should be taken: + +* the associated order is set to cancelled +* the product itself is deleted + +[above steps to be reviewed] + +## How to delete a product (rest api) + +http -a nicoschottelius:$(pass +ungleich.ch/nico.schottelius@ungleich.ch) +http://localhost:8000/net/vpn/43c83088-f4d6-49b9-86c7-40251ac07ada/ + +-> does not delete the reservation. + + +### Deleting a VPN + +When the product is deleted, the network must be marked as free. diff --git a/doc/README-vpn.org b/doc/README-vpn.org new file mode 100644 index 0000000..7d041cb --- /dev/null +++ b/doc/README-vpn.org @@ -0,0 +1,34 @@ +* How to add a new VPN Host +** Install wireguard to the host +** Install uncloud to the host +** Add `python manage.py vpn --hostname fqdn-of-this-host` to the crontab +** Use the CLI to configure one or more VPN Networks for this host +* Example of adding a VPN host at ungleich +** Create a new dual stack alpine VM +** Add it to DNS as vpn-XXX.ungleich.ch +** Route a /40 network to its IPv6 address +** Install wireguard on it +** TODO Enable wireguard on boot +** TODO Create a new VPNPool on uncloud with +*** the network address (selecting from our existing pool) +*** the network size (/...) +*** the vpn host that provides the network (selecting the created VM) +*** the wireguard private key of the vpn host (using wg genkey) +*** http command +``` +http -a nicoschottelius:$(pass + ungleich.ch/nico.schottelius@ungleich.ch) + http://localhost:8000/admin/vpnpool/ network=2a0a:e5c1:200:: \ + network_size=40 subnetwork_size=48 + vpn_hostname=vpn-2a0ae5c1200.ungleich.ch + wireguard_private_key=... +``` +* Example http commands / REST calls +** creating a new vpn pool + http -a nicoschottelius:$(pass + ungleich.ch/nico.schottelius@ungleich.ch) + http://localhost:8000/admin/vpnpool/ network_size=40 + subnetwork_size=48 network=2a0a:e5c1:200:: + vpn_hostname=vpn-2a0ae5c1200.ungleich.ch wireguard_private_key=$(wg + genkey) +** Creating a new vpn network diff --git a/doc/README.md b/doc/README.md new file mode 100644 index 0000000..3bded7e --- /dev/null +++ b/doc/README.md @@ -0,0 +1,135 @@ +## Install + +### OS package requirements + +Alpine: + +``` +apk add openldap-dev postgresql-dev +``` + +Debian/Devuan: + +``` +apt install postgresql-server-dev-all +``` + + +### Python requirements + +If you prefer using a venv, use: + +``` +python -m venv venv +. ./venv/bin/activate +``` + +Then install the requirements + +``` +pip install -r requirements.txt +``` + +### Database requirements + +Due to the use of the JSONField, postgresql is required. + +First create a role to be used: + +``` +postgres=# create role nico login; +``` + +Then create the database owner by the new role: + +``` +postgres=# create database uncloud owner nico; +``` + +Installing the postgresql service is os dependent, but some hints: + +* Alpine: `apk add postgresql-server && rc-update add postgresql && rc-service postgresql start` +* Debian/Devuan: `apt install postgresql` + +After postresql is started, apply the migrations: + +``` +python manage.py migrate +``` + +## Flows / Orders + + +### Creating a VMHost + +### Creating a VM + +* Create a VMHost +* Create a VM on a VMHost + + +### Creating a VM Snapshot + + +## Working Beta APIs + +These APIs can be used for internal testing. + +### URL Overview + +``` +http -a nicoschottelius:$(pass ungleich.ch/nico.schottelius@ungleich.ch) http://localhost:8000 +``` + +### Snapshotting + +``` +http -a nicoschottelius:$(pass ungleich.ch/nico.schottelius@ungleich.ch) http://localhost:8000/vm/snapshot/ vm_uuid=$(uuidgen) +``` + +## VPNs + +VPNs consist of VPNPools ("networks of networks") which are handled by +VPNHosts. Users can requests VPNs with specific sizes. + +VPNs support both IPv6 and IPv4. However only IPv6 support has not been + +### Managing VPNPools + +``` +http -a nico:$(pass ldap/nico) https://uncloud.place7.ungleich.ch/v1/admin/vpnpool/ network=2a0a:e5c1:200:: network_size=40 subnetwork_size=48 vpn_hostname=vpn-2a0ae5c1200.ungleich.ch wireguard_private_key=$(wg genkey) +``` + +This will create the VPNPool 2a0a:e5c1:200::/40 from which /48 +networks will be used for clients. + +VPNPools can only be managed by staff. + +### Managing VPNNetworks + + +To request a network as a client, use the following call: + +``` +http -a nicoschottelius:$(pass ungleich.ch/nico.schottelius@ungleich.ch)https://uncloud.place7.ungleich.ch/v1/net/vpn/ network_size=48 wireguard_public_key=$(wg genkey | tee privatekey | wg pubkey) +``` + +VPNNetworks can be managed by all authenticated users. + + +## Proposed (uncoded) flows + +### Changing the disk size of a VM + +* GET on ../vm/vm/ should list uuids of disks +* UPDATE on ../vm/disk/ with size=newsize + * Newsize > oldsize! +* Triggers shutdown of VM +* Resizes disk +* Starts VM +* Maybe confirm flag? + + +### Adding a disk to a VM + +(TBD) diff --git a/manage.py b/manage.py new file mode 100755 index 0000000..b050590 --- /dev/null +++ b/manage.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" +import os +import sys + + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + try: + from django.core.management import execute_from_command_line + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc + execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/models.dot b/models.dot new file mode 100644 index 0000000..0adfba8 --- /dev/null +++ b/models.dot @@ -0,0 +1,1482 @@ +digraph model_graph { + // Dotfile by Django-Extensions graph_models + // Created: 2020-03-17 12:30 + // Cli Options: -a + + fontname = "Roboto" + fontsize = 8 + splines = true + + node [ + fontname = "Roboto" + fontsize = 8 + shape = "plaintext" + ] + + edge [ + fontname = "Roboto" + fontsize = 8 + ] + + // Labels + + + django_contrib_admin_models_LogEntry [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + LogEntry +
+ id + + AutoField +
+ content_type + + ForeignKey (id) +
+ user + + ForeignKey (id) +
+ action_flag + + PositiveSmallIntegerField +
+ action_time + + DateTimeField +
+ change_message + + TextField +
+ object_id + + TextField +
+ object_repr + + CharField +
+ >] + + + + + django_contrib_auth_models_Permission [label=< + + + + + + + + + + + + + + + + + + + +
+ + Permission +
+ id + + AutoField +
+ content_type + + ForeignKey (id) +
+ codename + + CharField +
+ name + + CharField +
+ >] + + django_contrib_auth_models_Group [label=< + + + + + + + + + + + +
+ + Group +
+ id + + AutoField +
+ name + + CharField +
+ >] + + + + + django_contrib_contenttypes_models_ContentType [label=< + + + + + + + + + + + + + + + +
+ + ContentType +
+ id + + AutoField +
+ app_label + + CharField +
+ model + + CharField +
+ >] + + + + + django_contrib_sessions_base_session_AbstractBaseSession [label=< + + + + + + + + + + + +
+ + AbstractBaseSession +
+ expire_date + + DateTimeField +
+ session_data + + TextField +
+ >] + + django_contrib_sessions_models_Session [label=< + + + + + + + + + + + + + + + +
+ + Session
<AbstractBaseSession> +
+ session_key + + CharField +
+ expire_date + + DateTimeField +
+ session_data + + TextField +
+ >] + + + + + uncloud_pay_models_StripeCustomer [label=< + + + + + + + + + + + +
+ + StripeCustomer +
+ owner + + OneToOneField (id) +
+ stripe_id + + CharField +
+ >] + + uncloud_pay_models_Payment [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + Payment +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ amount + + DecimalField +
+ source + + CharField +
+ timestamp + + DateTimeField +
+ >] + + uncloud_pay_models_PaymentMethod [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + PaymentMethod +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ description + + TextField +
+ primary + + BooleanField +
+ source + + CharField +
+ stripe_card_id + + CharField +
+ >] + + uncloud_pay_models_Bill [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Bill +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ creation_date + + DateTimeField +
+ due_date + + DateField +
+ ending_date + + DateTimeField +
+ starting_date + + DateTimeField +
+ valid + + BooleanField +
+ >] + + uncloud_pay_models_Order [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + Order +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ creation_date + + DateTimeField +
+ ending_date + + DateTimeField +
+ recurring_period + + CharField +
+ starting_date + + DateTimeField +
+ >] + + uncloud_pay_models_OrderRecord [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + OrderRecord +
+ id + + AutoField +
+ order + + ForeignKey (uuid) +
+ description + + TextField +
+ one_time_price + + DecimalField +
+ recurring_price + + DecimalField +
+ >] + + + + + django_contrib_auth_models_AbstractUser [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + AbstractUser
<AbstractBaseUser,PermissionsMixin> +
+ date_joined + + DateTimeField +
+ email + + EmailField +
+ first_name + + CharField +
+ is_active + + BooleanField +
+ is_staff + + BooleanField +
+ is_superuser + + BooleanField +
+ last_login + + DateTimeField +
+ last_name + + CharField +
+ password + + CharField +
+ username + + CharField +
+ >] + + uncloud_auth_models_User [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + User
<AbstractUser> +
+ id + + AutoField +
+ date_joined + + DateTimeField +
+ email + + EmailField +
+ first_name + + CharField +
+ is_active + + BooleanField +
+ is_staff + + BooleanField +
+ is_superuser + + BooleanField +
+ last_login + + DateTimeField +
+ last_name + + CharField +
+ password + + CharField +
+ username + + CharField +
+ >] + + + + + uncloud_pay_models_Product [label=< + + + + + + + + + + + + + + + +
+ + Product +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ status + + CharField +
+ >] + + uncloud_vm_models_VMHost [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMHost +
+ uuid + + UUIDField +
+ hostname + + CharField +
+ physical_cores + + IntegerField +
+ status + + CharField +
+ usable_cores + + IntegerField +
+ usable_ram_in_gb + + FloatField +
+ >] + + uncloud_vm_models_VMProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vmhost + + ForeignKey (uuid) +
+ cores + + IntegerField +
+ name + + CharField +
+ ram_in_gb + + FloatField +
+ status + + CharField +
+ vmid + + IntegerField +
+ >] + + uncloud_vm_models_VMWithOSProduct [label=< + + + + + + + +
+ + VMWithOSProduct +
+ vmproduct_ptr + + OneToOneField (uuid) +
+ >] + + uncloud_vm_models_VMDiskImageProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMDiskImageProduct +
+ uuid + + UUIDField +
+ owner + + ForeignKey (id) +
+ image_source + + CharField +
+ image_source_type + + CharField +
+ import_url + + URLField +
+ is_os_image + + BooleanField +
+ is_public + + BooleanField +
+ name + + CharField +
+ size_in_gb + + FloatField +
+ status + + CharField +
+ storage_class + + CharField +
+ >] + + uncloud_vm_models_VMDiskProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMDiskProduct +
+ uuid + + UUIDField +
+ image + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ size_in_gb + + FloatField +
+ >] + + uncloud_vm_models_VMNetworkCard [label=< + + + + + + + + + + + + + + + + + + + +
+ + VMNetworkCard +
+ id + + AutoField +
+ vm + + ForeignKey (uuid) +
+ ip_address + + GenericIPAddressField +
+ mac_address + + BigIntegerField +
+ >] + + uncloud_vm_models_VMSnapshotProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + VMSnapshotProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ gb_hdd + + FloatField +
+ gb_ssd + + FloatField +
+ status + + CharField +
+ >] + + + + + uncloud_pay_models_Product [label=< + + + + + + + + + + + + + + + +
+ + Product +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ status + + CharField +
+ >] + + ungleich_service_models_MatrixServiceProduct [label=< + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ + MatrixServiceProduct
<Product> +
+ uuid + + UUIDField +
+ order + + ForeignKey (uuid) +
+ owner + + ForeignKey (id) +
+ vm + + ForeignKey (uuid) +
+ domain + + CharField +
+ status + + CharField +
+ >] + + + + + opennebula_models_VM [label=< + + + + + + + + + + + + + + + +
+ + VM +
+ vmid + + IntegerField +
+ owner + + ForeignKey (id) +
+ data + + JSONField +
+ >] + + + + + // Relations + + django_contrib_admin_models_LogEntry -> uncloud_auth_models_User + [label=" user (logentry)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_admin_models_LogEntry -> django_contrib_contenttypes_models_ContentType + [label=" content_type (logentry)"] [arrowhead=none, arrowtail=dot, dir=both]; + + + django_contrib_auth_models_Permission -> django_contrib_contenttypes_models_ContentType + [label=" content_type (permission)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_auth_models_Group -> django_contrib_auth_models_Permission + [label=" permissions (group)"] [arrowhead=dot arrowtail=dot, dir=both]; + + + + django_contrib_sessions_models_Session -> django_contrib_sessions_base_session_AbstractBaseSession + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_StripeCustomer -> uncloud_auth_models_User + [label=" owner (stripecustomer)"] [arrowhead=none, arrowtail=none, dir=both]; + + uncloud_pay_models_Payment -> uncloud_auth_models_User + [label=" owner (payment)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_PaymentMethod -> uncloud_auth_models_User + [label=" owner (paymentmethod)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Bill -> uncloud_auth_models_User + [label=" owner (bill)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Order -> uncloud_auth_models_User + [label=" owner (order)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Order -> uncloud_pay_models_Bill + [label=" bill (order)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_pay_models_OrderRecord -> uncloud_pay_models_Order + [label=" order (orderrecord)"] [arrowhead=none, arrowtail=dot, dir=both]; + + django_contrib_auth_base_user_AbstractBaseUser [label=< + + +
+ AbstractBaseUser +
+ >] + django_contrib_auth_models_AbstractUser -> django_contrib_auth_base_user_AbstractBaseUser + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + django_contrib_auth_models_PermissionsMixin [label=< + + +
+ PermissionsMixin +
+ >] + django_contrib_auth_models_AbstractUser -> django_contrib_auth_models_PermissionsMixin + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_Group + [label=" groups (user)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_Permission + [label=" user_permissions (user)"] [arrowhead=dot arrowtail=dot, dir=both]; + + uncloud_auth_models_User -> django_contrib_auth_models_AbstractUser + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_Product -> uncloud_auth_models_User + [label=" owner (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Product -> uncloud_pay_models_Order + [label=" order (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMProduct -> uncloud_vm_models_VMHost + [label=" vmhost (vmproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_vm_models_VMWithOSProduct -> uncloud_vm_models_VMProduct + [label=" multi-table\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + uncloud_vm_models_VMDiskImageProduct -> uncloud_auth_models_User + [label=" owner (vmdiskimageproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_auth_models_User + [label=" owner (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_vm_models_VMProduct + [label=" vm (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMDiskProduct -> uncloud_vm_models_VMDiskImageProduct + [label=" image (vmdiskproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMNetworkCard -> uncloud_vm_models_VMProduct + [label=" vm (vmnetworkcard)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMSnapshotProduct -> uncloud_vm_models_VMProduct + [label=" vm (vmsnapshotproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_vm_models_VMSnapshotProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + uncloud_pay_models_Product -> uncloud_auth_models_User + [label=" owner (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + uncloud_pay_models_Product -> uncloud_pay_models_Order + [label=" order (product)"] [arrowhead=none, arrowtail=dot, dir=both]; + + ungleich_service_models_MatrixServiceProduct -> uncloud_vm_models_VMProduct + [label=" vm (matrixserviceproduct)"] [arrowhead=none, arrowtail=dot, dir=both]; + + ungleich_service_models_MatrixServiceProduct -> uncloud_pay_models_Product + [label=" abstract\ninheritance"] [arrowhead=empty, arrowtail=none, dir=both]; + + + opennebula_models_VM -> uncloud_auth_models_User + [label=" owner (vm)"] [arrowhead=none, arrowtail=dot, dir=both]; + + +} diff --git a/models.png b/models.png new file mode 100644 index 0000000..f9d0c2e Binary files /dev/null and b/models.png differ diff --git a/opennebula/__init__.py b/opennebula/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/opennebula/admin.py b/opennebula/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/opennebula/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/opennebula/apps.py b/opennebula/apps.py new file mode 100644 index 0000000..0750576 --- /dev/null +++ b/opennebula/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class OpennebulaConfig(AppConfig): + name = 'opennebula' diff --git a/opennebula/management/commands/opennebula-synchosts.py b/opennebula/management/commands/opennebula-synchosts.py new file mode 100644 index 0000000..29f9ac1 --- /dev/null +++ b/opennebula/management/commands/opennebula-synchosts.py @@ -0,0 +1,74 @@ +import json + +import uncloud.secrets as secrets + +from xmlrpc.client import ServerProxy as RPCClient + +from django.core.management.base import BaseCommand +from xmltodict import parse +from enum import IntEnum +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost +from django_auth_ldap.backend import LDAPBackend + + +class HostStates(IntEnum): + """ + The following flags are copied from + https://docs.opennebula.org/5.8/integration/system_interfaces/api.html#schemas-for-host + """ + INIT = 0 # Initial state for enabled hosts + MONITORING_MONITORED = 1 # Monitoring the host (from monitored) + MONITORED = 2 # The host has been successfully monitored + ERROR = 3 # An error ocurrer while monitoring the host + DISABLED = 4 # The host is disabled + MONITORING_ERROR = 5 # Monitoring the host (from error) + MONITORING_INIT = 6 # Monitoring the host (from init) + MONITORING_DISABLED = 7 # Monitoring the host (from disabled) + OFFLINE = 8 # The host is totally offline + + +class Command(BaseCommand): + help = 'Syncronize Host information from OpenNebula' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + with RPCClient(secrets.OPENNEBULA_URL) as rpc_client: + success, response, *_ = rpc_client.one.hostpool.info(secrets.OPENNEBULA_USER_PASS) + if success: + response = json.loads(json.dumps(parse(response))) + host_pool = response.get('HOST_POOL', {}).get('HOST', {}) + for host in host_pool: + host_share = host.get('HOST_SHARE', {}) + + host_name = host.get('NAME') + state = int(host.get('STATE', HostStates.OFFLINE.value)) + + if state == HostStates.MONITORED: + status = 'active' + elif state == HostStates.DISABLED: + status = 'disabled' + else: + status = 'unusable' + + usable_cores = host_share.get('TOTAL_CPU') + usable_ram_in_kb = int(host_share.get('TOTAL_MEM', 0)) + usable_ram_in_gb = int(usable_ram_in_kb / 2 ** 20) + + # vms cannot be created like this -- Nico, 2020-03-17 + # vms = host.get('VMS', {}) or {} + # vms = vms.get('ID', []) or [] + # vms = ','.join(vms) + + VMHost.objects.update_or_create( + hostname=host_name, + defaults={ + 'usable_cores': usable_cores, + 'usable_ram_in_gb': usable_ram_in_gb, + 'status': status + } + ) + else: + print(response) diff --git a/opennebula/management/commands/opennebula-syncvms.py b/opennebula/management/commands/opennebula-syncvms.py new file mode 100644 index 0000000..458528b --- /dev/null +++ b/opennebula/management/commands/opennebula-syncvms.py @@ -0,0 +1,47 @@ +import json + +import uncloud.secrets as secrets + + +from xmlrpc.client import ServerProxy as RPCClient + +from django_auth_ldap.backend import LDAPBackend +from django.core.management.base import BaseCommand +from xmltodict import parse + +from opennebula.models import VM as VMModel + + +class Command(BaseCommand): + help = 'Syncronize VM information from OpenNebula' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + with RPCClient(secrets.OPENNEBULA_URL) as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + secrets.OPENNEBULA_USER_PASS, -2, -1, -1, -1 + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + unknown_user = set() + + backend = LDAPBackend() + + for vm in vms: + vm_id = vm['ID'] + vm_owner = vm['UNAME'] + + user = backend.populate_user(username=vm_owner) + + if not user: + unknown_user.add(vm_owner) + else: + VMModel.objects.update_or_create( + vmid=vm_id, + defaults={'data': vm, 'owner': user} + ) + print('User not found in ldap:', unknown_user) + else: + print(response) diff --git a/opennebula/management/commands/opennebula-to-uncloud.py b/opennebula/management/commands/opennebula-to-uncloud.py new file mode 100644 index 0000000..230159a --- /dev/null +++ b/opennebula/management/commands/opennebula-to-uncloud.py @@ -0,0 +1,193 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand +from django.utils import timezone +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct + +from uncloud_pay.models import Order + +import logging + +log = logging.getLogger(__name__) + +def convert_mac_to_int(mac_address: str): + # Remove octet connecting characters + mac_address = mac_address.replace(':', '') + mac_address = mac_address.replace('.', '') + mac_address = mac_address.replace('-', '') + mac_address = mac_address.replace(' ', '') + + # Parse the resulting number as hexadecimal + mac_address = int(mac_address, base=16) + + return mac_address + + +def get_vm_price(core, ram, ssd_size, hdd_size, n_of_ipv4, n_of_ipv6): + total = 3 * core + 4 * ram + (3.5 * ssd_size/10.) + (1.5 * hdd_size/100.) + 8 * n_of_ipv4 + 0 * n_of_ipv6 + + # TODO: Find some reason about the following magical subtraction. + total -= 8 + + return total + + +def create_nics(one_vm, vm_product): + for nic in one_vm.nics: + mac_address = convert_mac_to_int(nic.get('MAC')) + ip_address = nic.get('IP', None) or nic.get('IP6_GLOBAL', None) + + VMNetworkCard.objects.update_or_create( + mac_address=mac_address, vm=vm_product, defaults={'ip_address': ip_address} + ) + + +def sync_disk_and_image(one_vm, vm_product, disk_owner): + """ + a) Check all opennebula disk if they are in the uncloud VM, if not add + b) Check all uncloud disks and remove them if they are not in the opennebula VM + """ + + vmdisknum = 0 + + one_disks_extra_data = [] + + for disk in one_vm.disks: + vmowner = one_vm.owner + name = disk.get('image') + vmdisknum += 1 + + log.info("Checking disk {} for VM {}".format(name, one_vm)) + + is_os_image, is_public, status = True, False, 'active' + + image_size_in_gb = disk.get('image_size_in_gb') + disk_size_in_gb = disk.get('size_in_gb') + storage_class = disk.get('storage_class') + image_source = disk.get('source') + image_source_type = disk.get('source_type') + + image, _ = VMDiskImageProduct.objects.update_or_create( + name=name, + defaults={ + 'owner': disk_owner, + 'is_os_image': is_os_image, + 'is_public': is_public, + 'size_in_gb': image_size_in_gb, + 'storage_class': storage_class, + 'image_source': image_source, + 'image_source_type': image_source_type, + 'status': status + } + ) + + # identify vmdisk from opennebula - primary mapping key + extra_data = { + 'opennebula_vm': one_vm.vmid, + 'opennebula_size_in_gb': disk_size_in_gb, + 'opennebula_source': disk.get('opennebula_source'), + 'opennebula_disk_num': vmdisknum + } + # Save for comparing later + one_disks_extra_data.append(extra_data) + + try: + vm_disk = VMDiskProduct.objects.get(extra_data=extra_data) + except VMDiskProduct.DoesNotExist: + vm_disk = VMDiskProduct.objects.create( + owner=vmowner, + vm=vm_product, + image=image, + size_in_gb=disk_size_in_gb, + extra_data=extra_data + ) + + # Now remove all disks that are not in above extra_data list + for disk in VMDiskProduct.objects.filter(vm=vm_product): + extra_data = disk.extra_data + if not extra_data in one_disks_extra_data: + log.info("Removing disk {} from VM {}".format(disk, vm_product)) + disk.delete() + + disks = [ disk.extra_data for disk in VMDiskProduct.objects.filter(vm=vm_product) ] + log.info("VM {} has disks: {}".format(vm_product, disks)) + +class Command(BaseCommand): + help = 'Migrate Opennebula VM to regular (uncloud) vm' + + def add_arguments(self, parser): + parser.add_argument('--disk-owner', required=True, help="The user who owns the the opennebula disks") + + def handle(self, *args, **options): + log.debug("{} {}".format(args, options)) + + disk_owner = get_user_model().objects.get(username=options['disk_owner']) + + for one_vm in VMModel.objects.all(): + + if not one_vm.last_host: + log.warning("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid)) + continue + + try: + vmhost = VMHost.objects.get(hostname=one_vm.last_host) + except VMHost.DoesNotExist: + log.error("VMHost {} does not exist, aborting".format(one_vm.last_host)) + raise + + cores = one_vm.cores + ram_in_gb = one_vm.ram_in_gb + owner = one_vm.owner + status = 'active' + + ssd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['ssd', 'one'] ]) + hdd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['hdd'] ]) + + # List of IPv4 addresses and Global IPv6 addresses + ipv4, ipv6 = one_vm.ips + + # TODO: Insert actual/real creation_date, starting_date, ending_date + # instead of pseudo one we are putting currently + creation_date = starting_date = datetime.now(tz=timezone.utc) + + # Price calculation based on datacenterlight.ch + one_time_price = 0 + recurring_period = 'per_month' + recurring_price = get_vm_price(cores, ram_in_gb, + ssd_size, hdd_size, + len(ipv4), len(ipv6)) + + try: + vm_product = VMProduct.objects.get(extra_data__opennebula_id=one_vm.vmid) + except VMProduct.DoesNotExist: + order = Order.objects.create( + owner=owner, + creation_date=creation_date, + starting_date=starting_date + ) + vm_product = VMProduct( + extra_data={ 'opennebula_id': one_vm.vmid }, + name=one_vm.uncloud_name, + order=order + ) + + # we don't use update_or_create, as filtering by json AND setting json + # at the same time does not work + + vm_product.vmhost = vmhost + vm_product.owner = owner + vm_product.cores = cores + vm_product.ram_in_gb = ram_in_gb + vm_product.status = status + + vm_product.save() + + # Create VMNetworkCards + create_nics(one_vm, vm_product) + + # Create VMDiskImageProduct and VMDiskProduct + sync_disk_and_image(one_vm, vm_product, disk_owner=disk_owner) diff --git a/opennebula/migrations/0001_initial.py b/opennebula/migrations/0001_initial.py new file mode 100644 index 0000000..4c0527a --- /dev/null +++ b/opennebula/migrations/0001_initial.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.3 on 2020-02-23 17:12 + +from django.conf import settings +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='VM', + fields=[ + ('vmid', models.IntegerField(primary_key=True, serialize=False)), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, unique=True)), + ('data', django.contrib.postgres.fields.jsonb.JSONField()), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/opennebula/migrations/0002_auto_20200225_1335.py b/opennebula/migrations/0002_auto_20200225_1335.py new file mode 100644 index 0000000..1554aa6 --- /dev/null +++ b/opennebula/migrations/0002_auto_20200225_1335.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.3 on 2020-02-25 13:35 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('opennebula', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='vm', + name='uuid', + ), + migrations.RemoveField( + model_name='vm', + name='vmid', + ), + migrations.AddField( + model_name='vm', + name='id', + field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True), + ), + ] diff --git a/opennebula/migrations/0003_auto_20200225_1428.py b/opennebula/migrations/0003_auto_20200225_1428.py new file mode 100644 index 0000000..8bb3d8d --- /dev/null +++ b/opennebula/migrations/0003_auto_20200225_1428.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-02-25 14:28 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('opennebula', '0002_auto_20200225_1335'), + ] + + operations = [ + migrations.AlterField( + model_name='vm', + name='id', + field=models.CharField(default=uuid.uuid4, max_length=64, primary_key=True, serialize=False, unique=True), + ), + ] diff --git a/opennebula/migrations/0004_auto_20200225_1816.py b/opennebula/migrations/0004_auto_20200225_1816.py new file mode 100644 index 0000000..5b39f26 --- /dev/null +++ b/opennebula/migrations/0004_auto_20200225_1816.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-02-25 18:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('opennebula', '0003_auto_20200225_1428'), + ] + + operations = [ + migrations.RemoveField( + model_name='vm', + name='id', + ), + migrations.AddField( + model_name='vm', + name='vmid', + field=models.IntegerField(default=42, primary_key=True, serialize=False), + preserve_default=False, + ), + ] diff --git a/opennebula/migrations/__init__.py b/opennebula/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/opennebula/models.py b/opennebula/models.py new file mode 100644 index 0000000..826b615 --- /dev/null +++ b/opennebula/models.py @@ -0,0 +1,91 @@ +import uuid +from django.db import models +from django.contrib.auth import get_user_model +from django.contrib.postgres.fields import JSONField + +# ungleich specific +storage_class_mapping = { + 'one': 'ssd', + 'ssd': 'ssd', + 'hdd': 'hdd' +} + +class VM(models.Model): + vmid = models.IntegerField(primary_key=True) + owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + data = JSONField() + + @property + def uncloud_name(self): + return "opennebula-{}".format(self.vmid) + + @property + def cores(self): + return int(self.data['TEMPLATE']['VCPU']) + + @property + def ram_in_gb(self): + return int(self.data['TEMPLATE']['MEMORY'])/1024 + + @property + def disks(self): + """ + If there is no disk then the key DISK does not exist. + + If there is only one disk, we have a dictionary in the database. + + If there are multiple disks, we have a list of dictionaries in the database. + """ + + disks = [] + + if 'DISK' in self.data['TEMPLATE']: + if type(self.data['TEMPLATE']['DISK']) is dict: + disks = [self.data['TEMPLATE']['DISK']] + else: + disks = self.data['TEMPLATE']['DISK'] + + disks = [ + { + 'size_in_gb': int(d['SIZE'])/1024, + 'opennebula_source': d['SOURCE'], + 'opennebula_name': d['IMAGE'], + 'image_size_in_gb': int(d['ORIGINAL_SIZE'])/1024, + 'pool_name': d['POOL_NAME'], + 'image': d['IMAGE'], + 'source': d['SOURCE'], + 'source_type': d['TM_MAD'], + 'storage_class': storage_class_mapping[d['POOL_NAME']] + + } + for d in disks + ] + + return disks + + @property + def last_host(self): + return ((self.data.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None) + + @property + def graphics(self): + return self.data.get('TEMPLATE', {}).get('GRAPHICS', {}) + + @property + def nics(self): + _nics = self.data.get('TEMPLATE', {}).get('NIC', {}) + if isinstance(_nics, dict): + _nics = [_nics] + return _nics + + @property + def ips(self): + ipv4, ipv6 = [], [] + for nic in self.nics: + ip = nic.get('IP') + ip6 = nic.get('IP6_GLOBAL') + if ip: + ipv4.append(ip) + if ip6: + ipv6.append(ip6) + return ipv4, ipv6 diff --git a/opennebula/serializers.py b/opennebula/serializers.py new file mode 100644 index 0000000..cd00622 --- /dev/null +++ b/opennebula/serializers.py @@ -0,0 +1,10 @@ +from rest_framework import serializers +from opennebula.models import VM + + +class OpenNebulaVMSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = VM + fields = [ 'vmid', 'owner', 'data', + 'uncloud_name', 'cores', 'ram_in_gb', + 'disks', 'nics', 'ips' ] diff --git a/opennebula/tests.py b/opennebula/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/opennebula/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/opennebula/views.py b/opennebula/views.py new file mode 100644 index 0000000..89b1a52 --- /dev/null +++ b/opennebula/views.py @@ -0,0 +1,16 @@ +from rest_framework import viewsets, permissions + +from .models import VM +from .serializers import OpenNebulaVMSerializer + +class VMViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = OpenNebulaVMSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VM.objects.all() + else: + obj = VM.objects.filter(owner=self.request.user) + + return obj diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a7fc9f2 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,24 @@ +django +djangorestframework +django-auth-ldap +stripe +xmltodict +psycopg2 + +parsedatetime + +# Follow are for creating graph models +pyparsing +pydot +django-extensions + +# PDF creating +django-hardcopy + +# schema support +pyyaml +uritemplate + +# Comprehensive interface to validate VAT numbers, making use of the VIES +# service for European countries. +vat-validator diff --git a/resources/ci/.lock b/resources/ci/.lock new file mode 100644 index 0000000..e69de29 diff --git a/resources/ci/Dockerfile b/resources/ci/Dockerfile new file mode 100644 index 0000000..020b66e --- /dev/null +++ b/resources/ci/Dockerfile @@ -0,0 +1,3 @@ +FROM fedora:latest + +RUN dnf install -y python3-devel python3-pip python3-coverage libpq-devel openldap-devel gcc chromium diff --git a/resources/vat-rates.csv b/resources/vat-rates.csv new file mode 100644 index 0000000..17bdb99 --- /dev/null +++ b/resources/vat-rates.csv @@ -0,0 +1,325 @@ +start_date,stop_date,territory_codes,currency_code,rate,rate_type,description +2011-01-04,,AI,XCD,0,standard,Anguilla (British overseas territory) is exempted of VAT. +1984-01-01,,AT,EUR,0.2,standard,Austria (member state) standard VAT rate. +1976-01-01,1984-01-01,AT,EUR,0.18,standard, +1973-01-01,1976-01-01,AT,EUR,0.16,standard, +1984-01-01,,"AT-6691 +DE-87491",EUR,0.19,standard,Jungholz (Austrian town) special VAT rate. +1984-01-01,,"AT-6991 +AT-6992 +AT-6993 +DE-87567 +DE-87568 +DE-87569",EUR,0.19,standard,Mittelberg (Austrian town) special VAT rate. +1996-01-01,,BE,EUR,0.21,standard,Belgium (member state) standard VAT rate. +1994-01-01,1996-01-01,BE,EUR,0.205,standard, +1992-04-01,1994-01-01,BE,EUR,0.195,standard, +1983-01-01,1992-04-01,BE,EUR,0.19,standard, +1981-07-01,1983-01-01,BE,EUR,0.17,standard, +1978-07-01,1981-07-01,BE,EUR,0.16,standard, +1971-07-01,1978-07-01,BE,EUR,0.18,standard, +1999-01-01,,BG,BGN,0.2,standard,Bulgaria (member state) standard VAT rate. +1996-07-01,1999-01-01,BG,BGN,0.22,standard, +1994-04-01,1996-07-01,BG,BGN,0.18,standard, +2011-01-04,,BM,BMD,0,standard,Bermuda (British overseas territory) is exempted of VAT. +2014-01-13,,"CY +GB-BFPO 57 +GB-BFPO 58 +GB-BFPO 59 +UK-BFPO 57 +UK-BFPO 58 +UK-BFPO 59",EUR,0.19,standard,"Cyprus (member state) standard VAT rate. +Akrotiri and Dhekelia (British overseas territory) is subjected to Cyprus' standard VAT rate." +2013-01-14,2014-01-13,CY,EUR,0.18,standard, +2012-03-01,2013-01-14,CY,EUR,0.17,standard, +2003-01-01,2012-03-01,CY,EUR,0.15,standard, +2002-07-01,2003-01-01,CY,EUR,0.13,standard, +2000-07-01,2002-07-01,CY,EUR,0.1,standard, +1993-10-01,2000-07-01,CY,EUR,0.08,standard, +1992-07-01,1993-10-01,CY,EUR,0.05,standard, +2013-01-01,,CZ,CZK,0.21,standard,Czech Republic (member state) standard VAT rate. +2010-01-01,2013-01-01,CZ,CZK,0.2,standard, +2004-05-01,2010-01-01,CZ,CZK,0.19,standard, +1995-01-01,2004-05-01,CZ,CZK,0.22,standard, +1993-01-01,1995-01-01,CZ,CZK,0.23,standard, +2007-01-01,,DE,EUR,0.19,standard,Germany (member state) standard VAT rate. +1998-04-01,2007-01-01,DE,EUR,0.16,standard, +1993-01-01,1998-04-01,DE,EUR,0.15,standard, +1983-07-01,1993-01-01,DE,EUR,0.14,standard, +1979-07-01,1983-07-01,DE,EUR,0.13,standard, +1978-01-01,1979-07-01,DE,EUR,0.12,standard, +1968-07-01,1978-01-01,DE,EUR,0.11,standard, +1968-01-01,1968-07-01,DE,EUR,0.1,standard, +2007-01-01,,DE-27498,EUR,0,standard,Heligoland (German island) is exempted of VAT. +2007-01-01,,"DE-78266 +CH-8238",EUR,0,standard,Busingen am Hochrhein (German territory) is exempted of VAT. +1992-01-01,,DK,DKK,0.25,standard,Denmark (member state) standard VAT rate. +1980-06-30,1992-01-01,DK,DKK,0.22,standard, +1978-10-30,1980-06-30,DK,DKK,0.2025,standard, +1977-10-03,1978-10-30,DK,DKK,0.18,standard, +1970-06-29,1977-10-03,DK,DKK,0.15,standard, +1968-04-01,1970-06-29,DK,DKK,0.125,standard, +1967-07-03,1968-04-01,DK,DKK,0.1,standard, +2009-07-01,,EE,EUR,0.2,standard,Estonia (member state) standard VAT rate. +1993-01-01,2009-07-01,EE,EUR,0.18,standard, +1991-01-01,1993-01-01,EE,EUR,0.1,standard, +2016-06-01,,"GR +EL",EUR,0.24,standard,Greece (member state) standard VAT rate. +2010-07-01,2016-06-01,"GR +EL",EUR,0.23,standard, +2010-03-15,2010-07-01,"GR +EL",EUR,0.21,standard, +2005-04-01,2010-03-15,"GR +EL",EUR,0.19,standard, +1990-04-28,2005-04-01,"GR +EL",EUR,0.18,standard, +1988-01-01,1990-04-28,"GR +EL",EUR,0.16,standard, +1987-01-01,1988-01-01,"GR +EL",EUR,0.18,standard, +2012-09-01,,ES,EUR,0.21,standard,Spain (member state) standard VAT rate. +2010-07-01,2012-09-01,ES,EUR,0.18,standard, +1995-01-01,2010-07-01,ES,EUR,0.16,standard, +1992-08-01,1995-01-01,ES,EUR,0.15,standard, +1992-01-01,1992-08-01,ES,EUR,0.13,standard, +1986-01-01,1992-01-01,ES,EUR,0.12,standard, +2012-09-01,,"ES-CN +ES-GC +ES-TF +IC",EUR,0,standard,Canary Islands (Spanish autonomous community) is exempted of VAT. +2012-09-01,,"ES-ML +ES-CE +EA",EUR,0,standard,Ceuta and Melilla (Spanish autonomous cities) is exempted of VAT. +2013-01-01,,FI,EUR,0.24,standard,Finland (member state) standard VAT rate. +2010-07-01,2013-01-01,FI,EUR,0.23,standard, +1994-06-01,2010-07-01,FI,EUR,0.22,standard, +2013-01-01,,"FI-01 +AX",EUR,0,standard,Aland Islands (Finish autonomous region) is exempted of VAT. +2011-01-04,,FK,FKP,0,standard,Falkland Islands (British overseas territory) is exempted of VAT. +1992-01-01,,FO,DKK,0,standard,Faroe Islands (Danish autonomous country) is exempted of VAT. +2014-01-01,,"FR +MC",EUR,0.2,standard,"France (member state) standard VAT rate. +Monaco (sovereign city-state) is member of the EU VAT area and subjected to France's standard VAT rate." +2000-04-01,2014-01-01,"FR +MC",EUR,0.196,standard, +1995-08-01,2000-04-01,"FR +MC",EUR,0.206,standard, +1982-07-01,1995-08-01,"FR +MC",EUR,0.186,standard, +1977-01-01,1982-07-01,"FR +MC",EUR,0.176,standard, +1973-01-01,1977-01-01,"FR +MC",EUR,0.2,standard, +1970-01-01,1973-01-01,"FR +MC",EUR,0.23,standard, +1968-12-01,1970-01-01,"FR +MC",EUR,0.19,standard, +1968-01-01,1968-12-01,"FR +MC",EUR,0.1666,standard, +2014-01-01,,"FR-BL +BL",EUR,0,standard,Saint Barthelemy (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-GF +GF",EUR,0,standard,Guiana (French overseas department) is exempted of VAT. +2014-01-01,,"FR-GP +GP",EUR,0.085,standard,Guadeloupe (French overseas department) special VAT rate. +2014-01-01,,"FR-MF +MF",EUR,0,standard,Saint Martin (French overseas collectivity) is subjected to France's standard VAT rate. +2014-01-01,,"FR-MQ +MQ",EUR,0.085,standard,Martinique (French overseas department) special VAT rate. +2014-01-01,,"FR-NC +NC",XPF,0,standard,New Caledonia (French special collectivity) is exempted of VAT. +2014-01-01,,"FR-PF +PF",XPF,0,standard,French Polynesia (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-PM +PM",EUR,0,standard,Saint Pierre and Miquelon (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-RE +RE",EUR,0.085,standard,Reunion (French overseas department) special VAT rate. +2014-01-01,,"FR-TF +TF",EUR,0,standard,French Southern and Antarctic Lands (French overseas territory) is exempted of VAT. +2014-01-01,,"FR-WF +WF",XPF,0,standard,Wallis and Futuna (French overseas collectivity) is exempted of VAT. +2014-01-01,,"FR-YT +YT",EUR,0,standard,Mayotte (French overseas department) is exempted of VAT. +2011-01-04,,GG,GBP,0,standard,Guernsey (British Crown dependency) is exempted of VAT. +2011-01-04,,GI,GIP,0,standard,Gibraltar (British overseas territory) is exempted of VAT. +1992-01-01,,GL,DKK,0,standard,Greenland (Danish autonomous country) is exempted of VAT. +2010-07-01,2016-06-01,"GR-34007 +EL-34007",EUR,0.16,standard,Skyros (Greek island) special VAT rate. +2010-07-01,2016-06-01,"GR-37002 +GR-37003 +GR-37005 +EL-37002 +EL-37003 +EL-37005",EUR,0.16,standard,Northern Sporades (Greek islands) special VAT rate. +2010-07-01,2016-06-01,"GR-64004 +EL-64004",EUR,0.16,standard,Thasos (Greek island) special VAT rate. +2010-07-01,2016-06-01,"GR-68002 +EL-68002",EUR,0.16,standard,Samothrace (Greek island) special VAT rate. +2010-07-01,,"GR-69 +EL-69",EUR,0,standard,Mount Athos (Greek self-governed part) is exempted of VAT. +2010-07-01,2016-06-01,"GR-81 +EL-81",EUR,0.16,standard,Dodecanese (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-82 +EL-82",EUR,0.16,standard,Cyclades (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-83 +EL-83",EUR,0.16,standard,Lesbos (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-84 +EL-84",EUR,0.16,standard,Samos (Greek department) special VAT rate. +2010-07-01,2016-06-01,"GR-85 +EL-85",EUR,0.16,standard,Chios (Greek department) special VAT rate. +2011-01-04,,GS,GBP,0,standard,South Georgia and the South Sandwich Islands (British overseas territory) is exempted of VAT. +2012-03-01,,HR,HRK,0.25,standard,Croatia (member state) standard VAT rate. +2009-08-01,2012-03-01,HR,HRK,0.23,standard, +1998-08-01,2009-08-01,HR,HRK,0.22,standard, +2012-01-01,,HU,HUF,0.27,standard,Hungary (member state) standard VAT rate. +2009-07-01,2012-01-01,HU,HUF,0.25,standard, +2006-01-01,2009-07-01,HU,HUF,0.2,standard, +1988-01-01,2006-01-01,HU,HUF,0.25,standard, +2012-01-01,,IE,EUR,0.23,standard,Republic of Ireland (member state) standard VAT rate. +2010-01-01,2012-01-01,IE,EUR,0.21,standard, +2008-12-01,2010-01-01,IE,EUR,0.215,standard, +2002-03-01,2008-12-01,IE,EUR,0.21,standard, +2001-01-01,2002-03-01,IE,EUR,0.2,standard, +1991-03-01,2001-01-01,IE,EUR,0.21,standard, +1990-03-01,1991-03-01,IE,EUR,0.23,standard, +1986-03-01,1990-03-01,IE,EUR,0.25,standard, +1983-05-01,1986-03-01,IE,EUR,0.23,standard, +1983-03-01,1983-05-01,IE,EUR,0.35,standard, +1982-05-01,1983-03-01,IE,EUR,0.3,standard, +1980-05-01,1982-05-01,IE,EUR,0.25,standard, +1976-03-01,1980-05-01,IE,EUR,0.2,standard, +1973-09-03,1976-03-01,IE,EUR,0.195,standard, +1972-11-01,1973-09-03,IE,EUR,0.1637,standard, +2011-01-04,,IO,GBP,0,standard,British Indian Ocean Territory (British overseas territory) is exempted of VAT. +2013-10-01,,IT,EUR,0.22,standard,Italy (member state) standard VAT rate. +2011-09-17,2013-10-01,IT,EUR,0.21,standard, +1997-10-01,2011-09-17,IT,EUR,0.2,standard, +1988-08-01,1997-10-01,IT,EUR,0.19,standard, +1982-08-05,1988-08-01,IT,EUR,0.18,standard, +1981-01-01,1982-08-05,IT,EUR,0.15,standard, +1980-11-01,1981-01-01,IT,EUR,0.14,standard, +1980-07-03,1980-11-01,IT,EUR,0.15,standard, +1977-02-08,1980-07-03,IT,EUR,0.14,standard, +1973-01-01,1977-02-08,IT,EUR,0.12,standard, +2013-10-01,,"IT-22060 +CH-6911",CHF,0,standard,Campione (Italian town) is exempted of VAT. +2013-10-01,,IT-23030,EUR,0,standard,Livigno (Italian town) is exempted of VAT. +2011-01-04,,JE,GBP,0,standard,Jersey (British Crown dependency) is exempted of VAT. +2011-01-04,,KY,KYD,0,standard,Cayman Islands (British overseas territory) is exempted of VAT. +2009-09-01,,LT,EUR,0.21,standard,Lithuania (member state) standard VAT rate. +2009-01-01,2009-09-01,LT,EUR,0.19,standard, +1994-05-01,2009-01-01,LT,EUR,0.18,standard, +2015-01-01,,LU,EUR,0.17,standard,Luxembourg (member state) standard VAT rate. +1992-01-01,2015-01-01,LU,EUR,0.15,standard, +1983-07-01,1992-01-01,LU,EUR,0.12,standard, +1971-01-01,1983-07-01,LU,EUR,0.1,standard, +1970-01-01,1971-01-01,LU,EUR,0.8,standard, +2012-07-01,,LV,EUR,0.21,standard,Latvia (member state) standard VAT rate. +2011-01-01,2012-07-01,LV,EUR,0.22,standard, +2009-01-01,2011-01-01,LV,EUR,0.21,standard, +1995-05-01,2009-01-01,LV,EUR,0.18,standard, +2011-01-04,,MS,XCD,0,standard,Montserrat (British overseas territory) is exempted of VAT. +2004-01-01,,MT,EUR,0.18,standard,Malta (member state) standard VAT rate. +1995-01-01,2004-01-01,MT,EUR,0.15,standard, +2012-10-01,,NL,EUR,0.21,standard,Netherlands (member state) standard VAT rate. +2001-01-01,2012-10-01,NL,EUR,0.19,standard, +1992-10-01,2001-01-01,NL,EUR,0.175,standard, +1989-01-01,1992-10-01,NL,EUR,0.185,standard, +1986-10-01,1989-01-01,NL,EUR,0.2,standard, +1984-01-01,1986-10-01,NL,EUR,0.19,standard, +1976-01-01,1984-01-01,NL,EUR,0.18,standard, +1973-01-01,1976-01-01,NL,EUR,0.16,standard, +1971-01-01,1973-01-01,NL,EUR,0.14,standard, +1969-01-01,1971-01-01,NL,EUR,0.12,standard, +2012-10-01,,"NL-AW +AW",AWG,0,standard,Aruba (Dutch country) are exempted of VAT. +2012-10-01,,"NL-CW +NL-SX +CW +SX",ANG,0,standard,Curacao and Sint Maarten (Dutch countries) are exempted of VAT. +2012-10-01,,"NL-BQ1 +NL-BQ2 +NL-BQ3 +BQ +BQ-BO +BQ-SA +BQ-SE",USD,0,standard,"Bonaire, Saba and Sint Eustatius (Dutch special municipalities) are exempted of VAT." +2011-01-01,,PL,PLN,0.23,standard,Poland (member state) standard VAT rate. +1993-01-08,2011-01-01,PL,PLN,0.22,standard, +2011-01-04,,PN,NZD,0,standard,Pitcairn Islands (British overseas territory) is exempted of VAT. +2011-01-01,,PT,EUR,0.23,standard,Portugal (member state) standard VAT rate. +2010-07-01,2011-01-01,PT,EUR,0.21,standard, +2008-07-01,2010-07-01,PT,EUR,0.2,standard, +2005-07-01,2008-07-01,PT,EUR,0.21,standard, +2002-06-05,2005-07-01,PT,EUR,0.19,standard, +1995-01-01,2002-06-05,PT,EUR,0.17,standard, +1992-03-24,1995-01-01,PT,EUR,0.16,standard, +1988-02-01,1992-03-24,PT,EUR,0.17,standard, +1986-01-01,1988-02-01,PT,EUR,0.16,standard, +2011-01-01,,PT-20,EUR,0.18,standard,Azores (Portuguese autonomous region) special VAT rate. +2011-01-01,,PT-30,EUR,0.22,standard,Madeira (Portuguese autonomous region) special VAT rate. +2017-01-01,,RO,RON,0.19,standard,Romania (member state) standard VAT rate. +2016-01-01,2017-01-01,RO,RON,0.2,standard,Romania (member state) standard VAT rate. +2010-07-01,2016-01-01,RO,RON,0.24,standard, +2000-01-01,2010-07-01,RO,RON,0.19,standard, +1998-02-01,2000-01-01,RO,RON,0.22,standard, +1993-07-01,1998-02-01,RO,RON,0.18,standard, +1990-07-01,,SE,SEK,0.25,standard,Sweden (member state) standard VAT rate. +1983-01-01,1990-07-01,SE,SEK,0.2346,standard, +1981-11-16,1983-01-01,SE,SEK,0.2151,standard, +1980-09-08,1981-11-16,SE,SEK,0.2346,standard, +1977-06-01,1980-09-08,SE,SEK,0.2063,standard, +1971-01-01,1977-06-01,SE,SEK,0.1765,standard, +1969-01-01,1971-01-01,SE,SEK,0.1111,standard, +2011-01-04,,"AC +SH +SH-AC +SH-HL",SHP,0,standard,Ascension and Saint Helena (British overseas territory) is exempted of VAT. +2011-01-04,,"TA +SH-TA",GBP,0,standard,Tristan da Cunha (British oversea territory) is exempted of VAT. +2013-07-01,,SI,EUR,0.22,standard,Slovenia (member state) standard VAT rate. +2002-01-01,2013-07-01,SI,EUR,0.2,standard, +1999-07-01,2002-01-01,SI,EUR,0.19,standard, +2011-01-01,,SK,EUR,0.2,standard,Slovakia (member state) standard VAT rate. +2004-01-01,2011-01-01,SK,EUR,0.19,standard, +2003-01-01,2004-01-01,SK,EUR,0.2,standard, +1996-01-01,2003-01-01,SK,EUR,0.23,standard, +1993-08-01,1996-01-01,SK,EUR,0.25,standard, +1993-01-01,1993-08-01,SK,EUR,0.23,standard, +2011-01-04,,TC,USD,0,standard,Turks and Caicos Islands (British overseas territory) is exempted of VAT. +2011-01-04,,"GB +UK +IM",GBP,0.2,standard,"United Kingdom (member state) standard VAT rate. +Isle of Man (British self-governing dependency) is member of the EU VAT area and subjected to UK's standard VAT rate." +2010-01-01,2011-01-04,"GB +UK +IM",GBP,0.175,standard, +2008-12-01,2010-01-01,"GB +UK +IM",GBP,0.15,standard, +1991-04-01,2008-12-01,"GB +UK +IM",GBP,0.175,standard, +1979-06-18,1991-04-01,"GB +UK +IM",GBP,0.15,standard, +1974-07-29,1979-06-18,"GB +UK +IM",GBP,0.08,standard, +1973-04-01,1974-07-29,"GB +UK +IM",GBP,0.1,standard, +2011-01-04,,VG,USD,0,standard,British Virgin Islands (British overseas territory) is exempted of VAT. +2014-01-01,,CP,EUR,0,standard,Clipperton Island (French overseas possession) is exempted of VAT. +2019-11-15,,CH,CHF,0.077,standard,Switzerland standard VAT (added manually) +2019-11-15,,MC,EUR,0.196,standard,Monaco standard VAT (added manually) +2019-11-15,,FR,EUR,0.2,standard,France standard VAT (added manually) +2019-11-15,,GR,EUR,0.24,standard,Greece standard VAT (added manually) +2019-11-15,,GB,EUR,0.2,standard,UK standard VAT (added manually) +2019-12-17,,AD,EUR,0.045,standard,Andorra standard VAT (added manually) +2019-12-17,,TK,EUR,0.18,standard,Turkey standard VAT (added manually) +2019-12-17,,IS,EUR,0.24,standard,Iceland standard VAT (added manually) +2019-12-17,,FX,EUR,0.20,standard,France metropolitan standard VAT (added manually) +2020-01-04,,CY,EUR,0.19,standard,Cyprus standard VAT (added manually) +2019-01-04,,IL,EUR,0.23,standard,Ireland standard VAT (added manually) +2019-01-04,,LI,EUR,0.077,standard,Liechtenstein standard VAT (added manually) diff --git a/scripts/uncloud b/scripts/uncloud deleted file mode 100755 index a6e61aa..0000000 --- a/scripts/uncloud +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 -import logging -import sys -import importlib -import argparse - -from uncloud import UncloudException - -# the components that use etcd -ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner', 'imagescanner', 'metadata', 'configure'] - -ALL_COMPONENTS = ETCD_COMPONENTS.copy() -ALL_COMPONENTS.append('cli') - - -def exception_hook(exc_type, exc_value, exc_traceback): - logging.getLogger(__name__).error( - 'Uncaught exception', - exc_info=(exc_type, exc_value, exc_traceback) - ) - - -sys.excepthook = exception_hook - -if __name__ == '__main__': - # Setting up root logger - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - - arg_parser = argparse.ArgumentParser() - subparsers = arg_parser.add_subparsers(dest='command') - - parent_parser = argparse.ArgumentParser(add_help=False) - parent_parser.add_argument('--debug', '-d', - action='store_true', - default=False, - help='More verbose logging') - parent_parser.add_argument('--conf-dir', '-c', - help='Configuration directory') - - etcd_parser = argparse.ArgumentParser(add_help=False) - etcd_parser.add_argument('--etcd-host') - etcd_parser.add_argument('--etcd-port') - etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate') - etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate') - etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key') - - for component in ALL_COMPONENTS: - mod = importlib.import_module('uncloud.{}.main'.format(component)) - parser = getattr(mod, 'arg_parser') - - if component in ETCD_COMPONENTS: - subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser]) - else: - subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser]) - - args = arg_parser.parse_args() - if not args.command: - arg_parser.print_help() - else: - arguments = vars(args) - name = arguments.pop('command') - mod = importlib.import_module('uncloud.{}.main'.format(name)) - main = getattr(mod, 'main') - - # If the component requires etcd3, we import it and catch the - # etcd3.exceptions.ConnectionFailedError - if name in ETCD_COMPONENTS: - import etcd3 - - try: - main(arguments) - except UncloudException as err: - logger.error(err) - sys.exit(1) - except Exception as err: - logger.exception(err) - sys.exit(1) diff --git a/uncloud/.gitignore b/uncloud/.gitignore new file mode 100644 index 0000000..6a07bff --- /dev/null +++ b/uncloud/.gitignore @@ -0,0 +1 @@ +local_settings.py diff --git a/uncloud/__init__.py b/uncloud/__init__.py index 2920f47..e69de29 100644 --- a/uncloud/__init__.py +++ b/uncloud/__init__.py @@ -1,2 +0,0 @@ -class UncloudException(Exception): - pass diff --git a/uncloud/asgi.py b/uncloud/asgi.py new file mode 100644 index 0000000..2b5a7a3 --- /dev/null +++ b/uncloud/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for uncloud project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + +application = get_asgi_application() diff --git a/uncloud/cli/helper.py b/uncloud/cli/helper.py deleted file mode 100644 index 3c63073..0000000 --- a/uncloud/cli/helper.py +++ /dev/null @@ -1,51 +0,0 @@ -import requests -import json -import argparse -import binascii - -from pyotp import TOTP -from os.path import join as join_path -from uncloud.common.settings import settings - - -def get_otp_parser(): - otp_parser = argparse.ArgumentParser('otp') - try: - name = settings['client']['name'] - realm = settings['client']['realm'] - seed = settings['client']['seed'] - except Exception: - otp_parser.add_argument('--name', required=True) - otp_parser.add_argument('--realm', required=True) - otp_parser.add_argument('--seed', required=True, type=get_token, dest='token', metavar='SEED') - else: - otp_parser.add_argument('--name', default=name) - otp_parser.add_argument('--realm', default=realm) - otp_parser.add_argument('--seed', default=seed, type=get_token, dest='token', metavar='SEED') - - return otp_parser - - -def load_dump_pretty(content): - if isinstance(content, bytes): - content = content.decode('utf-8') - parsed = json.loads(content) - return json.dumps(parsed, indent=4, sort_keys=True) - - -def make_request(*args, data=None, request_method=requests.post): - r = request_method(join_path(settings['client']['api_server'], *args), json=data) - try: - print(load_dump_pretty(r.content)) - except Exception: - print('Error occurred while getting output from api server.') - - -def get_token(seed): - if seed is not None: - try: - token = TOTP(seed).now() - except binascii.Error: - raise argparse.ArgumentTypeError('Invalid seed') - else: - return token diff --git a/uncloud/common/shared.py b/uncloud/common/shared.py deleted file mode 100644 index 918dd0c..0000000 --- a/uncloud/common/shared.py +++ /dev/null @@ -1,34 +0,0 @@ -from uncloud.common.settings import settings -from uncloud.common.vm import VmPool -from uncloud.common.host import HostPool -from uncloud.common.request import RequestPool -from uncloud.common.storage_handlers import get_storage_handler - - -class Shared: - @property - def etcd_client(self): - return settings.get_etcd_client() - - @property - def host_pool(self): - return HostPool( - self.etcd_client, settings["etcd"]["host_prefix"] - ) - - @property - def vm_pool(self): - return VmPool(self.etcd_client, settings["etcd"]["vm_prefix"]) - - @property - def request_pool(self): - return RequestPool( - self.etcd_client, settings["etcd"]["request_prefix"] - ) - - @property - def storage_handler(self): - return get_storage_handler() - - -shared = Shared() diff --git a/uncloud/docs/source/hacking.rst b/uncloud/docs/source/hacking.rst deleted file mode 100644 index 2df42a7..0000000 --- a/uncloud/docs/source/hacking.rst +++ /dev/null @@ -1,17 +0,0 @@ -Hacking -======= -How to hack on the code. - -[ to be done by Balazs: - -* make nice -* indent with shell script mode - -] - -* git clone the repo -* cd to the repo -* Setup your venv: python -m venv venv -* . ./venv/bin/activate # you need the leading dot for sourcing! -* Run ./bin/ucloud-run-reinstall - it should print you an error - message on how to use ucloud diff --git a/uncloud/management/commands/uncloud.py b/uncloud/management/commands/uncloud.py new file mode 100644 index 0000000..bd47c6b --- /dev/null +++ b/uncloud/management/commands/uncloud.py @@ -0,0 +1,28 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand + +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster + +import logging +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'General uncloud commands' + + def add_arguments(self, parser): + parser.add_argument('--bootstrap', action='store_true', help='Bootstrap a typical uncloud installation') + + def handle(self, *args, **options): + + if options['bootstrap']: + self.bootstrap() + + def bootstrap(self): + default_cluster = VMCluster.objects.get_or_create(name="default") +# local_host = diff --git a/uncloud/models.py b/uncloud/models.py new file mode 100644 index 0000000..bd7a931 --- /dev/null +++ b/uncloud/models.py @@ -0,0 +1,35 @@ +from django.db import models +from django.contrib.postgres.fields import JSONField +from django.utils.translation import gettext_lazy as _ + +class UncloudModel(models.Model): + """ + This class extends the standard model with an + extra_data field that can be used to include public, + but internal information. + + For instance if you migrate from an existing virtualisation + framework to uncloud. + + The extra_data attribute should be considered a hack and whenever + data is necessary for running uncloud, it should **not** be stored + in there. + + """ + + extra_data = JSONField(editable=False, blank=True, null=True) + + class Meta: + abstract = True + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class UncloudStatus(models.TextChoices): + PENDING = 'PENDING', _('Pending') + AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment') + BEING_CREATED = 'BEING_CREATED', _('Being created') + SCHEDULED = 'SCHEDULED', _('Scheduled') # resource selected, waiting for dispatching + ACTIVE = 'ACTIVE', _('Active') + MODIFYING = 'MODIFYING', _('Modifying') # Resource is being changed + DELETED = 'DELETED', _('Deleted') # Resource has been deleted + DISABLED = 'DISABLED', _('Disabled') # Is usable, but cannot be used for new things + UNUSABLE = 'UNUSABLE', _('Unusable'), # Has some kind of error diff --git a/uncloud/settings.py b/uncloud/settings.py new file mode 100644 index 0000000..884c370 --- /dev/null +++ b/uncloud/settings.py @@ -0,0 +1,189 @@ +""" +Django settings for uncloud project. + +Generated by 'django-admin startproject' using Django 3.0.3. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os +import ldap + +from django.core.management.utils import get_random_secret_key +from django_auth_ldap.config import LDAPSearch, LDAPSearchUnion + + +LOGGING = {} + + + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), + } +} + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'django_extensions', + 'rest_framework', + 'uncloud', + 'uncloud_pay', + 'uncloud_auth', + 'uncloud_net', + 'uncloud_storage', + 'uncloud_vm', + 'uncloud_service', + 'opennebula' +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'uncloud.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'uncloud.wsgi.application' + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + +################################################################################ +# AUTH/LDAP + +AUTH_LDAP_SERVER_URI = "" +AUTH_LDAP_BIND_DN = "" +AUTH_LDAP_BIND_PASSWORD = "" +AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=example,dc=com", + ldap.SCOPE_SUBTREE, + "(uid=%(user)s)") + +AUTH_LDAP_USER_ATTR_MAP = { + "first_name": "givenName", + "last_name": "sn", + "email": "mail" +} + +################################################################################ +# AUTH/Django +AUTHENTICATION_BACKENDS = [ + "django_auth_ldap.backend.LDAPBackend", + "django.contrib.auth.backends.ModelBackend" +] + +AUTH_USER_MODEL = 'uncloud_auth.User' + + +################################################################################ +# AUTH/REST +REST_FRAMEWORK = { + 'DEFAULT_AUTHENTICATION_CLASSES': [ + 'rest_framework.authentication.BasicAuthentication', + 'rest_framework.authentication.SessionAuthentication', + ] +} + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ +STATIC_URL = '/static/' +STATICFILES_DIRS = [ os.path.join(BASE_DIR, "static") ] + +# XML-RPC interface of opennebula +OPENNEBULA_URL = 'https://opennebula.example.com:2634/RPC2' + +# user:pass for accessing opennebula +OPENNEBULA_USER_PASS = 'user:password' + + +# Stripe (Credit Card payments) +STRIPE_KEY="" +STRIPE_PUBLIC_KEY="" + +# The django secret key +SECRET_KEY=get_random_secret_key() + +ALLOWED_HOSTS = [] + +# Overwrite settings with local settings, if existing +try: + from uncloud.local_settings import * +except (ModuleNotFoundError, ImportError): + pass diff --git a/uncloud/urls.py b/uncloud/urls.py new file mode 100644 index 0000000..723ef45 --- /dev/null +++ b/uncloud/urls.py @@ -0,0 +1,89 @@ +"""uncloud URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" + +from django.contrib import admin +from django.urls import path, include +from django.conf import settings +from django.conf.urls.static import static + +from rest_framework import routers +from rest_framework.schemas import get_schema_view + +from opennebula import views as oneviews +from uncloud_auth import views as authviews +from uncloud_net import views as netviews +from uncloud_pay import views as payviews +from uncloud_vm import views as vmviews +from uncloud_service import views as serviceviews + +router = routers.DefaultRouter() + +# Beta endpoints +router.register(r'beta/vm', vmviews.NicoVMProductViewSet, basename='nicovmproduct') + +# VM +router.register(r'v1/vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct') +router.register(r'v1/vm/diskimage', vmviews.VMDiskImageProductViewSet, basename='vmdiskimageproduct') +router.register(r'v1/vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct') +router.register(r'v1/vm/vm', vmviews.VMProductViewSet, basename='vmproduct') + + +# creates VM from os image +#router.register(r'vm/ipv6onlyvm', vmviews.VMProductViewSet, basename='vmproduct') +# ... AND adds IPv4 mapping +#router.register(r'vm/dualstackvm', vmviews.VMProductViewSet, basename='vmproduct') + +# Services +router.register(r'v1/service/matrix', serviceviews.MatrixServiceProductViewSet, basename='matrixserviceproduct') +router.register(r'v1/service/generic', serviceviews.GenericServiceProductViewSet, basename='genericserviceproduct') + + +# Net +router.register(r'v1/net/vpn', netviews.VPNNetworkViewSet, basename='vpnnet') +router.register(r'v1/admin/vpnreservation', netviews.VPNNetworkReservationViewSet, basename='vpnnetreservation') + + +# Pay +router.register(r'v1/my/address', payviews.BillingAddressViewSet, basename='address') +router.register(r'v1/my/bill', payviews.BillViewSet, basename='bill') +router.register(r'v1/my/order', payviews.OrderViewSet, basename='order') +router.register(r'v1/my/payment', payviews.PaymentViewSet, basename='payment') +router.register(r'v1/my/payment-method', payviews.PaymentMethodViewSet, basename='payment-method') + +# admin/staff urls +router.register(r'v1/admin/bill', payviews.AdminBillViewSet, basename='admin/bill') +router.register(r'v1/admin/payment', payviews.AdminPaymentViewSet, basename='admin/payment') +router.register(r'v1/admin/order', payviews.AdminOrderViewSet, basename='admin/order') +router.register(r'v1/admin/vmhost', vmviews.VMHostViewSet) +router.register(r'v1/admin/vmcluster', vmviews.VMClusterViewSet) +router.register(r'v1/admin/vpnpool', netviews.VPNPoolViewSet) +router.register(r'v1/admin/opennebula', oneviews.VMViewSet, basename='opennebula') + +# User/Account +router.register(r'v1/my/user', authviews.UserViewSet, basename='user') +router.register(r'v1/admin/user', authviews.AdminUserViewSet, basename='useradmin') + +urlpatterns = [ + path('', include(router.urls)), + # web/ = stuff to view in the browser + + path('api-auth/', include('rest_framework.urls', namespace='rest_framework')), # for login to REST API + path('openapi', get_schema_view( + title="uncloud", + description="uncloud API", + version="1.0.0" + ), name='openapi-schema'), +] diff --git a/uncloud/wsgi.py b/uncloud/wsgi.py new file mode 100644 index 0000000..c4a07b8 --- /dev/null +++ b/uncloud/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for uncloud project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'uncloud.settings') + +application = get_wsgi_application() diff --git a/uncloud_auth/__init__.py b/uncloud_auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_auth/admin.py b/uncloud_auth/admin.py new file mode 100644 index 0000000..f91be8f --- /dev/null +++ b/uncloud_auth/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin +from .models import User + +admin.site.register(User, UserAdmin) diff --git a/uncloud_auth/apps.py b/uncloud_auth/apps.py new file mode 100644 index 0000000..c16bd7a --- /dev/null +++ b/uncloud_auth/apps.py @@ -0,0 +1,4 @@ +from django.apps import AppConfig + +class AuthConfig(AppConfig): + name = 'uncloud_auth' diff --git a/uncloud_auth/management/commands/make-admin.py b/uncloud_auth/management/commands/make-admin.py new file mode 100644 index 0000000..b750bc3 --- /dev/null +++ b/uncloud_auth/management/commands/make-admin.py @@ -0,0 +1,16 @@ +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model +import sys + +class Command(BaseCommand): + help = 'Give Admin rights to existing user' + + def add_arguments(self, parser): + parser.add_argument('username', type=str) + + def handle(self, *args, **options): + user = get_user_model().objects.get(username=options['username']) + user.is_staff = True + user.save() + + print("{} is now admin.".format(user.username)) diff --git a/uncloud_auth/migrations/0001_initial.py b/uncloud_auth/migrations/0001_initial.py new file mode 100644 index 0000000..a1f8d00 --- /dev/null +++ b/uncloud_auth/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 3.0.3 on 2020-03-03 16:49 + +import django.contrib.auth.models +import django.contrib.auth.validators +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0011_update_proxy_permissions'), + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=30, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/uncloud_auth/migrations/0002_auto_20200318_1343.py b/uncloud_auth/migrations/0002_auto_20200318_1343.py new file mode 100644 index 0000000..ad2654f --- /dev/null +++ b/uncloud_auth/migrations/0002_auto_20200318_1343.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.3 on 2020-03-18 13:43 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_auth', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='user', + name='amount', + field=models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)]), + ), + migrations.AddField( + model_name='user', + name='maximum_credit', + field=models.FloatField(default=0), + preserve_default=False, + ), + ] diff --git a/uncloud_auth/migrations/0003_auto_20200318_1345.py b/uncloud_auth/migrations/0003_auto_20200318_1345.py new file mode 100644 index 0000000..31b1717 --- /dev/null +++ b/uncloud_auth/migrations/0003_auto_20200318_1345.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-03-18 13:45 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_auth', '0002_auto_20200318_1343'), + ] + + operations = [ + migrations.RemoveField( + model_name='user', + name='amount', + ), + migrations.AlterField( + model_name='user', + name='maximum_credit', + field=models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)]), + ), + ] diff --git a/uncloud_auth/migrations/0004_user_primary_billing_address.py b/uncloud_auth/migrations/0004_user_primary_billing_address.py new file mode 100644 index 0000000..640c9c5 --- /dev/null +++ b/uncloud_auth/migrations/0004_user_primary_billing_address.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.6 on 2020-05-10 17:31 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0014_paymentsettings'), + ('uncloud_auth', '0003_auto_20200318_1345'), + ] + + operations = [ + migrations.AddField( + model_name='user', + name='primary_billing_address', + field=models.OneToOneField(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='uncloud_pay.BillingAddress'), + ), + ] diff --git a/uncloud_auth/migrations/0005_auto_20200510_1736.py b/uncloud_auth/migrations/0005_auto_20200510_1736.py new file mode 100644 index 0000000..38c303e --- /dev/null +++ b/uncloud_auth/migrations/0005_auto_20200510_1736.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.6 on 2020-05-10 17:36 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0014_paymentsettings'), + ('uncloud_auth', '0004_user_primary_billing_address'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='primary_billing_address', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='uncloud_pay.BillingAddress'), + ), + ] diff --git a/uncloud_auth/migrations/__init__.py b/uncloud_auth/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_auth/models.py b/uncloud_auth/models.py new file mode 100644 index 0000000..c456648 --- /dev/null +++ b/uncloud_auth/models.py @@ -0,0 +1,28 @@ +from django.contrib.auth.models import AbstractUser +from django.db import models +from django.core.validators import MinValueValidator + +from uncloud_pay import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS +from uncloud_pay.models import get_balance_for_user + +class User(AbstractUser): + """ + We use the standard user and add a maximum credit that is allowed + to be accumulated. After that we need to have warnings, cancellation, etc. + """ + + maximum_credit = models.DecimalField( + default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + # Need to use the string here to prevent a circular import + primary_billing_address = models.ForeignKey('uncloud_pay.BillingAddress', + on_delete=models.PROTECT, + blank=True, + null=True) + + @property + def balance(self): + return get_balance_for_user(self) diff --git a/uncloud_auth/serializers.py b/uncloud_auth/serializers.py new file mode 100644 index 0000000..92bbf01 --- /dev/null +++ b/uncloud_auth/serializers.py @@ -0,0 +1,25 @@ +from django.contrib.auth import get_user_model +from rest_framework import serializers + +from uncloud_pay import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS +from uncloud_pay.models import BillingAddress + +class UserSerializer(serializers.ModelSerializer): + class Meta: + model = get_user_model() + read_only_fields = [ 'username', 'balance', 'maximum_credit' ] + fields = read_only_fields + [ 'email', 'primary_billing_address' ] + + def validate(self, data): + """ + Ensure that the primary billing address belongs to the user + """ + + if 'primary_billing_address' in data: + if not data['primary_billing_address'].owner == self.instance: + raise serializers.ValidationError("Invalid data") + + return data + +class ImportUserSerializer(serializers.Serializer): + username = serializers.CharField() diff --git a/uncloud_auth/views.py b/uncloud_auth/views.py new file mode 100644 index 0000000..77f0a0f --- /dev/null +++ b/uncloud_auth/views.py @@ -0,0 +1,54 @@ +from rest_framework import viewsets, permissions, status +from .serializers import * +from django_auth_ldap.backend import LDAPBackend +from rest_framework.decorators import action +from rest_framework.response import Response + +class UserViewSet(viewsets.GenericViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = UserSerializer + + def get_queryset(self): + return self.request.user + + def list(self, request, format=None): + # This is a bit stupid: we have a user, we create a queryset by + # matching on the username. But I don't know a "nicer" way. + # Nico, 2020-03-18 + user = request.user + serializer = self.get_serializer(user, context = {'request': request}) + return Response(serializer.data) + + def create(self, request): + """ + Modify existing user data + """ + + user = request.user + serializer = self.get_serializer(user, + context = {'request': request}, + data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(serializer.data) + +class AdminUserViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAdminUser] + + def get_serializer_class(self): + if self.action == 'import_from_ldap': + return ImportUserSerializer + else: + return UserSerializer + + def get_queryset(self): + return get_user_model().objects.all() + + @action(detail=False, methods=['post'], url_path='import_from_ldap') + def import_from_ldap(self, request, pk=None): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + ldap_username = serializer.validated_data.pop("username") + user = LDAPBackend().populate_user(ldap_username) + + return Response(UserSerializer(user, context = {'request': request}).data) diff --git a/uncloud_net/__init__.py b/uncloud_net/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_net/admin.py b/uncloud_net/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_net/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_net/apps.py b/uncloud_net/apps.py new file mode 100644 index 0000000..489beb1 --- /dev/null +++ b/uncloud_net/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudNetConfig(AppConfig): + name = 'uncloud_net' diff --git a/uncloud_net/management/commands/vpn.py b/uncloud_net/management/commands/vpn.py new file mode 100644 index 0000000..9fdc80d --- /dev/null +++ b/uncloud_net/management/commands/vpn.py @@ -0,0 +1,44 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand + +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster + +import logging +log = logging.getLogger(__name__) + + + +peer_template=""" +# {username} +[Peer] +PublicKey = {public_key} +AllowedIPs = {vpnnetwork} +""" + +class Command(BaseCommand): + help = 'General uncloud commands' + + def add_arguments(self, parser): + parser.add_argument('--hostname', + action='store_true', + help='Name of this VPN Host', + required=True) + + def handle(self, *args, **options): + if options['bootstrap']: + self.bootstrap() + + self.create_vpn_config(options['hostname']) + + def create_vpn_config(self, hostname): + configs = [] + + for pool in VPNPool.objects.filter(vpn_hostname=hostname): + configs.append(pool_config) + + print(configs) diff --git a/uncloud_net/migrations/0001_initial.py b/uncloud_net/migrations/0001_initial.py new file mode 100644 index 0000000..940d63f --- /dev/null +++ b/uncloud_net/migrations/0001_initial.py @@ -0,0 +1,68 @@ +# Generated by Django 3.0.5 on 2020-04-06 21:38 + +from django.conf import settings +import django.contrib.postgres.fields.jsonb +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='MACAdress', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ], + ), + migrations.CreateModel( + name='VPNPool', + fields=[ + ('extra_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True)), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('network', models.GenericIPAddressField(unique=True)), + ('network_size', models.IntegerField(validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(128)])), + ('subnetwork_size', models.IntegerField(validators=[django.core.validators.MinValueValidator(0), django.core.validators.MaxValueValidator(128)])), + ('vpn_hostname', models.CharField(max_length=256)), + ('wireguard_private_key', models.CharField(max_length=48)), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VPNNetworkReservation', + fields=[ + ('extra_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True)), + ('address', models.GenericIPAddressField(primary_key=True, serialize=False)), + ('vpnpool', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_net.VPNPool')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VPNNetwork', + fields=[ + ('extra_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True)), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32)), + ('wireguard_public_key', models.CharField(max_length=48)), + ('network', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_net.VPNNetworkReservation')), + ('order', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order')), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/uncloud_net/migrations/0002_auto_20200409_1225.py b/uncloud_net/migrations/0002_auto_20200409_1225.py new file mode 100644 index 0000000..fcc2374 --- /dev/null +++ b/uncloud_net/migrations/0002_auto_20200409_1225.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.5 on 2020-04-09 12:25 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='vpnnetworkreservation', + name='status', + field=models.CharField(choices=[('used', 'used'), ('free', 'free')], default='used', max_length=256), + ), + migrations.AlterField( + model_name='vpnnetwork', + name='network', + field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to='uncloud_net.VPNNetworkReservation'), + ), + ] diff --git a/uncloud_net/migrations/0003_auto_20200417_0551.py b/uncloud_net/migrations/0003_auto_20200417_0551.py new file mode 100644 index 0000000..24f4a7f --- /dev/null +++ b/uncloud_net/migrations/0003_auto_20200417_0551.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.5 on 2020-04-17 05:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_net', '0002_auto_20200409_1225'), + ] + + operations = [ + migrations.AlterField( + model_name='vpnnetwork', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='AWAITING_PAYMENT', max_length=32), + ), + ] diff --git a/uncloud_net/migrations/__init__.py b/uncloud_net/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_net/models.py b/uncloud_net/models.py new file mode 100644 index 0000000..b5a181e --- /dev/null +++ b/uncloud_net/models.py @@ -0,0 +1,187 @@ +import uuid +import ipaddress + +from django.db import models +from django.contrib.auth import get_user_model +from django.core.validators import MinValueValidator, MaxValueValidator + + +from uncloud_pay.models import Product, RecurringPeriod +from uncloud.models import UncloudModel, UncloudStatus + + +class MACAdress(models.Model): + default_prefix = 0x420000000000 + +class VPNPool(UncloudModel): + """ + Network address pools from which VPNs can be created + """ + + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + + network = models.GenericIPAddressField(unique=True) + network_size = models.IntegerField(validators=[MinValueValidator(0), + MaxValueValidator(128)]) + + subnetwork_size = models.IntegerField(validators=[ + MinValueValidator(0), + MaxValueValidator(128) + ]) + + vpn_hostname = models.CharField(max_length=256) + + wireguard_private_key = models.CharField(max_length=48) + + @property + def num_maximum_networks(self): + """ + sample: + network_size = 40 + subnetwork_size = 48 + maximum_networks = 2^(48-40) + + 2nd sample: + network_size = 8 + subnetwork_size = 24 + maximum_networks = 2^(24-8) + """ + + return 2**(self.subnetwork_size - self.network_size) + + @property + def used_networks(self): + return self.vpnnetworkreservation_set.filter(vpnpool=self, status='used') + + @property + def free_networks(self): + return self.vpnnetworkreservation_set.filter(vpnpool=self, status='free') + + @property + def num_used_networks(self): + return len(self.used_networks) + + @property + def num_free_networks(self): + return self.num_maximum_networks - self.num_used_networks + len(self.free_networks) + + @property + def next_free_network(self): + if self.num_free_networks == 0: + # FIXME: use right exception + raise Exception("No free networks") + + if len(self.free_networks) > 0: + return self.free_networks[0].address + + if len(self.used_networks) > 0: + """ + sample: + + pool = 2a0a:e5c1:200::/40 + last_used = 2a0a:e5c1:204::/48 + + next: + """ + + last_net = ipaddress.ip_network(self.used_networks.last().address) + last_net_ip = last_net[0] + + if last_net_ip.version == 6: + offset_to_next = 2**(128 - self.subnetwork_size) + elif last_net_ip.version == 4: + offset_to_next = 2**(32 - self.subnetwork_size) + + next_net_ip = last_net_ip + offset_to_next + + return str(next_net_ip) + else: + # first network to be created + return self.network + + @property + def wireguard_config_filename(self): + return '/etc/wireguard/{}.conf'.format(self.network) + + @property + def wireguard_config(self): + wireguard_config = [ + """ +[Interface] +ListenPort = 51820 +PrivateKey = {privatekey} +""".format(privatekey=self.wireguard_private_key) ] + + peers = [] + + for reservation in self.vpnnetworkreservation_set.filter(status='used'): + public_key = reservation.vpnnetwork_set.first().wireguard_public_key + peer_network = "{}/{}".format(reservation.address, self.subnetwork_size) + owner = reservation.vpnnetwork_set.first().owner + + peers.append(""" +# Owner: {owner} +[Peer] +PublicKey = {public_key} +AllowedIPs = {peer_network} +""".format( + owner=owner, + public_key=public_key, + peer_network=peer_network)) + + wireguard_config.extend(peers) + + return "\n".join(wireguard_config) + + + def configure_wireguard_vpnserver(self): + """ + This method is designed to run as a celery task and should + not be called directly from the web + """ + + # subprocess, ssh + + pass + + +class VPNNetworkReservation(UncloudModel): + """ + This class tracks the used VPN networks. It will be deleted, when the product is cancelled. + """ + vpnpool = models.ForeignKey(VPNPool, + on_delete=models.CASCADE) + + address = models.GenericIPAddressField(primary_key=True) + + status = models.CharField(max_length=256, + default='used', + choices = ( + ('used', 'used'), + ('free', 'free') + ) + ) + + +class VPNNetwork(Product): + """ + A selected network. Used for tracking reservations / used networks + """ + network = models.ForeignKey(VPNNetworkReservation, + on_delete=models.CASCADE, + editable=False) + + wireguard_public_key = models.CharField(max_length=48) + + default_recurring_period = RecurringPeriod.PER_YEAR + + @property + def recurring_price(self): + return 120 + + + def delete(self, *args, **kwargs): + self.network.status = 'free' + self.network.save() + super().save(*args, **kwargs) + print("deleted {}".format(self)) diff --git a/uncloud_net/serializers.py b/uncloud_net/serializers.py new file mode 100644 index 0000000..dc4866e --- /dev/null +++ b/uncloud_net/serializers.py @@ -0,0 +1,100 @@ +import base64 + +from django.contrib.auth import get_user_model +from django.utils.translation import gettext_lazy as _ +from rest_framework import serializers + +from .models import * + +class VPNPoolSerializer(serializers.ModelSerializer): + class Meta: + model = VPNPool + fields = '__all__' + +class VPNNetworkReservationSerializer(serializers.ModelSerializer): + class Meta: + model = VPNNetworkReservation + fields = '__all__' + + +class VPNNetworkSerializer(serializers.ModelSerializer): + class Meta: + model = VPNNetwork + fields = '__all__' + + # This is required for finding the VPN pool, but does not + # exist in the model + network_size = serializers.IntegerField(min_value=0, + max_value=128, + write_only=True) + + def validate_wireguard_public_key(self, value): + msg = _("Supplied key is not a valid wireguard public key") + + """ FIXME: verify that this does not create broken wireguard config files, + i.e. contains \n or similar! + We might even need to be more strict to not break wireguard... + """ + + try: + base64.standard_b64decode(value) + except Exception as e: + raise serializers.ValidationError(msg) + + if '\n' in value: + raise serializers.ValidationError(msg) + + return value + + def validate(self, data): + + # FIXME: filter for status = active or similar + all_pools = VPNPool.objects.all() + sizes = [ p.subnetwork_size for p in all_pools ] + + pools = VPNPool.objects.filter(subnetwork_size=data['network_size']) + + if len(pools) == 0: + msg = _("No pool available for networks with size = {}. Available are: {}".format(data['network_size'], sizes)) + raise serializers.ValidationError(msg) + + return data + + def create(self, validated_data): + """ + Creating a new vpnnetwork - there are a couple of race conditions, + especially when run in parallel. + + What we should be doing: + + - create a reservation race free + - map the reservation to a network (?) + """ + + pools = VPNPool.objects.filter(subnetwork_size=validated_data['network_size']) + + vpn_network = None + + for pool in pools: + if pool.num_free_networks > 0: + next_address = pool.next_free_network + + reservation, created = VPNNetworkReservation.objects.update_or_create( + vpnpool=pool, address=next_address, + defaults = { + 'status': 'used' + }) + + vpn_network = VPNNetwork.objects.create( + owner=self.context['request'].user, + network=reservation, + wireguard_public_key=validated_data['wireguard_public_key'] + ) + + break + if not vpn_network: + # FIXME: use correct exception + raise Exception("Did not find any free pool") + + + return vpn_network diff --git a/uncloud_net/tests.py b/uncloud_net/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/uncloud_net/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/uncloud_net/views.py b/uncloud_net/views.py new file mode 100644 index 0000000..dc86959 --- /dev/null +++ b/uncloud_net/views.py @@ -0,0 +1,33 @@ + +from django.shortcuts import render + +from rest_framework import viewsets, permissions + + +from .models import * +from .serializers import * + + +class VPNPoolViewSet(viewsets.ModelViewSet): + serializer_class = VPNPoolSerializer + permission_classes = [permissions.IsAdminUser] + queryset = VPNPool.objects.all() + +class VPNNetworkReservationViewSet(viewsets.ModelViewSet): + serializer_class = VPNNetworkReservationSerializer + permission_classes = [permissions.IsAdminUser] + queryset = VPNNetworkReservation.objects.all() + + +class VPNNetworkViewSet(viewsets.ModelViewSet): + serializer_class = VPNNetworkSerializer +# permission_classes = [permissions.IsAdminUser] + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VPNNetwork.objects.all() + else: + obj = VPNNetwork.objects.filter(owner=self.request.user) + + return obj diff --git a/uncloud_pay/__init__.py b/uncloud_pay/__init__.py new file mode 100644 index 0000000..4bda45f --- /dev/null +++ b/uncloud_pay/__init__.py @@ -0,0 +1,250 @@ +from django.utils.translation import gettext_lazy as _ +import decimal + +# Define DecimalField properties, used to represent amounts of money. +AMOUNT_MAX_DIGITS=10 +AMOUNT_DECIMALS=2 + +decimal.getcontext().prec = AMOUNT_DECIMALS + +# http://xml.coverpages.org/country3166.html +COUNTRIES = ( + ('AD', _('Andorra')), + ('AE', _('United Arab Emirates')), + ('AF', _('Afghanistan')), + ('AG', _('Antigua & Barbuda')), + ('AI', _('Anguilla')), + ('AL', _('Albania')), + ('AM', _('Armenia')), + ('AN', _('Netherlands Antilles')), + ('AO', _('Angola')), + ('AQ', _('Antarctica')), + ('AR', _('Argentina')), + ('AS', _('American Samoa')), + ('AT', _('Austria')), + ('AU', _('Australia')), + ('AW', _('Aruba')), + ('AZ', _('Azerbaijan')), + ('BA', _('Bosnia and Herzegovina')), + ('BB', _('Barbados')), + ('BD', _('Bangladesh')), + ('BE', _('Belgium')), + ('BF', _('Burkina Faso')), + ('BG', _('Bulgaria')), + ('BH', _('Bahrain')), + ('BI', _('Burundi')), + ('BJ', _('Benin')), + ('BM', _('Bermuda')), + ('BN', _('Brunei Darussalam')), + ('BO', _('Bolivia')), + ('BR', _('Brazil')), + ('BS', _('Bahama')), + ('BT', _('Bhutan')), + ('BV', _('Bouvet Island')), + ('BW', _('Botswana')), + ('BY', _('Belarus')), + ('BZ', _('Belize')), + ('CA', _('Canada')), + ('CC', _('Cocos (Keeling) Islands')), + ('CF', _('Central African Republic')), + ('CG', _('Congo')), + ('CH', _('Switzerland')), + ('CI', _('Ivory Coast')), + ('CK', _('Cook Iislands')), + ('CL', _('Chile')), + ('CM', _('Cameroon')), + ('CN', _('China')), + ('CO', _('Colombia')), + ('CR', _('Costa Rica')), + ('CU', _('Cuba')), + ('CV', _('Cape Verde')), + ('CX', _('Christmas Island')), + ('CY', _('Cyprus')), + ('CZ', _('Czech Republic')), + ('DE', _('Germany')), + ('DJ', _('Djibouti')), + ('DK', _('Denmark')), + ('DM', _('Dominica')), + ('DO', _('Dominican Republic')), + ('DZ', _('Algeria')), + ('EC', _('Ecuador')), + ('EE', _('Estonia')), + ('EG', _('Egypt')), + ('EH', _('Western Sahara')), + ('ER', _('Eritrea')), + ('ES', _('Spain')), + ('ET', _('Ethiopia')), + ('FI', _('Finland')), + ('FJ', _('Fiji')), + ('FK', _('Falkland Islands (Malvinas)')), + ('FM', _('Micronesia')), + ('FO', _('Faroe Islands')), + ('FR', _('France')), + ('FX', _('France, Metropolitan')), + ('GA', _('Gabon')), + ('GB', _('United Kingdom (Great Britain)')), + ('GD', _('Grenada')), + ('GE', _('Georgia')), + ('GF', _('French Guiana')), + ('GH', _('Ghana')), + ('GI', _('Gibraltar')), + ('GL', _('Greenland')), + ('GM', _('Gambia')), + ('GN', _('Guinea')), + ('GP', _('Guadeloupe')), + ('GQ', _('Equatorial Guinea')), + ('GR', _('Greece')), + ('GS', _('South Georgia and the South Sandwich Islands')), + ('GT', _('Guatemala')), + ('GU', _('Guam')), + ('GW', _('Guinea-Bissau')), + ('GY', _('Guyana')), + ('HK', _('Hong Kong')), + ('HM', _('Heard & McDonald Islands')), + ('HN', _('Honduras')), + ('HR', _('Croatia')), + ('HT', _('Haiti')), + ('HU', _('Hungary')), + ('ID', _('Indonesia')), + ('IE', _('Ireland')), + ('IL', _('Israel')), + ('IN', _('India')), + ('IO', _('British Indian Ocean Territory')), + ('IQ', _('Iraq')), + ('IR', _('Islamic Republic of Iran')), + ('IS', _('Iceland')), + ('IT', _('Italy')), + ('JM', _('Jamaica')), + ('JO', _('Jordan')), + ('JP', _('Japan')), + ('KE', _('Kenya')), + ('KG', _('Kyrgyzstan')), + ('KH', _('Cambodia')), + ('KI', _('Kiribati')), + ('KM', _('Comoros')), + ('KN', _('St. Kitts and Nevis')), + ('KP', _('Korea, Democratic People\'s Republic of')), + ('KR', _('Korea, Republic of')), + ('KW', _('Kuwait')), + ('KY', _('Cayman Islands')), + ('KZ', _('Kazakhstan')), + ('LA', _('Lao People\'s Democratic Republic')), + ('LB', _('Lebanon')), + ('LC', _('Saint Lucia')), + ('LI', _('Liechtenstein')), + ('LK', _('Sri Lanka')), + ('LR', _('Liberia')), + ('LS', _('Lesotho')), + ('LT', _('Lithuania')), + ('LU', _('Luxembourg')), + ('LV', _('Latvia')), + ('LY', _('Libyan Arab Jamahiriya')), + ('MA', _('Morocco')), + ('MC', _('Monaco')), + ('MD', _('Moldova, Republic of')), + ('MG', _('Madagascar')), + ('MH', _('Marshall Islands')), + ('ML', _('Mali')), + ('MN', _('Mongolia')), + ('MM', _('Myanmar')), + ('MO', _('Macau')), + ('MP', _('Northern Mariana Islands')), + ('MQ', _('Martinique')), + ('MR', _('Mauritania')), + ('MS', _('Monserrat')), + ('MT', _('Malta')), + ('MU', _('Mauritius')), + ('MV', _('Maldives')), + ('MW', _('Malawi')), + ('MX', _('Mexico')), + ('MY', _('Malaysia')), + ('MZ', _('Mozambique')), + ('NA', _('Namibia')), + ('NC', _('New Caledonia')), + ('NE', _('Niger')), + ('NF', _('Norfolk Island')), + ('NG', _('Nigeria')), + ('NI', _('Nicaragua')), + ('NL', _('Netherlands')), + ('NO', _('Norway')), + ('NP', _('Nepal')), + ('NR', _('Nauru')), + ('NU', _('Niue')), + ('NZ', _('New Zealand')), + ('OM', _('Oman')), + ('PA', _('Panama')), + ('PE', _('Peru')), + ('PF', _('French Polynesia')), + ('PG', _('Papua New Guinea')), + ('PH', _('Philippines')), + ('PK', _('Pakistan')), + ('PL', _('Poland')), + ('PM', _('St. Pierre & Miquelon')), + ('PN', _('Pitcairn')), + ('PR', _('Puerto Rico')), + ('PT', _('Portugal')), + ('PW', _('Palau')), + ('PY', _('Paraguay')), + ('QA', _('Qatar')), + ('RE', _('Reunion')), + ('RO', _('Romania')), + ('RU', _('Russian Federation')), + ('RW', _('Rwanda')), + ('SA', _('Saudi Arabia')), + ('SB', _('Solomon Islands')), + ('SC', _('Seychelles')), + ('SD', _('Sudan')), + ('SE', _('Sweden')), + ('SG', _('Singapore')), + ('SH', _('St. Helena')), + ('SI', _('Slovenia')), + ('SJ', _('Svalbard & Jan Mayen Islands')), + ('SK', _('Slovakia')), + ('SL', _('Sierra Leone')), + ('SM', _('San Marino')), + ('SN', _('Senegal')), + ('SO', _('Somalia')), + ('SR', _('Suriname')), + ('ST', _('Sao Tome & Principe')), + ('SV', _('El Salvador')), + ('SY', _('Syrian Arab Republic')), + ('SZ', _('Swaziland')), + ('TC', _('Turks & Caicos Islands')), + ('TD', _('Chad')), + ('TF', _('French Southern Territories')), + ('TG', _('Togo')), + ('TH', _('Thailand')), + ('TJ', _('Tajikistan')), + ('TK', _('Tokelau')), + ('TM', _('Turkmenistan')), + ('TN', _('Tunisia')), + ('TO', _('Tonga')), + ('TP', _('East Timor')), + ('TR', _('Turkey')), + ('TT', _('Trinidad & Tobago')), + ('TV', _('Tuvalu')), + ('TW', _('Taiwan, Province of China')), + ('TZ', _('Tanzania, United Republic of')), + ('UA', _('Ukraine')), + ('UG', _('Uganda')), + ('UM', _('United States Minor Outlying Islands')), + ('US', _('United States of America')), + ('UY', _('Uruguay')), + ('UZ', _('Uzbekistan')), + ('VA', _('Vatican City State (Holy See)')), + ('VC', _('St. Vincent & the Grenadines')), + ('VE', _('Venezuela')), + ('VG', _('British Virgin Islands')), + ('VI', _('United States Virgin Islands')), + ('VN', _('Viet Nam')), + ('VU', _('Vanuatu')), + ('WF', _('Wallis & Futuna Islands')), + ('WS', _('Samoa')), + ('YE', _('Yemen')), + ('YT', _('Mayotte')), + ('YU', _('Yugoslavia')), + ('ZA', _('South Africa')), + ('ZM', _('Zambia')), + ('ZR', _('Zaire')), + ('ZW', _('Zimbabwe')), +) diff --git a/uncloud_pay/admin.py b/uncloud_pay/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_pay/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_pay/apps.py b/uncloud_pay/apps.py new file mode 100644 index 0000000..051ffb4 --- /dev/null +++ b/uncloud_pay/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudPayConfig(AppConfig): + name = 'uncloud_pay' diff --git a/uncloud_pay/helpers.py b/uncloud_pay/helpers.py new file mode 100644 index 0000000..f791564 --- /dev/null +++ b/uncloud_pay/helpers.py @@ -0,0 +1,26 @@ +from functools import reduce +from datetime import datetime +from rest_framework import mixins +from rest_framework.viewsets import GenericViewSet +from django.utils import timezone +from calendar import monthrange + +def beginning_of_month(year, month): + tz = timezone.get_current_timezone() + return datetime(year=year, month=month, day=1, tzinfo=tz) + +def end_of_month(year, month): + (_, days) = monthrange(year, month) + tz = timezone.get_current_timezone() + return datetime(year=year, month=month, day=days, + hour=23, minute=59, second=59, tzinfo=tz) + +class ProductViewSet(mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.ListModelMixin, + GenericViewSet): + """ + A customer-facing viewset that provides default `create()`, `retrieve()` + and `list()`. + """ + pass diff --git a/uncloud_pay/management/commands/charge-negative-balance.py b/uncloud_pay/management/commands/charge-negative-balance.py new file mode 100644 index 0000000..8ee8736 --- /dev/null +++ b/uncloud_pay/management/commands/charge-negative-balance.py @@ -0,0 +1,31 @@ +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Order, Bill, PaymentMethod, get_balance_for_user + +from datetime import timedelta +from django.utils import timezone + +class Command(BaseCommand): + help = 'Generate bills and charge customers if necessary.' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + users = User.objects.all() + print("Processing {} users.".format(users.count())) + for user in users: + balance = get_balance_for_user(user) + if balance < 0: + print("User {} has negative balance ({}), charging.".format(user.username, balance)) + payment_method = PaymentMethod.get_primary_for(user) + if payment_method != None: + amount_to_be_charged = abs(balance) + charge_ok = payment_method.charge(amount_to_be_charged) + if not charge_ok: + print("ERR: charging {} with method {} failed" + .format(user.username, payment_method.uuid) + ) + else: + print("ERR: no payment method registered for {}".format(user.username)) + print("=> Done.") diff --git a/uncloud_pay/management/commands/generate-bills.py b/uncloud_pay/management/commands/generate-bills.py new file mode 100644 index 0000000..5bd4519 --- /dev/null +++ b/uncloud_pay/management/commands/generate-bills.py @@ -0,0 +1,35 @@ +import logging + +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Order, Bill +from django.core.exceptions import ObjectDoesNotExist + +from datetime import timedelta, date +from django.utils import timezone +from uncloud_pay.models import Bill + +logger = logging.getLogger(__name__) + +class Command(BaseCommand): + help = 'Generate bills and charge customers if necessary.' + + def add_arguments(self, parser): + pass + + # TODO: use logger.* + def handle(self, *args, **options): + # Iterate over all 'active' users. + # TODO: filter out inactive users. + users = User.objects.all() + print("Processing {} users.".format(users.count())) + + for user in users: + now = timezone.now() + Bill.generate_for( + year=now.year, + month=now.month, + user=user) + + # We're done for this round :-) + print("=> Done.") diff --git a/uncloud_pay/management/commands/handle-overdue-bills.py b/uncloud_pay/management/commands/handle-overdue-bills.py new file mode 100644 index 0000000..595fbc2 --- /dev/null +++ b/uncloud_pay/management/commands/handle-overdue-bills.py @@ -0,0 +1,23 @@ +from django.core.management.base import BaseCommand +from uncloud_auth.models import User +from uncloud_pay.models import Bill + +from datetime import timedelta +from django.utils import timezone + +class Command(BaseCommand): + help = 'Take action on overdue bills.' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + users = User.objects.all() + print("Processing {} users.".format(users.count())) + for user in users: + for bill in Bill.get_overdue_for(user): + print("/!\ Overdue bill for {}, {} with amount {}" + .format(user.username, bill.uuid, bill.amount)) + # TODO: take action? + + print("=> Done.") diff --git a/uncloud_pay/management/commands/import-vat-rates.py b/uncloud_pay/management/commands/import-vat-rates.py new file mode 100644 index 0000000..32938e4 --- /dev/null +++ b/uncloud_pay/management/commands/import-vat-rates.py @@ -0,0 +1,44 @@ +from django.core.management.base import BaseCommand +from uncloud_pay.models import VATRate +import csv + + +class Command(BaseCommand): + help = '''Imports VAT Rates. Assume vat rates of format https://github.com/kdeldycke/vat-rates/blob/master/vat_rates.csv''' + + def add_arguments(self, parser): + parser.add_argument('csv_file', nargs='+', type=str) + + def handle(self, *args, **options): + try: + for c_file in options['csv_file']: + print("c_file = %s" % c_file) + with open(c_file, mode='r') as csv_file: + csv_reader = csv.DictReader(csv_file) + line_count = 0 + for row in csv_reader: + if line_count == 0: + line_count += 1 + obj, created = VATRate.objects.get_or_create( + start_date=row["start_date"], + stop_date=row["stop_date"] if row["stop_date"] is not "" else None, + territory_codes=row["territory_codes"], + currency_code=row["currency_code"], + rate=row["rate"], + rate_type=row["rate_type"], + description=row["description"] + ) + if created: + self.stdout.write(self.style.SUCCESS( + '%s. %s - %s - %s - %s' % ( + line_count, + obj.start_date, + obj.stop_date, + obj.territory_codes, + obj.rate + ) + )) + line_count+=1 + + except Exception as e: + print(" *** Error occurred. Details {}".format(str(e))) diff --git a/uncloud_pay/migrations/0001_initial.py b/uncloud_pay/migrations/0001_initial.py new file mode 100644 index 0000000..89fa586 --- /dev/null +++ b/uncloud_pay/migrations/0001_initial.py @@ -0,0 +1,85 @@ +# Generated by Django 3.0.3 on 2020-03-05 10:17 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_auth', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Bill', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField()), + ('ending_date', models.DateTimeField()), + ('due_date', models.DateField()), + ('valid', models.BooleanField(default=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='Order', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField(auto_now_add=True)), + ('ending_date', models.DateTimeField(blank=True, null=True)), + ('recurring_period', models.CharField(choices=[('ONCE', 'Onetime'), ('YEAR', 'Per Year'), ('MONTH', 'Per Month'), ('MINUTE', 'Per Minute'), ('DAY', 'Per Day'), ('HOUR', 'Per Hour'), ('SECOND', 'Per Second')], default='MONTH', max_length=32)), + ('bill', models.ManyToManyField(blank=True, editable=False, to='uncloud_pay.Bill')), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='StripeCustomer', + fields=[ + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to=settings.AUTH_USER_MODEL)), + ('stripe_id', models.CharField(max_length=32)), + ], + ), + migrations.CreateModel( + name='Payment', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('amount', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('source', models.CharField(choices=[('wire', 'Wire Transfer'), ('stripe', 'Stripe'), ('voucher', 'Voucher'), ('referral', 'Referral'), ('unknown', 'Unknown')], default='unknown', max_length=256)), + ('timestamp', models.DateTimeField(auto_now_add=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='OrderRecord', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('one_time_price', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('recurring_price', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('description', models.TextField()), + ('order', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order')), + ], + ), + migrations.CreateModel( + name='PaymentMethod', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('source', models.CharField(choices=[('stripe', 'Stripe'), ('unknown', 'Unknown')], default='stripe', max_length=256)), + ('description', models.TextField()), + ('primary', models.BooleanField(default=True)), + ('stripe_card_id', models.CharField(blank=True, max_length=32, null=True)), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('owner', 'primary')}, + }, + ), + ] diff --git a/uncloud_pay/migrations/0002_auto_20200305_1524.py b/uncloud_pay/migrations/0002_auto_20200305_1524.py new file mode 100644 index 0000000..0768dd0 --- /dev/null +++ b/uncloud_pay/migrations/0002_auto_20200305_1524.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.3 on 2020-03-05 15:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0001_initial'), + ] + + operations = [ + migrations.RenameField( + model_name='paymentmethod', + old_name='stripe_card_id', + new_name='stripe_payment_method_id', + ), + migrations.AddField( + model_name='paymentmethod', + name='stripe_setup_intent_id', + field=models.CharField(blank=True, max_length=32, null=True), + ), + migrations.AlterUniqueTogether( + name='paymentmethod', + unique_together=set(), + ), + ] diff --git a/uncloud_pay/migrations/0003_auto_20200305_1354.py b/uncloud_pay/migrations/0003_auto_20200305_1354.py new file mode 100644 index 0000000..4157732 --- /dev/null +++ b/uncloud_pay/migrations/0003_auto_20200305_1354.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-05 13:54 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0002_auto_20200305_1524'), + ] + + operations = [ + migrations.AlterField( + model_name='paymentmethod', + name='primary', + field=models.BooleanField(default=False, editable=False), + ), + ] diff --git a/uncloud_pay/migrations/0004_auto_20200409_1225.py b/uncloud_pay/migrations/0004_auto_20200409_1225.py new file mode 100644 index 0000000..32aac87 --- /dev/null +++ b/uncloud_pay/migrations/0004_auto_20200409_1225.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.5 on 2020-04-09 12:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0003_auto_20200305_1354'), + ] + + operations = [ + migrations.AlterField( + model_name='order', + name='recurring_period', + field=models.CharField(choices=[('ONCE', 'Onetime'), ('YEAR', 'Per Year'), ('MONTH', 'Per Month'), ('MINUTE', 'Per Minute'), ('WEEK', 'Per Week'), ('DAY', 'Per Day'), ('HOUR', 'Per Hour'), ('SECOND', 'Per Second')], default='MONTH', max_length=32), + ), + migrations.AlterField( + model_name='order', + name='starting_date', + field=models.DateTimeField(), + ), + ] diff --git a/uncloud_pay/migrations/0005_auto_20200413_0924.py b/uncloud_pay/migrations/0005_auto_20200413_0924.py new file mode 100644 index 0000000..3f6a646 --- /dev/null +++ b/uncloud_pay/migrations/0005_auto_20200413_0924.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.5 on 2020-04-13 09:24 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0004_auto_20200409_1225'), + ] + + operations = [ + migrations.AlterField( + model_name='order', + name='recurring_period', + field=models.CharField(choices=[('ONCE', 'Onetime'), ('YEAR', 'Per Year'), ('MONTH', 'Per Month'), ('WEEK', 'Per Week'), ('DAY', 'Per Day'), ('HOUR', 'Per Hour'), ('MINUTE', 'Per Minute'), ('SECOND', 'Per Second')], default='MONTH', max_length=32), + ), + ] diff --git a/uncloud_pay/migrations/0006_auto_20200415_1003.py b/uncloud_pay/migrations/0006_auto_20200415_1003.py new file mode 100644 index 0000000..1f37eae --- /dev/null +++ b/uncloud_pay/migrations/0006_auto_20200415_1003.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.5 on 2020-04-15 10:03 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0005_auto_20200413_0924'), + ] + + operations = [ + migrations.CreateModel( + name='VATRate', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('start_date', models.DateField(blank=True, null=True)), + ('stop_date', models.DateField(blank=True, null=True)), + ('territory_codes', models.TextField(blank=True, default='')), + ('currency_code', models.CharField(max_length=10)), + ('rate', models.FloatField()), + ('rate_type', models.TextField(blank=True, default='')), + ('description', models.TextField(blank=True, default='')), + ], + ), + migrations.AlterField( + model_name='order', + name='recurring_period', + field=models.CharField(choices=[('ONCE', 'Onetime'), ('YEAR', 'Per Year'), ('MONTH', 'Per Month'), ('WEEK', 'Per Week'), ('DAY', 'Per Day'), ('HOUR', 'Per Hour'), ('MINUTE', 'Per Minute'), ('SECOND', 'Per Second')], default='MONTH', max_length=32), + ), + ] diff --git a/uncloud_pay/migrations/0006_billingaddress.py b/uncloud_pay/migrations/0006_billingaddress.py new file mode 100644 index 0000000..79b25ab --- /dev/null +++ b/uncloud_pay/migrations/0006_billingaddress.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.5 on 2020-04-15 12:29 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uncloud_pay.models + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0006_auto_20200415_1003'), + ] + + operations = [ + migrations.CreateModel( + name='BillingAddress', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('street', models.CharField(max_length=100)), + ('city', models.CharField(max_length=50)), + ('postal_code', models.CharField(max_length=50)), + ('country', uncloud_pay.models.CountryField(choices=[('AD', 'Andorra'), ('AE', 'United Arab Emirates'), ('AF', 'Afghanistan'), ('AG', 'Antigua & Barbuda'), ('AI', 'Anguilla'), ('AL', 'Albania'), ('AM', 'Armenia'), ('AN', 'Netherlands Antilles'), ('AO', 'Angola'), ('AQ', 'Antarctica'), ('AR', 'Argentina'), ('AS', 'American Samoa'), ('AT', 'Austria'), ('AU', 'Australia'), ('AW', 'Aruba'), ('AZ', 'Azerbaijan'), ('BA', 'Bosnia and Herzegovina'), ('BB', 'Barbados'), ('BD', 'Bangladesh'), ('BE', 'Belgium'), ('BF', 'Burkina Faso'), ('BG', 'Bulgaria'), ('BH', 'Bahrain'), ('BI', 'Burundi'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BN', 'Brunei Darussalam'), ('BO', 'Bolivia'), ('BR', 'Brazil'), ('BS', 'Bahama'), ('BT', 'Bhutan'), ('BV', 'Bouvet Island'), ('BW', 'Botswana'), ('BY', 'Belarus'), ('BZ', 'Belize'), ('CA', 'Canada'), ('CC', 'Cocos (Keeling) Islands'), ('CF', 'Central African Republic'), ('CG', 'Congo'), ('CH', 'Switzerland'), ('CI', 'Ivory Coast'), ('CK', 'Cook Iislands'), ('CL', 'Chile'), ('CM', 'Cameroon'), ('CN', 'China'), ('CO', 'Colombia'), ('CR', 'Costa Rica'), ('CU', 'Cuba'), ('CV', 'Cape Verde'), ('CX', 'Christmas Island'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DE', 'Germany'), ('DJ', 'Djibouti'), ('DK', 'Denmark'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('DZ', 'Algeria'), ('EC', 'Ecuador'), ('EE', 'Estonia'), ('EG', 'Egypt'), ('EH', 'Western Sahara'), ('ER', 'Eritrea'), ('ES', 'Spain'), ('ET', 'Ethiopia'), ('FI', 'Finland'), ('FJ', 'Fiji'), ('FK', 'Falkland Islands (Malvinas)'), ('FM', 'Micronesia'), ('FO', 'Faroe Islands'), ('FR', 'France'), ('FX', 'France, Metropolitan'), ('GA', 'Gabon'), ('GB', 'United Kingdom (Great Britain)'), ('GD', 'Grenada'), ('GE', 'Georgia'), ('GF', 'French Guiana'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GL', 'Greenland'), ('GM', 'Gambia'), ('GN', 'Guinea'), ('GP', 'Guadeloupe'), ('GQ', 'Equatorial Guinea'), ('GR', 'Greece'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GT', 'Guatemala'), ('GU', 'Guam'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HK', 'Hong Kong'), ('HM', 'Heard & McDonald Islands'), ('HN', 'Honduras'), ('HR', 'Croatia'), ('HT', 'Haiti'), ('HU', 'Hungary'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('IL', 'Israel'), ('IN', 'India'), ('IO', 'British Indian Ocean Territory'), ('IQ', 'Iraq'), ('IR', 'Islamic Republic of Iran'), ('IS', 'Iceland'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JO', 'Jordan'), ('JP', 'Japan'), ('KE', 'Kenya'), ('KG', 'Kyrgyzstan'), ('KH', 'Cambodia'), ('KI', 'Kiribati'), ('KM', 'Comoros'), ('KN', 'St. Kitts and Nevis'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('KW', 'Kuwait'), ('KY', 'Cayman Islands'), ('KZ', 'Kazakhstan'), ('LA', "Lao People's Democratic Republic"), ('LB', 'Lebanon'), ('LC', 'Saint Lucia'), ('LI', 'Liechtenstein'), ('LK', 'Sri Lanka'), ('LR', 'Liberia'), ('LS', 'Lesotho'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('LV', 'Latvia'), ('LY', 'Libyan Arab Jamahiriya'), ('MA', 'Morocco'), ('MC', 'Monaco'), ('MD', 'Moldova, Republic of'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('ML', 'Mali'), ('MN', 'Mongolia'), ('MM', 'Myanmar'), ('MO', 'Macau'), ('MP', 'Northern Mariana Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MS', 'Monserrat'), ('MT', 'Malta'), ('MU', 'Mauritius'), ('MV', 'Maldives'), ('MW', 'Malawi'), ('MX', 'Mexico'), ('MY', 'Malaysia'), ('MZ', 'Mozambique'), ('NA', 'Namibia'), ('NC', 'New Caledonia'), ('NE', 'Niger'), ('NF', 'Norfolk Island'), ('NG', 'Nigeria'), ('NI', 'Nicaragua'), ('NL', 'Netherlands'), ('NO', 'Norway'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('NU', 'Niue'), ('NZ', 'New Zealand'), ('OM', 'Oman'), ('PA', 'Panama'), ('PE', 'Peru'), ('PF', 'French Polynesia'), ('PG', 'Papua New Guinea'), ('PH', 'Philippines'), ('PK', 'Pakistan'), ('PL', 'Poland'), ('PM', 'St. Pierre & Miquelon'), ('PN', 'Pitcairn'), ('PR', 'Puerto Rico'), ('PT', 'Portugal'), ('PW', 'Palau'), ('PY', 'Paraguay'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SA', 'Saudi Arabia'), ('SB', 'Solomon Islands'), ('SC', 'Seychelles'), ('SD', 'Sudan'), ('SE', 'Sweden'), ('SG', 'Singapore'), ('SH', 'St. Helena'), ('SI', 'Slovenia'), ('SJ', 'Svalbard & Jan Mayen Islands'), ('SK', 'Slovakia'), ('SL', 'Sierra Leone'), ('SM', 'San Marino'), ('SN', 'Senegal'), ('SO', 'Somalia'), ('SR', 'Suriname'), ('ST', 'Sao Tome & Principe'), ('SV', 'El Salvador'), ('SY', 'Syrian Arab Republic'), ('SZ', 'Swaziland'), ('TC', 'Turks & Caicos Islands'), ('TD', 'Chad'), ('TF', 'French Southern Territories'), ('TG', 'Togo'), ('TH', 'Thailand'), ('TJ', 'Tajikistan'), ('TK', 'Tokelau'), ('TM', 'Turkmenistan'), ('TN', 'Tunisia'), ('TO', 'Tonga'), ('TP', 'East Timor'), ('TR', 'Turkey'), ('TT', 'Trinidad & Tobago'), ('TV', 'Tuvalu'), ('TW', 'Taiwan, Province of China'), ('TZ', 'Tanzania, United Republic of'), ('UA', 'Ukraine'), ('UG', 'Uganda'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VA', 'Vatican City State (Holy See)'), ('VC', 'St. Vincent & the Grenadines'), ('VE', 'Venezuela'), ('VG', 'British Virgin Islands'), ('VI', 'United States Virgin Islands'), ('VN', 'Viet Nam'), ('VU', 'Vanuatu'), ('WF', 'Wallis & Futuna Islands'), ('WS', 'Samoa'), ('YE', 'Yemen'), ('YT', 'Mayotte'), ('YU', 'Yugoslavia'), ('ZA', 'South Africa'), ('ZM', 'Zambia'), ('ZR', 'Zaire'), ('ZW', 'Zimbabwe')], default='CH', max_length=2)), + ('vat_number', models.CharField(blank=True, default='', max_length=100)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/uncloud_pay/migrations/0007_auto_20200418_0737.py b/uncloud_pay/migrations/0007_auto_20200418_0737.py new file mode 100644 index 0000000..c9c2342 --- /dev/null +++ b/uncloud_pay/migrations/0007_auto_20200418_0737.py @@ -0,0 +1,42 @@ +# Generated by Django 3.0.5 on 2020-04-18 07:37 + +from django.db import migrations, models +import django.db.models.deletion +import uncloud_pay.models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0006_billingaddress'), + ] + + operations = [ + migrations.RemoveField( + model_name='billingaddress', + name='id', + ), + migrations.AddField( + model_name='billingaddress', + name='name', + field=models.CharField(default='unknown', max_length=100), + preserve_default=False, + ), + migrations.AddField( + model_name='billingaddress', + name='uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False), + ), + migrations.AddField( + model_name='order', + name='billing_address', + field=models.ForeignKey(default=None, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.BillingAddress'), + preserve_default=False, + ), + migrations.AlterField( + model_name='billingaddress', + name='country', + field=uncloud_pay.models.CountryField(blank=True, choices=[('AD', 'Andorra'), ('AE', 'United Arab Emirates'), ('AF', 'Afghanistan'), ('AG', 'Antigua & Barbuda'), ('AI', 'Anguilla'), ('AL', 'Albania'), ('AM', 'Armenia'), ('AN', 'Netherlands Antilles'), ('AO', 'Angola'), ('AQ', 'Antarctica'), ('AR', 'Argentina'), ('AS', 'American Samoa'), ('AT', 'Austria'), ('AU', 'Australia'), ('AW', 'Aruba'), ('AZ', 'Azerbaijan'), ('BA', 'Bosnia and Herzegovina'), ('BB', 'Barbados'), ('BD', 'Bangladesh'), ('BE', 'Belgium'), ('BF', 'Burkina Faso'), ('BG', 'Bulgaria'), ('BH', 'Bahrain'), ('BI', 'Burundi'), ('BJ', 'Benin'), ('BM', 'Bermuda'), ('BN', 'Brunei Darussalam'), ('BO', 'Bolivia'), ('BR', 'Brazil'), ('BS', 'Bahama'), ('BT', 'Bhutan'), ('BV', 'Bouvet Island'), ('BW', 'Botswana'), ('BY', 'Belarus'), ('BZ', 'Belize'), ('CA', 'Canada'), ('CC', 'Cocos (Keeling) Islands'), ('CF', 'Central African Republic'), ('CG', 'Congo'), ('CH', 'Switzerland'), ('CI', 'Ivory Coast'), ('CK', 'Cook Iislands'), ('CL', 'Chile'), ('CM', 'Cameroon'), ('CN', 'China'), ('CO', 'Colombia'), ('CR', 'Costa Rica'), ('CU', 'Cuba'), ('CV', 'Cape Verde'), ('CX', 'Christmas Island'), ('CY', 'Cyprus'), ('CZ', 'Czech Republic'), ('DE', 'Germany'), ('DJ', 'Djibouti'), ('DK', 'Denmark'), ('DM', 'Dominica'), ('DO', 'Dominican Republic'), ('DZ', 'Algeria'), ('EC', 'Ecuador'), ('EE', 'Estonia'), ('EG', 'Egypt'), ('EH', 'Western Sahara'), ('ER', 'Eritrea'), ('ES', 'Spain'), ('ET', 'Ethiopia'), ('FI', 'Finland'), ('FJ', 'Fiji'), ('FK', 'Falkland Islands (Malvinas)'), ('FM', 'Micronesia'), ('FO', 'Faroe Islands'), ('FR', 'France'), ('FX', 'France, Metropolitan'), ('GA', 'Gabon'), ('GB', 'United Kingdom (Great Britain)'), ('GD', 'Grenada'), ('GE', 'Georgia'), ('GF', 'French Guiana'), ('GH', 'Ghana'), ('GI', 'Gibraltar'), ('GL', 'Greenland'), ('GM', 'Gambia'), ('GN', 'Guinea'), ('GP', 'Guadeloupe'), ('GQ', 'Equatorial Guinea'), ('GR', 'Greece'), ('GS', 'South Georgia and the South Sandwich Islands'), ('GT', 'Guatemala'), ('GU', 'Guam'), ('GW', 'Guinea-Bissau'), ('GY', 'Guyana'), ('HK', 'Hong Kong'), ('HM', 'Heard & McDonald Islands'), ('HN', 'Honduras'), ('HR', 'Croatia'), ('HT', 'Haiti'), ('HU', 'Hungary'), ('ID', 'Indonesia'), ('IE', 'Ireland'), ('IL', 'Israel'), ('IN', 'India'), ('IO', 'British Indian Ocean Territory'), ('IQ', 'Iraq'), ('IR', 'Islamic Republic of Iran'), ('IS', 'Iceland'), ('IT', 'Italy'), ('JM', 'Jamaica'), ('JO', 'Jordan'), ('JP', 'Japan'), ('KE', 'Kenya'), ('KG', 'Kyrgyzstan'), ('KH', 'Cambodia'), ('KI', 'Kiribati'), ('KM', 'Comoros'), ('KN', 'St. Kitts and Nevis'), ('KP', "Korea, Democratic People's Republic of"), ('KR', 'Korea, Republic of'), ('KW', 'Kuwait'), ('KY', 'Cayman Islands'), ('KZ', 'Kazakhstan'), ('LA', "Lao People's Democratic Republic"), ('LB', 'Lebanon'), ('LC', 'Saint Lucia'), ('LI', 'Liechtenstein'), ('LK', 'Sri Lanka'), ('LR', 'Liberia'), ('LS', 'Lesotho'), ('LT', 'Lithuania'), ('LU', 'Luxembourg'), ('LV', 'Latvia'), ('LY', 'Libyan Arab Jamahiriya'), ('MA', 'Morocco'), ('MC', 'Monaco'), ('MD', 'Moldova, Republic of'), ('MG', 'Madagascar'), ('MH', 'Marshall Islands'), ('ML', 'Mali'), ('MN', 'Mongolia'), ('MM', 'Myanmar'), ('MO', 'Macau'), ('MP', 'Northern Mariana Islands'), ('MQ', 'Martinique'), ('MR', 'Mauritania'), ('MS', 'Monserrat'), ('MT', 'Malta'), ('MU', 'Mauritius'), ('MV', 'Maldives'), ('MW', 'Malawi'), ('MX', 'Mexico'), ('MY', 'Malaysia'), ('MZ', 'Mozambique'), ('NA', 'Namibia'), ('NC', 'New Caledonia'), ('NE', 'Niger'), ('NF', 'Norfolk Island'), ('NG', 'Nigeria'), ('NI', 'Nicaragua'), ('NL', 'Netherlands'), ('NO', 'Norway'), ('NP', 'Nepal'), ('NR', 'Nauru'), ('NU', 'Niue'), ('NZ', 'New Zealand'), ('OM', 'Oman'), ('PA', 'Panama'), ('PE', 'Peru'), ('PF', 'French Polynesia'), ('PG', 'Papua New Guinea'), ('PH', 'Philippines'), ('PK', 'Pakistan'), ('PL', 'Poland'), ('PM', 'St. Pierre & Miquelon'), ('PN', 'Pitcairn'), ('PR', 'Puerto Rico'), ('PT', 'Portugal'), ('PW', 'Palau'), ('PY', 'Paraguay'), ('QA', 'Qatar'), ('RE', 'Reunion'), ('RO', 'Romania'), ('RU', 'Russian Federation'), ('RW', 'Rwanda'), ('SA', 'Saudi Arabia'), ('SB', 'Solomon Islands'), ('SC', 'Seychelles'), ('SD', 'Sudan'), ('SE', 'Sweden'), ('SG', 'Singapore'), ('SH', 'St. Helena'), ('SI', 'Slovenia'), ('SJ', 'Svalbard & Jan Mayen Islands'), ('SK', 'Slovakia'), ('SL', 'Sierra Leone'), ('SM', 'San Marino'), ('SN', 'Senegal'), ('SO', 'Somalia'), ('SR', 'Suriname'), ('ST', 'Sao Tome & Principe'), ('SV', 'El Salvador'), ('SY', 'Syrian Arab Republic'), ('SZ', 'Swaziland'), ('TC', 'Turks & Caicos Islands'), ('TD', 'Chad'), ('TF', 'French Southern Territories'), ('TG', 'Togo'), ('TH', 'Thailand'), ('TJ', 'Tajikistan'), ('TK', 'Tokelau'), ('TM', 'Turkmenistan'), ('TN', 'Tunisia'), ('TO', 'Tonga'), ('TP', 'East Timor'), ('TR', 'Turkey'), ('TT', 'Trinidad & Tobago'), ('TV', 'Tuvalu'), ('TW', 'Taiwan, Province of China'), ('TZ', 'Tanzania, United Republic of'), ('UA', 'Ukraine'), ('UG', 'Uganda'), ('UM', 'United States Minor Outlying Islands'), ('US', 'United States of America'), ('UY', 'Uruguay'), ('UZ', 'Uzbekistan'), ('VA', 'Vatican City State (Holy See)'), ('VC', 'St. Vincent & the Grenadines'), ('VE', 'Venezuela'), ('VG', 'British Virgin Islands'), ('VI', 'United States Virgin Islands'), ('VN', 'Viet Nam'), ('VU', 'Vanuatu'), ('WF', 'Wallis & Futuna Islands'), ('WS', 'Samoa'), ('YE', 'Yemen'), ('YT', 'Mayotte'), ('YU', 'Yugoslavia'), ('ZA', 'South Africa'), ('ZM', 'Zambia'), ('ZR', 'Zaire'), ('ZW', 'Zimbabwe')], default='CH', max_length=2), + ), + ] diff --git a/uncloud_pay/migrations/0008_auto_20200502_1921.py b/uncloud_pay/migrations/0008_auto_20200502_1921.py new file mode 100644 index 0000000..c244357 --- /dev/null +++ b/uncloud_pay/migrations/0008_auto_20200502_1921.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.5 on 2020-05-02 19:21 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0007_auto_20200418_0737'), + ] + + operations = [ + migrations.AlterField( + model_name='order', + name='starting_date', + field=models.DateTimeField(default=django.utils.timezone.now), + ), + ] diff --git a/uncloud_pay/migrations/0009_auto_20200502_2047.py b/uncloud_pay/migrations/0009_auto_20200502_2047.py new file mode 100644 index 0000000..cb9cd78 --- /dev/null +++ b/uncloud_pay/migrations/0009_auto_20200502_2047.py @@ -0,0 +1,47 @@ +# Generated by Django 3.0.5 on 2020-05-02 20:47 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django.utils.timezone +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0008_auto_20200502_1921'), + ] + + operations = [ + migrations.AddField( + model_name='order', + name='one_time_price', + field=models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)]), + ), + migrations.AddField( + model_name='order', + name='recurring_price', + field=models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)]), + ), + migrations.AddField( + model_name='order', + name='replaced_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, to='uncloud_pay.Order'), + ), + migrations.CreateModel( + name='OrderTimothee', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField(default=django.utils.timezone.now)), + ('ending_date', models.DateTimeField(blank=True, null=True)), + ('recurring_period', models.CharField(choices=[('ONCE', 'Onetime'), ('YEAR', 'Per Year'), ('MONTH', 'Per Month'), ('WEEK', 'Per Week'), ('DAY', 'Per Day'), ('HOUR', 'Per Hour'), ('MINUTE', 'Per Minute'), ('SECOND', 'Per Second')], default='MONTH', max_length=32)), + ('bill', models.ManyToManyField(blank=True, editable=False, to='uncloud_pay.Bill')), + ('billing_address', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.BillingAddress')), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/uncloud_pay/migrations/0010_order_description.py b/uncloud_pay/migrations/0010_order_description.py new file mode 100644 index 0000000..2613bff --- /dev/null +++ b/uncloud_pay/migrations/0010_order_description.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.6 on 2020-05-07 10:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0009_auto_20200502_2047'), + ] + + operations = [ + migrations.AddField( + model_name='order', + name='description', + field=models.TextField(default=''), + preserve_default=False, + ), + ] diff --git a/uncloud_pay/migrations/0011_billingaddress_organization.py b/uncloud_pay/migrations/0011_billingaddress_organization.py new file mode 100644 index 0000000..ac36eee --- /dev/null +++ b/uncloud_pay/migrations/0011_billingaddress_organization.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.6 on 2020-05-07 13:07 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0010_order_description'), + ] + + operations = [ + migrations.AddField( + model_name='billingaddress', + name='organization', + field=models.CharField(default='', max_length=100), + preserve_default=False, + ), + ] diff --git a/uncloud_pay/migrations/0012_billnico.py b/uncloud_pay/migrations/0012_billnico.py new file mode 100644 index 0000000..f69241d --- /dev/null +++ b/uncloud_pay/migrations/0012_billnico.py @@ -0,0 +1,29 @@ +# Generated by Django 3.0.6 on 2020-05-08 07:06 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0011_billingaddress_organization'), + ] + + operations = [ + migrations.CreateModel( + name='BillNico', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('creation_date', models.DateTimeField(auto_now_add=True)), + ('starting_date', models.DateTimeField()), + ('ending_date', models.DateTimeField()), + ('due_date', models.DateField()), + ('valid', models.BooleanField(default=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/uncloud_pay/migrations/0013_auto_20200508_1446.py b/uncloud_pay/migrations/0013_auto_20200508_1446.py new file mode 100644 index 0000000..dcf7675 --- /dev/null +++ b/uncloud_pay/migrations/0013_auto_20200508_1446.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.6 on 2020-05-08 14:46 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0012_billnico'), + ] + + operations = [ + migrations.AddField( + model_name='order', + name='depends_on', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='parent_of', to='uncloud_pay.Order'), + ), + migrations.AlterField( + model_name='order', + name='replaced_by', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.PROTECT, related_name='supersede', to='uncloud_pay.Order'), + ), + ] diff --git a/uncloud_pay/migrations/0014_paymentsettings.py b/uncloud_pay/migrations/0014_paymentsettings.py new file mode 100644 index 0000000..2a4f9a0 --- /dev/null +++ b/uncloud_pay/migrations/0014_paymentsettings.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.6 on 2020-05-10 13:53 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0013_auto_20200508_1446'), + ] + + operations = [ + migrations.CreateModel( + name='PaymentSettings', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('owner', models.OneToOneField(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('primary_billing_address', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.BillingAddress')), + ], + ), + ] diff --git a/uncloud_pay/migrations/__init__.py b/uncloud_pay/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_pay/models.py b/uncloud_pay/models.py new file mode 100644 index 0000000..efb7b07 --- /dev/null +++ b/uncloud_pay/models.py @@ -0,0 +1,911 @@ +from django.db import models +from django.db.models import Q +from django.contrib.auth import get_user_model +from django.utils.translation import gettext_lazy as _ +from django.core.validators import MinValueValidator +from django.utils import timezone +from django.core.exceptions import ObjectDoesNotExist, ValidationError + +import uuid +import logging +from functools import reduce +import itertools +from math import ceil +from datetime import timedelta +from calendar import monthrange +from decimal import Decimal + +import uncloud_pay.stripe +from uncloud_pay.helpers import beginning_of_month, end_of_month +from uncloud_pay import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS, COUNTRIES +from uncloud.models import UncloudModel, UncloudStatus + +from decimal import Decimal +import decimal + +# Used to generate bill due dates. +BILL_PAYMENT_DELAY=timedelta(days=10) + +# Initialize logger. +logger = logging.getLogger(__name__) + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class RecurringPeriod(models.TextChoices): + ONE_TIME = 'ONCE', _('Onetime') + PER_YEAR = 'YEAR', _('Per Year') + PER_MONTH = 'MONTH', _('Per Month') + PER_WEEK = 'WEEK', _('Per Week') + PER_DAY = 'DAY', _('Per Day') + PER_HOUR = 'HOUR', _('Per Hour') + PER_MINUTE = 'MINUTE', _('Per Minute') + PER_SECOND = 'SECOND', _('Per Second') + +class CountryField(models.CharField): + def __init__(self, *args, **kwargs): + kwargs.setdefault('choices', COUNTRIES) + kwargs.setdefault('default', 'CH') + kwargs.setdefault('max_length', 2) + + super(CountryField, self).__init__(*args, **kwargs) + + def get_internal_type(self): + return "CharField" + +def get_balance_for_user(user): + bills = reduce( + lambda acc, entry: acc + entry.total, + Bill.objects.filter(owner=user), + 0) + payments = reduce( + lambda acc, entry: acc + entry.amount, + Payment.objects.filter(owner=user), + 0) + return payments - bills + +class StripeCustomer(models.Model): + owner = models.OneToOneField( get_user_model(), + primary_key=True, + on_delete=models.CASCADE) + stripe_id = models.CharField(max_length=32) + +### +# Payments and Payment Methods. + +class Payment(models.Model): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + amount = models.DecimalField( + default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + source = models.CharField(max_length=256, + choices = ( + ('wire', 'Wire Transfer'), + ('stripe', 'Stripe'), + ('voucher', 'Voucher'), + ('referral', 'Referral'), + ('unknown', 'Unknown') + ), + default='unknown') + timestamp = models.DateTimeField(editable=False, auto_now_add=True) + + # We override save() in order to active products awaiting payment. + def save(self, *args, **kwargs): + # _state.adding is switched to false after super(...) call. + being_created = self._state.adding + + unpaid_bills_before_payment = Bill.get_unpaid_for(self.owner) + super(Payment, self).save(*args, **kwargs) # Save payment in DB. + unpaid_bills_after_payment = Bill.get_unpaid_for(self.owner) + + newly_paid_bills = list( + set(unpaid_bills_before_payment) - set(unpaid_bills_after_payment)) + for bill in newly_paid_bills: + bill.activate_products() + +class PaymentMethod(models.Model): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE, + editable=False) + source = models.CharField(max_length=256, + choices = ( + ('stripe', 'Stripe'), + ('unknown', 'Unknown'), + ), + default='stripe') + description = models.TextField() + primary = models.BooleanField(default=False, editable=False) + + # Only used for "Stripe" source + stripe_payment_method_id = models.CharField(max_length=32, blank=True, null=True) + stripe_setup_intent_id = models.CharField(max_length=32, blank=True, null=True) + + @property + def stripe_card_last4(self): + if self.source == 'stripe' and self.active: + payment_method = uncloud_pay.stripe.get_payment_method( + self.stripe_payment_method_id) + return payment_method.card.last4 + else: + return None + + @property + def active(self): + if self.source == 'stripe' and self.stripe_payment_method_id != None: + return True + else: + return False + + def charge(self, amount): + if not self.active: + raise Exception('This payment method is inactive.') + + if amount < 0: # Make sure we don't charge negative amount by errors... + raise Exception('Cannot charge negative amount.') + + if self.source == 'stripe': + stripe_customer = StripeCustomer.objects.get(owner=self.owner).stripe_id + stripe_payment = uncloud_pay.stripe.charge_customer( + amount, stripe_customer, self.stripe_payment_method_id) + if 'paid' in stripe_payment and stripe_payment['paid'] == False: + raise Exception(stripe_payment['error']) + else: + payment = Payment.objects.create( + owner=self.owner, source=self.source, amount=amount) + + return payment + else: + raise Exception('This payment method is unsupported/cannot be charged.') + + def set_as_primary_for(self, user): + methods = PaymentMethod.objects.filter(owner=user, primary=True) + for method in methods: + print(method) + method.primary = False + method.save() + + self.primary = True + self.save() + + def get_primary_for(user): + methods = PaymentMethod.objects.filter(owner=user) + for method in methods: + # Do we want to do something with non-primary method? + if method.active and method.primary: + return method + + return None + + class Meta: + # TODO: limit to one primary method per user. + # unique_together is no good since it won't allow more than one + # non-primary method. + pass + +### +# Bills. + +class BillingAddress(models.Model): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) + + organization = models.CharField(max_length=100) + name = models.CharField(max_length=100) + street = models.CharField(max_length=100) + city = models.CharField(max_length=50) + postal_code = models.CharField(max_length=50) + country = CountryField(blank=True) + vat_number = models.CharField(max_length=100, default="", blank=True) + + @staticmethod + def get_addresses_for(user): + return BillingAddress.objects.filter(owner=user) + + @classmethod + def get_preferred_address_for(cls, user): + addresses = cls.get_addresses_for(user) + if len(addresses) == 0: + return None + else: + # TODO: allow user to set primary/preferred address + return addresses[0] + + def __str__(self): + return "{}, {}, {} {}, {}".format( + self.name, self.street, self.postal_code, self.city, + self.country) + +# Populated with the import-vat-numbers django command. +class VATRate(models.Model): + start_date = models.DateField(blank=True, null=True) + stop_date = models.DateField(blank=True, null=True) + territory_codes = models.TextField(blank=True, default='') + currency_code = models.CharField(max_length=10) + rate = models.FloatField() + rate_type = models.TextField(blank=True, default='') + description = models.TextField(blank=True, default='') + + @staticmethod + def get_for_country(country_code): + vat_rate = None + try: + vat_rate = VATRate.objects.get( + territory_codes=country_code, start_date__isnull=False, stop_date=None + ) + return vat_rate.rate + except VATRate.DoesNotExist as dne: + logger.debug(str(dne)) + logger.debug("Did not find VAT rate for %s, returning 0" % country_code) + return 0 + +class BillNico(models.Model): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField() + ending_date = models.DateTimeField() + due_date = models.DateField() + + valid = models.BooleanField(default=True) + +class Bill(models.Model): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE) + + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField() + ending_date = models.DateTimeField() + due_date = models.DateField() + + valid = models.BooleanField(default=True) + + # Trigger product activation if bill paid at creation (from balance). + def save(self, *args, **kwargs): + super(Bill, self).save(*args, **kwargs) + if not self in Bill.get_unpaid_for(self.owner): + self.activate_products() + + @property + def reference(self): + return "{}-{}".format( + self.owner.username, + self.creation_date.strftime("%Y-%m-%d-%H%M")) + + @property + def records(self): + bill_records = [] + orders = Order.objects.filter(bill=self) + for order in orders: + bill_record = BillRecord(self, order) + bill_records.append(bill_record) + + return bill_records + + @property + def amount(self): + return reduce(lambda acc, record: acc + record.amount, self.records, 0) + + @property + def vat_amount(self): + return reduce(lambda acc, record: acc + record.vat_amount, self.records, 0) + + @property + def total(self): + return self.amount + self.vat_amount + + @property + def final(self): + # A bill is final when its ending date is passed, or when all of its + # orders have been terminated. + every_order_terminated = True + billing_period_is_over = self.ending_date < timezone.now() + for order in self.order_set.all(): + every_order_terminated = every_order_terminated and order.is_terminated + + return billing_period_is_over or every_order_terminated + + def activate_products(self): + for order in self.order_set.all(): + # FIXME: using __something might not be a good idea. + for product_class in Product.__subclasses__(): + for product in product_class.objects.filter(order=order): + if product.status == UncloudStatus.AWAITING_PAYMENT: + product.status = UncloudStatus.PENDING + product.save() + + @property + def billing_address(self): + orders = Order.objects.filter(bill=self) + # The genrate_for method makes sure all the orders of a bill share the + # same billing address. TODO: It would be nice to enforce that somehow... + if orders: + return orders[0].billing_address + else: + return None + + # TODO: split this huuuge method! + @staticmethod + def generate_for(year, month, user): + # /!\ We exclusively work on the specified year and month. + generated_bills = [] + + # Default values for next bill (if any). + starting_date=beginning_of_month(year, month) + ending_date=end_of_month(year, month) + creation_date=timezone.now() + + # Select all orders active on the request period (i.e. starting on or after starting_date). + orders = Order.objects.filter( + Q(ending_date__gte=starting_date) | Q(ending_date__isnull=True), + owner=user) + + # Check if there is already a bill covering the order and period pair: + # * Get latest bill by ending_date: previous_bill.ending_date + # * For monthly bills: if previous_bill.ending_date is before + # (next_bill) ending_date, a new bill has to be generated. + # * For yearly bill: if previous_bill.ending_date is on working + # month, generate new bill. + unpaid_orders = { 'monthly_or_less': [], 'yearly': {} } + for order in orders: + try: + previous_bill = order.bill.latest('ending_date') + except ObjectDoesNotExist: + previous_bill = None + + # FIXME: control flow is confusing in this block. + if order.recurring_period == RecurringPeriod.PER_YEAR: + # We ignore anything smaller than a day in here. + next_yearly_bill_start_on = None + if previous_bill == None: + next_yearly_bill_start_on = order.starting_date + elif previous_bill.ending_date <= ending_date: + next_yearly_bill_start_on = (previous_bill.ending_date + timedelta(days=1)) + + # Store for bill generation. One bucket per day of month with a starting bill. + # bucket is a reference here, no need to reassign. + if next_yearly_bill_start_on: + # We want to group orders by date but keep using datetimes. + next_yearly_bill_start_on = next_yearly_bill_start_on.replace( + minute=0, hour=0, second=0, microsecond=0) + bucket = unpaid_orders['yearly'].get(next_yearly_bill_start_on) + if bucket == None: + unpaid_orders['yearly'][next_yearly_bill_start_on] = [order] + else: + unpaid_orders['yearly'][next_yearly_bill_start_on] = bucket + [order] + else: + if previous_bill == None or previous_bill.ending_date < ending_date: + unpaid_orders['monthly_or_less'].append(order) + + # Handle working month's billing. + if len(unpaid_orders['monthly_or_less']) > 0: + # TODO: PREPAID billing is not supported yet. + prepaid_due_date = min(creation_date, starting_date) + BILL_PAYMENT_DELAY + postpaid_due_date = max(creation_date, ending_date) + BILL_PAYMENT_DELAY + + # There should not be any bill linked to orders with different + # billing addresses. + per_address_orders = itertools.groupby( + unpaid_orders['monthly_or_less'], + lambda o: o.billing_address) + + for addr, bill_orders in per_address_orders: + next_monthly_bill = Bill.objects.create(owner=user, + creation_date=creation_date, + starting_date=starting_date, # FIXME: this is a hack! + ending_date=ending_date, + due_date=postpaid_due_date) + + # It is not possible to register many-to-many relationship before + # the two end-objects are saved in database. + for order in bill_orders: + order.bill.add(next_monthly_bill) + + logger.info("Generated monthly bill {} (amount: {}) for user {}." + .format(next_monthly_bill.uuid, next_monthly_bill.total, user)) + + # Add to output. + generated_bills.append(next_monthly_bill) + + # Handle yearly bills starting on working month. + if len(unpaid_orders['yearly']) > 0: + # For every starting date, generate new bill. + for next_yearly_bill_start_on in unpaid_orders['yearly']: + # No postpaid for yearly payments. + prepaid_due_date = min(creation_date, next_yearly_bill_start_on) + BILL_PAYMENT_DELAY + # Bump by one year, remove one day. + ending_date = next_yearly_bill_start_on.replace( + year=next_yearly_bill_start_on.year+1) - timedelta(days=1) + + # There should not be any bill linked to orders with different + # billing addresses. + per_address_orders = itertools.groupby( + unpaid_orders['yearly'][next_yearly_bill_start_on], + lambda o: o.billing_address) + + for addr, bill_orders in per_address_orders: + next_yearly_bill = Bill.objects.create(owner=user, + creation_date=creation_date, + starting_date=next_yearly_bill_start_on, + ending_date=ending_date, + due_date=prepaid_due_date) + + # It is not possible to register many-to-many relationship before + # the two end-objects are saved in database. + for order in bill_orders: + order.bill.add(next_yearly_bill) + + logger.info("Generated yearly bill {} (amount: {}) for user {}." + .format(next_yearly_bill.uuid, next_yearly_bill.total, user)) + + # Add to output. + generated_bills.append(next_yearly_bill) + + # Return generated (monthly + yearly) bills. + return generated_bills + + @staticmethod + def get_unpaid_for(user): + balance = get_balance_for_user(user) + unpaid_bills = [] + # No unpaid bill if balance is positive. + if balance >= 0: + return unpaid_bills + else: + bills = Bill.objects.filter( + owner=user, + ).order_by('-creation_date') + + # Amount to be paid by the customer. + unpaid_balance = abs(balance) + for bill in bills: + if unpaid_balance <= 0: + break + + unpaid_balance -= bill.total + unpaid_bills.append(bill) + + return unpaid_bills + + @staticmethod + def get_overdue_for(user): + unpaid_bills = Bill.get_unpaid_for(user) + return list(filter(lambda bill: bill.due_date > timezone.now(), unpaid_bills)) + +class BillRecord(): + """ + Entry of a bill, dynamically generated from an order. + """ + + def __init__(self, bill, order): + self.bill = bill + self.order = order + self.recurring_price = order.recurring_price + self.recurring_period = order.recurring_period + self.description = order.description + + if self.order.starting_date >= self.bill.starting_date: + self.one_time_price = order.one_time_price + else: + self.one_time_price = 0 + + # Set decimal context for amount computations. + # XXX: understand why we need +1 here. + decimal.getcontext().prec = AMOUNT_DECIMALS + 1 + + @property + def recurring_count(self): + # Compute billing delta. + billed_until = self.bill.ending_date + if self.order.ending_date != None and self.order.ending_date <= self.bill.ending_date: + billed_until = self.order.ending_date + + billed_from = self.bill.starting_date + if self.order.starting_date > self.bill.starting_date: + billed_from = self.order.starting_date + + if billed_from > billed_until: + # TODO: think about and check edge cases. This should not be + # possible. + raise Exception('Impossible billing delta!') + + billed_delta = billed_until - billed_from + + # TODO: refactor this thing? + # TODO: weekly + if self.recurring_period == RecurringPeriod.PER_YEAR: + # XXX: Should always be one => we do not bill for more than one year. + # TODO: check billed_delta is ~365 days. + return 1 + elif self.recurring_period == RecurringPeriod.PER_MONTH: + days = ceil(billed_delta / timedelta(days=1)) + + # Monthly bills always cover one single month. + if (self.bill.starting_date.year != self.bill.starting_date.year or + self.bill.starting_date.month != self.bill.ending_date.month): + raise Exception('Bill {} covers more than one month. Cannot bill PER_MONTH.'. + format(self.bill.uuid)) + + # XXX: minumal length of monthly order is to be enforced somewhere else. + (_, days_in_month) = monthrange( + self.bill.starting_date.year, + self.bill.starting_date.month) + return round(days / days_in_month, AMOUNT_DECIMALS) + elif self.recurring_period == RecurringPeriod.PER_WEEK: + weeks = ceil(billed_delta / timedelta(week=1)) + return weeks + elif self.recurring_period == RecurringPeriod.PER_DAY: + days = ceil(billed_delta / timedelta(days=1)) + return days + elif self.recurring_period == RecurringPeriod.PER_HOUR: + hours = ceil(billed_delta / timedelta(hours=1)) + return hours + elif self.recurring_period == RecurringPeriod.PER_SECOND: + seconds = ceil(billed_delta / timedelta(seconds=1)) + return seconds + elif self.recurring_period == RecurringPeriod.ONE_TIME: + return 0 + else: + raise Exception('Unsupported recurring period: {}.'. + format(self.order.recurring_period)) + + @property + def vat_rate(self): + return Decimal(VATRate.get_for_country(self.bill.billing_address.country)) + + @property + def vat_amount(self): + return self.amount * self.vat_rate + + @property + def amount(self): + return Decimal(float(self.recurring_price) * self.recurring_count) + self.one_time_price + + @property + def total(self): + return self.amount + self.vat_amount + +### +# Orders. + +# Order are assumed IMMUTABLE and used as SOURCE OF TRUST for generating +# bills. Do **NOT** mutate then! +class Order(models.Model): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE, + editable=False) + billing_address = models.ForeignKey(BillingAddress, on_delete=models.CASCADE) + description = models.TextField() + replaced_by = models.ForeignKey('self', on_delete=models.CASCADE, blank=True, null=True) + + # TODO: enforce ending_date - starting_date to be larger than recurring_period. + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField(default=timezone.now) + ending_date = models.DateTimeField(blank=True, + null=True) + + bill = models.ManyToManyField(Bill, + editable=False, + blank=True) + + recurring_period = models.CharField(max_length=32, + choices = RecurringPeriod.choices, + default = RecurringPeriod.PER_MONTH) + + one_time_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + recurring_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + replaced_by = models.ForeignKey('self', + related_name='supersede', + on_delete=models.PROTECT, + blank=True, + null=True) + + depends_on = models.ForeignKey('self', + related_name='parent_of', + on_delete=models.PROTECT, + blank=True, + null=True) + + @property + def is_terminated(self): + return self.ending_date != None and self.ending_date < timezone.now() + + def terminate(self): + if not self.is_terminated: + self.ending_date = timezone.now() + self.save() + + # Trigger initial bill generation at order creation. + def save(self, *args, **kwargs): + if self.ending_date and self.ending_date < self.starting_date: + raise ValidationError("End date cannot be before starting date") + + super().save(*args, **kwargs) + + def generate_initial_bill(self): + return Bill.generate_for(self.starting_date.year, self.starting_date.month, self.owner) + + # Used by uncloud_pay tests. + @property + def bills(self): + return Bill.objects.filter(order=self) + + @staticmethod + def from_product(product, **kwargs): + # FIXME: this is only a workaround. + billing_address = BillingAddress.get_preferred_address_for(product.owner) + if billing_address == None: + raise Exception("Owner does not have a billing address!") + + return Order(description=product.description, + one_time_price=product.one_time_price, + recurring_price=product.recurring_price, + billing_address=billing_address, + owner=product.owner, + **kwargs) + + def __str__(self): + return "Order {} created at {}, {}->{}, recurring period {}. One time price {}, recurring price {}".format( + self.uuid, self.creation_date, + self.starting_date, self.ending_date, + self.recurring_period, + self.one_time_price, + self.recurring_price) + +class OrderTimothee(models.Model): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE, + editable=False) + billing_address = models.ForeignKey(BillingAddress, on_delete=models.CASCADE) + + # TODO: enforce ending_date - starting_date to be larger than recurring_period. + creation_date = models.DateTimeField(auto_now_add=True) + starting_date = models.DateTimeField(default=timezone.now) + ending_date = models.DateTimeField(blank=True, + null=True) + + bill = models.ManyToManyField(Bill, + editable=False, + blank=True) + + recurring_period = models.CharField(max_length=32, + choices = RecurringPeriod.choices, + default = RecurringPeriod.PER_MONTH) + + # Trigger initial bill generation at order creation. + def save(self, *args, **kwargs): + if self.ending_date and self.ending_date < self.starting_date: + raise ValidationError("End date cannot be before starting date") + + super().save(*args, **kwargs) + + Bill.generate_for(self.starting_date.year, self.starting_date.month, self.owner) + + @property + def records(self): + return OrderRecord.objects.filter(order=self) + + @property + def one_time_price(self): + return reduce(lambda acc, record: acc + record.one_time_price, self.records, 0) + + @property + def recurring_price(self): + return reduce(lambda acc, record: acc + record.recurring_price, self.records, 0) + + # Used by uncloud_pay tests. + @property + def bills(self): + return Bill.objects.filter(order=self) + + def add_record(self, one_time_price, recurring_price, description): + OrderRecord.objects.create(order=self, + one_time_price=one_time_price, + recurring_price=recurring_price, + description=description) + + def __str__(self): + return "Order {} created at {}, {}->{}, recurring period {}. Price one time {}, recurring {}".format( + self.uuid, self.creation_date, + self.starting_date, self.ending_date, + self.recurring_period, + self.one_time_price, + self.recurring_price) + + + +class OrderRecord(models.Model): + """ + Order records store billing informations for products: the actual product + might be mutated and/or moved to another order but we do not want to loose + the details of old orders. + + Used as source of trust to dynamically generate bill entries. + """ + + order = models.ForeignKey(Order, on_delete=models.CASCADE) + one_time_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + recurring_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + description = models.TextField() + + + @property + def recurring_period(self): + return self.order.recurring_period + + @property + def starting_date(self): + return self.order.starting_date + + @property + def ending_date(self): + return self.order.ending_date + + +### +# Products + +# Abstract (= no database representation) class used as parent for products +# (e.g. uncloud_vm.models.VMProduct). +class Product(UncloudModel): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + owner = models.ForeignKey(get_user_model(), + on_delete=models.CASCADE, + editable=False) + + description = "Generic Product" + + status = models.CharField(max_length=32, + choices=UncloudStatus.choices, + default=UncloudStatus.AWAITING_PAYMENT) + + order = models.ForeignKey(Order, + on_delete=models.CASCADE, + editable=False, + null=True) + + # Default period for all products + default_recurring_period = RecurringPeriod.PER_MONTH + + # Used to save records. + def save(self, *args, **kwargs): + # _state.adding is switched to false after super(...) call. + being_created = self._state.adding + + # First time saving - create an order + if not self.order: + billing_address = BillingAddress.get_preferred_address_for(self.owner) + print(billing_address) + + if not billing_address: + raise ValidationError("Cannot order without a billing address") + + # FIXME: allow user to choose recurring_period + self.order = Order.objects.create(owner=self.owner, + billing_address=billing_address, + one_time_price=self.one_time_price, + recurring_period=self.default_recurring_period, + recurring_price=self.recurring_price) + + super().save(*args, **kwargs) + + # # Make sure we only create records on creation. + # if being_created: + # record = OrderRecord( + # one_time_price=self.one_time_price, + # recurring_price=self.recurring_price, + # description=self.description) + # self.order.orderrecord_set.add(record, bulk=False) + + @property + def recurring_price(self): + pass # To be implemented in child. + + @property + def one_time_price(self): + return 0 + + @property + def billing_address(self): + return self.order.billing_address + + @staticmethod + def allowed_recurring_periods(): + return RecurringPeriod.choices + + class Meta: + abstract = True + + def discounted_price_by_period(self, requested_period): + """ + Each product has a standard recurring period for which + we define a pricing. I.e. VPN is usually year, VM is usually monthly. + + The user can opt-in to use a different period, which influences the price: + The longer a user commits, the higher the discount. + + Products can also be limited in the available periods. For instance + a VPN only makes sense to be bought for at least one day. + + Rules are as follows: + + given a standard recurring period of ..., changing to ... modifies price ... + + + # One month for free if buying / year, compared to a month: about 8.33% discount + per_year -> per_month -> /11 + per_month -> per_year -> *11 + + # Month has 30.42 days on average. About 7.9% discount to go monthly + per_month -> per_day -> /28 + per_day -> per_month -> *28 + + # Day has 24h, give one for free + per_day -> per_hour -> /23 + per_hour -> per_day -> /23 + + + Examples + + VPN @ 120CHF/y becomes + - 10.91 CHF/month (130.91 CHF/year) + - 0.39 CHF/day (142.21 CHF/year) + + VM @ 15 CHF/month becomes + - 165 CHF/month (13.75 CHF/month) + - 0.54 CHF/day (16.30 CHF/month) + + """ + + + if self.default_recurring_period == RecurringPeriod.PER_YEAR: + if requested_period == RecurringPeriod.PER_YEAR: + return self.recurring_price + if requested_period == RecurringPeriod.PER_MONTH: + return self.recurring_price/11. + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price/11./28. + + elif self.default_recurring_period == RecurringPeriod.PER_MONTH: + if requested_period == RecurringPeriod.PER_YEAR: + return self.recurring_price*11 + if requested_period == RecurringPeriod.PER_MONTH: + return self.recurring_price + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price/28. + + elif self.default_recurring_period == RecurringPeriod.PER_DAY: + if requested_period == RecurringPeriod.PER_YEAR: + return self.recurring_price*11*28 + if requested_period == RecurringPeriod.PER_MONTH: + return self.recurring_price*28 + if requested_period == RecurringPeriod.PER_DAY: + return self.recurring_price + else: + # FIXME: use the right type of exception here! + raise Exception("Did not implement the discounter for this case") diff --git a/uncloud_pay/serializers.py b/uncloud_pay/serializers.py new file mode 100644 index 0000000..5ee5ad5 --- /dev/null +++ b/uncloud_pay/serializers.py @@ -0,0 +1,118 @@ +from django.contrib.auth import get_user_model +from rest_framework import serializers +from uncloud_auth.serializers import UserSerializer +from django.utils.translation import gettext_lazy as _ + +from .models import * + +### +# Payments and Payment Methods. + +class PaymentSerializer(serializers.ModelSerializer): + class Meta: + model = Payment + fields = '__all__' + +class PaymentMethodSerializer(serializers.ModelSerializer): + stripe_card_last4 = serializers.IntegerField() + + class Meta: + model = PaymentMethod + fields = ['uuid', 'source', 'description', 'primary', 'stripe_card_last4', 'active'] + +class UpdatePaymentMethodSerializer(serializers.ModelSerializer): + class Meta: + model = PaymentMethod + fields = ['description', 'primary'] + +class ChargePaymentMethodSerializer(serializers.Serializer): + amount = serializers.DecimalField(max_digits=10, decimal_places=2) + +class CreatePaymentMethodSerializer(serializers.ModelSerializer): + please_visit = serializers.CharField(read_only=True) + class Meta: + model = PaymentMethod + fields = ['source', 'description', 'primary', 'please_visit'] + +### +# Orders & Products. + +class OrderRecordSerializer(serializers.ModelSerializer): + class Meta: + model = OrderRecord + fields = ['one_time_price', 'recurring_price', 'description'] + + +class OrderSerializer(serializers.ModelSerializer): + owner = serializers.PrimaryKeyRelatedField(queryset=get_user_model().objects.all()) + + def __init__(self, *args, **kwargs): + # Don't pass the 'fields' arg up to the superclass + admin = kwargs.pop('admin', None) + + # Instantiate the superclass normally + super(OrderSerializer, self).__init__(*args, **kwargs) + + # Only allows owner in admin mode. + if not admin: + self.fields.pop('owner') + + def create(self, validated_data): + billing_address = BillingAddress.get_preferred_address_for(validated_data["owner"]) + instance = Order(billing_address=billing_address, **validated_data) + instance.save() + + return instance + + def validate_owner(self, value): + if BillingAddress.get_preferred_address_for(value) == None: + raise serializers.ValidationError("Owner does not have a valid billing address.") + + return value + + class Meta: + model = Order + read_only_fields = ['replaced_by', 'depends_on'] + fields = ['uuid', 'owner', 'description', 'creation_date', 'starting_date', 'ending_date', + 'bill', 'recurring_period', 'recurring_price', 'one_time_price'] + read_only_fields + + +### +# Bills + +# TODO: remove magic numbers for decimal fields +class BillRecordSerializer(serializers.Serializer): + order = serializers.HyperlinkedRelatedField( + view_name='order-detail', + read_only=True) + description = serializers.CharField() + one_time_price = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + recurring_price = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + recurring_period = serializers.ChoiceField(choices=RecurringPeriod.choices) + recurring_count = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + vat_rate = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + vat_amount = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + amount = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + total = serializers.DecimalField(AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS) + +class BillingAddressSerializer(serializers.ModelSerializer): + class Meta: + model = BillingAddress + fields = ['uuid', 'organization', 'name', 'street', 'city', 'postal_code', 'country', 'vat_number'] + +class BillSerializer(serializers.ModelSerializer): + billing_address = BillingAddressSerializer(read_only=True) + records = BillRecordSerializer(many=True, read_only=True) + + class Meta: + model = Bill + fields = ['uuid', 'reference', 'owner', 'amount', 'vat_amount', 'total', + 'due_date', 'creation_date', 'starting_date', 'ending_date', + 'records', 'final', 'billing_address'] + +# We do not want users to mutate the country / VAT number of an address, as it +# will change VAT on existing bills. +class UpdateBillingAddressSerializer(serializers.ModelSerializer): + class Meta: + model = BillingAddress + fields = ['uuid', 'street', 'city', 'postal_code'] diff --git a/uncloud_pay/stripe.py b/uncloud_pay/stripe.py new file mode 100644 index 0000000..2ed4ef2 --- /dev/null +++ b/uncloud_pay/stripe.py @@ -0,0 +1,114 @@ +import stripe +import stripe.error +import logging + +from django.core.exceptions import ObjectDoesNotExist +from django.conf import settings + +import uncloud_pay.models + +# Static stripe configuration used below. +CURRENCY = 'chf' + +# README: We use the Payment Intent API as described on +# https://stripe.com/docs/payments/save-and-reuse + +# For internal use only. +stripe.api_key = settings.STRIPE_KEY + +# Helper (decorator) used to catch errors raised by stripe logic. +# Catch errors that should not be displayed to the end user, raise again. +def handle_stripe_error(f): + def handle_problems(*args, **kwargs): + response = { + 'paid': False, + 'response_object': None, + 'error': None + } + + common_message = "Currently it is not possible to make payments. Please try agin later." + try: + response_object = f(*args, **kwargs) + return response_object + except stripe.error.CardError as e: + # Since it's a decline, stripe.error.CardError will be caught + body = e.json_body + logging.error(str(e)) + + raise e # For error handling. + except stripe.error.RateLimitError: + logging.error("Too many requests made to the API too quickly.") + raise Exception(common_message) + except stripe.error.InvalidRequestError as e: + logging.error(str(e)) + raise Exception('Invalid parameters.') + except stripe.error.AuthenticationError as e: + # Authentication with Stripe's API failed + # (maybe you changed API keys recently) + logging.error(str(e)) + raise Exception(common_message) + except stripe.error.APIConnectionError as e: + logging.error(str(e)) + raise Exception(common_message) + except stripe.error.StripeError as e: + # XXX: maybe send email + logging.error(str(e)) + raise Exception(common_message) + except Exception as e: + # maybe send email + logging.error(str(e)) + raise Exception(common_message) + + return handle_problems + +# Actual Stripe logic. + +def public_api_key(): + return settings.STRIPE_PUBLIC_KEY + +def get_customer_id_for(user): + try: + # .get() raise if there is no matching entry. + return uncloud_pay.models.StripeCustomer.objects.get(owner=user).stripe_id + except ObjectDoesNotExist: + # No entry yet - making a new one. + try: + customer = create_customer(user.username, user.email) + uncloud_stripe_mapping = uncloud_pay.models.StripeCustomer.objects.create( + owner=user, stripe_id=customer.id) + return uncloud_stripe_mapping.stripe_id + except Exception as e: + return None + +@handle_stripe_error +def create_setup_intent(customer_id): + return stripe.SetupIntent.create(customer=customer_id) + +@handle_stripe_error +def get_setup_intent(setup_intent_id): + return stripe.SetupIntent.retrieve(setup_intent_id) + +def get_payment_method(payment_method_id): + return stripe.PaymentMethod.retrieve(payment_method_id) + +@handle_stripe_error +def charge_customer(amount, customer_id, card_id): + # Amount is in CHF but stripes requires smallest possible unit. + # https://stripe.com/docs/api/payment_intents/create#create_payment_intent-amount + adjusted_amount = int(amount * 100) + return stripe.PaymentIntent.create( + amount=adjusted_amount, + currency=CURRENCY, + customer=customer_id, + payment_method=card_id, + off_session=True, + confirm=True, + ) + +@handle_stripe_error +def create_customer(name, email): + return stripe.Customer.create(name=name, email=email) + +@handle_stripe_error +def get_customer(customer_id): + return stripe.Customer.retrieve(customer_id) diff --git a/uncloud_pay/templates/bill.html.j2 b/uncloud_pay/templates/bill.html.j2 new file mode 100644 index 0000000..0ea7089 --- /dev/null +++ b/uncloud_pay/templates/bill.html.j2 @@ -0,0 +1,1080 @@ +{% load static %} + + + + + + + + + {{ bill.reference }} | {{ bill.uuid }} + + + + + + +
+ +
+
+ ungleich glarus ag +
Bahnhofstrasse 1 +
8783 Linthal +
Switzerland +
+
+
+ {% if bill.billing_address.organization != "" %} + {{ bill.billing_address.organization }} +
{{ bill.billing_address.name }} + {% else %} + {{ bill.billing_address.name }} + {% endif %} +
{{ bill.billing_address.street }} +
{{ bill.billing_address.postal_code }} {{ bill.billing_address.city }} +
{{ bill.billing_address.country }} +
+
+
+
+ Rechnungsdatum: +
Rechnungsnummer +
Zahlbar bis + +
+
+ {{ bill.creation_date.date }}
+ {% if bill.billing_address.vat_number != "" %} + {{ bill.billing_address.vat_number %}
+ {% else %} + None
+ {% endif %} + {{ bill.billing_address.vat_number }}
+ {{ bill.due_date }} +
+
+
+
+

RECHNUNG

+
+ + + + + + + + + + + + {% for record in bill.records %} + + + + + + + + {% endfor %} + +
BeschreibungDetailAmountVATTotal
{{ record.description }} + {{ record.recurring_price }} * {{ record.recurring_count }} + {{ record.recurring_period }} + {% if record.one_time_price != 0 %} + + one time {{ record.one_time_price }} + {% endif %} + {{ record.amount }}{{ record.vat_amount }} ({{ record.vat_rate }}){{ record.total }}
+
+

+ Total + {{ bill.amount }} +

+

+ VAT + {{ bill.vat_amount }} +

+
+
+

+ Gesamtbetrag + {{ bill.total }} +

+
+
+
+

+ +41 (0) 55 505-62-66

+ +

+ buchhaltung-ag@ungleich.ch

+ +

+ https://ungleich.ch

+

+ @ungleich

+
+
+

Glarner Kantonalbank

+

+ IBAN: CH 4300 7730 0055 5931 177 +

+

+ BIC: GLKBCH22 +

+

Mwst-Nummer: CHE-156.970.649 MWST

+
+
+ + + diff --git a/uncloud_pay/templates/error.html.j2 b/uncloud_pay/templates/error.html.j2 new file mode 100644 index 0000000..ba9209c --- /dev/null +++ b/uncloud_pay/templates/error.html.j2 @@ -0,0 +1,18 @@ + + + + Error + + + +
+

Error

+

{{ error }}

+
+ + diff --git a/uncloud_pay/templates/stripe-payment.html.j2 b/uncloud_pay/templates/stripe-payment.html.j2 new file mode 100644 index 0000000..6c59740 --- /dev/null +++ b/uncloud_pay/templates/stripe-payment.html.j2 @@ -0,0 +1,76 @@ + + + + Stripe Card Registration + + + + + + + + +
+

Registering Stripe Credit Card

+ + + +
+
+ +
+ + +
+
+ + + + + + + + diff --git a/uncloud_pay/tests.py b/uncloud_pay/tests.py new file mode 100644 index 0000000..00ee294 --- /dev/null +++ b/uncloud_pay/tests.py @@ -0,0 +1,238 @@ +from django.test import TestCase +from django.contrib.auth import get_user_model +from datetime import datetime, date, timedelta + +from .models import * +from uncloud_service.models import GenericServiceProduct + +class BillingTestCase(TestCase): + def setUp(self): + self.user = get_user_model().objects.create( + username='jdoe', + email='john.doe@domain.tld') + self.billing_address = BillingAddress.objects.create( + owner=self.user, + street="unknown", + city="unknown", + postal_code="unknown") + + def test_basic_monthly_billing(self): + one_time_price = 10 + recurring_price = 20 + description = "Test Product 1" + + # Three months: full, full, partial. + starting_date = datetime.fromisoformat('2020-03-01') + ending_date = datetime.fromisoformat('2020-05-08') + + # Create order to be billed. + order = Order.objects.create( + owner=self.user, + starting_date=starting_date, + ending_date=ending_date, + recurring_period=RecurringPeriod.PER_MONTH, + recurring_price=recurring_price, + one_time_price=one_time_price, + description=description, + billing_address=self.billing_address) + + # Generate & check bill for first month: full recurring_price + setup. + first_month_bills = order.generate_initial_bill() + self.assertEqual(len(first_month_bills), 1) + self.assertEqual(first_month_bills[0].amount, one_time_price + recurring_price) + + # Generate & check bill for second month: full recurring_price. + second_month_bills = Bill.generate_for(2020, 4, self.user) + self.assertEqual(len(second_month_bills), 1) + self.assertEqual(second_month_bills[0].amount, recurring_price) + + # Generate & check bill for third and last month: partial recurring_price. + third_month_bills = Bill.generate_for(2020, 5, self.user) + self.assertEqual(len(third_month_bills), 1) + # 31 days in May. + self.assertEqual(float(third_month_bills[0].amount), + round(round((7/31), AMOUNT_DECIMALS) * recurring_price, AMOUNT_DECIMALS)) + + # Check that running Bill.generate_for() twice does not create duplicates. + self.assertEqual(len(Bill.generate_for(2020, 3, self.user)), 0) + + def test_basic_yearly_billing(self): + one_time_price = 10 + recurring_price = 150 + description = "Test Product 1" + + starting_date = datetime.fromisoformat('2020-03-31T08:05:23') + + # Create order to be billed. + order = Order.objects.create( + owner=self.user, + starting_date=starting_date, + recurring_period=RecurringPeriod.PER_YEAR, + recurring_price=recurring_price, + one_time_price=one_time_price, + description=description, + billing_address=self.billing_address) + + # Generate & check bill for first year: recurring_price + setup. + first_year_bills = order.generate_initial_bill() + self.assertEqual(len(first_year_bills), 1) + self.assertEqual(first_year_bills[0].starting_date.date(), + date.fromisoformat('2020-03-31')) + self.assertEqual(first_year_bills[0].ending_date.date(), + date.fromisoformat('2021-03-30')) + self.assertEqual(first_year_bills[0].amount, + recurring_price + one_time_price) + + # Generate & check bill for second year: recurring_price. + second_year_bills = Bill.generate_for(2021, 3, self.user) + self.assertEqual(len(second_year_bills), 1) + self.assertEqual(second_year_bills[0].starting_date.date(), + date.fromisoformat('2021-03-31')) + self.assertEqual(second_year_bills[0].ending_date.date(), + date.fromisoformat('2022-03-30')) + self.assertEqual(second_year_bills[0].amount, recurring_price) + + # Check that running Bill.generate_for() twice does not create duplicates. + self.assertEqual(len(Bill.generate_for(2020, 3, self.user)), 0) + self.assertEqual(len(Bill.generate_for(2020, 4, self.user)), 0) + self.assertEqual(len(Bill.generate_for(2020, 2, self.user)), 0) + self.assertEqual(len(Bill.generate_for(2021, 3, self.user)), 0) + + def test_basic_hourly_billing(self): + one_time_price = 10 + recurring_price = 1.4 + description = "Test Product 1" + + starting_date = datetime.fromisoformat('2020-03-31T08:05:23') + ending_date = datetime.fromisoformat('2020-04-01T11:13:32') + + # Create order to be billed. + order = Order.objects.create( + owner=self.user, + starting_date=starting_date, + ending_date=ending_date, + recurring_period=RecurringPeriod.PER_HOUR, + recurring_price=recurring_price, + one_time_price=one_time_price, + description=description, + billing_address=self.billing_address) + + # Generate & check bill for first month: recurring_price + setup. + first_month_bills = order.generate_initial_bill() + self.assertEqual(len(first_month_bills), 1) + self.assertEqual(float(first_month_bills[0].amount), + round(16 * recurring_price, AMOUNT_DECIMALS) + one_time_price) + + # Generate & check bill for first month: recurring_price. + second_month_bills = Bill.generate_for(2020, 4, self.user) + self.assertEqual(len(second_month_bills), 1) + self.assertEqual(float(second_month_bills[0].amount), + round(12 * recurring_price, AMOUNT_DECIMALS)) + +class ProductActivationTestCase(TestCase): + def setUp(self): + self.user = get_user_model().objects.create( + username='jdoe', + email='john.doe@domain.tld') + + self.billing_address = BillingAddress.objects.create( + owner=self.user, + street="unknown", + city="unknown", + postal_code="unknown") + + def test_product_activation(self): + starting_date = datetime.fromisoformat('2020-03-01') + one_time_price = 0 + recurring_price = 1 + description = "Test Product" + + order = Order.objects.create( + owner=self.user, + starting_date=starting_date, + recurring_period=RecurringPeriod.PER_MONTH, + recurring_price=recurring_price, + one_time_price=one_time_price, + description=description, + billing_address=self.billing_address) + + product = GenericServiceProduct( + custom_description=description, + custom_one_time_price=one_time_price, + custom_recurring_price=recurring_price, + owner=self.user, + order=order) + product.save() + + # Validate initial state: must be awaiting payment. + self.assertEqual(product.status, UncloudStatus.AWAITING_PAYMENT) + + # Pay initial bill, check that product is activated. + order.generate_initial_bill() + amount = product.order.bills[0].amount + payment = Payment(owner=self.user, amount=amount) + payment.save() + self.assertEqual( + GenericServiceProduct.objects.get(uuid=product.uuid).status, + UncloudStatus.PENDING + ) + +class BillingAddressTestCase(TestCase): + def setUp(self): + self.user = get_user_model().objects.create( + username='jdoe', + email='john.doe@domain.tld') + + self.billing_address_01 = BillingAddress.objects.create( + owner=self.user, + street="unknown1", + city="unknown1", + postal_code="unknown1", + country="CH") + + self.billing_address_02 = BillingAddress.objects.create( + owner=self.user, + street="unknown2", + city="unknown2", + postal_code="unknown2", + country="CH") + + def test_billing_with_single_address(self): + # Create new orders somewhere in the past so that we do not encounter + # auto-created initial bills. + starting_date = datetime.fromisoformat('2020-03-01') + + order_01 = Order.objects.create( + owner=self.user, + starting_date=starting_date, + recurring_period=RecurringPeriod.PER_MONTH, + billing_address=self.billing_address_01) + order_02 = Order.objects.create( + owner=self.user, + starting_date=starting_date, + recurring_period=RecurringPeriod.PER_MONTH, + billing_address=self.billing_address_01) + + # We need a single bill since we work with a single address. + bills = Bill.generate_for(2020, 4, self.user) + self.assertEqual(len(bills), 1) + + def test_billing_with_multiple_addresses(self): + # Create new orders somewhere in the past so that we do not encounter + # auto-created initial bills. + starting_date = datetime.fromisoformat('2020-03-01') + + order_01 = Order.objects.create( + owner=self.user, + starting_date=starting_date, + recurring_period=RecurringPeriod.PER_MONTH, + billing_address=self.billing_address_01) + order_02 = Order.objects.create( + owner=self.user, + starting_date=starting_date, + recurring_period=RecurringPeriod.PER_MONTH, + billing_address=self.billing_address_02) + + # We need different bills since we work with different addresses. + bills = Bill.generate_for(2020, 4, self.user) + self.assertEqual(len(bills), 2) diff --git a/uncloud_pay/views.py b/uncloud_pay/views.py new file mode 100644 index 0000000..1144b49 --- /dev/null +++ b/uncloud_pay/views.py @@ -0,0 +1,371 @@ +from django.shortcuts import render +from django.db import transaction +from django.contrib.auth import get_user_model +from rest_framework import viewsets, mixins, permissions, status, views +from rest_framework.renderers import TemplateHTMLRenderer +from rest_framework.response import Response +from rest_framework.decorators import action +from rest_framework.reverse import reverse +from rest_framework.decorators import renderer_classes +from vat_validator import validate_vat, vies +from vat_validator.countries import EU_COUNTRY_CODES +from hardcopy import bytestring_to_pdf +from django.core.files.temp import NamedTemporaryFile +from django.http import FileResponse +from django.template.loader import render_to_string +from copy import deepcopy + +import json +import logging + +from .models import * +from .serializers import * +from datetime import datetime +from vat_validator import sanitize_vat +import uncloud_pay.stripe as uncloud_stripe + +logger = logging.getLogger(__name__) + +### +# Payments and Payment Methods. + +class PaymentViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = PaymentSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Payment.objects.filter(owner=self.request.user) + +class OrderViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = OrderSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Order.objects.filter(owner=self.request.user) + +class PaymentMethodViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_serializer_class(self): + if self.action == 'create': + return CreatePaymentMethodSerializer + elif self.action == 'update': + return UpdatePaymentMethodSerializer + elif self.action == 'charge': + return ChargePaymentMethodSerializer + else: + return PaymentMethodSerializer + + def get_queryset(self): + return PaymentMethod.objects.filter(owner=self.request.user) + + # XXX: Handling of errors is far from great down there. + @transaction.atomic + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Set newly created method as primary if no other method is. + if PaymentMethod.get_primary_for(request.user) == None: + serializer.validated_data['primary'] = True + + if serializer.validated_data['source'] == "stripe": + # Retrieve Stripe customer ID for user. + customer_id = uncloud_stripe.get_customer_id_for(request.user) + if customer_id == None: + return Response( + {'error': 'Could not resolve customer stripe ID.'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + try: + setup_intent = uncloud_stripe.create_setup_intent(customer_id) + except Exception as e: + return Response({'error': str(e)}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + payment_method = PaymentMethod.objects.create( + owner=request.user, + stripe_setup_intent_id=setup_intent.id, + **serializer.validated_data) + + # TODO: find a way to use reverse properly: + # https://www.django-rest-framework.org/api-guide/reverse/ + path = "payment-method/{}/register-stripe-cc".format( + payment_method.uuid) + stripe_registration_url = reverse('api-root', request=request) + path + return Response({'please_visit': stripe_registration_url}) + else: + serializer.save(owner=request.user, **serializer.validated_data) + return Response(serializer.data) + + @action(detail=True, methods=['post']) + def charge(self, request, pk=None): + payment_method = self.get_object() + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + amount = serializer.validated_data['amount'] + try: + payment = payment_method.charge(amount) + output_serializer = PaymentSerializer(payment) + return Response(output_serializer.data) + except Exception as e: + return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + @action(detail=True, methods=['get'], url_path='register-stripe-cc', renderer_classes=[TemplateHTMLRenderer]) + def register_stripe_cc(self, request, pk=None): + payment_method = self.get_object() + + if payment_method.source != 'stripe': + return Response( + {'error': 'This is not a Stripe-based payment method.'}, + template_name='error.html.j2') + + if payment_method.active: + return Response( + {'error': 'This payment method is already active'}, + template_name='error.html.j2') + + try: + setup_intent = uncloud_stripe.get_setup_intent( + payment_method.stripe_setup_intent_id) + except Exception as e: + return Response( + {'error': str(e)}, + template_name='error.html.j2') + + # TODO: find a way to use reverse properly: + # https://www.django-rest-framework.org/api-guide/reverse/ + callback_path= "payment-method/{}/activate-stripe-cc/".format( + payment_method.uuid) + callback = reverse('api-root', request=request) + callback_path + + # Render stripe card registration form. + template_args = { + 'client_secret': setup_intent.client_secret, + 'stripe_pk': uncloud_stripe.public_api_key, + 'callback': callback + } + return Response(template_args, template_name='stripe-payment.html.j2') + + @action(detail=True, methods=['post'], url_path='activate-stripe-cc') + def activate_stripe_cc(self, request, pk=None): + payment_method = self.get_object() + try: + setup_intent = uncloud_stripe.get_setup_intent( + payment_method.stripe_setup_intent_id) + except Exception as e: + return Response({'error': str(e)}, status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + # Card had been registered, fetching payment method. + print(setup_intent) + if setup_intent.payment_method: + payment_method.stripe_payment_method_id = setup_intent.payment_method + payment_method.save() + + return Response({ + 'uuid': payment_method.uuid, + 'activated': payment_method.active}) + else: + error = 'Could not fetch payment method from stripe. Please try again.' + return Response({'error': error}) + + @action(detail=True, methods=['post'], url_path='set-as-primary') + def set_as_primary(self, request, pk=None): + payment_method = self.get_object() + payment_method.set_as_primary_for(request.user) + + serializer = self.get_serializer(payment_method) + return Response(serializer.data) + +### +# Bills and Orders. + +class BillViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = BillSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Bill.objects.filter(owner=self.request.user) + + + @action(detail=False, methods=['get']) + def unpaid(self, request): + serializer = self.get_serializer( + Bill.get_unpaid_for(self.request.user), + many=True) + return Response(serializer.data) + + @action(detail=True, methods=['get']) + def download(self, *args, **kwargs): + bill = self.get_object() + output_file = NamedTemporaryFile() + bill_html = render_to_string("bill.html.j2", {'bill': bill}) + + bytestring_to_pdf(bill_html.encode('utf-8'), output_file) + response = FileResponse(output_file, content_type="application/pdf") + response['Content-Disposition'] = 'filename="{}_{}.pdf"'.format( + bill.reference, bill.uuid + ) + + return response + + +class OrderViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = OrderSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return Order.objects.filter(owner=self.request.user) + +class BillingAddressViewSet(mixins.CreateModelMixin, + mixins.RetrieveModelMixin, + mixins.UpdateModelMixin, + mixins.ListModelMixin, + viewsets.GenericViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_serializer_class(self): + if self.action == 'update': + return UpdateBillingAddressSerializer + else: + return BillingAddressSerializer + + def get_queryset(self): + return self.request.user.billingaddress_set.all() + + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + + # Validate VAT numbers. + country = serializer.validated_data["country"] + + # We ignore empty VAT numbers. + if 'vat_number' in serializer.validated_data and serializer.validated_data["vat_number"] != "": + vat_number = serializer.validated_data["vat_number"] + + if not validate_vat(country, vat_number): + return Response( + {'error': 'Malformed VAT number.'}, + status=status.HTTP_400_BAD_REQUEST) + elif country in EU_COUNTRY_CODES: + # XXX: make a synchroneous call to a third patry API here might not be a good idea.. + try: + vies_state = vies.check_vat(country, vat_number) + if not vies_state.valid: + return Response( + {'error': 'European VAT number does not exist in VIES.'}, + status=status.HTTP_400_BAD_REQUEST) + except Exception as e: + logger.warning(e) + return Response( + {'error': 'Could not validate EU VAT number against VIES. Try again later..'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + + serializer.save(owner=request.user) + return Response(serializer.data) + +### +# Admin stuff. + +class AdminPaymentViewSet(viewsets.ModelViewSet): + serializer_class = PaymentSerializer + permission_classes = [permissions.IsAdminUser] + + def get_queryset(self): + return Payment.objects.all() + + def create(self, request): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save(timestamp=datetime.now()) + + headers = self.get_success_headers(serializer.data) + return Response(serializer.data, status=status.HTTP_201_CREATED, headers=headers) + +# Bills are generated from orders and should not be created or updated by hand. +class AdminBillViewSet(BillViewSet): + serializer_class = BillSerializer + permission_classes = [permissions.IsAdminUser] + + def get_queryset(self): + return Bill.objects.all() + + @action(detail=False, methods=['get']) + def unpaid(self, request): + unpaid_bills = [] + # XXX: works but we can do better than number of users + 1 SQL requests... + for user in get_user_model().objects.all(): + unpaid_bills = unpaid_bills + Bill.get_unpaid_for(self.request.user) + + serializer = self.get_serializer(unpaid_bills, many=True) + return Response(serializer.data) + + @action(detail=False, methods=['post']) + def generate(self, request): + users = get_user_model().objects.all() + + generated_bills = [] + for user in users: + now = timezone.now() + generated_bills = generated_bills + Bill.generate_for( + year=now.year, + month=now.month, + user=user) + + return Response( + map(lambda b: b.reference, generated_bills), + status=status.HTTP_200_OK) + +class AdminOrderViewSet(mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.CreateModelMixin, + mixins.UpdateModelMixin, + viewsets.GenericViewSet): + serializer_class = OrderSerializer + permission_classes = [permissions.IsAdminUser] + + def get_serializer(self, *args, **kwargs): + return self.serializer_class(*args, **kwargs, admin=True) + + def get_queryset(self): + return Order.objects.all() + + # Updates create a new order and terminate the 'old' one. + @transaction.atomic + def update(self, request, *args, **kwargs): + order = self.get_object() + partial = kwargs.pop('partial', False) + serializer = self.get_serializer(order, data=request.data, partial=partial) + serializer.is_valid(raise_exception=True) + + # Clone existing order for replacement. + replacing_order = deepcopy(order) + + # Yes, that's how you make a new entry in DB: + # https://docs.djangoproject.com/en/3.0/topics/db/queries/#copying-model-instances + replacing_order.pk = None + + for attr, value in serializer.validated_data.items(): + setattr(replacing_order, attr, value) + + # Save replacing order and terminate 'previous' one. + replacing_order.save() + order.replaced_by = replacing_order + order.save() + order.terminate() + + return Response(replacing_order) + + @action(detail=True, methods=['post']) + def terminate(self, request, pk): + order = self.get_object() + if order.is_terminated: + return Response( + {'error': 'Order is already terminated.'}, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) + + else: + order.terminate() + return Response({}, status=status.HTTP_200_OK) diff --git a/uncloud_service/__init__.py b/uncloud_service/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_service/admin.py b/uncloud_service/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_service/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_service/apps.py b/uncloud_service/apps.py new file mode 100644 index 0000000..184e181 --- /dev/null +++ b/uncloud_service/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UngleichServiceConfig(AppConfig): + name = 'ungleich_service' diff --git a/uncloud_service/migrations/0001_initial.py b/uncloud_service/migrations/0001_initial.py new file mode 100644 index 0000000..f0f5535 --- /dev/null +++ b/uncloud_service/migrations/0001_initial.py @@ -0,0 +1,36 @@ +# Generated by Django 3.0.5 on 2020-04-13 09:38 + +from django.conf import settings +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('uncloud_pay', '0005_auto_20200413_0924'), + ('uncloud_vm', '0010_auto_20200413_0924'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='MatrixServiceProduct', + fields=[ + ('extra_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True)), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32)), + ('domain', models.CharField(default='domain.tld', max_length=255)), + ('order', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order')), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMProduct')), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/uncloud_service/migrations/0002_auto_20200418_0641.py b/uncloud_service/migrations/0002_auto_20200418_0641.py new file mode 100644 index 0000000..717f163 --- /dev/null +++ b/uncloud_service/migrations/0002_auto_20200418_0641.py @@ -0,0 +1,41 @@ +# Generated by Django 3.0.5 on 2020-04-18 06:41 + +from django.conf import settings +import django.contrib.postgres.fields.jsonb +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('uncloud_pay', '0005_auto_20200413_0924'), + ('uncloud_service', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='matrixserviceproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='AWAITING_PAYMENT', max_length=32), + ), + migrations.CreateModel( + name='GenericServiceProduct', + fields=[ + ('extra_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True)), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='AWAITING_PAYMENT', max_length=32)), + ('custom_description', models.TextField()), + ('custom_recurring_price', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('custom_one_time_price', models.DecimalField(decimal_places=2, default=0.0, max_digits=10, validators=[django.core.validators.MinValueValidator(0)])), + ('order', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order')), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'abstract': False, + }, + ), + ] diff --git a/uncloud_service/migrations/__init__.py b/uncloud_service/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_service/models.py b/uncloud_service/models.py new file mode 100644 index 0000000..35a479e --- /dev/null +++ b/uncloud_service/models.py @@ -0,0 +1,64 @@ +import uuid + +from django.db import models +from uncloud_pay.models import Product, RecurringPeriod, AMOUNT_MAX_DIGITS, AMOUNT_DECIMALS +from uncloud_vm.models import VMProduct, VMDiskImageProduct +from django.core.validators import MinValueValidator + +class MatrixServiceProduct(Product): + monthly_managment_fee = 20 + + description = "Managed Matrix HomeServer" + + # Specific to Matrix-as-a-Service + vm = models.ForeignKey( + VMProduct, on_delete=models.CASCADE + ) + domain = models.CharField(max_length=255, default='domain.tld') + + # Default recurring price is PER_MONT, see Product class. + def recurring_price(self, recurring_period=RecurringPeriod.PER_MONTH): + return self.monthly_managment_fee + + @staticmethod + def base_image(): + # TODO: find a way to safely reference debian 10 image. + return VMDiskImageProduct.objects.get(uuid="93e564c5-adb3-4741-941f-718f76075f02") + + @staticmethod + def allowed_recurring_periods(): + return list(filter( + lambda pair: pair[0] in [RecurringPeriod.PER_MONTH], + RecurringPeriod.choices)) + + @property + def one_time_price(self): + return 30 + +class GenericServiceProduct(Product): + custom_description = models.TextField() + custom_recurring_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + custom_one_time_price = models.DecimalField(default=0.0, + max_digits=AMOUNT_MAX_DIGITS, + decimal_places=AMOUNT_DECIMALS, + validators=[MinValueValidator(0)]) + + @property + def recurring_price(self): + # FIXME: handle recurring_period somehow. + return self.custom_recurring_price + + @property + def description(self): + return self.custom_description + + @property + def one_time_price(self): + return self.custom_one_time_price + + @staticmethod + def allowed_recurring_periods(): + return RecurringPeriod.choices diff --git a/uncloud_service/serializers.py b/uncloud_service/serializers.py new file mode 100644 index 0000000..6666a15 --- /dev/null +++ b/uncloud_service/serializers.py @@ -0,0 +1,60 @@ +from rest_framework import serializers +from .models import * +from uncloud_vm.serializers import ManagedVMProductSerializer +from uncloud_vm.models import VMProduct +from uncloud_pay.models import RecurringPeriod, BillingAddress + +# XXX: the OrderSomethingSomthingProductSerializer classes add a lot of +# boilerplate: can we reduce it somehow? + +class MatrixServiceProductSerializer(serializers.ModelSerializer): + vm = ManagedVMProductSerializer() + + class Meta: + model = MatrixServiceProduct + fields = ['uuid', 'order', 'owner', 'status', 'vm', 'domain', + 'recurring_period'] + read_only_fields = ['uuid', 'order', 'owner', 'status'] + +class OrderMatrixServiceProductSerializer(MatrixServiceProductSerializer): + recurring_period = serializers.ChoiceField( + choices=MatrixServiceProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(OrderMatrixServiceProductSerializer, self).__init__(*args, **kwargs) + self.fields['billing_address'] = serializers.ChoiceField( + choices=BillingAddress.get_addresses_for( + self.context['request'].user) + ) + + class Meta: + model = MatrixServiceProductSerializer.Meta.model + fields = MatrixServiceProductSerializer.Meta.fields + [ + 'recurring_period', 'billing_address' + ] + read_only_fields = MatrixServiceProductSerializer.Meta.read_only_fields + +class GenericServiceProductSerializer(serializers.ModelSerializer): + class Meta: + model = GenericServiceProduct + fields = ['uuid', 'order', 'owner', 'status', 'custom_recurring_price', + 'custom_description', 'custom_one_time_price'] + read_only_fields = ['uuid', 'order', 'owner', 'status'] + +class OrderGenericServiceProductSerializer(GenericServiceProductSerializer): + recurring_period = serializers.ChoiceField( + choices=GenericServiceProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(OrderGenericServiceProductSerializer, self).__init__(*args, **kwargs) + self.fields['billing_address'] = serializers.ChoiceField( + choices=BillingAddress.get_addresses_for( + self.context['request'].user) + ) + + class Meta: + model = GenericServiceProductSerializer.Meta.model + fields = GenericServiceProductSerializer.Meta.fields + [ + 'recurring_period', 'billing_address' + ] + read_only_fields = GenericServiceProductSerializer.Meta.read_only_fields diff --git a/uncloud_service/tests.py b/uncloud_service/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/uncloud_service/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/uncloud_service/views.py b/uncloud_service/views.py new file mode 100644 index 0000000..abd4a05 --- /dev/null +++ b/uncloud_service/views.py @@ -0,0 +1,128 @@ +from rest_framework import viewsets, permissions +from rest_framework.response import Response +from django.db import transaction +from django.utils import timezone + +from .models import * +from .serializers import * + +from uncloud_pay.helpers import ProductViewSet +from uncloud_pay.models import Order +from uncloud_vm.models import VMProduct, VMDiskProduct + +def create_managed_vm(cores, ram, disk_size, image, order): + # Create VM + disk = VMDiskProduct( + owner=order.owner, + order=order, + size_in_gb=disk_size, + image=image) + vm = VMProduct( + name="Managed Service Host", + owner=order.owner, + cores=cores, + ram_in_gb=ram, + primary_disk=disk) + disk.vm = vm + + vm.save() + disk.save() + + return vm + + +class MatrixServiceProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = MatrixServiceProductSerializer + + def get_queryset(self): + return MatrixServiceProduct.objects.filter(owner=self.request.user) + + def get_serializer_class(self): + if self.action == 'create': + return OrderMatrixServiceProductSerializer + else: + return MatrixServiceProductSerializer + + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + order_billing_address = serializer.validated_data.pop("billing_address") + + # Create base order.) + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user, + billing_address=order_billing_address, + starting_date=timezone.now() + ) + order.save() + + # Create unerderlying VM. + data = serializer.validated_data.pop('vm') + vm = create_managed_vm( + order=order, + cores=data['cores'], + ram=data['ram_in_gb'], + disk_size=data['primary_disk']['size_in_gb'], + image=MatrixServiceProduct.base_image()) + + # Create service. + service = serializer.save( + order=order, + owner=request.user, + vm=vm) + + return Response(serializer.data) + +class GenericServiceProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return GenericServiceProduct.objects.filter(owner=self.request.user) + + def get_serializer_class(self): + if self.action == 'create': + return OrderGenericServiceProductSerializer + else: + return GenericServiceProductSerializer + + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + order_billing_address = serializer.validated_data.pop("billing_address") + + # Create base order. + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user, + billing_address=order_billing_address, + starting_date=timezone.now() + ) + order.save() + + # Create service. + print(serializer.validated_data) + service = serializer.save(order=order, owner=request.user) + + # XXX: Move this to some kind of on_create hook in parent + # Product class? + order.add_record( + service.one_time_price, + service.recurring_price, + service.description) + + # XXX: Move this to some kind of on_create hook in parent + # Product class? + order.add_record( + service.one_time_price, + service.recurring_price, + service.description) + + return Response(serializer.data) diff --git a/uncloud_storage/__init__.py b/uncloud_storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_storage/admin.py b/uncloud_storage/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_storage/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_storage/apps.py b/uncloud_storage/apps.py new file mode 100644 index 0000000..38b2301 --- /dev/null +++ b/uncloud_storage/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudStorageConfig(AppConfig): + name = 'uncloud_storage' diff --git a/uncloud_storage/models.py b/uncloud_storage/models.py new file mode 100644 index 0000000..0dac5c2 --- /dev/null +++ b/uncloud_storage/models.py @@ -0,0 +1,7 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class StorageClass(models.TextChoices): + HDD = 'HDD', _('HDD') + SSD = 'SSD', _('SSD') diff --git a/uncloud_storage/tests.py b/uncloud_storage/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/uncloud_storage/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/uncloud_storage/views.py b/uncloud_storage/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/uncloud_storage/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/uncloud_vm/__init__.py b/uncloud_vm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_vm/admin.py b/uncloud_vm/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/uncloud_vm/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/uncloud_vm/apps.py b/uncloud_vm/apps.py new file mode 100644 index 0000000..c5e94a5 --- /dev/null +++ b/uncloud_vm/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class UncloudVmConfig(AppConfig): + name = 'uncloud_vm' diff --git a/uncloud_vm/management/commands/vm.py b/uncloud_vm/management/commands/vm.py new file mode 100644 index 0000000..667c5ad --- /dev/null +++ b/uncloud_vm/management/commands/vm.py @@ -0,0 +1,119 @@ +import json + +import uncloud.secrets as secrets + +from django.core.management.base import BaseCommand +from django.contrib.auth import get_user_model + +from uncloud_vm.models import VMSnapshotProduct, VMProduct, VMHost +from datetime import datetime + +class Command(BaseCommand): + help = 'Select VM Host for VMs' + + def add_arguments(self, parser): + parser.add_argument('--this-hostname', required=True) + parser.add_argument('--this-cluster', required=True) + + parser.add_argument('--create-vm-snapshots', action='store_true') + parser.add_argument('--schedule-vms', action='store_true') + parser.add_argument('--start-vms', action='store_true') + + + def handle(self, *args, **options): + for cmd in [ 'create_vm_snapshots', 'schedule_vms', 'start_vms' ]: + if options[cmd]: + f = getattr(self, cmd) + f(args, options) + + def schedule_vms(self, *args, **options): + for pending_vm in VMProduct.objects.filter(status='PENDING'): + cores_needed = pending_vm.cores + ram_needed = pending_vm.ram_in_gb + + # Database filtering + possible_vmhosts = VMHost.objects.filter(physical_cores__gte=cores_needed) + + # Logical filtering + possible_vmhosts = [ vmhost for vmhost in possible_vmhosts + if vmhost.available_cores >=cores_needed + and vmhost.available_ram_in_gb >= ram_needed ] + + if not possible_vmhosts: + log.error("No suitable Host found - cannot schedule VM {}".format(pending_vm)) + continue + + vmhost = possible_vmhosts[0] + pending_vm.vmhost = vmhost + pending_vm.status = 'SCHEDULED' + pending_vm.save() + + print("Scheduled VM {} on VMHOST {}".format(pending_vm, pending_vm.vmhost)) + + print(self) + + def start_vms(self, *args, **options): + vmhost = VMHost.objects.get(hostname=options['this_hostname']) + + if not vmhost: + raise Exception("No vmhost {} exists".format(options['vmhostname'])) + + # not active? done here + if not vmhost.status = 'ACTIVE': + return + + vms_to_start = VMProduct.objects.filter(vmhost=vmhost, + status='SCHEDULED') + for vm in vms_to_start: + """ run qemu: + check if VM is not already active / qemu running + prepare / create the Qemu arguments + """ + print("Starting VM {}".format(VM)) + + def check_vms(self, *args, **options): + """ + Check if all VMs that are supposed to run are running + """ + + def modify_vms(self, *args, **options): + """ + Check all VMs that are requested to be modified and restart them + """ + + def create_vm_snapshots(self, *args, **options): + this_cluster = VMCluster(option['this_cluster']) + + for snapshot in VMSnapshotProduct.objects.filter(status='PENDING', + cluster=this_cluster): + if not snapshot.extra_data: + snapshot.extra_data = {} + + # TODO: implement locking here + if 'creating_hostname' in snapshot.extra_data: + pass + + snapshot.extra_data['creating_hostname'] = options['this_hostname'] + snapshot.extra_data['creating_start'] = str(datetime.now()) + snapshot.save() + + # something on the line of: + # for disk im vm.disks: + # rbd snap create pool/image-name@snapshot name + # snapshot.extra_data['snapshots'] + # register the snapshot names in extra_data (?) + + print(snapshot) + + def check_health(self, *args, **options): + pending_vms = VMProduct.objects.filter(status='PENDING') + vmhosts = VMHost.objects.filter(status='active') + + # 1. Check that all active hosts reported back N seconds ago + # 2. Check that no VM is running on a dead host + # 3. Migrate VMs if necessary + # 4. Check that no VMs have been pending for longer than Y seconds + + # If VM snapshots exist without a VM -> notify user (?) + + print("Nothing is good, you should implement me") diff --git a/uncloud_vm/migrations/0001_initial.py b/uncloud_vm/migrations/0001_initial.py new file mode 100644 index 0000000..f9f40d8 --- /dev/null +++ b/uncloud_vm/migrations/0001_initial.py @@ -0,0 +1,108 @@ +# Generated by Django 3.0.3 on 2020-03-05 10:34 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('uncloud_pay', '0001_initial'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='VMDiskImageProduct', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=256)), + ('is_os_image', models.BooleanField(default=False)), + ('is_public', models.BooleanField(default=False)), + ('size_in_gb', models.FloatField(blank=True, null=True)), + ('import_url', models.URLField(blank=True, null=True)), + ('image_source', models.CharField(max_length=128, null=True)), + ('image_source_type', models.CharField(max_length=128, null=True)), + ('storage_class', models.CharField(choices=[('hdd', 'HDD'), ('ssd', 'SSD')], default='ssd', max_length=32)), + ('status', models.CharField(choices=[('pending', 'Pending'), ('creating', 'Creating'), ('active', 'Active'), ('disabled', 'Disabled'), ('unusable', 'Unusable'), ('deleted', 'Deleted')], default='pending', max_length=32)), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.CreateModel( + name='VMHost', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('hostname', models.CharField(max_length=253, unique=True)), + ('physical_cores', models.IntegerField(default=0)), + ('usable_cores', models.IntegerField(default=0)), + ('usable_ram_in_gb', models.FloatField(default=0)), + ('status', models.CharField(choices=[('pending', 'Pending'), ('creating', 'Creating'), ('active', 'Active'), ('disabled', 'Disabled'), ('unusable', 'Unusable'), ('deleted', 'Deleted')], default='pending', max_length=32)), + ('vms', models.TextField(default='')), + ], + ), + migrations.CreateModel( + name='VMProduct', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted')], default='PENDING', max_length=32)), + ('name', models.CharField(max_length=32)), + ('cores', models.IntegerField()), + ('ram_in_gb', models.FloatField()), + ('vmid', models.IntegerField(null=True)), + ('order', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order')), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('vmhost', models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMHost')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VMWithOSProduct', + fields=[ + ('vmproduct_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='uncloud_vm.VMProduct')), + ], + options={ + 'abstract': False, + }, + bases=('uncloud_vm.vmproduct',), + ), + migrations.CreateModel( + name='VMSnapshotProduct', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted')], default='PENDING', max_length=32)), + ('gb_ssd', models.FloatField(editable=False)), + ('gb_hdd', models.FloatField(editable=False)), + ('order', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order')), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMProduct')), + ], + options={ + 'abstract': False, + }, + ), + migrations.CreateModel( + name='VMNetworkCard', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('mac_address', models.BigIntegerField()), + ('ip_address', models.GenericIPAddressField(blank=True, null=True)), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMProduct')), + ], + ), + migrations.CreateModel( + name='VMDiskProduct', + fields=[ + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('size_in_gb', models.FloatField(blank=True)), + ('image', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMDiskImageProduct')), + ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ('vm', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMProduct')), + ], + ), + ] diff --git a/uncloud_vm/migrations/0002_auto_20200305_1321.py b/uncloud_vm/migrations/0002_auto_20200305_1321.py new file mode 100644 index 0000000..2711b33 --- /dev/null +++ b/uncloud_vm/migrations/0002_auto_20200305_1321.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-03-05 13:21 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='vmdiskimageproduct', + name='storage_class', + field=models.CharField(choices=[('HDD', 'HDD'), ('SSD', 'SSD')], default='SSD', max_length=32), + ), + migrations.AlterField( + model_name='vmproduct', + name='name', + field=models.CharField(blank=True, max_length=32, null=True), + ), + ] diff --git a/uncloud_vm/migrations/0003_remove_vmhost_vms.py b/uncloud_vm/migrations/0003_remove_vmhost_vms.py new file mode 100644 index 0000000..70ee863 --- /dev/null +++ b/uncloud_vm/migrations/0003_remove_vmhost_vms.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-03-05 13:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0002_auto_20200305_1321'), + ] + + operations = [ + migrations.RemoveField( + model_name='vmhost', + name='vms', + ), + ] diff --git a/uncloud_vm/migrations/0004_remove_vmproduct_vmid.py b/uncloud_vm/migrations/0004_remove_vmproduct_vmid.py new file mode 100644 index 0000000..5f44b57 --- /dev/null +++ b/uncloud_vm/migrations/0004_remove_vmproduct_vmid.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-03-17 14:40 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0003_remove_vmhost_vms'), + ] + + operations = [ + migrations.RemoveField( + model_name='vmproduct', + name='vmid', + ), + ] diff --git a/uncloud_vm/migrations/0004_vmproduct_primary_disk.py b/uncloud_vm/migrations/0004_vmproduct_primary_disk.py new file mode 100644 index 0000000..c78acc1 --- /dev/null +++ b/uncloud_vm/migrations/0004_vmproduct_primary_disk.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-09 12:43 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0004_remove_vmproduct_vmid'), + ] + + operations = [ + migrations.AddField( + model_name='vmproduct', + name='primary_disk', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMDiskProduct'), + ), + ] diff --git a/uncloud_vm/migrations/0005_auto_20200309_1258.py b/uncloud_vm/migrations/0005_auto_20200309_1258.py new file mode 100644 index 0000000..0356558 --- /dev/null +++ b/uncloud_vm/migrations/0005_auto_20200309_1258.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.3 on 2020-03-09 12:58 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_pay', '0001_initial'), + ('uncloud_vm', '0004_vmproduct_primary_disk'), + ] + + operations = [ + migrations.AddField( + model_name='vmdiskproduct', + name='order', + field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order'), + ), + migrations.AddField( + model_name='vmdiskproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted')], default='PENDING', max_length=32), + ), + ] diff --git a/uncloud_vm/migrations/0005_auto_20200321_1058.py b/uncloud_vm/migrations/0005_auto_20200321_1058.py new file mode 100644 index 0000000..40eface --- /dev/null +++ b/uncloud_vm/migrations/0005_auto_20200321_1058.py @@ -0,0 +1,50 @@ +# Generated by Django 3.0.3 on 2020-03-21 10:58 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0005_auto_20200309_1258'), + ] + + operations = [ + migrations.AddField( + model_name='vmdiskimageproduct', + name='extra_data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True), + ), + migrations.AddField( + model_name='vmdiskproduct', + name='extra_data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True), + ), + migrations.AddField( + model_name='vmhost', + name='extra_data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True), + ), + migrations.AddField( + model_name='vmproduct', + name='extra_data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True), + ), + migrations.AddField( + model_name='vmsnapshotproduct', + name='extra_data', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True), + ), + migrations.AlterField( + model_name='vmdiskproduct', + name='vm', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='disks', to='uncloud_vm.VMProduct'), + ), + migrations.AlterField( + model_name='vmsnapshotproduct', + name='vm', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='snapshots', to='uncloud_vm.VMProduct'), + ), + ] diff --git a/uncloud_vm/migrations/0006_auto_20200322_1758.py b/uncloud_vm/migrations/0006_auto_20200322_1758.py new file mode 100644 index 0000000..7726c9b --- /dev/null +++ b/uncloud_vm/migrations/0006_auto_20200322_1758.py @@ -0,0 +1,57 @@ +# Generated by Django 3.0.3 on 2020-03-22 17:58 + +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models +import django.db.models.deletion +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0005_auto_20200321_1058'), + ] + + operations = [ + migrations.CreateModel( + name='VMCluster', + fields=[ + ('extra_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True)), + ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)), + ('name', models.CharField(max_length=128, unique=True)), + ], + options={ + 'abstract': False, + }, + ), + migrations.AlterField( + model_name='vmdiskimageproduct', + name='is_public', + field=models.BooleanField(default=False, editable=False), + ), + migrations.AlterField( + model_name='vmdiskimageproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + migrations.AlterField( + model_name='vmhost', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + migrations.AlterField( + model_name='vmproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + migrations.AlterField( + model_name='vmsnapshotproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + migrations.AddField( + model_name='vmproduct', + name='vmcluster', + field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMCluster'), + ), + ] diff --git a/uncloud_vm/migrations/0007_vmhost_vmcluster.py b/uncloud_vm/migrations/0007_vmhost_vmcluster.py new file mode 100644 index 0000000..6766dd7 --- /dev/null +++ b/uncloud_vm/migrations/0007_vmhost_vmcluster.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-22 18:09 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0006_auto_20200322_1758'), + ] + + operations = [ + migrations.AddField( + model_name='vmhost', + name='vmcluster', + field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMCluster'), + ), + ] diff --git a/uncloud_vm/migrations/0008_auto_20200403_1727.py b/uncloud_vm/migrations/0008_auto_20200403_1727.py new file mode 100644 index 0000000..5f4b494 --- /dev/null +++ b/uncloud_vm/migrations/0008_auto_20200403_1727.py @@ -0,0 +1,33 @@ +# Generated by Django 3.0.5 on 2020-04-03 17:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0007_vmhost_vmcluster'), + ] + + operations = [ + migrations.AlterField( + model_name='vmdiskimageproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + migrations.AlterField( + model_name='vmhost', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + migrations.AlterField( + model_name='vmproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + migrations.AlterField( + model_name='vmsnapshotproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + ] diff --git a/uncloud_vm/migrations/0009_auto_20200417_0551.py b/uncloud_vm/migrations/0009_auto_20200417_0551.py new file mode 100644 index 0000000..641f849 --- /dev/null +++ b/uncloud_vm/migrations/0009_auto_20200417_0551.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.5 on 2020-04-17 05:51 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0008_auto_20200403_1727'), + ] + + operations = [ + migrations.AlterField( + model_name='vmproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='AWAITING_PAYMENT', max_length=32), + ), + migrations.AlterField( + model_name='vmsnapshotproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='AWAITING_PAYMENT', max_length=32), + ), + ] diff --git a/uncloud_vm/migrations/0009_merge_20200413_0857.py b/uncloud_vm/migrations/0009_merge_20200413_0857.py new file mode 100644 index 0000000..2a9d70c --- /dev/null +++ b/uncloud_vm/migrations/0009_merge_20200413_0857.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.5 on 2020-04-13 08:57 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0004_remove_vmproduct_vmid'), + ('uncloud_vm', '0008_auto_20200403_1727'), + ] + + operations = [ + ] diff --git a/uncloud_vm/migrations/0010_auto_20200413_0924.py b/uncloud_vm/migrations/0010_auto_20200413_0924.py new file mode 100644 index 0000000..8883277 --- /dev/null +++ b/uncloud_vm/migrations/0010_auto_20200413_0924.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.5 on 2020-04-13 09:24 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0009_merge_20200413_0857'), + ] + + operations = [ + migrations.AlterField( + model_name='vmdiskproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32), + ), + migrations.AlterField( + model_name='vmdiskproduct', + name='vm', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMProduct'), + ), + ] diff --git a/uncloud_vm/migrations/0011_merge_20200418_0641.py b/uncloud_vm/migrations/0011_merge_20200418_0641.py new file mode 100644 index 0000000..c0d4c32 --- /dev/null +++ b/uncloud_vm/migrations/0011_merge_20200418_0641.py @@ -0,0 +1,14 @@ +# Generated by Django 3.0.5 on 2020-04-18 06:41 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0009_auto_20200417_0551'), + ('uncloud_vm', '0010_auto_20200413_0924'), + ] + + operations = [ + ] diff --git a/uncloud_vm/migrations/0012_auto_20200418_0641.py b/uncloud_vm/migrations/0012_auto_20200418_0641.py new file mode 100644 index 0000000..9af8649 --- /dev/null +++ b/uncloud_vm/migrations/0012_auto_20200418_0641.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.5 on 2020-04-18 06:41 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0011_merge_20200418_0641'), + ] + + operations = [ + migrations.AlterField( + model_name='vmdiskproduct', + name='status', + field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='AWAITING_PAYMENT', max_length=32), + ), + ] diff --git a/uncloud_vm/migrations/0013_remove_vmproduct_primary_disk.py b/uncloud_vm/migrations/0013_remove_vmproduct_primary_disk.py new file mode 100644 index 0000000..849012d --- /dev/null +++ b/uncloud_vm/migrations/0013_remove_vmproduct_primary_disk.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.5 on 2020-05-02 19:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0012_auto_20200418_0641'), + ] + + operations = [ + migrations.RemoveField( + model_name='vmproduct', + name='primary_disk', + ), + ] diff --git a/uncloud_vm/migrations/0014_vmwithosproduct_primary_disk.py b/uncloud_vm/migrations/0014_vmwithosproduct_primary_disk.py new file mode 100644 index 0000000..4747f60 --- /dev/null +++ b/uncloud_vm/migrations/0014_vmwithosproduct_primary_disk.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.6 on 2020-05-08 14:01 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0013_remove_vmproduct_primary_disk'), + ] + + operations = [ + migrations.AddField( + model_name='vmwithosproduct', + name='primary_disk', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMDiskProduct'), + ), + ] diff --git a/uncloud_vm/migrations/__init__.py b/uncloud_vm/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/uncloud_vm/models.py b/uncloud_vm/models.py new file mode 100644 index 0000000..cc07986 --- /dev/null +++ b/uncloud_vm/models.py @@ -0,0 +1,198 @@ +import uuid + +from django.db import models +from django.contrib.auth import get_user_model + +from uncloud_pay.models import Product, RecurringPeriod +from uncloud.models import UncloudModel, UncloudStatus + +import uncloud_pay.models as pay_models +import uncloud_storage.models + +class VMCluster(UncloudModel): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=128, unique=True) + + +class VMHost(UncloudModel): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + + # 253 is the maximum DNS name length + hostname = models.CharField(max_length=253, unique=True) + + vmcluster = models.ForeignKey( + VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + # indirectly gives a maximum number of cores / VM - f.i. 32 + physical_cores = models.IntegerField(default=0) + + # determines the maximum usable cores - f.i. 320 if you overbook by a factor of 10 + usable_cores = models.IntegerField(default=0) + + # ram that can be used of the server + usable_ram_in_gb = models.FloatField(default=0) + + status = models.CharField( + max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING + ) + + @property + def vms(self): + return VMProduct.objects.filter(vmhost=self) + + @property + def used_ram_in_gb(self): + return sum([vm.ram_in_gb for vm in VMProduct.objects.filter(vmhost=self)]) + + @property + def available_ram_in_gb(self): + return self.usable_ram_in_gb - self.used_ram_in_gb + + @property + def available_cores(self): + return self.usable_cores - sum([vm.cores for vm in self.vms ]) + + +class VMProduct(Product): + vmhost = models.ForeignKey( + VMHost, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + vmcluster = models.ForeignKey( + VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + + # VM-specific. The name is only intended for customers: it's a pain to + # remember IDs (speaking from experience as ungleich customer)! + name = models.CharField(max_length=32, blank=True, null=True) + cores = models.IntegerField() + ram_in_gb = models.FloatField() + + # Default recurring price is PER_MONTH, see uncloud_pay.models.Product. + @property + def recurring_price(self): + return self.cores * 3 + self.ram_in_gb * 4 + + def __str__(self): + return "VM {} ({}): {} cores {} gb ram".format(self.uuid, + self.name, + self.cores, + self.ram_in_gb) + + @property + def description(self): + return "Virtual machine '{}': {} core(s), {}GB memory".format( + self.name, self.cores, self.ram_in_gb) + + @staticmethod + def allowed_recurring_periods(): + return list(filter( + lambda pair: pair[0] in [RecurringPeriod.PER_YEAR, + RecurringPeriod.PER_MONTH, RecurringPeriod.PER_HOUR], + RecurringPeriod.choices)) + + def __str__(self): + return "VM {} ({} Cores/{} GB RAM) running on {} in cluster {}".format( + self.uuid, self.cores, self.ram_in_gb, + self.vmhost, self.vmcluster) + + +class VMWithOSProduct(VMProduct): + primary_disk = models.ForeignKey('VMDiskProduct', on_delete=models.CASCADE, null=True) + + +class VMDiskImageProduct(UncloudModel): + """ + Images are used for cloning/linking. + + They are the base for images. + + """ + + uuid = models.UUIDField( + primary_key=True, default=uuid.uuid4, editable=False + ) + owner = models.ForeignKey( + get_user_model(), on_delete=models.CASCADE, editable=False + ) + + name = models.CharField(max_length=256) + is_os_image = models.BooleanField(default=False) + is_public = models.BooleanField(default=False, editable=False) # only allow admins to set this + + size_in_gb = models.FloatField(null=True, blank=True) + import_url = models.URLField(null=True, blank=True) + image_source = models.CharField(max_length=128, null=True) + image_source_type = models.CharField(max_length=128, null=True) + + storage_class = models.CharField(max_length=32, + choices = uncloud_storage.models.StorageClass.choices, + default = uncloud_storage.models.StorageClass.SSD) + + status = models.CharField( + max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING + ) + + def __str__(self): + return "VMDiskImage {} ({}): {} gb".format(self.uuid, + self.name, + self.size_in_gb) + + + +class VMDiskProduct(Product): + """ + The VMDiskProduct is attached to a VM. + + It is based on a VMDiskImageProduct that will be used as a basis. + + It can be enlarged, but not shrinked compared to the VMDiskImageProduct. + """ + + vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE) + image = models.ForeignKey(VMDiskImageProduct, on_delete=models.CASCADE) + + size_in_gb = models.FloatField(blank=True) + + @property + def description(self): + return "Disk for VM '{}': {}GB".format(self.vm.name, self.size_in_gb) + + @property + def recurring_price(self): + return (self.size_in_gb / 10) * 3.5 + + # Sample code for clean method + + # Ensures that a VMDiskProduct can only be created from a VMDiskImageProduct + # that is in status 'active' + + # def clean(self): + # if self.image.status != 'active': + # raise ValidationError({ + # 'image': 'VM Disk must be created from an active disk image.' + # }) + + def save(self, *args, **kwargs): + self.full_clean() + super().save(*args, **kwargs) + + + +class VMNetworkCard(models.Model): + vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE) + + mac_address = models.BigIntegerField() + + ip_address = models.GenericIPAddressField(blank=True, + null=True) + + +class VMSnapshotProduct(Product): + gb_ssd = models.FloatField(editable=False) + gb_hdd = models.FloatField(editable=False) + + vm = models.ForeignKey(VMProduct, + related_name='snapshots', + on_delete=models.CASCADE) diff --git a/uncloud_vm/serializers.py b/uncloud_vm/serializers.py new file mode 100644 index 0000000..19fb872 --- /dev/null +++ b/uncloud_vm/serializers.py @@ -0,0 +1,143 @@ +from django.contrib.auth import get_user_model + +from rest_framework import serializers + +from .models import * +from uncloud_pay.models import RecurringPeriod, BillingAddress + +# XXX: does not seem to be used? + +GB_SSD_PER_DAY=0.012 +GB_HDD_PER_DAY=0.0006 + +GB_SSD_PER_DAY=0.012 +GB_HDD_PER_DAY=0.0006 + +### +# Admin views. + +class VMHostSerializer(serializers.HyperlinkedModelSerializer): + vms = serializers.PrimaryKeyRelatedField(many=True, read_only=True) + + class Meta: + model = VMHost + fields = '__all__' + read_only_fields = [ 'vms' ] + +class VMClusterSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = VMCluster + fields = '__all__' + + +### +# Disks. + +class VMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = '__all__' + +class CreateVMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = ['size_in_gb', 'image'] + +class CreateManagedVMDiskProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskProduct + fields = ['size_in_gb'] + +class VMDiskImageProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMDiskImageProduct + fields = '__all__' + +class VMSnapshotProductSerializer(serializers.ModelSerializer): + class Meta: + model = VMSnapshotProduct + fields = '__all__' + + + # verify that vm.owner == user.request + def validate_vm(self, value): + if not value.owner == self.context['request'].user: + raise serializers.ValidationError("VM {} not found for owner {}.".format(value, + self.context['request'].user)) + disks = VMDiskProduct.objects.filter(vm=value) + + if len(disks) == 0: + raise serializers.ValidationError("VM {} does not have any disks, cannot snapshot".format(value.uuid)) + + return value + + pricing = {} + pricing['per_gb_ssd'] = 0.012 + pricing['per_gb_hdd'] = 0.0006 + pricing['recurring_period'] = 'per_day' + +### +# VMs + +# Helper used in uncloud_service for services allocating VM. +class ManagedVMProductSerializer(serializers.ModelSerializer): + """ + Managed VM serializer used in ungleich_service app. + """ + primary_disk = CreateManagedVMDiskProductSerializer() + class Meta: + model = VMWithOSProduct + fields = [ 'cores', 'ram_in_gb', 'primary_disk'] + +class VMProductSerializer(serializers.ModelSerializer): + primary_disk = CreateVMDiskProductSerializer() + snapshots = VMSnapshotProductSerializer(many=True, read_only=True) + disks = VMDiskProductSerializer(many=True, read_only=True) + + class Meta: + model = VMWithOSProduct + fields = ['uuid', 'order', 'owner', 'status', 'name', 'cores', + 'ram_in_gb', 'primary_disk', 'snapshots', 'disks', 'extra_data'] + read_only_fields = ['uuid', 'order', 'owner', 'status'] + +class OrderVMProductSerializer(VMProductSerializer): + recurring_period = serializers.ChoiceField( + choices=VMWithOSProduct.allowed_recurring_periods()) + + def __init__(self, *args, **kwargs): + super(VMProductSerializer, self).__init__(*args, **kwargs) + + class Meta: + model = VMProductSerializer.Meta.model + fields = VMProductSerializer.Meta.fields + [ 'recurring_period' ] + read_only_fields = VMProductSerializer.Meta.read_only_fields + +# Nico's playground. +class NicoVMProductSerializer(serializers.ModelSerializer): + snapshots = VMSnapshotProductSerializer(many=True, read_only=True) + order = serializers.StringRelatedField() + + class Meta: + model = VMProduct + read_only_fields = ['uuid', 'order', 'owner', 'status', + 'vmhost', 'vmcluster', 'snapshots', + 'extra_data' ] + fields = read_only_fields + [ 'name', + 'cores', + 'ram_in_gb' + ] + +class DCLVMProductSerializer(serializers.HyperlinkedModelSerializer): + """ + Create an interface similar to standard DCL + """ + + # Custom field used at creation (= ordering) only. + recurring_period = serializers.ChoiceField( + choices=VMProduct.allowed_recurring_periods()) + + os_disk_uuid = serializers.UUIDField() + # os_disk_size = + + class Meta: + model = VMProduct diff --git a/uncloud_vm/tests.py b/uncloud_vm/tests.py new file mode 100644 index 0000000..1f47001 --- /dev/null +++ b/uncloud_vm/tests.py @@ -0,0 +1,114 @@ +import datetime + +import parsedatetime + +from django.test import TestCase +from django.contrib.auth import get_user_model +from django.utils import timezone +from django.core.exceptions import ValidationError + +from uncloud_vm.models import VMDiskImageProduct, VMDiskProduct, VMProduct, VMHost +from uncloud_pay.models import Order, RecurringPeriod + +User = get_user_model() +cal = parsedatetime.Calendar() + + +# If you want to check the test database using some GUI/cli tool +# then use the following connecting parameters + +# host: localhost +# database: test_uncloud +# user: root +# password: +# port: 5432 + +class VMTestCase(TestCase): + @classmethod + def setUpClass(cls): + # Setup vm host + cls.vm_host, created = VMHost.objects.get_or_create( + hostname='serverx.placey.ungleich.ch', physical_cores=32, usable_cores=320, + usable_ram_in_gb=512.0, status='active' + ) + super().setUpClass() + + def setUp(self) -> None: + # Setup two users as it is common to test with different user + self.user = User.objects.create_user( + username='testuser', email='test@test.com', first_name='Test', last_name='User' + ) + self.user2 = User.objects.create_user( + username='Meow', email='meow123@test.com', first_name='Meow', last_name='Cat' + ) + super().setUp() + + def create_sample_vm(self, owner): + one_month_later, parse_status = cal.parse("1 month later") + return VMProduct.objects.create( + vmhost=self.vm_host, cores=2, ram_in_gb=4, owner=owner, + order=Order.objects.create( + owner=owner, + creation_date=datetime.datetime.now(tz=timezone.utc), + starting_date=datetime.datetime.now(tz=timezone.utc), + ending_date=datetime.datetime(*one_month_later[:6], tzinfo=timezone.utc), + recurring_period=RecurringPeriod.PER_MONTH + ) + ) + +# TODO: the logic tested by this test is not implemented yet. +# def test_disk_product(self): +# """Ensures that a VMDiskProduct can only be created from a VMDiskImageProduct +# that is in status 'active'""" +# +# vm = self.create_sample_vm(owner=self.user) +# +# pending_disk_image = VMDiskImageProduct.objects.create( +# owner=self.user, name='pending_disk_image', is_os_image=True, is_public=True, size_in_gb=10, +# status='pending' +# ) +# try: +# vm_disk_product = VMDiskProduct.objects.create( +# owner=self.user, vm=vm, image=pending_disk_image, size_in_gb=10 +# ) +# except ValidationError: +# vm_disk_product = None +# +# self.assertIsNone( +# vm_disk_product, +# msg='VMDiskProduct created with disk image whose status is not active.' +# ) + + def test_vm_disk_product_creation(self): + """Ensure that a user can only create a VMDiskProduct for an existing VM""" + + disk_image = VMDiskImageProduct.objects.create( + owner=self.user, name='disk_image', is_os_image=True, is_public=True, size_in_gb=10, + status='active' + ) + + with self.assertRaises(ValidationError, msg='User created a VMDiskProduct for non-existing VM'): + # Create VMProduct object but don't save it in database + vm = VMProduct() + + vm_disk_product = VMDiskProduct.objects.create( + owner=self.user, vm=vm, image=disk_image, size_in_gb=10 + ) + +# TODO: the logic tested by this test is not implemented yet. +# def test_vm_disk_product_creation_for_someone_else(self): +# """Ensure that a user can only create a VMDiskProduct for his/her own VM""" +# +# # Create a VM which is ownership of self.user2 +# someone_else_vm = self.create_sample_vm(owner=self.user2) +# +# # 'self.user' would try to create a VMDiskProduct for 'user2's VM +# with self.assertRaises(ValidationError, msg='User created a VMDiskProduct for someone else VM.'): +# vm_disk_product = VMDiskProduct.objects.create( +# owner=self.user, vm=someone_else_vm, +# size_in_gb=10, +# image=VMDiskImageProduct.objects.create( +# owner=self.user, name='disk_image', is_os_image=True, is_public=True, size_in_gb=10, +# status='active' +# ) +# ) diff --git a/uncloud_vm/views.py b/uncloud_vm/views.py new file mode 100644 index 0000000..67f8656 --- /dev/null +++ b/uncloud_vm/views.py @@ -0,0 +1,261 @@ +from django.db import transaction +from django.shortcuts import render +from django.utils import timezone + +from django.contrib.auth.models import User +from django.shortcuts import get_object_or_404 + +from rest_framework import viewsets, permissions +from rest_framework.response import Response +from rest_framework.exceptions import ValidationError + +from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster +from uncloud_pay.models import Order, BillingAddress + +from .serializers import * +from uncloud_pay.helpers import ProductViewSet + +import datetime + +### +# Generic disk image views. Do not require orders / billing. + +class VMDiskImageProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskImageProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMDiskImageProduct.objects.all() + else: + obj = VMDiskImageProduct.objects.filter(owner=self.request.user) | VMDiskImageProduct.objects.filter(is_public=True) + + return obj + + + def create(self, request): + serializer = VMDiskImageProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + + # did not specify size NOR import url? + if not serializer.validated_data['size_in_gb']: + if not serializer.validated_data['import_url']: + raise ValidationError(detail={ 'error_mesage': 'Specify either import_url or size_in_gb' }) + + serializer.save(owner=request.user) + return Response(serializer.data) + +class VMDiskImageProductPublicViewSet(viewsets.ReadOnlyModelViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskImageProductSerializer + + def get_queryset(self): + return VMDiskImageProduct.objects.filter(is_public=True) + +### +# User VM disk and snapshots. + +class VMDiskProductViewSet(viewsets.ModelViewSet): + """ + Let a user modify their own VMDisks + """ + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMDiskProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMDiskProduct.objects.all() + else: + obj = VMDiskProduct.objects.filter(owner=self.request.user) + + return obj + + def create(self, request): + serializer = VMDiskProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + + # get disk size from image, if not specified + if not 'size_in_gb' in serializer.validated_data: + size_in_gb = serializer.validated_data['image'].size_in_gb + else: + size_in_gb = serializer.validated_data['size_in_gb'] + + if size_in_gb < serializer.validated_data['image'].size_in_gb: + raise ValidationError(detail={ 'error_mesage': 'Size is smaller than original image' }) + + serializer.save(owner=request.user, size_in_gb=size_in_gb) + return Response(serializer.data) + +class VMSnapshotProductViewSet(viewsets.ModelViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = VMSnapshotProductSerializer + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMSnapshotProduct.objects.all() + else: + obj = VMSnapshotProduct.objects.filter(owner=self.request.user) + + return obj + + def create(self, request): + serializer = VMSnapshotProductSerializer(data=request.data, context={'request': request}) + + # This verifies that the VM belongs to the request user + serializer.is_valid(raise_exception=True) + + vm = vm=serializer.validated_data['vm'] + disks = VMDiskProduct.objects.filter(vm=vm) + ssds_size = sum([d.size_in_gb for d in disks if d.image.storage_class == 'ssd']) + hdds_size = sum([d.size_in_gb for d in disks if d.image.storage_class == 'hdd']) + + recurring_price = serializer.pricing['per_gb_ssd'] * ssds_size + serializer.pricing['per_gb_hdd'] * hdds_size + recurring_period = serializer.pricing['recurring_period'] + + # Create order + now = datetime.datetime.now() + order = Order(owner=request.user, + recurring_period=recurring_period) + order.save() + order.add_record(one_time_price=0, + recurring_price=recurring_price, + description="Snapshot of VM {} from {}".format(vm, now)) + + serializer.save(owner=request.user, + order=order, + gb_ssd=ssds_size, + gb_hdd=hdds_size) + + return Response(serializer.data) + +### +# User VMs. + +class VMProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + if self.request.user.is_superuser: + obj = VMWithOSProduct.objects.all() + else: + obj = VMWithOSProduct.objects.filter(owner=self.request.user) + + return obj + + def get_serializer_class(self): + if self.action == 'create': + return OrderVMProductSerializer + else: + return VMProductSerializer + + # Use a database transaction so that we do not get half-created structure + # if something goes wrong. + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + + # Create disk image. + disk = VMDiskProduct(owner=request.user, + **serializer.validated_data.pop("primary_disk")) + vm = VMWithOSProduct(owner=request.user, primary_disk=disk, + **serializer.validated_data) + disk.vm = vm # XXX: Is this really needed? + + # Create VM and Disk orders. + vm_order = Order.from_product( + vm, + recurring_period=order_recurring_period, + starting_date=timezone.now() + ) + + disk_order = Order.from_product( + disk, + recurring_period=order_recurring_period, + starting_date=timezone.now(), + depends_on=vm_order + ) + + + # Commit to DB. + vm.order = vm_order + vm.save() + vm_order.save() + + disk.order = disk_order + disk_order.save() + disk.save() + + return Response(VMProductSerializer(vm, context={'request': request}).data) + +class NicoVMProductViewSet(ProductViewSet): + permission_classes = [permissions.IsAuthenticated] + serializer_class = NicoVMProductSerializer + + def get_queryset(self): + obj = VMProduct.objects.filter(owner=self.request.user) + return obj + + def create(self, request): + serializer = self.serializer_class(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + vm = serializer.save(owner=request.user) + + return Response(serializer.data) + + +### +# Admin stuff. + +class VMHostViewSet(viewsets.ModelViewSet): + serializer_class = VMHostSerializer + queryset = VMHost.objects.all() + permission_classes = [permissions.IsAdminUser] + +class VMClusterViewSet(viewsets.ModelViewSet): + serializer_class = VMClusterSerializer + queryset = VMCluster.objects.all() + permission_classes = [permissions.IsAdminUser] + +## +# Nico's playground. + +# Also create: +# - /dcl/available_os +# Basically a view of public and my disk images +# - +class DCLCreateVMProductViewSet(ProductViewSet): + """ + This view resembles the way how DCL VMs are created by default. + + The user chooses an OS, os disk size, ram, cpu and whether or not to have a mapped IPv4 address + """ + + permission_classes = [permissions.IsAuthenticated] + serializer_class = DCLVMProductSerializer + + def get_queryset(self): + return VMProduct.objects.filter(owner=self.request.user) + + # Use a database transaction so that we do not get half-created structure + # if something goes wrong. + @transaction.atomic + def create(self, request): + # Extract serializer data. + serializer = VMProductSerializer(data=request.data, context={'request': request}) + serializer.is_valid(raise_exception=True) + order_recurring_period = serializer.validated_data.pop("recurring_period") + + # Create base order. + order = Order.objects.create( + recurring_period=order_recurring_period, + owner=request.user + ) + order.save() + + # Create VM. + vm = serializer.save(owner=request.user, order=order) + + return Response(serializer.data)