Merge branch 'master' into 'master'

- Implement a complete cycle for buying a Matrix Chat Host

See merge request uncloud/uncloud!11

.gitignore

@@ -1,18 +1,27 @@
-.idea
-.vscode
+.idea/
+.vscode/
+__pycache__/
 
+pay.conf
+log.txt
+test.py
+STRIPE
+venv/
 
-ucloud/docs/build
+uncloud/docs/build
 logs.txt
 
-ucloud.egg-info
+uncloud.egg-info
 
 # run artefacts
 default.etcd
 __pycache__
 
 # build artefacts
-ucloud/version.py
+uncloud/version.py
 build/
 venv/
 dist/
+.history/
+*.iso
+*.sqlite3

.gitlab-ci.yml

@@ -0,0 +1,18 @@
+stages:
+  - lint
+  - test
+
+run-tests:
+  stage: test
+  image: code.ungleich.ch:5050/uncloud/uncloud/uncloud-ci:latest
+  services:
+    - postgres:latest
+  variables:
+    DATABASE_HOST: postgres
+    DATABASE_USER: postgres
+    POSTGRES_HOST_AUTH_METHOD: trust
+  coverage: /^TOTAL.+?(\d+\%)$/
+  script:
+    - pip install -r requirements.txt
+    - coverage run --source='.' ./manage.py test
+    - coverage report

README.md

@@ -1,3 +1,70 @@
-# ucloud
+# Uncloud
 
-Checkout https://ungleich.ch/ucloud/ for the documentation of ucloud.
+Cloud management platform, the ungleich way.
+
+
+[![pipeline status](https://code.ungleich.ch/uncloud/uncloud/badges/master/pipeline.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master)
+[![coverage report](https://code.ungleich.ch/uncloud/uncloud/badges/master/coverage.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master)
+
+## Useful commands
+
+* `./manage.py import-vat-rates path/to/csv`
+* `./manage.py createsuperuser`
+
+## Development setup
+
+Install system dependencies:
+
+* On Fedora, you will need the following packages: `python3-virtualenv python3-devel openldap-devel gcc chromium`
+* sudo apt-get install libpq-dev python-dev libxml2-dev libxslt1-dev libldap2-dev libsasl2-dev libffi-dev
+
+
+NOTE: you will need to configure a LDAP server and credentials for authentication. See `uncloud/settings.py`.
+
+```
+# Initialize virtualenv.
+» virtualenv .venv
+Using base prefix '/usr'
+New python executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python3
+Also creating executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python
+Installing setuptools, pip, wheel...
+done.
+
+# Enter virtualenv.
+» source .venv/bin/activate
+
+# Install dependencies.
+» pip install -r requirements.txt
+[...]
+
+# Run migrations.
+» ./manage.py migrate
+Operations to perform:
+  Apply all migrations: admin, auth, contenttypes, opennebula, sessions, uncloud_auth, uncloud_net, uncloud_pay, uncloud_service, uncloud_vm
+Running migrations:
+  [...]
+
+# Run webserver.
+» ./manage.py runserver
+Watching for file changes with StatReloader
+Performing system checks...
+
+System check identified no issues (0 silenced).
+May 07, 2020 - 10:17:08
+Django version 3.0.6, using settings 'uncloud.settings'
+Starting development server at http://127.0.0.1:8000/
+Quit the server with CONTROL-C.
+```
+### Run Background Job Queue
+We use Django Q to handle the asynchronous code and Background Cron jobs
+To start the workers make sure first that Redis or the Django Q broker is working and you can edit it's settings in the settings file.
+```
+./manage.py qcluster
+```
+
+### Note on PGSQL
+
+If you want to use Postgres:
+
+* Install on configure PGSQL on your base system.
+* OR use a container! `podman run --rm -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -it postgres:latest`

archive/issues.org

@@ -0,0 +1,6 @@
+* Intro
+  This file lists issues that should be handled, are small and likely
+  not yet high prio.
+* Issues
+** TODO Register prefered address in User model
+** TODO Allow to specify different recurring periods

archive/uncloud_django_based/hacks/abk-hacks.py

@@ -0,0 +1,55 @@
+"""
+investigate into a simple python function that maps an ldap user to a vat percentage. Basically you need to
+lookup the customer address, check if she is a business/registered tax number and if not apply the local
+vat
+"""
+
+import iso3166
+import datetime
+
+from csv import DictReader
+
+
+def get_vat(street_address, city, postal_code, country, vat_number=None):
+    vat = {
+        'Austria': [
+            {'period': '1984-01-01/', 'rate': 0.2},
+            {'period': '1976-01-01/1984-01-01', 'rate': 0.18},
+            {'period': '1973-01-01/1976-01-01', 'rate': 0.16},
+            ]
+    }
+    return iso3166.countries.get(country)
+
+    # return iso3166.countries_by_name[country]
+
+
+def main():
+    # vat = get_vat(
+    #     street_address='82 Nasheman-e-Iqbal near Wapda Town',
+    #     city='Lahore',
+    #     postal_code=53700,
+    #     country='Pakistan',
+    # )
+    # print(vat)
+    vat_rates = {}
+    with open('vat_rates.csv', newline='') as csvfile:
+        reader = DictReader(csvfile)
+        for row in reader:
+            territory_codes = row['territory_codes'].split('\n')
+            for code in territory_codes:
+                if code not in vat_rates:
+                    vat_rates[code] = {}
+
+                start_date = row['start_date']
+                stop_data = row['stop_date']
+                time_period = f'{start_date}|{stop_data}'
+                r = row.copy()
+                del r['start_date']
+                del r['stop_date']
+                del r['territory_codes']
+                vat_rates[code][time_period] = r
+    print(vat_rates)
+
+
+if __name__ == '__main__':
+    main()

archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py

@@ -0,0 +1,46 @@
+import importlib
+import sys
+import os
+
+from os.path import join as join_path
+from xmlrpc.client import ServerProxy as RPCClient
+
+root = os.path.dirname(os.getcwd())
+sys.path.append(join_path(root, 'uncloud'))
+secrets = importlib.import_module('uncloud.secrets')
+
+
+class OpenNebula:
+    def __init__(self, url, session_string):
+        self.session_string = session_string
+        self.client = RPCClient(secrets.OPENNEBULA_URL)
+
+    def create_user(self, username, password, authentication_driver='', group_id=None):
+        # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-user-allocate
+
+        if group_id is None:
+            group_id = []
+
+        return self.client.one.user.allocate(
+            self.session_string,
+            username,
+            password,
+            authentication_driver,
+            group_id
+        )
+
+    def chmod(self, vm_id, user_id=-1, group_id=-1):
+        # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-vm-chown
+
+        return self.client.one.vm.chown(self.session_string, vm_id, user_id, group_id)
+
+
+one = OpenNebula(secrets.OPENNEBULA_URL, secrets.OPENNEBULA_USER_PASS)
+
+# Create User in OpenNebula
+# success, response, *_ = one.create_user(username='meow12345', password='hello_world')
+# print(success, response)
+
+# Change owner of a VM
+# success, response, *_ = one.chmod(vm_id=25589, user_id=706)
+# print(success, response)

archive/uncloud_django_based/hacks/command-wrapper.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+dbhost=$1; shift
+
+ssh -L5432:localhost:5432 "$dbhost" &
+
+python manage.py "$@"
+
+
+
+# command only needs to be active while manage command is running
+
+# -T no pseudo terminal
+
+
+# alternatively: commands output shell code
+
+# ssh uncloud@dbhost "python manage.py --hostname xxx ..."

archive/uncloud_django_based/meow-payv1/README.md

@@ -0,0 +1,51 @@
+# uncloud-pay
+
+The generic product/payment system.
+
+## Installation
+
+```shell script
+pip3 install -r requirements.txt
+```
+
+## Getting Started
+
+```shell script
+python ucloud_pay.py
+```
+
+## Usage
+
+#### 1. Adding of products
+```shell script
+http --json http://[::]:5000/product/add username=your_username_here password=your_password_here specs:=@ipv6-only-vm.json
+```
+
+#### 2. Listing of products
+```shell script
+http --json http://[::]:5000/product/list
+```
+
+#### 3. Registering user's payment method (credit card for now using Stripe)
+
+```shell script
+http --json http://[::]:5000/user/register_payment card_number=4111111111111111 cvc=123 expiry_year=2020 expiry_month=8 card_holder_name="The test user" username=your_username_here password=your_password_here line1="your_billing_address" city="your_city" country="your_country"
+```
+
+#### 4. Ordering products
+
+First of all, user have to buy the membership first.
+
+```shell script
+http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=membership pay=True
+```
+
+```shell script
+http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=ipv6-only-vm cpu=1 ram=1 os-disk-space=10 os=alpine pay=True
+```
+
+#### 5. Listing users orders
+
+```shell script
+http --json POST http://[::]:5000/order/list username=your_username_here password=your_password_here
+```

archive/uncloud_django_based/meow-payv1/config.py

@@ -0,0 +1,21 @@
+import os
+
+from ungleich_common.ldap.ldap_manager import LdapManager
+from ungleich_common.std.configparser import StrictConfigParser
+from ungleich_common.etcd.etcd_wrapper import EtcdWrapper
+
+config_file = os.environ.get('meow-pay-config-file', default='pay.conf')
+
+config = StrictConfigParser(allow_no_value=True)
+config.read(config_file)
+
+etcd_client = EtcdWrapper(
+    host=config.get('etcd', 'host'), port=config.get('etcd', 'port'),
+    ca_cert=config.get('etcd', 'ca_cert'), cert_key=config.get('etcd', 'cert_key'),
+    cert_cert=config.get('etcd', 'cert_cert')
+)
+
+ldap_manager = LdapManager(
+    server=config.get('ldap', 'server'), admin_dn=config.get('ldap', 'admin_dn'),
+    admin_password=config.get('ldap', 'admin_password')
+)

archive/uncloud_django_based/meow-payv1/hack-a-vpn.py

@@ -0,0 +1,213 @@
+from flask import Flask, request
+from flask_restful import Resource, Api
+import etcd3
+import json
+import logging
+from functools import wraps
+
+from ldaptest import is_valid_ldap_user
+
+def authenticate(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        if not getattr(func, 'authenticated', True):
+            return func(*args, **kwargs)
+
+        # pass in username/password !
+        acct = basic_authentication()  # custom account lookup function
+
+        if acct:
+            return func(*args, **kwargs)
+
+        flask_restful.abort(401)
+    return wrapper
+
+def readable_errors(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except etcd3.exceptions.ConnectionFailedError as e:
+            raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
+        except etcd3.exceptions.ConnectionTimeoutError as e:
+            raise UncloudException('etcd connection timeout. {}'.format(e))
+
+    return wrapper
+
+
+class DB(object):
+    def __init__(self, config, prefix="/"):
+        self.config = config
+
+        # Root for everything
+        self.base_prefix= '/nicohack'
+
+        # Can be set from outside
+        self.prefix = prefix
+
+        self.connect()
+
+    @readable_errors
+    def connect(self):
+        self._db_clients = []
+        for endpoint in self.config.etcd_hosts:
+            client = etcd3.client(host=endpoint, **self.config.etcd_args)
+            self._db_clients.append(client)
+
+    def realkey(self, key):
+        return "{}{}/{}".format(self.base_prefix,
+                                self.prefix,
+                                key)
+
+    @readable_errors
+    def get(self, key, as_json=False, **kwargs):
+        value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
+
+        if as_json:
+            value = json.loads(value)
+
+        return value
+
+
+    @readable_errors
+    def set(self, key, value, as_json=False, **kwargs):
+        if as_json:
+            value = json.dumps(value)
+
+        # FIXME: iterate over clients in case of failure ?
+        return self._db_clients[0].put(self.realkey(key), value, **kwargs)
+
+
+class Membership(Resource):
+    def __init__(self, config):
+        self.config = config
+
+    def get(self):
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+        return {'message': 'Order successful' }, 200
+
+    def post(self):
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+        return {'message': 'Order 2x successful' }, 200
+
+
+class Order(Resource):
+    def __init__(self, config):
+        self.config = config
+
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+
+
+class Product(Resource):
+    def __init__(self, config):
+        self.config = config
+
+        self.products = []
+        self.products.append(
+            { "name": "membership-free",
+              "description": """
+This membership gives you access to the API and includes a VPN
+with 1 IPv6 address.
+See https://redmine.ungleich.ch/issues/7747?
+""",
+              "uuid": "a3883466-0012-4d01-80ff-cbf7469957af",
+              "recurring": True,
+              "recurring_time_frame": "per_year",
+              "features": [
+                  { "name": "membership",
+                    "price_one_time": 0,
+                    "price_recurring": 0
+                  }
+              ]
+            }
+        )
+        self.products.append(
+            { "name": "membership-standard",
+              "description": """
+This membership gives you access to the API and includes an IPv6-VPN with
+one IPv6 address ("Road warrior")
+See https://redmine.ungleich.ch/issues/7747?
+""",
+              "uuid": "1d85296b-0863-4dd6-a543-a6d5a4fbe4a6",
+              "recurring": True,
+              "recurring_time_frame": "per_month",
+              "features": [
+                  { "name": "membership",
+                    "price_one_time": 0,
+                    "price_recurring": 5
+                  }
+
+              ]
+            }
+        )
+        self.products.append(
+            { "name": "membership-premium",
+              "description": """
+This membership gives you access to the API and includes an
+IPv6-VPN with a /48 IPv6 network.
+See https://redmine.ungleich.ch/issues/7747?
+""",
+              "uuid": "bfd63fd2-d227-436f-a8b8-600de74dd6ce",
+              "recurring": True,
+              "recurring_time_frame": "per_month",
+              "features": [
+                  { "name": "membership",
+                    "price_one_time": 0,
+                    "price_recurring": 5
+                  }
+
+              ]
+            }
+        )
+        self.products.append(
+            { "name": "ipv6-vpn-with-/48",
+              "description": """
+An IPv6 VPN with a /48 network included.
+""",
+              "uuid": "fe5753f8-6fe1-4dc4-9b73-7b803de4c597",
+              "recurring": True,
+              "recurring_time_frame": "per_year",
+              "features": [
+                  { "name": "vpn",
+                    "price_one_time": 0,
+                    "price_recurring": 120
+                  }
+              ]
+            }
+        )
+
+
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+
+    def get(self):
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+
+        return self.products
+
+
+
+
+
+if __name__ == '__main__':
+    app = Flask(__name__)
+
+    config = {}
+
+    config['etcd_url']="https://etcd1.ungleich.ch"
+    config['ldap_url']="ldaps://ldap1.ungleich.ch"
+
+    api = Api(app)
+    api.add_resource(Order, '/orders', resource_class_args=( config, ))
+    api.add_resource(Product, '/products', resource_class_args=( config, ))
+    api.add_resource(Membership, '/membership',  resource_class_args=( config, ))
+
+    app.run(host='::', port=5000, debug=True)

archive/uncloud_django_based/meow-payv1/helper.py

@@ -0,0 +1,87 @@
+import logging
+
+import parsedatetime
+
+from datetime import datetime
+from stripe_utils import StripeUtils
+
+
+def get_plan_id_from_product(product):
+    plan_id = 'ucloud-v1-'
+    plan_id += product['name'].strip().replace(' ', '-')
+    return plan_id
+
+
+def get_pricing(price_in_chf_cents, product_type, recurring_period):
+    if product_type == 'recurring':
+        return 'CHF {}/{}'.format(price_in_chf_cents/100, recurring_period)
+    elif product_type == 'one-time':
+        return 'CHF {} (One time charge)'.format(price_in_chf_cents/100)
+
+
+def get_user_friendly_product(product_dict):
+    uf_product = {
+        'name': product_dict['name'],
+        'description': product_dict['description'],
+        'product_id': product_dict['usable-id'],
+        'pricing': get_pricing(
+            product_dict['price'], product_dict['type'], product_dict['recurring_period']
+        )
+    }
+    if product_dict['type'] == 'recurring':
+        uf_product['minimum_subscription_period'] = product_dict['minimum_subscription_period']
+    return uf_product
+
+
+def get_token(card_number, cvc, exp_month, exp_year):
+    stripe_utils = StripeUtils()
+    token_response = stripe_utils.get_token_from_card(
+        card_number, cvc, exp_month, exp_year
+    )
+    if token_response['response_object']:
+        return token_response['response_object'].id
+    else:
+        return None
+
+
+def resolve_product(usable_id, etcd_client):
+    products = etcd_client.get_prefix('/v1/products/', value_in_json=True)
+    for p in products:
+        if p.value['usable-id'] == usable_id:
+            return p.value
+    return None
+
+
+def calculate_charges(specification, data):
+    logging.debug('Calculating charges for specs:{} and data:{}'.format(specification, data))
+    one_time_charge = 0
+    recurring_charge = 0
+    for feature_name, feature_detail in specification['features'].items():
+        if feature_detail['constant']:
+            data[feature_name] = 1
+
+        if feature_detail['unit']['type'] != 'str':
+            one_time_charge += feature_detail['one_time_fee']
+            recurring_charge += (
+                    feature_detail['price_per_unit_per_period'] * data[feature_name] /
+                    feature_detail['unit']['value']
+            )
+    return one_time_charge, recurring_charge
+
+
+def is_order_valid(order_timestamp, renewal_period):
+    """
+    Sample Code Usage
+
+    >> current_datetime, status = cal.parse('Now')
+    >> current_datetime = datetime(*current_datetime[:6])
+
+    >> print('Is order valid: ', is_order_valid(current_datetime, '1 month'))
+    >> True
+    """
+    cal = parsedatetime.Calendar()
+
+    renewal_datetime, status = cal.parse(renewal_period)
+    renewal_datetime = datetime(*renewal_datetime[:6])
+
+    return order_timestamp <= renewal_datetime

archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json

@@ -0,0 +1,28 @@
+{
+  "usable-id": "ipv6-only-django-hosting",
+  "active": true,
+  "name": "IPv6 Only Django Hosting",
+  "description": "Host your Django application on our shiny IPv6 Only VM",
+  "recurring_period": "month",
+  "quantity": "inf",
+  "features": {
+    "cpu": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 3,
+      "one_time_fee": 0,
+      "constant": false
+    },
+    "ram": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 4,
+      "one_time_fee": 0,
+      "constant": false
+    },
+    "os-disk-space": {
+      "unit": {"value": 10, "type":"int"},
+      "one_time_fee": 0,
+      "price_per_unit_per_period": 3.5,
+      "constant": false
+    }
+  }
+}

archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json

@@ -0,0 +1,34 @@
+{
+  "usable-id": "ipv6-only-vm",
+  "active": true,
+  "name": "IPv6 Only VM",
+  "description": "IPv6 Only VM are accessible to only those having IPv6 for themselves",
+  "recurring_period": "month",
+  "quantity": "inf",
+  "features": {
+    "cpu": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 3,
+      "one_time_fee": 0,
+      "constant": false
+    },
+    "ram": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 4,
+      "one_time_fee": 0,
+      "constant": false
+    },
+    "os-disk-space": {
+      "unit": {"value": 10, "type":"int"},
+      "one_time_fee": 0,
+      "price_per_unit_per_period": 4,
+      "constant": false
+    },
+    "os": {
+      "unit": {"value": 1, "type":"str"},
+      "one_time_fee": 0,
+      "price_per_unit_per_period": 0,
+      "constant": false
+    }
+  }
+}

archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json

@@ -0,0 +1,16 @@
+{
+  "usable-id": "ipv6-only-vpn",
+  "active": true,
+  "name": "IPv6 Only VPN",
+  "description": "IPv6 VPN enable you to access IPv6 only websites and more",
+  "recurring_period": "month",
+  "quantity": "inf",
+  "features": {
+    "vpn": {
+      "unit": {"value": 1, "type": "int"},
+      "price_per_unit_per_period": 10,
+      "one_time_fee": 0,
+      "constant": true
+    }
+  }
+}

archive/uncloud_django_based/meow-payv1/products/ipv6box.json

@@ -0,0 +1,16 @@
+{
+  "usable-id": "ipv6-box",
+  "active": true,
+  "name": "IPv6 Box",
+  "description": "A ready-to-go IPv6 Box: it creates a VPN to ungleich and distributes IPv6 addresses to all your computers.",
+  "recurring_period": "eternity",
+  "quantity": 4,
+  "features": {
+    "ipv6-box": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 0,
+      "one_time_fee": 250,
+      "constant": true
+    }
+  }
+}

archive/uncloud_django_based/meow-payv1/products/membership.json

@@ -0,0 +1,17 @@
+{
+  "usable-id": "membership",
+  "active": true,
+  "name": "Membership",
+  "description": "Membership to use uncloud-pay",
+  "recurring_period": "month",
+  "quantity": "inf",
+  "features": {
+    "membership": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 5,
+      "one_time_fee": 0,
+      "constant": true
+    }
+  },
+  "max_per_user": "1"
+}

archive/uncloud_django_based/meow-payv1/requirements.txt

@@ -0,0 +1,7 @@
+stripe
+flask
+Flask-RESTful
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-etcd&subdirectory=etcd
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-std&subdirectory=std
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-schemas&subdirectory=schemas

archive/uncloud_django_based/meow-payv1/sample-pay.conf

@@ -0,0 +1,17 @@
+[etcd]
+host = 127.0.0.1
+port = 2379
+ca_cert
+cert_cert
+cert_key
+
+[stripe]
+private_key=stripe_private_key
+
+[app]
+port = 5000
+
+[ldap]
+server = ldap_server_url
+admin_dn = ldap_admin_dn
+admin_password = ldap_admin_password

archive/uncloud_django_based/meow-payv1/schemas.py

@@ -0,0 +1,136 @@
+import logging
+import config
+import json
+import math
+
+from config import ldap_manager, etcd_client
+from helper import resolve_product
+from ungleich_common.schemas.schemas import BaseSchema, Field, ValidationException
+
+
+class AddProductSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.add_schema(UserCredentialSchema, data)
+        self.specs = Field('specs', dict, **self.get(data, 'specs'))
+        self.update = Field('update', bool, **self.get(data, 'update', return_default=True, default=False))
+
+    def validation(self):
+        user = self.objects['user']
+        user = json.loads(user.entry_to_json())
+        uid, ou, *dc = user['dn'].replace('ou=', '').replace('dc=', '').replace('uid=', '').split(',')
+        if ou != config.config.get('ldap', 'internal_user_ou', fallback='users'):
+            raise ValidationException('You do not have access to create product.')
+
+        product = resolve_product(self.specs.value['usable-id'], etcd_client)
+        if product:
+            self.objects['product'] = product
+
+
+class AddressSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.line1 = Field('line1', str, **self.get(data, 'line1'))
+        self.line2 = Field('line2', str, **self.get(data, 'line2', return_default=True))
+        self.city = Field('city', str, **self.get(data, 'city'))
+        self.country = Field('country', str, **self.get(data, 'country'))
+        self.state = Field('state', str, **self.get(data, 'state', return_default=True))
+        self.postal_code = Field('postal_code', str, **self.get(data, 'postal_code', return_default=True))
+
+
+class UserRegisterPaymentSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+
+        self.add_schema(UserCredentialSchema, data)
+        self.add_schema(AddressSchema, data, under_field_name='address')
+
+        self.card_number = Field('card_number', str, **self.get(data, 'card_number'))
+        self.cvc = Field('cvc', str, **self.get(data, 'cvc'))
+        self.expiry_year = Field('expiry_year', int, **self.get(data, 'expiry_year'))
+        self.expiry_month = Field('expiry_month', int, **self.get(data, 'expiry_month'))
+        self.card_holder_name = Field('card_holder_name', str, **self.get(data, 'card_holder_name'))
+
+
+class UserCredentialSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.username = Field('username', str, **self.get(data, 'username'))
+        self.password = Field('password', str, **self.get(data, 'password'))
+
+    def validation(self):
+        try:
+            entry = ldap_manager.is_password_valid(self.username.value, self.password.value, query_key='uid')
+        except ValueError:
+            raise ValidationException('No user with \'{}\' username found. You can create account at '
+                                      'https://account.ungleich.ch'.format(self.username.value))
+        except Exception:
+            raise ValidationException('Invalid username/password.')
+        else:
+            self.objects['user'] = entry
+
+
+class ProductOrderSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.product_id = Field(
+            'product_id', str, **self.get(data, 'product_id'), validators=[self.product_id_validation]
+        )
+        self.pay_consent = Field('pay', bool, **self.get(data, 'pay', return_default=True, default=False))
+        self.add_schema(UserCredentialSchema, data)
+
+    def product_id_validation(self):
+        product = resolve_product(self.product_id.value, etcd_client)
+        if product:
+            product['quantity'] = float(product['quantity'])
+            self.product_id.value = product['uuid']
+            self.objects['product'] = product
+            logging.debug('Got product {}'.format(product))
+
+            if not product['active']:
+                raise ValidationException('Product is not active at the moment.')
+
+            if product['quantity'] <= 0:
+                raise ValidationException('Out of stock.')
+        else:
+            raise ValidationException('No such product exists.')
+
+    def validation(self):
+        username = self.objects['user'].uid
+        customer_previous_orders = etcd_client.get_prefix('/v1/user/{}'.format(username), value_in_json=True)
+        customer_previous_orders = [o.value for o in customer_previous_orders]
+        membership = next(filter(lambda o: o['product'] == 'membership', customer_previous_orders), None)
+        if membership is None and self.objects['product']['usable-id'] != 'membership':
+            raise ValidationException('Please buy membership first to use this facility')
+        max_quantity_user_can_order = float(self.objects['product'].get('max_per_user', math.inf))
+        previous_order_of_same_product = [
+            o for o in customer_previous_orders if o['product'] == self.objects['product']['usable-id']
+        ]
+        if len(previous_order_of_same_product) >= max_quantity_user_can_order:
+            raise ValidationException(
+                'You cannot buy {} more than {} times'.format(
+                    self.objects['product']['name'], int(max_quantity_user_can_order)
+                )
+            )
+
+
+class OrderListSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.add_schema(UserCredentialSchema, data)
+
+
+def make_return_message(err, status_code=200):
+    logging.debug('message: {}'.format(str(err)))
+    return {'message': str(err)}, status_code
+
+
+def create_schema(specification, data):
+    fields = {}
+    for feature_name, feature_detail in specification['features'].items():
+        if not feature_detail['constant']:
+            fields[feature_name] = Field(
+                feature_name, eval(feature_detail['unit']['type']), **BaseSchema.get(data, feature_name)
+            )
+
+    return type('{}Schema'.format(specification['name']), (BaseSchema,), fields)

archive/uncloud_django_based/meow-payv1/stripe_hack.py

@@ -0,0 +1,7 @@
+import stripe_utils
+import os
+
+
+if __name__ == '__main__':
+    s = stripe_utils.StripeUtils(os.environ['STRIPE_PRIVATE_KEY'])
+    print(s.get_stripe_customer_from_email('coder.purple+2002@gmail.com'))

archive/uncloud_django_based/meow-payv1/stripe_utils.py

@@ -0,0 +1,491 @@
+import re
+import stripe
+import stripe.error
+import logging
+
+from config import etcd_client as client, config as config
+
+stripe.api_key = config.get('stripe', 'private_key')
+
+
+def handle_stripe_error(f):
+    def handle_problems(*args, **kwargs):
+        response = {
+            'paid': False,
+            'response_object': None,
+            'error': None
+        }
+
+        common_message = "Currently it's not possible to make payments."
+        try:
+            response_object = f(*args, **kwargs)
+            response = {
+                'response_object': response_object,
+                'error': None
+            }
+            return response
+        except stripe.error.CardError as e:
+            # Since it's a decline, stripe.error.CardError will be caught
+            body = e.json_body
+            err = body['error']
+            response.update({'error': err['message']})
+            logging.error(str(e))
+            return response
+        except stripe.error.RateLimitError:
+            response.update(
+                {'error': "Too many requests made to the API too quickly"})
+            return response
+        except stripe.error.InvalidRequestError as e:
+            logging.error(str(e))
+            response.update({'error': "Invalid parameters"})
+            return response
+        except stripe.error.AuthenticationError as e:
+            # Authentication with Stripe's API failed
+            # (maybe you changed API keys recently)
+            logging.error(str(e))
+            response.update({'error': common_message})
+            return response
+        except stripe.error.APIConnectionError as e:
+            logging.error(str(e))
+            response.update({'error': common_message})
+            return response
+        except stripe.error.StripeError as e:
+            # maybe send email
+            logging.error(str(e))
+            response.update({'error': common_message})
+            return response
+        except Exception as e:
+            # maybe send email
+            logging.error(str(e))
+            response.update({'error': common_message})
+            return response
+
+    return handle_problems
+
+
+class StripeUtils(object):
+    CURRENCY = 'chf'
+    INTERVAL = 'month'
+    SUCCEEDED_STATUS = 'succeeded'
+    STRIPE_PLAN_ALREADY_EXISTS = 'Plan already exists'
+    STRIPE_NO_SUCH_PLAN = 'No such plan'
+    PLAN_EXISTS_ERROR_MSG = 'Plan {} exists already.\nCreating a local StripePlan now.'
+    PLAN_DOES_NOT_EXIST_ERROR_MSG = 'Plan {} does not exist.'
+
+    def __init__(self, private_key):
+        self.stripe = stripe
+        stripe.api_key = private_key
+
+    @handle_stripe_error
+    def card_exists(self, customer, cc_number, exp_month, exp_year, cvc):
+        token_obj = stripe.Token.create(
+            card={
+                'number': cc_number,
+                'exp_month': exp_month,
+                'exp_year': exp_year,
+                'cvc': cvc,
+            },
+        )
+        cards = stripe.Customer.list_sources(
+            customer,
+            limit=20,
+            object='card'
+        )
+
+        for card in cards.data:
+            if (card.fingerprint == token_obj.card.fingerprint and
+                    int(card.exp_month) == int(exp_month) and int(card.exp_year) == int(exp_year)):
+                return True
+        return False
+
+    @staticmethod
+    def get_stripe_customer_from_email(email):
+        customer = stripe.Customer.list(limit=1, email=email)
+        return customer.data[0] if len(customer.data) == 1 else None
+
+    @staticmethod
+    def update_customer_token(customer, token):
+        customer.source = token
+        customer.save()
+
+    @handle_stripe_error
+    def get_token_from_card(self, cc_number, cvc, expiry_month, expiry_year):
+        token_obj = stripe.Token.create(
+            card={
+                'number': cc_number,
+                'exp_month': expiry_month,
+                'exp_year': expiry_year,
+                'cvc': cvc,
+            },
+        )
+        return token_obj
+
+    @handle_stripe_error
+    def associate_customer_card(self, stripe_customer_id, token,
+                                set_as_default=False):
+        customer = stripe.Customer.retrieve(stripe_customer_id)
+        card = customer.sources.create(source=token)
+        if set_as_default:
+            customer.default_source = card.id
+            customer.save()
+        return True
+
+    @handle_stripe_error
+    def dissociate_customer_card(self, stripe_customer_id, card_id):
+        customer = stripe.Customer.retrieve(stripe_customer_id)
+        card = customer.sources.retrieve(card_id)
+        card.delete()
+
+    @handle_stripe_error
+    def update_customer_card(self, customer_id, token):
+        customer = stripe.Customer.retrieve(customer_id)
+        current_card_token = customer.default_source
+        customer.sources.retrieve(current_card_token).delete()
+        customer.source = token
+        customer.save()
+        credit_card_raw_data = customer.sources.data.pop()
+        new_card_data = {
+            'last4': credit_card_raw_data.last4,
+            'brand': credit_card_raw_data.brand
+        }
+        return new_card_data
+
+    @handle_stripe_error
+    def get_card_details(self, customer_id):
+        customer = stripe.Customer.retrieve(customer_id)
+        credit_card_raw_data = customer.sources.data.pop()
+        card_details = {
+            'last4': credit_card_raw_data.last4,
+            'brand': credit_card_raw_data.brand,
+            'exp_month': credit_card_raw_data.exp_month,
+            'exp_year': credit_card_raw_data.exp_year,
+            'fingerprint': credit_card_raw_data.fingerprint,
+            'card_id': credit_card_raw_data.id
+        }
+        return card_details
+
+    @handle_stripe_error
+    def get_all_invoices(self, customer_id, created_gt):
+        return_list = []
+        has_more_invoices = True
+        starting_after = False
+        while has_more_invoices:
+            if starting_after:
+                invoices = stripe.Invoice.list(
+                    limit=10, customer=customer_id, created={'gt': created_gt},
+                    starting_after=starting_after
+                )
+            else:
+                invoices = stripe.Invoice.list(
+                    limit=10, customer=customer_id, created={'gt': created_gt}
+                )
+            has_more_invoices = invoices.has_more
+            for invoice in invoices.data:
+                sub_ids = []
+                for line in invoice.lines.data:
+                    if line.type == 'subscription':
+                        sub_ids.append(line.id)
+                    elif line.type == 'invoiceitem':
+                        sub_ids.append(line.subscription)
+                    else:
+                        sub_ids.append('')
+                invoice_details = {
+                    'created': invoice.created,
+                    'receipt_number': invoice.receipt_number,
+                    'invoice_number': invoice.number,
+                    'paid_at': invoice.status_transitions.paid_at if invoice.paid else 0,
+                    'period_start': invoice.period_start,
+                    'period_end': invoice.period_end,
+                    'billing_reason': invoice.billing_reason,
+                    'discount': invoice.discount.coupon.amount_off if invoice.discount else 0,
+                    'total': invoice.total,
+                    # to see how many line items we have in this invoice and
+                    # then later check if we have more than 1
+                    'lines_data_count': len(invoice.lines.data) if invoice.lines.data is not None else 0,
+                    'invoice_id': invoice.id,
+                    'lines_meta_data_csv': ','.join(
+                        [line.metadata.VM_ID if hasattr(line.metadata, 'VM_ID') else '' for line in invoice.lines.data]
+                    ),
+                    'subscription_ids_csv': ','.join(sub_ids),
+                    'line_items': invoice.lines.data
+                }
+                starting_after = invoice.id
+                return_list.append(invoice_details)
+        return return_list
+
+    @handle_stripe_error
+    def get_cards_details_from_token(self, token):
+        stripe_token = stripe.Token.retrieve(token)
+        card_details = {
+            'last4': stripe_token.card.last4,
+            'brand': stripe_token.card.brand,
+            'exp_month': stripe_token.card.exp_month,
+            'exp_year': stripe_token.card.exp_year,
+            'fingerprint': stripe_token.card.fingerprint,
+            'card_id': stripe_token.card.id
+        }
+        return card_details
+
+    def check_customer(self, stripe_cus_api_id, user, token):
+        try:
+            customer = stripe.Customer.retrieve(stripe_cus_api_id)
+        except stripe.error.InvalidRequestError:
+            customer = self.create_customer(token, user.email, user.name)
+            user.stripecustomer.stripe_id = customer.get(
+                'response_object').get('id')
+            user.stripecustomer.save()
+        if type(customer) is dict:
+            customer = customer['response_object']
+        return customer
+
+    @handle_stripe_error
+    def get_customer(self, stripe_api_cus_id):
+        customer = stripe.Customer.retrieve(stripe_api_cus_id)
+        # data = customer.get('response_object')
+        return customer
+
+    @handle_stripe_error
+    def create_customer(self, token, email, name=None, address=None):
+        if name is None or name.strip() == "":
+            name = email
+        customer = self.stripe.Customer.create(
+            source=token,
+            description=name,
+            email=email,
+            address=address
+        )
+        return customer
+
+    @handle_stripe_error
+    def make_charge(self, amount=None, customer=None):
+        _amount = float(amount)
+        amount = int(_amount * 100)  # stripe amount unit, in cents
+        charge = self.stripe.Charge.create(
+            amount=amount,  # in cents
+            currency=self.CURRENCY,
+            customer=customer
+        )
+        return charge
+
+    @staticmethod
+    def _get_all_stripe_plans():
+        all_stripe_plans = client.get("/v1/stripe_plans")
+        all_stripe_plans_set = set()
+        if all_stripe_plans:
+            all_stripe_plans_obj = all_stripe_plans.value
+            if all_stripe_plans_obj and len(all_stripe_plans_obj['plans']) > 0:
+                all_stripe_plans_set = set(all_stripe_plans_obj["plans"])
+        return all_stripe_plans_set
+
+    @staticmethod
+    def _save_all_stripe_plans(stripe_plans):
+        client.put("/v1/stripe_plans", {"plans": list(stripe_plans)})
+
+    @handle_stripe_error
+    def get_or_create_stripe_plan(self, product_name, amount, stripe_plan_id,
+                                  interval=INTERVAL):
+        """
+        This function checks if a StripePlan with the given
+        stripe_plan_id already exists. If it exists then the function
+        returns this object otherwise it creates a new StripePlan and
+        returns the new object.
+
+        :param amount: The amount in CHF cents
+        :param product_name: The name of the Stripe plan (product) to be created.
+        :param stripe_plan_id: The id of the Stripe plan to be
+               created. Use get_stripe_plan_id_string function to
+               obtain the name of the plan to be created
+        :param interval: The interval for subscription {month, year}. Defaults
+               to month if not provided
+        :return: The StripePlan object if it exists else creates a
+               Plan object in Stripe and a local StripePlan and
+               returns it. Returns None in case of Stripe error
+        """
+        _amount = float(amount)
+        amount = int(_amount * 100)  # stripe amount unit, in cents
+        all_stripe_plans = self._get_all_stripe_plans()
+        if stripe_plan_id in all_stripe_plans:
+            logging.debug("{} plan exists in db.".format(stripe_plan_id))
+        else:
+            logging.debug(("{} plan DOES NOT exist in db. "
+                          "Creating").format(stripe_plan_id))
+            try:
+                plan_obj = self.stripe.Plan.retrieve(id=stripe_plan_id)
+                logging.debug("{} plan exists in Stripe".format(stripe_plan_id))
+                all_stripe_plans.add(stripe_plan_id)
+            except stripe.error.InvalidRequestError as e:
+                if "No such plan" in str(e):
+                    logging.debug("Plan {} does not exist in Stripe, Creating")
+                    plan_obj = self.stripe.Plan.create(
+                        amount=amount,
+                        product={'name': product_name},
+                        interval=interval,
+                        currency=self.CURRENCY,
+                        id=stripe_plan_id)
+                    logging.debug(self.PLAN_EXISTS_ERROR_MSG.format(stripe_plan_id))
+                    all_stripe_plans.add(stripe_plan_id)
+            self._save_all_stripe_plans(all_stripe_plans)
+        return stripe_plan_id
+
+    @handle_stripe_error
+    def delete_stripe_plan(self, stripe_plan_id):
+        """
+        Deletes the Plan in Stripe and also deletes the local db copy
+        of the plan if it exists
+
+        :param stripe_plan_id: The stripe plan id that needs to be
+               deleted
+        :return: True if the plan was deleted successfully from
+               Stripe, False otherwise.
+        """
+        return_value = False
+        try:
+            plan = self.stripe.Plan.retrieve(stripe_plan_id)
+            plan.delete()
+            return_value = True
+            all_stripe_plans = self._get_all_stripe_plans()
+            all_stripe_plans.remove(stripe_plan_id)
+            self._save_all_stripe_plans(all_stripe_plans)
+        except stripe.error.InvalidRequestError as e:
+            if self.STRIPE_NO_SUCH_PLAN in str(e):
+                logging.debug(
+                    self.PLAN_DOES_NOT_EXIST_ERROR_MSG.format(stripe_plan_id))
+        return return_value
+
+    @handle_stripe_error
+    def subscribe_customer_to_plan(self, customer, plans, trial_end=None):
+        """
+        Subscribes the given customer to the list of given plans
+
+        :param customer: The stripe customer identifier
+        :param plans: A list of stripe plans.
+        :param trial_end: An integer representing when the Stripe subscription
+               is supposed to end
+        Ref: https://stripe.com/docs/api/python#create_subscription-items
+              e.g.
+                    plans = [
+                                {
+                                  "plan": "dcl-v1-cpu-2-ram-5gb-ssd-10gb",
+                                },
+                            ]
+        :return: The subscription StripeObject
+        """
+
+        subscription_result = self.stripe.Subscription.create(
+            customer=customer, items=plans, trial_end=trial_end
+        )
+        return subscription_result
+
+    @handle_stripe_error
+    def set_subscription_metadata(self, subscription_id, metadata):
+        subscription = stripe.Subscription.retrieve(subscription_id)
+        subscription.metadata = metadata
+        subscription.save()
+
+    @handle_stripe_error
+    def unsubscribe_customer(self, subscription_id):
+        """
+        Cancels a given subscription
+
+        :param subscription_id: The Stripe subscription id string
+        :return:
+        """
+        sub = stripe.Subscription.retrieve(subscription_id)
+        return sub.delete()
+
+    @handle_stripe_error
+    def make_payment(self, customer, amount, token):
+        charge = self.stripe.Charge.create(
+            amount=amount,  # in cents
+            currency=self.CURRENCY,
+            customer=customer
+        )
+        return charge
+
+    @staticmethod
+    def get_stripe_plan_id(cpu, ram, ssd, version, app='dcl', hdd=None,
+                           price=None):
+        """
+        Returns the Stripe plan id string of the form
+        `dcl-v1-cpu-2-ram-5gb-ssd-10gb` based on the input parameters
+
+        :param cpu: The number of cores
+        :param ram: The size of the RAM in GB
+        :param ssd: The size of ssd storage in GB
+        :param hdd: The size of hdd storage in GB
+        :param version: The version of the Stripe plans
+        :param app: The application to which the stripe plan belongs
+        to. By default it is 'dcl'
+        :param price: The price for this plan
+        :return: A string of the form `dcl-v1-cpu-2-ram-5gb-ssd-10gb`
+        """
+        dcl_plan_string = 'cpu-{cpu}-ram-{ram}gb-ssd-{ssd}gb'.format(cpu=cpu,
+                                                                     ram=ram,
+                                                                     ssd=ssd)
+        if hdd is not None:
+            dcl_plan_string = '{dcl_plan_string}-hdd-{hdd}gb'.format(
+                dcl_plan_string=dcl_plan_string, hdd=hdd)
+        stripe_plan_id_string = '{app}-v{version}-{plan}'.format(
+            app=app,
+            version=version,
+            plan=dcl_plan_string
+        )
+        if price is not None:
+            stripe_plan_id_string_with_price = '{}-{}chf'.format(
+                stripe_plan_id_string,
+                round(price, 2)
+            )
+            return stripe_plan_id_string_with_price
+        else:
+            return stripe_plan_id_string
+
+    @staticmethod
+    def get_vm_config_from_stripe_id(stripe_id):
+        """
+        Given a string like "dcl-v1-cpu-2-ram-5gb-ssd-10gb" return different
+        configuration params as a dict
+
+        :param stripe_id|str
+        :return: dict
+        """
+        pattern = re.compile(r'^dcl-v(\d+)-cpu-(\d+)-ram-(\d+\.?\d*)gb-ssd-(\d+)gb-?(\d*\.?\d*)(chf)?$')
+        match_res = pattern.match(stripe_id)
+        if match_res is not None:
+            price = None
+            try:
+                price = match_res.group(5)
+            except IndexError:
+                logging.debug("Did not find price in {}".format(stripe_id))
+            return {
+                'version': match_res.group(1),
+                'cores': match_res.group(2),
+                'ram': match_res.group(3),
+                'ssd': match_res.group(4),
+                'price': price
+            }
+
+    @staticmethod
+    def get_stripe_plan_name(cpu, memory, disk_size, price):
+        """
+        Returns the Stripe plan name
+        :return:
+        """
+        return "{cpu} Cores, {memory} GB RAM, {disk_size} GB SSD, " \
+               "{price} CHF".format(
+                    cpu=cpu,
+                    memory=memory,
+                    disk_size=disk_size,
+                    price=round(price, 2)
+                )
+
+    @handle_stripe_error
+    def set_subscription_meta_data(self, subscription_id, meta_data):
+        """
+        Adds VM metadata to a subscription
+        :param subscription_id: Stripe identifier for the subscription
+        :param meta_data: A dict of meta data to be added
+        :return:
+        """
+        subscription = stripe.Subscription.retrieve(subscription_id)
+        subscription.metadata = meta_data
+        subscription.save()

archive/uncloud_django_based/meow-payv1/ucloud_pay.py

@@ -0,0 +1,338 @@
+import logging
+
+from datetime import datetime
+from uuid import uuid4
+
+from flask import Flask, request
+from flask_restful import Resource, Api
+from werkzeug.exceptions import HTTPException
+from config import etcd_client as client, config as config
+from stripe_utils import StripeUtils
+from schemas import (
+    make_return_message, ValidationException, UserRegisterPaymentSchema,
+    AddProductSchema, ProductOrderSchema, OrderListSchema, create_schema
+)
+from helper import get_plan_id_from_product, calculate_charges
+
+
+class ListProducts(Resource):
+    @staticmethod
+    def get():
+        products = client.get_prefix('/v1/products/')
+        products = [
+            product
+            for product in [p.value for p in products]
+            if product['active']
+        ]
+        prod_dict = {}
+        for p in products:
+            prod_dict[p['usable-id']] = {
+                'name': p['name'],
+                'description': p['description'],
+            }
+        logger.debug('Products = {}'.format(prod_dict))
+        return prod_dict, 200
+
+class AddProduct(Resource):
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+
+        try:
+            logger.debug('Got data: {}'.format(str(data)))
+            validator = AddProductSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            cleaned_values = validator.get_cleaned_values()
+            previous_product = cleaned_values.get('product', None)
+            if previous_product:
+                if not cleaned_values['update']:
+                    return make_return_message('Product already exists. Pass --update to update the product.')
+                else:
+                    product_uuid = previous_product.pop('uuid')
+            else:
+                product_uuid = uuid4().hex
+
+            product_value = cleaned_values['specs']
+
+            product_key = '/v1/products/{}'.format(product_uuid)
+            product_value['uuid'] = product_uuid
+
+            logger.debug('Adding product data: {}'.format(str(product_value)))
+            client.put(product_key, product_value)
+            if not previous_product:
+                return make_return_message('Product created.')
+            else:
+                return make_return_message('Product updated.')
+
+################################################################################
+# Nico-ok-marker
+
+
+class UserRegisterPayment(Resource):
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+
+        try:
+            logger.debug('Got data: {}'.format(str(data)))
+            validator = UserRegisterPaymentSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            cleaned_values = validator.get_cleaned_values()
+            last4 = data['card_number'].strip()[-4:]
+
+            stripe_utils = StripeUtils()
+
+            # Does customer already exist ?
+            stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail'])
+
+            # Does customer already exist ?
+            if stripe_customer is not None:
+                logger.debug('Customer {}-{} exists already'.format(
+                    cleaned_values['username'], cleaned_values['user']['mail'])
+                )
+
+                # Check if the card already exists
+                ce_response = stripe_utils.card_exists(
+                        stripe_customer.id, cc_number=data['card_number'],
+                        exp_month=int(data['expiry_month']),
+                        exp_year=int(data['expiry_year']),
+                        cvc=data['cvc'])
+
+                if ce_response['response_object']:
+                    message = 'The given card ending in {} exists already.'.format(last4)
+                    return make_return_message(message, 400)
+
+                elif ce_response['response_object'] is False:
+                    # Associate card with user
+                    logger.debug('Adding card ending in {}'.format(last4))
+                    token_response = stripe_utils.get_token_from_card(
+                        data['card_number'], data['cvc'], data['expiry_month'],
+                        data['expiry_year']
+                    )
+                    if token_response['response_object']:
+                        logger.debug('Token {}'.format(token_response['response_object'].id))
+                        resp = stripe_utils.associate_customer_card(
+                            stripe_customer.id, token_response['response_object'].id
+                        )
+                        if resp['response_object']:
+                            return make_return_message(
+                                'Card ending in {} registered as your payment source'.format(last4)
+                            )
+                    else:
+                        return make_return_message('Error with payment gateway. Contact support', 400)
+                else:
+                    return make_return_message('Error: {}'.format(ce_response['error']), 400)
+            else:
+                # Stripe customer does not exist, create a new one
+                logger.debug(
+                    'Customer {} does not exist, creating new'.format(cleaned_values['user']['mail'])
+                )
+                token_response = stripe_utils.get_token_from_card(
+                    cleaned_values['card_number'], cleaned_values['cvc'],
+                    cleaned_values['expiry_month'], cleaned_values['expiry_year']
+                )
+                if token_response['response_object']:
+                    logger.debug('Token {}'.format(token_response['response_object'].id))
+
+                    # Create stripe customer
+                    stripe_customer_resp = stripe_utils.create_customer(
+                        name=cleaned_values['card_holder_name'],
+                        token=token_response['response_object'].id,
+                        email=cleaned_values['user']['mail'],
+                        address=cleaned_values['address']
+                    )
+                    stripe_customer = stripe_customer_resp['response_object']
+
+                    if stripe_customer:
+                        logger.debug('Created stripe customer {}'.format(stripe_customer.id))
+                        return make_return_message(
+                            'Card ending in {} registered as your payment source'.format(last4)
+                        )
+                    else:
+                        return make_return_message('Error with card. Contact support', 400)
+                else:
+                    return make_return_message('Error with payment gateway. Contact support', 400)
+
+
+class ProductOrder(Resource):
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+
+        try:
+            validator = ProductOrderSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            cleaned_values = validator.get_cleaned_values()
+            stripe_utils = StripeUtils()
+
+            product = cleaned_values['product']
+
+            # Check the user has a payment source added
+            stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail'])
+
+            if not stripe_customer or len(stripe_customer.sources) == 0:
+                return make_return_message('Please register your payment method first.', 400)
+
+            try:
+                product_schema = create_schema(product, data)
+                product_schema = product_schema()
+                product_schema.is_valid()
+            except ValidationException as err:
+                return make_return_message(err, 400)
+            else:
+                transformed_data = product_schema.get_cleaned_values()
+                logger.debug('Tranformed data: {}'.format(transformed_data))
+                one_time_charge, recurring_charge = calculate_charges(product, transformed_data)
+                recurring_charge = int(recurring_charge)
+
+                if not cleaned_values['pay']:
+                    return make_return_message(
+                        'You would be charged {} CHF one time and {} CHF every {}. '
+                        'Add --pay to command to order.'.format(
+                            one_time_charge, recurring_charge, product['recurring_period']
+                        )
+                    )
+
+                with client.client.lock('product-order') as _:
+                    # Initiate a one-time/subscription based on product type
+                    if recurring_charge > 0:
+                        logger.debug('Product {} is recurring payment'.format(product['name']))
+                        plan_id = get_plan_id_from_product(product)
+                        res = stripe_utils.get_or_create_stripe_plan(
+                            product_name=product['name'],
+                            stripe_plan_id=plan_id, amount=recurring_charge,
+                            interval=product['recurring_period'],
+                        )
+                        if res['response_object']:
+                            logger.debug('Obtained plan {}'.format(plan_id))
+                            subscription_res = stripe_utils.subscribe_customer_to_plan(
+                                stripe_customer.id,
+                                [{'plan': plan_id}]
+                            )
+                            subscription_obj = subscription_res['response_object']
+                            if subscription_obj is None or subscription_obj.status != 'active':
+                                return make_return_message(
+                                    'Error subscribing to plan. Detail: {}'.format(subscription_res['error']), 400
+                                )
+                            else:
+                                order_obj = {
+                                    'order-id': uuid4().hex,
+                                    'ordered-at': datetime.now().isoformat(),
+                                    'product': product['usable-id'],
+                                    'one-time-price': one_time_charge,
+                                    'recurring-price': recurring_charge,
+                                    'recurring-period': product['recurring_period']
+                                }
+                                client.put(
+                                    '/v1/user/{}/orders/{}'.format(
+                                        cleaned_values['username'], order_obj['order-id']
+                                    ), order_obj
+                                )
+                                product['quantity'] -= 1
+                                client.put('/v1/products/{}'.format(product['uuid']), product)
+
+                                return {
+                                    'message': 'Order Successful.',
+                                    **order_obj
+                                }
+                        else:
+                            logger.error('Could not create plan {}'.format(plan_id))
+                            return make_return_message('Something wrong happened. Contact administrator', 400)
+
+                    elif recurring_charge == 0 and one_time_charge > 0:
+                        logger.debug('Product {} is one-time payment'.format(product['name']))
+                        charge_response = stripe_utils.make_charge(
+                            amount=one_time_charge,
+                            customer=stripe_customer.id
+                        )
+                        stripe_onetime_charge = charge_response.get('response_object')
+
+                        # Check if the payment was approved
+                        if not stripe_onetime_charge:
+                            msg = charge_response.get('error')
+                            return make_return_message('Error subscribing to plan. Details: {}'.format(msg), 400)
+
+                        order_obj = {
+                            'order-id': uuid4().hex,
+                            'ordered-at': datetime.now().isoformat(),
+                            'product': product['usable-id'],
+                            'one-time-price': one_time_charge,
+                        }
+                        client.put(
+                            '/v1/user/{}/orders/{}'.format(cleaned_values['username'], order_obj['order-id']),
+                            order_obj
+                        )
+                        product['quantity'] -= 1
+                        client.put('/v1/products/{}'.format(product['uuid']), product)
+
+                        return {'message': 'Order successful', **order_obj}, 200
+
+
+class OrderList(Resource):
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+
+        try:
+            validator = OrderListSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            cleaned_values = validator.get_cleaned_values()
+            orders = client.get_prefix('/v1/user/{}/orders'.format(cleaned_values['username']))
+            orders_dict = {
+                order.value['order-id']: {
+                    **order.value
+                }
+                for order in orders
+            }
+            logger.debug('Orders = {}'.format(orders_dict))
+            return {'orders': orders_dict}, 200
+
+
+if __name__ == '__main__':
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    log_formater = logging.Formatter('[%(filename)s:%(lineno)d] %(message)s')
+
+    stream_logger = logging.StreamHandler()
+    stream_logger.setFormatter(log_formater)
+
+    # file_logger = logging.FileHandler('log.txt')
+    # file_logger.setLevel(logging.DEBUG)
+    # file_logger.setFormatter(log_formater)
+
+    logger.addHandler(stream_logger)
+    # logger.addHandler(file_logger)
+
+    app = Flask(__name__)
+
+    api = Api(app)
+    api.add_resource(ListProducts, '/product/list')
+    api.add_resource(AddProduct, '/product/add')
+    api.add_resource(ProductOrder, '/product/order')
+    api.add_resource(UserRegisterPayment, '/user/register_payment')
+    api.add_resource(OrderList, '/order/list')
+
+    app.run(host='::', port=config.get('app', 'port', fallback=5000), debug=True)
+
+
+    @app.errorhandler(Exception)
+    def handle_exception(e):
+        app.logger.error(e)
+        # pass through HTTP errors
+        if isinstance(e, HTTPException):
+            return e
+
+        # now you're handling non-HTTP exceptions only
+        return {'message': 'Server Error'}, 500

archive/uncloud_django_based/notes-abk.md

@@ -0,0 +1,11 @@
+## TODO 2020-02-22
+
+* ~~move the current rest api to /opennebula~~
+* ~~make the /opennebula api only accessible by an admin account~~
+* ~~create a new filtered api on /vm/list that~~
+    * ~~a) requires authentication~~
+    * ~~b) only shows the VMs of the current user~~
+* ~~the new api should not contain all details, but: cpus (as read by the vcpu field), ram, ips, disks~~
+* ~~also make a (random) uuid the primary key for VMs - everything in this uncloud hack will use uuids as the id~~
+* ~~still expose the opennebula id as opennebula_id~~
+* ~~note put all secrets/configs into uncloud.secrets - I added a sample file into the repo~~

archive/uncloud_django_based/notes-nico.org

@@ -0,0 +1,102 @@
+* snapshot feature
+** product: vm-snapshot
+** flow
+*** list all my VMs
+**** get the uuid of the VM I want to take a snapshot of
+*** request a snapshot
+```
+vmuuid=$(http nicocustomer
+http -a nicocustomer:xxx http://uncloud.ch/vm/create_snapshot uuid=
+password=...
+```
+** backend realisation
+*** list snapshots
+    - have them in the DB
+    - create an entry on create
+*** creating snapshots
+    - vm sync / fsync?
+    - rbd snapshot
+      - host/cluster mapping?
+      - need image(s)
+
+* steps
+** DONE authenticate via ldap
+   CLOSED: [2020-02-20 Thu 19:05]
+** DONE Make classes / views require authentication
+   CLOSED: [2020-02-20 Thu 19:05]
+** TODO register credit card
+*** TODO find out what saving with us
+*** Info
+**** should not be fully saved in the DB
+**** model needs to be a bit different
+* Decide where to save sensitive data
+** stripe access key, etc.
+* python requirements (nicohack202002)
+  django djangorestframework django-auth-ldap stripe
+* os package requirements (alpine)
+  openldap-dev
+* VPN case
+** put on /orders with uuid
+** register cc
+* CC
+** TODO check whether we can register or not at stripe
+* membership
+** required for "smaller" / "shorter" products
+
+* TODO Membership missing
+* Flows to be implemented - see https://redmine.ungleich.ch/issues/7609
+** Membership
+*** 5 CHF
+** Django Hosting
+*** One time payment 35 CHF
+*** Monthly payment depends on VM size
+*** Parameters: same as IPv6 only VM
+** IPv6 VPN
+*** Parameters: none
+*** Is for free if the customer has an active VM
+** IPv6 only VM
+*** Parameters: cores, ram, os_disk_size, OS
+* Django rest framework
+** viewset: .list and .create
+** view: .get .post
+* TODO register CC
+* DONE list products
+  CLOSED: [2020-02-24 Mon 20:15]
+* An ungleich account - can be registered for free on
+  https://account.ungleich.ch
+* httpie installed (provides the http command)
+
+## Get a membership
+
+
+## Registering a payment method
+
+To be able to pay for the membership, you will need to register a
+credit card or apply for payment on bill (TO BE IMPLEMENTED).
+
+### Register credit card
+
+```
+http POST https://api.ungleich.ch/membership \
+    username=nico password=yourpassword \
+    cc_number=.. \
+    cc_
+
+```
+
+
+
+### Request payment via bill
+
+
+
+
+## Create the membership
+
+
+```
+http POST https://api.ungleich.ch/membership username=nico password=yourpassword
+
+```
+
+## List available products

archive/uncloud_django_based/plan.org

@@ -0,0 +1,6 @@
+* TODO register CC
+* TODO list products
+* ahmed
+** schemas
+*** field: is_valid? - used by schemas
+*** definition of a "schema"

archive/uncloud_django_based/uncloud/.gitignore

@@ -0,0 +1,4 @@
+db.sqlite3
+uncloud/secrets.py
+debug.log
+uncloud/local_settings.py

archive/uncloud_etcd_based/bin/gen-version

@@ -0,0 +1,29 @@
+#!/bin/sh
+# -*- coding: utf-8 -*-
+#
+# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org)
+#
+# This file is part of uncloud.
+#
+# uncloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uncloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+# Wrapper for real script to allow execution from checkout
+dir=${0%/*}
+
+# Ensure version is present - the bundled/shipped version contains a static version,
+# the git version contains a dynamic version
+printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py

archive/uncloud_etcd_based/bin/uncloud

@@ -25,9 +25,9 @@ dir=${0%/*}
 
 # Ensure version is present - the bundled/shipped version contains a static version,
 # the git version contains a dynamic version
-printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../ucloud/version.py
+${dir}/gen-version
 
 libdir=$(cd "${dir}/../" && pwd -P)
 export PYTHONPATH="${libdir}"
 
-"$dir/../scripts/ucloud" "$@"
+"$dir/../scripts/uncloud" "$@"

archive/uncloud_etcd_based/bin/uncloud-run-reinstall

@@ -0,0 +1,29 @@
+#!/bin/sh
+# -*- coding: utf-8 -*-
+#
+# 2012-2019 Nico Schottelius (nico-ucloud at schottelius.org)
+#
+# This file is part of ucloud.
+#
+# ucloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# ucloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with ucloud. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# Wrapper for real script to allow execution from checkout
+dir=${0%/*}
+
+${dir}/gen-version;
+pip uninstall -y uncloud >/dev/null
+python setup.py install  >/dev/null
+${dir}/uncloud "$@"

archive/uncloud_etcd_based/conf/uncloud.conf

@@ -0,0 +1,13 @@
+[etcd]
+url = localhost
+port = 2379
+base_prefix = /
+ca_cert
+cert_cert
+cert_key
+
+[client]
+name = replace_me
+realm = replace_me
+seed = replace_me
+api_server = http://localhost:5000

archive/uncloud_etcd_based/docs/Makefile

@@ -7,7 +7,7 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source/
 BUILDDIR      = build/
-DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/
+DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/
 
 .PHONY: all build clean
 

archive/uncloud_etcd_based/docs/README.md

@@ -0,0 +1,12 @@
+# uncloud docs
+
+## Requirements
+1. Python3
+2. Sphinx
+
+## Usage
+Run `make build` to build docs.
+
+Run `make clean` to remove build directory.
+
+Run `make publish` to push build dir to https://ungleich.ch/ucloud/

archive/uncloud_etcd_based/docs/source/admin-guide.rst

@@ -56,40 +56,13 @@ To start host we created earlier, execute the following command
 
     ucloud host ungleich.ch
 
-Create OS Image
----------------
+File & image scanners
+--------------------------
 
-Create ucloud-init ready OS image (Optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-This step is optional if you just want to test ucloud. However, sooner or later
-you want to create OS images with ucloud-init to properly
-contexualize VMs.
-
-1. Start a VM with OS image on which you want to install ucloud-init
-2. Execute the following command on the started VM
-
-   .. code-block:: sh
-
-    apk add git
-    git clone https://code.ungleich.ch/ucloud/ucloud-init.git
-    cd ucloud-init
-    sh ./install.sh
-3. Congratulations. Your image is now ucloud-init ready.
-
-
-Upload Sample OS Image
-~~~~~~~~~~~~~~~~~~~~~~
-Execute the following to get the sample OS image file.
-
-.. code-block:: sh
-
-    mkdir /var/www/admin
-    (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download)
-
-Run File Scanner and Image Scanner
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make
-images from tracked files. So, we need to track the file by running File Scanner
+Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our
+uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by
+ucloud. We can only make images from tracked files. So, we need to track the
+file by running File Scanner
 
 .. code-block:: sh
 

archive/uncloud_etcd_based/docs/source/conf.py

@@ -17,9 +17,9 @@
 
 # -- Project information -----------------------------------------------------
 
-project = 'ucloud'
-copyright = '2019, ungleich'
-author = 'ungleich'
+project = "uncloud"
+copyright = "2019, ungleich"
+author = "ungleich"
 
 # -- General configuration ---------------------------------------------------
 
@@ -27,12 +27,12 @@ author = 'ungleich'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
-    'sphinx.ext.autodoc',
-    'sphinx_rtd_theme',
+    "sphinx.ext.autodoc",
+    "sphinx_rtd_theme",
 ]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
@@ -50,4 +50,4 @@ html_theme = "sphinx_rtd_theme"
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]

archive/uncloud_etcd_based/docs/source/hacking.rst

@@ -0,0 +1,36 @@
+Hacking
+=======
+Using uncloud in hacking (aka development) mode.
+
+
+Get the code
+------------
+.. code-block:: sh
+    :linenos:
+
+     git clone https://code.ungleich.ch/uncloud/uncloud.git
+
+
+
+Install python requirements
+---------------------------
+You need to have python3 installed.
+
+.. code-block:: sh
+    :linenos:
+
+    cd uncloud!
+    python -m venv venv
+    . ./venv/bin/activate
+    ./bin/uncloud-run-reinstall
+
+
+
+Install os requirements
+-----------------------
+Install the following software packages: **dnsmasq**.
+
+If you already have a working IPv6 SLAAC and DNS setup,
+this step can be skipped.
+
+Note that you need at least one /64 IPv6 network to run uncloud.

archive/uncloud_etcd_based/docs/source/index.rst

@@ -11,12 +11,12 @@ Welcome to ucloud's documentation!
    :caption: Contents:
 
    introduction
-   user-guide
    setup-install
+   vm-images
+   user-guide
    admin-guide
-   user-guide/how-to-create-an-os-image-for-ucloud
    troubleshooting
-
+   hacking
 
 Indices and tables
 ==================

archive/uncloud_etcd_based/docs/source/vm-images.rst

@@ -0,0 +1,66 @@
+VM images
+==================================
+
+Overview
+---------
+
+ucloud tries to be least invasise towards VMs and only require
+strictly necessary changes for running in a virtualised
+environment. This includes configurations for:
+
+* Configuring the network
+* Managing access via ssh keys
+* Resizing the attached disk(s)
+
+Upstream images
+---------------
+
+The 'official' uncloud images are defined in the `uncloud/images
+<https://code.ungleich.ch/uncloud/images>`_ repository.
+
+How to make you own Uncloud images
+----------------------------------
+
+.. note::
+  It is fairly easy to create your own images for uncloud, as the common
+  operations (which are detailed below) can be automatically handled by the
+  `uncloud/uncloud-init <https://code.ungleich.ch/uncloud/uncloud-init>`_ tool.
+
+Network configuration
+~~~~~~~~~~~~~~~~~~~~~
+All VMs in ucloud are required to support IPv6. The primary network
+configuration is always done using SLAAC. A VM thus needs only to be
+configured to
+
+* accept router advertisements on all network interfaces
+* use the router advertisements to configure the network interfaces
+* accept the DNS entries from the router advertisements
+
+
+Configuring SSH keys
+~~~~~~~~~~~~~~~~~~~~
+
+To be able to access the VM, ucloud support provisioning SSH keys.
+
+To accept ssh keys in your VM, request the URL
+*http://metadata/ssh_keys*. Add the content to the appropriate user's
+**authorized_keys** file. Below you find sample code to accomplish
+this task:
+
+.. code-block:: sh
+
+    tmp=$(mktemp)
+    curl -s http://metadata/ssk_keys > "$tmp"
+    touch ~/.ssh/authorized_keys # ensure it exists
+    cat ~/.ssh/authorized_keys >> "$tmp"
+    sort "$tmp" | uniq > ~/.ssh/authorized_keys
+
+
+Disk resize
+~~~~~~~~~~~
+In virtualised environments, the disk sizes might grow. The operating
+system should detect disks that are bigger than the existing partition
+table and resize accordingly. This task is os specific.
+
+ucloud does not support shrinking disks due to the complexity and
+intra OS dependencies.

archive/uncloud_etcd_based/scripts/uncloud

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+import logging
+import sys
+import importlib
+import argparse
+import os
+
+from etcd3.exceptions import ConnectionFailedError
+
+from uncloud.common import settings
+from uncloud import UncloudException
+from uncloud.common.cli import resolve_otp_credentials
+
+# Components that use etcd
+ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner',
+                   'imagescanner', 'metadata', 'configure', 'hack']
+
+ALL_COMPONENTS = ETCD_COMPONENTS.copy()
+ALL_COMPONENTS.append('oneshot')
+#ALL_COMPONENTS.append('cli')
+
+
+if __name__ == '__main__':
+    arg_parser = argparse.ArgumentParser()
+    subparsers = arg_parser.add_subparsers(dest='command')
+
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
+                               help='More verbose logging')
+    parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
+                               default=os.path.expanduser('~/uncloud'))
+
+    etcd_parser = argparse.ArgumentParser(add_help=False)
+    etcd_parser.add_argument('--etcd-host')
+    etcd_parser.add_argument('--etcd-port')
+    etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
+    etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
+    etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
+
+    for component in ALL_COMPONENTS:
+        mod = importlib.import_module('uncloud.{}.main'.format(component))
+        parser = getattr(mod, 'arg_parser')
+
+        if component in ETCD_COMPONENTS:
+            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser])
+        else:
+            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
+
+    arguments = vars(arg_parser.parse_args())
+    etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
+    etcd_arguments = {
+        'etcd': {
+            key.replace('etcd_', ''): arguments[key]
+            for key in etcd_arguments
+        }
+    }
+    if not arguments['command']:
+        arg_parser.print_help()
+    else:
+        # Initializing Settings and resolving otp_credentials
+        # It is neccessary to resolve_otp_credentials after argument parsing is done because
+        # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
+        # providing the default values for --name, --realm and --seed arguments from the values
+        # we read from file. But, now we are asking user about where the config file lives. So,
+        # to providing default value is not possible before parsing arguments. So, we are doing
+        # it after..
+#        settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
+#        resolve_otp_credentials(arguments)
+
+        name = arguments.pop('command')
+        mod = importlib.import_module('uncloud.{}.main'.format(name))
+        main = getattr(mod, 'main')
+
+        if arguments['debug']:
+            logging.basicConfig(level=logging.DEBUG)
+        else:
+            logging.basicConfig(level=logging.INFO)
+
+        log = logging.getLogger()
+
+        try:
+            main(arguments)
+        except UncloudException as err:
+            log.error(err)
+            sys.exit(1)
+#        except ConnectionFailedError as err:
+#            log.error('Cannot connect to etcd: {}'.format(err))
+        except Exception as err:
+            log.exception(err)

archive/uncloud_etcd_based/setup.py

@@ -0,0 +1,51 @@
+import os
+
+from setuptools import setup, find_packages
+
+with open("README.md", "r") as fh:
+    long_description = fh.read()
+
+try:
+    import uncloud.version
+
+    version = uncloud.version.VERSION
+except:
+    import subprocess
+
+    c = subprocess.check_output(["git", "describe"])
+    version = c.decode("utf-8").strip()
+
+
+setup(
+    name="uncloud",
+    version=version,
+    description="uncloud cloud management",
+    url="https://code.ungleich.ch/uncloud/uncloud",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    classifiers=[
+        "Development Status :: 3 - Alpha",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Programming Language :: Python :: 3",
+    ],
+    author="ungleich",
+    author_email="technik@ungleich.ch",
+    packages=find_packages(),
+    install_requires=[
+        "requests",
+        "Flask>=1.1.1",
+        "flask-restful",
+        "bitmath",
+        "pyotp",
+        "pynetbox",
+        "colorama",
+        "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3",
+        "marshmallow",
+        "ldap3"
+    ],
+    scripts=["scripts/uncloud"],
+    data_files=[
+        (os.path.expanduser("~/uncloud/"), ["conf/uncloud.conf"])
+    ],
+    zip_safe=False,
+)

archive/uncloud_etcd_based/test/test_mac_local.py

@@ -0,0 +1,37 @@
+import unittest
+from unittest.mock import Mock
+
+from uncloud.hack.mac import MAC
+from uncloud import UncloudException
+
+class TestMacLocal(unittest.TestCase):
+    def setUp(self):
+        self.config = Mock()
+        self.config.arguments = {"no_db":True}
+        self.mac = MAC(self.config)
+        self.mac.create()
+
+    def testMacInt(self):
+        self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong first MAC index")
+
+    def testMacRepr(self):
+        self.assertEqual(self.mac.__repr__(), '420000000001', "wrong first MAC index")
+
+    def testMacStr(self):
+        self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong first MAC index")
+
+    def testValidationRaise(self):
+        with self.assertRaises(UncloudException):
+            self.mac.validate_mac("2")
+
+    def testValidation(self):
+        self.assertTrue(self.mac.validate_mac("42:00:00:00:00:01"), "Validation of a given MAC not working properly")
+
+    def testNextMAC(self):
+        self.mac.create()
+        self.assertEqual(self.mac.__repr__(), '420000000001', "wrong second MAC index")
+        self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong second MAC index")
+        self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong second MAC index")
+
+if __name__ == '__main__':
+    unittest.main()

archive/uncloud_etcd_based/uncloud/__init__.py

@@ -0,0 +1,2 @@
+class UncloudException(Exception):
+    pass

archive/uncloud_etcd_based/uncloud/api/common_fields.py

@@ -1,6 +1,6 @@
 import os
 
-from ucloud.config import etcd_client, env_vars
+from uncloud.common.shared import shared
 
 
 class Optional:
@@ -19,12 +19,16 @@ class Field:
 
     def is_valid(self):
         if self.value == KeyError:
-            self.add_error("'{}' field is a required field".format(self.name))
+            self.add_error(
+                "'{}' field is a required field".format(self.name)
+            )
         else:
             if isinstance(self.value, Optional):
                 pass
             elif not isinstance(self.value, self.type):
-                self.add_error("Incorrect Type for '{}' field".format(self.name))
+                self.add_error(
+                    "Incorrect Type for '{}' field".format(self.name)
+                )
             else:
                 self.validation()
 
@@ -48,6 +52,8 @@ class VmUUIDField(Field):
         self.validation = self.vm_uuid_validation
 
     def vm_uuid_validation(self):
-        r = etcd_client.get(os.path.join(env_vars.get('VM_PREFIX'), self.uuid))
+        r = shared.etcd_client.get(
+            os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
+        )
         if not r:
             self.add_error("VM with uuid {} does not exists".format(self.uuid))

archive/uncloud_etcd_based/uncloud/api/create_image_store.py

@@ -0,0 +1,19 @@
+import json
+import os
+
+from uuid import uuid4
+
+from uncloud.common.shared import shared
+
+data = {
+    'is_public': True,
+    'type': 'ceph',
+    'name': 'images',
+    'description': 'first ever public image-store',
+    'attributes': {'list': [], 'key': [], 'pool': 'images'},
+}
+
+shared.etcd_client.put(
+    os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
+    json.dumps(data),
+)

archive/uncloud_etcd_based/uncloud/api/helper.py

@@ -1,48 +1,51 @@
 import binascii
 import ipaddress
 import random
-import subprocess as sp
-
+import logging
 import requests
 
 from pyotp import TOTP
 
-from ucloud.config import vm_pool, env_vars
+from uncloud.common.shared import shared
+
+logger = logging.getLogger(__name__)
 
 
 def check_otp(name, realm, token):
     try:
         data = {
-            "auth_name": env_vars.get("AUTH_NAME"),
-            "auth_token": TOTP(env_vars.get("AUTH_SEED")).now(),
-            "auth_realm": env_vars.get("AUTH_REALM"),
+            "auth_name": shared.settings["otp"]["auth_name"],
+            "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
+            "auth_realm": shared.settings["otp"]["auth_realm"],
             "name": name,
             "realm": realm,
             "token": token,
         }
-    except binascii.Error:
+    except binascii.Error as err:
+        logger.error(
+            "Cannot compute OTP for seed: {}".format(
+                shared.settings["otp"]["auth_seed"]
+            )
+        )
         return 400
 
     response = requests.post(
-        "{OTP_SERVER}{OTP_VERIFY_ENDPOINT}".format(
-            OTP_SERVER=env_vars.get("OTP_SERVER", ""),
-            OTP_VERIFY_ENDPOINT=env_vars.get("OTP_VERIFY_ENDPOINT", "verify/"),
-        ),
-        json=data,
+        shared.settings["otp"]["verification_controller_url"], json=data
     )
     return response.status_code
 
 
 def resolve_vm_name(name, owner):
     """Return UUID of Virtual Machine of name == name and owner == owner
-    
+
     Input: name of vm, owner of vm.
     Output: uuid of vm if found otherwise None
     """
     result = next(
         filter(
-            lambda vm: vm.value["owner"] == owner and vm.value["name"] == name,
-            vm_pool.vms,
+            lambda vm: vm.value["owner"] == owner
+            and vm.value["name"] == name,
+            shared.vm_pool.vms,
         ),
         None,
     )
@@ -54,7 +57,7 @@ def resolve_vm_name(name, owner):
 
 def resolve_image_name(name, etcd_client):
     """Return image uuid given its name and its store
-       
+
        * If the provided name is not in correct format
          i.e {store_name}:{image_name} return ValueError
        * If no such image found then return KeyError
@@ -70,26 +73,35 @@ def resolve_image_name(name, etcd_client):
         """
         Examples, where it would work and where it would raise exception
         "images:alpine" --> ["images", "alpine"]
-        
+
         "images" --> ["images"] it would raise Exception as non enough value to unpack
-        
+
         "images:alpine:meow" --> ["images", "alpine", "meow"] it would raise Exception
                                                               as too many values to unpack
         """
         store_name, image_name = store_name_and_image_name
     except Exception:
-        raise ValueError("Image name not in correct format i.e {store_name}:{image_name}")
+        raise ValueError(
+            "Image name not in correct format i.e {store_name}:{image_name}"
+        )
 
-    images = etcd_client.get_prefix(env_vars.get('IMAGE_PREFIX'), value_in_json=True)
+    images = etcd_client.get_prefix(
+        shared.settings["etcd"]["image_prefix"], value_in_json=True
+    )
 
     # Try to find image with name == image_name and store_name == store_name
     try:
-        image = next(filter(lambda im: im.value['name'] == image_name
-                            and im.value['store_name'] == store_name, images))
+        image = next(
+            filter(
+                lambda im: im.value["name"] == image_name
+                and im.value["store_name"] == store_name,
+                images,
+            )
+        )
     except StopIteration:
         raise KeyError("No image with name {} found.".format(name))
     else:
-        image_uuid = image.key.split('/')[-1]
+        image_uuid = image.key.split("/")[-1]
 
     return image_uuid
 
@@ -98,7 +110,7 @@ def random_bytes(num=6):
     return [random.randrange(256) for _ in range(num)]
 
 
-def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='%02x'):
+def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
     mac = random_bytes()
     if oui:
         if type(oui) == str:
@@ -116,36 +128,6 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='
     return separator.join(byte_fmt % b for b in mac)
 
 
-def get_ip_addr(mac_address, device):
-    """Return IP address of a device provided its mac address / link local address
-       and the device with which it is connected.
-       
-       For Example, if we call get_ip_addr(mac_address="52:54:00:12:34:56", device="br0")
-       the following two scenarios can happen
-       1. It would return None if we can't be able to find device whose mac_address is equal
-          to the arg:mac_address or the mentioned arg:device does not exists or the ip address
-          we found is local.
-       2. It would return ip_address of device whose mac_address is equal to arg:mac_address
-          and is connected/neighbor of arg:device       
-    """
-    try:
-        output = sp.check_output(['ip', '-6', 'neigh', 'show', 'dev', device], stderr=sp.PIPE)
-    except sp.CalledProcessError:
-        return None
-    else:
-        result = []
-        output = output.strip().decode("utf-8")
-        output = output.split("\n")
-        for entry in output:
-            entry = entry.split()
-            if entry:
-                ip = ipaddress.ip_address(entry[0])
-                mac = entry[2]
-                if ip.is_global and mac_address == mac:
-                    result.append(ip)
-        return result
-
-
 def mac2ipv6(mac, prefix):
     # only accept MACs separated by a colon
     parts = mac.split(":")
@@ -158,8 +140,9 @@ def mac2ipv6(mac, prefix):
     # format output
     ipv6_parts = [str(0)] * 4
     for i in range(0, len(parts), 2):
-        ipv6_parts.append("".join(parts[i:i + 2]))
+        ipv6_parts.append("".join(parts[i : i + 2]))
 
     lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
     prefix = ipaddress.IPv6Address(prefix)
     return str(prefix + int(lower_part))
+

archive/uncloud_etcd_based/uncloud/api/main.py

@@ -0,0 +1,600 @@
+import json
+import pynetbox
+import logging
+import argparse
+
+from uuid import uuid4
+from os.path import join as join_path
+
+from flask import Flask, request
+from flask_restful import Resource, Api
+from werkzeug.exceptions import HTTPException
+
+from uncloud.common.shared import shared
+
+from uncloud.common import counters
+from uncloud.common.vm import VMStatus
+from uncloud.common.request import RequestEntry, RequestType
+from uncloud.api import schemas
+from uncloud.api.helper import generate_mac, mac2ipv6
+from uncloud import UncloudException
+
+logger = logging.getLogger(__name__)
+
+app = Flask(__name__)
+api = Api(app)
+app.logger.handlers.clear()
+
+arg_parser = argparse.ArgumentParser('api', add_help=False)
+arg_parser.add_argument('--port', '-p')
+
+
+@app.errorhandler(Exception)
+def handle_exception(e):
+    app.logger.error(e)
+    # pass through HTTP errors
+    if isinstance(e, HTTPException):
+        return e
+
+    # now you're handling non-HTTP exceptions only
+    return {'message': 'Server Error'}, 500
+
+
+class CreateVM(Resource):
+    """API Request to Handle Creation of VM"""
+
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.CreateVMSchema(data)
+        if validator.is_valid():
+            vm_uuid = uuid4().hex
+            vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
+            specs = {
+                'cpu': validator.specs['cpu'],
+                'ram': validator.specs['ram'],
+                'os-ssd': validator.specs['os-ssd'],
+                'hdd': validator.specs['hdd'],
+            }
+            macs = [generate_mac() for _ in range(len(data['network']))]
+            tap_ids = [
+                counters.increment_etcd_counter(
+                    shared.etcd_client, shared.settings['etcd']['tap_counter']
+                )
+                for _ in range(len(data['network']))
+            ]
+            vm_entry = {
+                'name': data['vm_name'],
+                'owner': data['name'],
+                'owner_realm': data['realm'],
+                'specs': specs,
+                'hostname': '',
+                'status': VMStatus.stopped,
+                'image_uuid': validator.image_uuid,
+                'log': [],
+                'vnc_socket': '',
+                'network': list(zip(data['network'], macs, tap_ids)),
+                'metadata': {'ssh-keys': []},
+                'in_migration': False,
+            }
+            shared.etcd_client.put(vm_key, vm_entry, value_in_json=True)
+
+            # Create ScheduleVM Request
+            r = RequestEntry.from_scratch(
+                type=RequestType.ScheduleVM,
+                uuid=vm_uuid,
+                request_prefix=shared.settings['etcd']['request_prefix'],
+            )
+            shared.request_pool.put(r)
+
+            return {'message': 'VM Creation Queued'}, 200
+        return validator.get_errors(), 400
+
+
+class VmStatus(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.VMStatusSchema(data)
+        if validator.is_valid():
+            vm = shared.vm_pool.get(
+                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
+            )
+            vm_value = vm.value.copy()
+            vm_value['ip'] = []
+            for network_mac_and_tap in vm.network:
+                network_name, mac, tap = network_mac_and_tap
+                network = shared.etcd_client.get(
+                    join_path(
+                        shared.settings['etcd']['network_prefix'],
+                        data['name'],
+                        network_name,
+                    ),
+                    value_in_json=True,
+                )
+                ipv6_addr = (
+                    network.value.get('ipv6').split('::')[0] + '::'
+                )
+                vm_value['ip'].append(mac2ipv6(mac, ipv6_addr))
+            vm.value = vm_value
+            return vm.value
+        else:
+            return validator.get_errors(), 400
+
+
+class CreateImage(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.CreateImageSchema(data)
+        if validator.is_valid():
+            file_entry = shared.etcd_client.get(
+                join_path(shared.settings['etcd']['file_prefix'], data['uuid'])
+            )
+            file_entry_value = json.loads(file_entry.value)
+
+            image_entry_json = {
+                'status': 'TO_BE_CREATED',
+                'owner': file_entry_value['owner'],
+                'filename': file_entry_value['filename'],
+                'name': data['name'],
+                'store_name': data['image_store'],
+                'visibility': 'public',
+            }
+            shared.etcd_client.put(
+                join_path(
+                    shared.settings['etcd']['image_prefix'], data['uuid']
+                ),
+                json.dumps(image_entry_json),
+            )
+
+            return {'message': 'Image queued for creation.'}
+        return validator.get_errors(), 400
+
+
+class ListPublicImages(Resource):
+    @staticmethod
+    def get():
+        images = shared.etcd_client.get_prefix(
+            shared.settings['etcd']['image_prefix'], value_in_json=True
+        )
+        r = {'images': []}
+        for image in images:
+            image_key = '{}:{}'.format(
+                image.value['store_name'], image.value['name']
+            )
+            r['images'].append(
+                {'name': image_key, 'status': image.value['status']}
+            )
+        return r, 200
+
+
+class VMAction(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.VmActionSchema(data)
+
+        if validator.is_valid():
+            vm_entry = shared.vm_pool.get(
+                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
+            )
+            action = data['action']
+
+            if action == 'start':
+                action = 'schedule'
+
+            if action == 'delete' and vm_entry.hostname == '':
+                if shared.storage_handler.is_vm_image_exists(
+                    vm_entry.uuid
+                ):
+                    r_status = shared.storage_handler.delete_vm_image(
+                        vm_entry.uuid
+                    )
+                    if r_status:
+                        shared.etcd_client.client.delete(vm_entry.key)
+                        return {'message': 'VM successfully deleted'}
+                    else:
+                        logger.error(
+                            'Some Error Occurred while deleting VM'
+                        )
+                        return {'message': 'VM deletion unsuccessfull'}
+                else:
+                    shared.etcd_client.client.delete(vm_entry.key)
+                    return {'message': 'VM successfully deleted'}
+
+            r = RequestEntry.from_scratch(
+                type='{}VM'.format(action.title()),
+                uuid=data['uuid'],
+                hostname=vm_entry.hostname,
+                request_prefix=shared.settings['etcd']['request_prefix'],
+            )
+            shared.request_pool.put(r)
+            return (
+                {'message': 'VM {} Queued'.format(action.title())},
+                200,
+            )
+        else:
+            return validator.get_errors(), 400
+
+
+class VMMigration(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.VmMigrationSchema(data)
+
+        if validator.is_valid():
+            vm = shared.vm_pool.get(data['uuid'])
+            r = RequestEntry.from_scratch(
+                type=RequestType.InitVMMigration,
+                uuid=vm.uuid,
+                hostname=join_path(
+                    shared.settings['etcd']['host_prefix'],
+                    validator.destination.value,
+                ),
+                request_prefix=shared.settings['etcd']['request_prefix'],
+            )
+
+            shared.request_pool.put(r)
+            return (
+                {'message': 'VM Migration Initialization Queued'},
+                200,
+            )
+        else:
+            return validator.get_errors(), 400
+
+
+class ListUserVM(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.OTPSchema(data)
+
+        if validator.is_valid():
+            vms = shared.etcd_client.get_prefix(
+                shared.settings['etcd']['vm_prefix'], value_in_json=True
+            )
+            return_vms = []
+            user_vms = filter(
+                lambda v: v.value['owner'] == data['name'], vms
+            )
+            for vm in user_vms:
+                return_vms.append(
+                    {
+                        'name': vm.value['name'],
+                        'vm_uuid': vm.key.split('/')[-1],
+                        'specs': vm.value['specs'],
+                        'status': vm.value['status'],
+                        'hostname': vm.value['hostname'],
+                        'vnc_socket': vm.value.get('vnc_socket', None),
+                    }
+                )
+            if return_vms:
+                return {'message': return_vms}, 200
+            return {'message': 'No VM found'}, 404
+
+        else:
+            return validator.get_errors(), 400
+
+
+class ListUserFiles(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.OTPSchema(data)
+
+        if validator.is_valid():
+            files = shared.etcd_client.get_prefix(
+                shared.settings['etcd']['file_prefix'], value_in_json=True
+            )
+            return_files = []
+            user_files = [f for f in files if f.value['owner'] == data['name']]
+            for file in user_files:
+                file_uuid = file.key.split('/')[-1]
+                file = file.value
+                file['uuid'] = file_uuid
+
+                file.pop('sha512sum', None)
+                file.pop('owner', None)
+
+                return_files.append(file)
+            return {'message': return_files}, 200
+        else:
+            return validator.get_errors(), 400
+
+
+class CreateHost(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.CreateHostSchema(data)
+        if validator.is_valid():
+            host_key = join_path(
+                shared.settings['etcd']['host_prefix'], uuid4().hex
+            )
+            host_entry = {
+                'specs': data['specs'],
+                'hostname': data['hostname'],
+                'status': 'DEAD',
+                'last_heartbeat': '',
+            }
+            shared.etcd_client.put(
+                host_key, host_entry, value_in_json=True
+            )
+
+            return {'message': 'Host Created'}, 200
+
+        return validator.get_errors(), 400
+
+
+class ListHost(Resource):
+    @staticmethod
+    def get():
+        hosts = shared.host_pool.hosts
+        r = {
+            host.key: {
+                'status': host.status,
+                'specs': host.specs,
+                'hostname': host.hostname,
+            }
+            for host in hosts
+        }
+        return r, 200
+
+
+class GetSSHKeys(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.GetSSHSchema(data)
+        if validator.is_valid():
+            if not validator.key_name.value:
+
+                # {user_prefix}/{realm}/{name}/key/
+                etcd_key = join_path(
+                    shared.settings['etcd']['user_prefix'],
+                    data['realm'],
+                    data['name'],
+                    'key',
+                )
+                etcd_entry = shared.etcd_client.get_prefix(
+                    etcd_key, value_in_json=True
+                )
+
+                keys = {
+                    key.key.split('/')[-1]: key.value
+                    for key in etcd_entry
+                }
+                return {'keys': keys}
+            else:
+
+                # {user_prefix}/{realm}/{name}/key/{key_name}
+                etcd_key = join_path(
+                    shared.settings['etcd']['user_prefix'],
+                    data['realm'],
+                    data['name'],
+                    'key',
+                    data['key_name'],
+                )
+                etcd_entry = shared.etcd_client.get(
+                    etcd_key, value_in_json=True
+                )
+
+                if etcd_entry:
+                    return {
+                        'keys': {
+                            etcd_entry.key.split('/')[
+                                -1
+                            ]: etcd_entry.value
+                        }
+                    }
+                else:
+                    return {'keys': {}}
+        else:
+            return validator.get_errors(), 400
+
+
+class AddSSHKey(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.AddSSHSchema(data)
+        if validator.is_valid():
+
+            # {user_prefix}/{realm}/{name}/key/{key_name}
+            etcd_key = join_path(
+                shared.settings['etcd']['user_prefix'],
+                data['realm'],
+                data['name'],
+                'key',
+                data['key_name'],
+            )
+            etcd_entry = shared.etcd_client.get(
+                etcd_key, value_in_json=True
+            )
+            if etcd_entry:
+                return {
+                    'message': 'Key with name "{}" already exists'.format(
+                        data['key_name']
+                    )
+                }
+            else:
+                # Key Not Found. It implies user' haven't added any key yet.
+                shared.etcd_client.put(
+                    etcd_key, data['key'], value_in_json=True
+                )
+                return {'message': 'Key added successfully'}
+        else:
+            return validator.get_errors(), 400
+
+
+class RemoveSSHKey(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.RemoveSSHSchema(data)
+        if validator.is_valid():
+
+            # {user_prefix}/{realm}/{name}/key/{key_name}
+            etcd_key = join_path(
+                shared.settings['etcd']['user_prefix'],
+                data['realm'],
+                data['name'],
+                'key',
+                data['key_name'],
+            )
+            etcd_entry = shared.etcd_client.get(
+                etcd_key, value_in_json=True
+            )
+            if etcd_entry:
+                shared.etcd_client.client.delete(etcd_key)
+                return {'message': 'Key successfully removed.'}
+            else:
+                return {
+                    'message': 'No Key with name "{}" Exists at all.'.format(
+                        data['key_name']
+                    )
+                }
+        else:
+            return validator.get_errors(), 400
+
+
+class CreateNetwork(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.CreateNetwork(data)
+
+        if validator.is_valid():
+
+            network_entry = {
+                'id': counters.increment_etcd_counter(
+                    shared.etcd_client, shared.settings['etcd']['vxlan_counter']
+                ),
+                'type': data['type'],
+            }
+            if validator.user.value:
+                try:
+                    nb = pynetbox.api(
+                        url=shared.settings['netbox']['url'],
+                        token=shared.settings['netbox']['token'],
+                    )
+                    nb_prefix = nb.ipam.prefixes.get(
+                        prefix=shared.settings['network']['prefix']
+                    )
+                    prefix = nb_prefix.available_prefixes.create(
+                        data={
+                            'prefix_length': int(
+                                shared.settings['network']['prefix_length']
+                            ),
+                            'description': '{}\'s network "{}"'.format(
+                                data['name'], data['network_name']
+                            ),
+                            'is_pool': True,
+                        }
+                    )
+                except Exception as err:
+                    app.logger.error(err)
+                    return {
+                        'message': 'Error occured while creating network.'
+                    }
+                else:
+                    network_entry['ipv6'] = prefix['prefix']
+            else:
+                network_entry['ipv6'] = 'fd00::/64'
+
+            network_key = join_path(
+                shared.settings['etcd']['network_prefix'],
+                data['name'],
+                data['network_name'],
+            )
+            shared.etcd_client.put(
+                network_key, network_entry, value_in_json=True
+            )
+            return {'message': 'Network successfully added.'}
+        else:
+            return validator.get_errors(), 400
+
+
+class ListUserNetwork(Resource):
+    @staticmethod
+    def post():
+        data = request.json
+        validator = schemas.OTPSchema(data)
+
+        if validator.is_valid():
+            prefix = join_path(
+                shared.settings['etcd']['network_prefix'], data['name']
+            )
+            networks = shared.etcd_client.get_prefix(
+                prefix, value_in_json=True
+            )
+            user_networks = []
+            for net in networks:
+                net.value['name'] = net.key.split('/')[-1]
+                user_networks.append(net.value)
+            return {'networks': user_networks}, 200
+        else:
+            return validator.get_errors(), 400
+
+
+api.add_resource(CreateVM, '/vm/create')
+api.add_resource(VmStatus, '/vm/status')
+
+api.add_resource(VMAction, '/vm/action')
+api.add_resource(VMMigration, '/vm/migrate')
+
+api.add_resource(CreateImage, '/image/create')
+api.add_resource(ListPublicImages, '/image/list-public')
+
+api.add_resource(ListUserVM, '/user/vms')
+api.add_resource(ListUserFiles, '/user/files')
+api.add_resource(ListUserNetwork, '/user/networks')
+
+api.add_resource(AddSSHKey, '/user/add-ssh')
+api.add_resource(RemoveSSHKey, '/user/remove-ssh')
+api.add_resource(GetSSHKeys, '/user/get-ssh')
+
+api.add_resource(CreateHost, '/host/create')
+api.add_resource(ListHost, '/host/list')
+
+api.add_resource(CreateNetwork, '/network/create')
+
+
+def main(arguments):
+    debug = arguments['debug']
+    port = arguments['port']
+
+    try:
+        image_stores = list(
+            shared.etcd_client.get_prefix(
+                shared.settings['etcd']['image_store_prefix'], value_in_json=True
+            )
+        )
+    except KeyError:
+        image_stores = False
+
+    # Do not inject default values that might be very wrong
+    # fail when required, not before
+    #
+    # if not image_stores:
+    #     data = {
+    #         'is_public': True,
+    #         'type': 'ceph',
+    #         'name': 'images',
+    #         'description': 'first ever public image-store',
+    #         'attributes': {'list': [], 'key': [], 'pool': 'images'},
+    #     }
+
+    #     shared.etcd_client.put(
+    #         join_path(
+    #             shared.settings['etcd']['image_store_prefix'], uuid4().hex
+    #         ),
+    #         json.dumps(data),
+    #     )
+
+    try:
+        app.run(host='::', port=port, debug=debug)
+    except OSError as e:
+        raise UncloudException('Failed to start Flask: {}'.format(e))

archive/uncloud_etcd_based/uncloud/api/schemas.py

@@ -1,6 +1,6 @@
 """
 This module contain classes thats validates and intercept/modify
-data coming from ucloud-cli (user)
+data coming from uncloud-cli (user)
 
 It was primarily developed as an alternative to argument parser
 of Flask_Restful which is going to be deprecated. I also tried
@@ -19,10 +19,10 @@ import os
 
 import bitmath
 
-from ucloud.common.host import HostStatus
-from ucloud.common.vm import VMStatus
-from ucloud.config import etcd_client, env_vars, vm_pool, host_pool
-from . import helper
+from uncloud.common.host import HostStatus
+from uncloud.common.vm import VMStatus
+from uncloud.common.shared import shared
+from . import helper, logger
 from .common_fields import Field, VmUUIDField
 from .helper import check_otp, resolve_vm_name
 
@@ -79,7 +79,12 @@ class OTPSchema(BaseSchema):
         super().__init__(data=data, fields=_fields)
 
     def validation(self):
-        if check_otp(self.name.value, self.realm.value, self.token.value) != 200:
+        if (
+            check_otp(
+                self.name.value, self.realm.value, self.token.value
+            )
+            != 200
+        ):
             self.add_error("Wrong Credentials")
 
 
@@ -91,7 +96,9 @@ class CreateImageSchema(BaseSchema):
         # Fields
         self.uuid = Field("uuid", str, data.get("uuid", KeyError))
         self.name = Field("name", str, data.get("name", KeyError))
-        self.image_store = Field("image_store", str, data.get("image_store", KeyError))
+        self.image_store = Field(
+            "image_store", str, data.get("image_store", KeyError)
+        )
 
         # Validations
         self.uuid.validation = self.file_uuid_validation
@@ -102,34 +109,51 @@ class CreateImageSchema(BaseSchema):
         super().__init__(data, fields)
 
     def file_uuid_validation(self):
-        file_entry = etcd_client.get(os.path.join(env_vars.get('FILE_PREFIX'), self.uuid.value))
+        file_entry = shared.etcd_client.get(
+            os.path.join(
+                shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
+            )
+        )
         if file_entry is None:
             self.add_error(
-                "Image File with uuid '{}' Not Found".format(self.uuid.value)
+                "Image File with uuid '{}' Not Found".format(
+                    self.uuid.value
+                )
             )
 
     def image_store_name_validation(self):
-        image_stores = list(etcd_client.get_prefix(env_vars.get('IMAGE_STORE_PREFIX')))
+        image_stores = list(
+            shared.etcd_client.get_prefix(
+                shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
+            )
+        )
 
         image_store = next(
             filter(
-                lambda s: json.loads(s.value)["name"] == self.image_store.value,
+                lambda s: json.loads(s.value)["name"]
+                == self.image_store.value,
                 image_stores,
             ),
             None,
         )
         if not image_store:
-            self.add_error("Store '{}' does not exists".format(self.image_store.value))
+            self.add_error(
+                "Store '{}' does not exists".format(
+                    self.image_store.value
+                )
+            )
 
 
 # Host Operations
 
+
 class CreateHostSchema(OTPSchema):
     def __init__(self, data):
-        self.parsed_specs = {}
         # Fields
         self.specs = Field("specs", dict, data.get("specs", KeyError))
-        self.hostname = Field("hostname", str, data.get("hostname", KeyError))
+        self.hostname = Field(
+            "hostname", str, data.get("hostname", KeyError)
+        )
 
         # Validation
         self.specs.validation = self.specs_validation
@@ -141,22 +165,28 @@ class CreateHostSchema(OTPSchema):
     def specs_validation(self):
         ALLOWED_BASE = 10
 
-        _cpu = self.specs.value.get('cpu', KeyError)
-        _ram = self.specs.value.get('ram', KeyError)
-        _os_ssd = self.specs.value.get('os-ssd', KeyError)
-        _hdd = self.specs.value.get('hdd', KeyError)
+        _cpu = self.specs.value.get("cpu", KeyError)
+        _ram = self.specs.value.get("ram", KeyError)
+        _os_ssd = self.specs.value.get("os-ssd", KeyError)
+        _hdd = self.specs.value.get("hdd", KeyError)
 
         if KeyError in [_cpu, _ram, _os_ssd, _hdd]:
-            self.add_error("You must specify CPU, RAM and OS-SSD in your specs")
+            self.add_error(
+                "You must specify CPU, RAM and OS-SSD in your specs"
+            )
             return None
         try:
             parsed_ram = bitmath.parse_string_unsafe(_ram)
             parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd)
 
             if parsed_ram.base != ALLOWED_BASE:
-                self.add_error("Your specified RAM is not in correct units")
+                self.add_error(
+                    "Your specified RAM is not in correct units"
+                )
             if parsed_os_ssd.base != ALLOWED_BASE:
-                self.add_error("Your specified OS-SSD is not in correct units")
+                self.add_error(
+                    "Your specified OS-SSD is not in correct units"
+                )
 
             if _cpu < 1:
                 self.add_error("CPU must be atleast 1")
@@ -171,7 +201,9 @@ class CreateHostSchema(OTPSchema):
             for hdd in _hdd:
                 _parsed_hdd = bitmath.parse_string_unsafe(hdd)
                 if _parsed_hdd.base != ALLOWED_BASE:
-                    self.add_error("Your specified HDD is not in correct units")
+                    self.add_error(
+                        "Your specified HDD is not in correct units"
+                    )
                     break
                 else:
                     parsed_hdd.append(str(_parsed_hdd))
@@ -182,15 +214,17 @@ class CreateHostSchema(OTPSchema):
         else:
             if self.get_errors():
                 self.specs = {
-                    'cpu': _cpu,
-                    'ram': str(parsed_ram),
-                    'os-ssd': str(parsed_os_ssd),
-                    'hdd': parsed_hdd
+                    "cpu": _cpu,
+                    "ram": str(parsed_ram),
+                    "os-ssd": str(parsed_os_ssd),
+                    "hdd": parsed_hdd,
                 }
 
     def validation(self):
         if self.realm.value != "ungleich-admin":
-            self.add_error("Invalid Credentials/Insufficient Permission")
+            self.add_error(
+                "Invalid Credentials/Insufficient Permission"
+            )
 
 
 # VM Operations
@@ -198,13 +232,15 @@ class CreateHostSchema(OTPSchema):
 
 class CreateVMSchema(OTPSchema):
     def __init__(self, data):
-        self.parsed_specs = {}
-
         # Fields
         self.specs = Field("specs", dict, data.get("specs", KeyError))
-        self.vm_name = Field("vm_name", str, data.get("vm_name", KeyError))
+        self.vm_name = Field(
+            "vm_name", str, data.get("vm_name", KeyError)
+        )
         self.image = Field("image", str, data.get("image", KeyError))
-        self.network = Field("network", list, data.get("network", KeyError))
+        self.network = Field(
+            "network", list, data.get("network", KeyError)
+        )
 
         # Validation
         self.image.validation = self.image_validation
@@ -218,16 +254,25 @@ class CreateVMSchema(OTPSchema):
 
     def image_validation(self):
         try:
-            image_uuid = helper.resolve_image_name(self.image.value, etcd_client)
+            image_uuid = helper.resolve_image_name(
+                self.image.value, shared.etcd_client
+            )
         except Exception as e:
+            logger.exception(
+                "Cannot resolve image name = %s", self.image.value
+            )
             self.add_error(str(e))
         else:
             self.image_uuid = image_uuid
 
     def vm_name_validation(self):
-        if resolve_vm_name(name=self.vm_name.value, owner=self.name.value):
+        if resolve_vm_name(
+            name=self.vm_name.value, owner=self.name.value
+        ):
             self.add_error(
-                'VM with same name "{}" already exists'.format(self.vm_name.value)
+                'VM with same name "{}" already exists'.format(
+                    self.vm_name.value
+                )
             )
 
     def network_validation(self):
@@ -235,34 +280,48 @@ class CreateVMSchema(OTPSchema):
 
         if _network:
             for net in _network:
-                network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'),
-                                                       self.name.value,
-                                                       net), value_in_json=True)
+                network = shared.etcd_client.get(
+                    os.path.join(
+                        shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
+                        self.name.value,
+                        net,
+                    ),
+                    value_in_json=True,
+                )
                 if not network:
-                    self.add_error("Network with name {} does not exists" \
-                                   .format(net))
+                    self.add_error(
+                        "Network with name {} does not exists".format(
+                            net
+                        )
+                    )
 
     def specs_validation(self):
         ALLOWED_BASE = 10
 
-        _cpu = self.specs.value.get('cpu', KeyError)
-        _ram = self.specs.value.get('ram', KeyError)
-        _os_ssd = self.specs.value.get('os-ssd', KeyError)
-        _hdd = self.specs.value.get('hdd', KeyError)
+        _cpu = self.specs.value.get("cpu", KeyError)
+        _ram = self.specs.value.get("ram", KeyError)
+        _os_ssd = self.specs.value.get("os-ssd", KeyError)
+        _hdd = self.specs.value.get("hdd", KeyError)
 
         if KeyError in [_cpu, _ram, _os_ssd, _hdd]:
-            self.add_error("You must specify CPU, RAM and OS-SSD in your specs")
+            self.add_error(
+                "You must specify CPU, RAM and OS-SSD in your specs"
+            )
             return None
         try:
             parsed_ram = bitmath.parse_string_unsafe(_ram)
             parsed_os_ssd = bitmath.parse_string_unsafe(_os_ssd)
 
             if parsed_ram.base != ALLOWED_BASE:
-                self.add_error("Your specified RAM is not in correct units")
+                self.add_error(
+                    "Your specified RAM is not in correct units"
+                )
             if parsed_os_ssd.base != ALLOWED_BASE:
-                self.add_error("Your specified OS-SSD is not in correct units")
+                self.add_error(
+                    "Your specified OS-SSD is not in correct units"
+                )
 
-            if _cpu < 1:
+            if int(_cpu) < 1:
                 self.add_error("CPU must be atleast 1")
 
             if parsed_ram < bitmath.GB(1):
@@ -275,7 +334,9 @@ class CreateVMSchema(OTPSchema):
             for hdd in _hdd:
                 _parsed_hdd = bitmath.parse_string_unsafe(hdd)
                 if _parsed_hdd.base != ALLOWED_BASE:
-                    self.add_error("Your specified HDD is not in correct units")
+                    self.add_error(
+                        "Your specified HDD is not in correct units"
+                    )
                     break
                 else:
                     parsed_hdd.append(str(_parsed_hdd))
@@ -286,21 +347,24 @@ class CreateVMSchema(OTPSchema):
         else:
             if self.get_errors():
                 self.specs = {
-                    'cpu': _cpu,
-                    'ram': str(parsed_ram),
-                    'os-ssd': str(parsed_os_ssd),
-                    'hdd': parsed_hdd
+                    "cpu": _cpu,
+                    "ram": str(parsed_ram),
+                    "os-ssd": str(parsed_os_ssd),
+                    "hdd": parsed_hdd,
                 }
 
 
 class VMStatusSchema(OTPSchema):
     def __init__(self, data):
         data["uuid"] = (
-                resolve_vm_name(
-                    name=data.get("vm_name", None),
-                    owner=(data.get("in_support_of", None) or data.get("name", None)),
-                )
-                or KeyError
+            resolve_vm_name(
+                name=data.get("vm_name", None),
+                owner=(
+                    data.get("in_support_of", None)
+                    or data.get("name", None)
+                ),
+            )
+            or KeyError
         )
         self.uuid = VmUUIDField(data)
 
@@ -309,9 +373,10 @@ class VMStatusSchema(OTPSchema):
         super().__init__(data, fields)
 
     def validation(self):
-        vm = vm_pool.get(self.uuid.value)
+        vm = shared.vm_pool.get(self.uuid.value)
         if not (
-                vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
+            vm.value["owner"] == self.name.value
+            or self.realm.value == "ungleich-admin"
         ):
             self.add_error("Invalid User")
 
@@ -319,11 +384,14 @@ class VMStatusSchema(OTPSchema):
 class VmActionSchema(OTPSchema):
     def __init__(self, data):
         data["uuid"] = (
-                resolve_vm_name(
-                    name=data.get("vm_name", None),
-                    owner=(data.get("in_support_of", None) or data.get("name", None)),
-                )
-                or KeyError
+            resolve_vm_name(
+                name=data.get("vm_name", None),
+                owner=(
+                    data.get("in_support_of", None)
+                    or data.get("name", None)
+                ),
+            )
+            or KeyError
         )
         self.uuid = VmUUIDField(data)
         self.action = Field("action", str, data.get("action", KeyError))
@@ -338,20 +406,23 @@ class VmActionSchema(OTPSchema):
         allowed_actions = ["start", "stop", "delete"]
         if self.action.value not in allowed_actions:
             self.add_error(
-                "Invalid Action. Allowed Actions are {}".format(allowed_actions)
+                "Invalid Action. Allowed Actions are {}".format(
+                    allowed_actions
+                )
             )
 
     def validation(self):
-        vm = vm_pool.get(self.uuid.value)
+        vm = shared.vm_pool.get(self.uuid.value)
         if not (
-                vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
+            vm.value["owner"] == self.name.value
+            or self.realm.value == "ungleich-admin"
         ):
             self.add_error("Invalid User")
 
         if (
-                self.action.value == "start"
-                and vm.status == VMStatus.running
-                and vm.hostname != ""
+            self.action.value == "start"
+            and vm.status == VMStatus.running
+            and vm.hostname != ""
         ):
             self.add_error("VM Already Running")
 
@@ -365,15 +436,20 @@ class VmActionSchema(OTPSchema):
 class VmMigrationSchema(OTPSchema):
     def __init__(self, data):
         data["uuid"] = (
-                resolve_vm_name(
-                    name=data.get("vm_name", None),
-                    owner=(data.get("in_support_of", None) or data.get("name", None)),
-                )
-                or KeyError
+            resolve_vm_name(
+                name=data.get("vm_name", None),
+                owner=(
+                    data.get("in_support_of", None)
+                    or data.get("name", None)
+                ),
+            )
+            or KeyError
         )
 
         self.uuid = VmUUIDField(data)
-        self.destination = Field("destination", str, data.get("destination", KeyError))
+        self.destination = Field(
+            "destination", str, data.get("destination", KeyError)
+        )
 
         self.destination.validation = self.destination_validation
 
@@ -382,31 +458,47 @@ class VmMigrationSchema(OTPSchema):
 
     def destination_validation(self):
         hostname = self.destination.value
-        host = next(filter(lambda h: h.hostname == hostname, host_pool.hosts), None)
+        host = next(
+            filter(
+                lambda h: h.hostname == hostname, shared.host_pool.hosts
+            ),
+            None,
+        )
         if not host:
-            self.add_error("No Such Host ({}) exists".format(self.destination.value))
+            self.add_error(
+                "No Such Host ({}) exists".format(
+                    self.destination.value
+                )
+            )
         elif host.status != HostStatus.alive:
             self.add_error("Destination Host is dead")
         else:
             self.destination.value = host.key
 
     def validation(self):
-        vm = vm_pool.get(self.uuid.value)
+        vm = shared.vm_pool.get(self.uuid.value)
         if not (
-                vm.value["owner"] == self.name.value or self.realm.value == "ungleich-admin"
+            vm.value["owner"] == self.name.value
+            or self.realm.value == "ungleich-admin"
         ):
             self.add_error("Invalid User")
 
         if vm.status != VMStatus.running:
             self.add_error("Can't migrate non-running VM")
 
-        if vm.hostname == os.path.join(env_vars.get('HOST_PREFIX'), self.destination.value):
-            self.add_error("Destination host couldn't be same as Source Host")
+        if vm.hostname == os.path.join(
+            shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
+        ):
+            self.add_error(
+                "Destination host couldn't be same as Source Host"
+            )
 
 
 class AddSSHSchema(OTPSchema):
     def __init__(self, data):
-        self.key_name = Field("key_name", str, data.get("key_name", KeyError))
+        self.key_name = Field(
+            "key_name", str, data.get("key_name", KeyError)
+        )
         self.key = Field("key", str, data.get("key_name", KeyError))
 
         fields = [self.key_name, self.key]
@@ -415,7 +507,9 @@ class AddSSHSchema(OTPSchema):
 
 class RemoveSSHSchema(OTPSchema):
     def __init__(self, data):
-        self.key_name = Field("key_name", str, data.get("key_name", KeyError))
+        self.key_name = Field(
+            "key_name", str, data.get("key_name", KeyError)
+        )
 
         fields = [self.key_name]
         super().__init__(data=data, fields=fields)
@@ -423,7 +517,9 @@ class RemoveSSHSchema(OTPSchema):
 
 class GetSSHSchema(OTPSchema):
     def __init__(self, data):
-        self.key_name = Field("key_name", str, data.get("key_name", None))
+        self.key_name = Field(
+            "key_name", str, data.get("key_name", None)
+        )
 
         fields = [self.key_name]
         super().__init__(data=data, fields=fields)
@@ -442,15 +538,20 @@ class CreateNetwork(OTPSchema):
         super().__init__(data, fields=fields)
 
     def network_name_validation(self):
-        network = etcd_client.get(os.path.join(env_vars.get('NETWORK_PREFIX'),
-                                               self.name.value,
-                                               self.network_name.value),
-                                  value_in_json=True)
+        key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
+        network = shared.etcd_client.get(key, value_in_json=True)
         if network:
-            self.add_error("Network with name {} already exists" \
-                           .format(self.network_name.value))
+            self.add_error(
+                "Network with name {} already exists".format(
+                    self.network_name.value
+                )
+            )
 
     def network_type_validation(self):
         supported_network_types = ["vxlan"]
         if self.type.value not in supported_network_types:
-            self.add_error("Unsupported Network Type. Supported network types are {}".format(supported_network_types))
+            self.add_error(
+                "Unsupported Network Type. Supported network types are {}".format(
+                    supported_network_types
+                )
+            )

archive/uncloud_etcd_based/uncloud/cli/helper.py

@@ -0,0 +1,46 @@
+import requests
+import json
+import argparse
+import binascii
+
+from pyotp import TOTP
+from os.path import join as join_path
+from uncloud.common.shared import shared
+
+
+def get_otp_parser():
+    otp_parser = argparse.ArgumentParser('otp')
+    otp_parser.add_argument('--name')
+    otp_parser.add_argument('--realm')
+    otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
+
+    return otp_parser
+
+
+def load_dump_pretty(content):
+    if isinstance(content, bytes):
+        content = content.decode('utf-8')
+    parsed = json.loads(content)
+    return json.dumps(parsed, indent=4, sort_keys=True)
+
+
+def make_request(*args, data=None, request_method=requests.post):
+    try:
+        r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
+    except requests.exceptions.RequestException:
+        print('Error occurred while connecting to API server.')
+    else:
+        try:
+            print(load_dump_pretty(r.content))
+        except Exception:
+            print('Error occurred while getting output from api server.')
+
+
+def get_token(seed):
+    if seed is not None:
+        try:
+            token = TOTP(seed).now()
+        except binascii.Error:
+            raise argparse.ArgumentTypeError('Invalid seed')
+        else:
+            return token

archive/uncloud_etcd_based/uncloud/cli/host.py

@@ -0,0 +1,45 @@
+import requests
+
+from uncloud.cli.helper import make_request, get_otp_parser
+from uncloud.common.parser import BaseParser
+
+
+class HostParser(BaseParser):
+    def __init__(self):
+        super().__init__('host')
+
+    def create(self, **kwargs):
+        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--hostname', required=True)
+        p.add_argument('--cpu', required=True, type=int)
+        p.add_argument('--ram', required=True)
+        p.add_argument('--os-ssd', required=True)
+        p.add_argument('--hdd', default=list())
+
+    def list(self, **kwargs):
+        self.subparser.add_parser('list', **kwargs)
+
+
+parser = HostParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('host_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        request_method = requests.post
+        data = None
+        if subcommand == 'create':
+            kwargs['specs'] = {
+                'cpu': kwargs.pop('cpu'),
+                'ram': kwargs.pop('ram'),
+                'os-ssd': kwargs.pop('os_ssd'),
+                'hdd': kwargs.pop('hdd')
+            }
+            data = kwargs
+        elif subcommand == 'list':
+            request_method = requests.get
+
+        make_request('host', subcommand, data=data, request_method=request_method)

archive/uncloud_etcd_based/uncloud/cli/image.py

@@ -0,0 +1,38 @@
+import requests
+
+from uncloud.cli.helper import make_request
+from uncloud.common.parser import BaseParser
+
+
+class ImageParser(BaseParser):
+    def __init__(self):
+        super().__init__('image')
+
+    def create(self, **kwargs):
+        p = self.subparser.add_parser('create', **kwargs)
+        p.add_argument('--name', required=True)
+        p.add_argument('--uuid', required=True)
+        p.add_argument('--image-store', required=True, dest='image_store')
+
+    def list(self, **kwargs):
+        self.subparser.add_parser('list', **kwargs)
+
+
+parser = ImageParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('image_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        data = None
+        request_method = requests.post
+        if subcommand == 'list':
+            subcommand = 'list-public'
+            request_method = requests.get
+        elif subcommand == 'create':
+            data = kwargs
+
+        make_request('image', subcommand, data=data, request_method=request_method)

archive/uncloud_etcd_based/uncloud/cli/main.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python3
+
+import argparse
+import importlib
+
+arg_parser = argparse.ArgumentParser('cli', add_help=False)
+subparser = arg_parser.add_subparsers(dest='subcommand')
+
+for component in ['user', 'host', 'image', 'network', 'vm']:
+    module = importlib.import_module('uncloud.cli.{}'.format(component))
+    parser = getattr(module, 'arg_parser')
+    subparser.add_parser(name=parser.prog, parents=[parser])
+
+
+def main(arguments):
+    if not arguments['subcommand']:
+        arg_parser.print_help()
+    else:
+        name = arguments.pop('subcommand')
+        arguments.pop('debug')
+        mod = importlib.import_module('uncloud.cli.{}'.format(name))
+        _main = getattr(mod, 'main')
+        _main(**arguments)

archive/uncloud_etcd_based/uncloud/cli/network.py

@@ -0,0 +1,32 @@
+import requests
+
+from uncloud.cli.helper import make_request, get_otp_parser
+from uncloud.common.parser import BaseParser
+
+
+class NetworkParser(BaseParser):
+    def __init__(self):
+        super().__init__('network')
+
+    def create(self, **kwargs):
+        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--network-name', required=True)
+        p.add_argument('--network-type', required=True, dest='type')
+        p.add_argument('--user', action='store_true')
+
+
+parser = NetworkParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('network_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        data = None
+        request_method = requests.post
+        if subcommand == 'create':
+            data = kwargs
+
+        make_request('network', subcommand, data=data, request_method=request_method)

archive/uncloud_etcd_based/uncloud/cli/user.py

@@ -0,0 +1,41 @@
+from uncloud.cli.helper import make_request, get_otp_parser
+from uncloud.common.parser import BaseParser
+
+
+class UserParser(BaseParser):
+    def __init__(self):
+        super().__init__('user')
+
+    def files(self, **kwargs):
+        self.subparser.add_parser('files', parents=[get_otp_parser()], **kwargs)
+
+    def vms(self, **kwargs):
+        self.subparser.add_parser('vms', parents=[get_otp_parser()], **kwargs)
+
+    def networks(self, **kwargs):
+        self.subparser.add_parser('networks', parents=[get_otp_parser()], **kwargs)
+
+    def add_ssh(self, **kwargs):
+        p = self.subparser.add_parser('add-ssh', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--key-name', required=True)
+        p.add_argument('--key', required=True)
+
+    def get_ssh(self, **kwargs):
+        p = self.subparser.add_parser('get-ssh', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--key-name', default='')
+
+    def remove_ssh(self, **kwargs):
+        p = self.subparser.add_parser('remove-ssh', parents=[get_otp_parser()], **kwargs)
+        p.add_argument('--key-name', required=True)
+
+
+parser = UserParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('user_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        make_request('user', subcommand, data=kwargs)

archive/uncloud_etcd_based/uncloud/cli/vm.py

@@ -0,0 +1,62 @@
+from uncloud.common.parser import BaseParser
+from uncloud.cli.helper import make_request, get_otp_parser
+
+
+class VMParser(BaseParser):
+    def __init__(self):
+        super().__init__('vm')
+
+    def start(self, **args):
+        p = self.subparser.add_parser('start', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+
+    def stop(self, **args):
+        p = self.subparser.add_parser('stop', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+
+    def status(self, **args):
+        p = self.subparser.add_parser('status', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+
+    def delete(self, **args):
+        p = self.subparser.add_parser('delete', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+
+    def migrate(self, **args):
+        p = self.subparser.add_parser('migrate', parents=[get_otp_parser()], **args)
+        p.add_argument('--vm-name', required=True)
+        p.add_argument('--destination', required=True)
+
+    def create(self, **args):
+        p = self.subparser.add_parser('create', parents=[get_otp_parser()], **args)
+        p.add_argument('--cpu', required=True)
+        p.add_argument('--ram', required=True)
+        p.add_argument('--os-ssd', required=True)
+        p.add_argument('--hdd', action='append', default=list())
+        p.add_argument('--image', required=True)
+        p.add_argument('--network', action='append', default=[])
+        p.add_argument('--vm-name', required=True)
+
+
+parser = VMParser()
+arg_parser = parser.arg_parser
+
+
+def main(**kwargs):
+    subcommand = kwargs.pop('vm_subcommand')
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        data = kwargs
+        endpoint = subcommand
+        if subcommand in ['start', 'stop', 'delete']:
+            endpoint = 'action'
+            data['action'] = subcommand
+        elif subcommand == 'create':
+            kwargs['specs'] = {
+                'cpu': kwargs.pop('cpu'),
+                'ram': kwargs.pop('ram'),
+                'os-ssd': kwargs.pop('os_ssd'),
+                'hdd': kwargs.pop('hdd')
+            }
+        make_request('vm', endpoint, data=data)

archive/uncloud_etcd_based/uncloud/client/main.py

@@ -0,0 +1,23 @@
+import argparse
+import etcd3
+from uncloud.common.etcd_wrapper import Etcd3Wrapper
+
+arg_parser = argparse.ArgumentParser('client', add_help=False)
+arg_parser.add_argument('--dump-etcd-contents-prefix', help="Dump contents below the given prefix")
+
+def dump_etcd_contents(prefix):
+    etcd = Etcd3Wrapper()
+    for k,v in etcd.get_prefix_raw(prefix):
+        k = k.decode('utf-8')
+        v = v.decode('utf-8')
+        print("{} = {}".format(k,v))
+#    print("{} = {}".format(k,v))
+
+#    for k,v in etcd.get_prefix(prefix):
+#
+    print("done")
+
+
+def main(arguments):
+    if 'dump_etcd_contents_prefix' in arguments:
+        dump_etcd_contents(prefix=arguments['dump_etcd_contents_prefix'])

archive/uncloud_etcd_based/uncloud/common/classes.py

@@ -1,4 +1,4 @@
-from etcd3_wrapper import EtcdEntry
+from .etcd_wrapper import EtcdEntry
 
 
 class SpecificEtcdEntryBase:

archive/uncloud_etcd_based/uncloud/common/cli.py

@@ -0,0 +1,26 @@
+from uncloud.common.shared import shared
+from pyotp import TOTP
+
+
+def get_token(seed):
+    if seed is not None:
+        try:
+            token = TOTP(seed).now()
+        except Exception:
+            raise Exception('Invalid seed')
+        else:
+            return token
+
+
+def resolve_otp_credentials(kwargs):
+    d = {
+        'name': shared.settings['client']['name'],
+        'realm': shared.settings['client']['realm'],
+        'token': get_token(shared.settings['client']['seed'])
+    }
+
+    for k, v in d.items():
+        if k in kwargs and kwargs[k] is None:
+            kwargs.update({k: v})
+
+    return d

archive/uncloud_etcd_based/uncloud/common/counters.py

@@ -1,4 +1,4 @@
-from etcd3_wrapper import Etcd3Wrapper
+from .etcd_wrapper import Etcd3Wrapper
 
 
 def increment_etcd_counter(etcd_client: Etcd3Wrapper, key):

archive/uncloud_etcd_based/uncloud/common/etcd_wrapper.py

@@ -0,0 +1,75 @@
+import etcd3
+import json
+
+from functools import wraps
+
+from uncloud import UncloudException
+from uncloud.common import logger
+
+
+class EtcdEntry:
+    def __init__(self, meta_or_key, value, value_in_json=False):
+        if hasattr(meta_or_key, 'key'):
+            # if meta has attr 'key' then get it
+            self.key = meta_or_key.key.decode('utf-8')
+        else:
+            # otherwise meta is the 'key'
+            self.key = meta_or_key
+        self.value = value.decode('utf-8')
+
+        if value_in_json:
+            self.value = json.loads(self.value)
+
+
+def readable_errors(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except etcd3.exceptions.ConnectionFailedError:
+            raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?')
+        except etcd3.exceptions.ConnectionTimeoutError as err:
+            raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err
+        except Exception:
+            logger.exception('Some etcd error occured. See syslog for details.')
+
+    return wrapper
+
+
+class Etcd3Wrapper:
+    @readable_errors
+    def __init__(self, *args, **kwargs):
+        self.client = etcd3.client(*args, **kwargs)
+
+    @readable_errors
+    def get(self, *args, value_in_json=False, **kwargs):
+        _value, _key = self.client.get(*args, **kwargs)
+        if _key is None or _value is None:
+            return None
+        return EtcdEntry(_key, _value, value_in_json=value_in_json)
+
+    @readable_errors
+    def put(self, *args, value_in_json=False, **kwargs):
+        _key, _value = args
+        if value_in_json:
+            _value = json.dumps(_value)
+
+        if not isinstance(_key, str):
+            _key = _key.decode('utf-8')
+
+        return self.client.put(_key, _value, **kwargs)
+
+    @readable_errors
+    def get_prefix(self, *args, value_in_json=False, raise_exception=True, **kwargs):
+        event_iterator = self.client.get_prefix(*args, **kwargs)
+        for e in event_iterator:
+            yield EtcdEntry(*e[::-1], value_in_json=value_in_json)
+
+    @readable_errors
+    def watch_prefix(self, key, raise_exception=True, value_in_json=False):
+        event_iterator, cancel = self.client.watch_prefix(key)
+        for e in event_iterator:
+            if hasattr(e, '_event'):
+                e = e._event
+                if e.type == e.PUT:
+                    yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)

archive/uncloud_etcd_based/uncloud/common/host.py

@@ -7,7 +7,7 @@ from .classes import SpecificEtcdEntryBase
 
 
 class HostStatus:
-    """Possible Statuses of ucloud host."""
+    """Possible Statuses of uncloud host."""
 
     alive = "ALIVE"
     dead = "DEAD"
@@ -26,11 +26,13 @@ class HostEntry(SpecificEtcdEntryBase):
 
     def update_heartbeat(self):
         self.status = HostStatus.alive
-        self.last_heartbeat = time.strftime("%Y-%m-%d %H:%M:%S")
+        self.last_heartbeat = datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
 
     def is_alive(self):
-        last_heartbeat = datetime.strptime(self.last_heartbeat, "%Y-%m-%d %H:%M:%S")
-        delta = datetime.now() - last_heartbeat
+        last_heartbeat = datetime.strptime(
+            self.last_heartbeat, "%Y-%m-%d %H:%M:%S"
+        )
+        delta = datetime.utcnow() - last_heartbeat
         if delta.total_seconds() > 60:
             return False
         return True

archive/uncloud_etcd_based/uncloud/common/network.py

@@ -0,0 +1,70 @@
+import subprocess as sp
+import random
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def random_bytes(num=6):
+    return [random.randrange(256) for _ in range(num)]
+
+
+def generate_mac(
+    uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
+):
+    mac = random_bytes()
+    if oui:
+        if type(oui) == str:
+            oui = [int(chunk) for chunk in oui.split(separator)]
+        mac = oui + random_bytes(num=6 - len(oui))
+    else:
+        if multicast:
+            mac[0] |= 1  # set bit 0
+        else:
+            mac[0] &= ~1  # clear bit 0
+        if uaa:
+            mac[0] &= ~(1 << 1)  # clear bit 1
+        else:
+            mac[0] |= 1 << 1  # set bit 1
+    return separator.join(byte_fmt % b for b in mac)
+
+
+def create_dev(script, _id, dev, ip=None):
+    command = [
+        "sudo",
+        "-p",
+        "Enter password to create network devices for vm: ",
+        script,
+        str(_id),
+        dev,
+    ]
+    if ip:
+        command.append(ip)
+    try:
+        output = sp.check_output(command, stderr=sp.PIPE)
+    except Exception:
+        logger.exception("Creation of interface %s failed.", dev)
+        return None
+    else:
+        return output.decode("utf-8").strip()
+
+
+def delete_network_interface(iface):
+    try:
+        sp.check_output(
+            [
+                "sudo",
+                "-p",
+                "Enter password to remove {} network device: ".format(
+                    iface
+                ),
+                "ip",
+                "link",
+                "del",
+                iface,
+            ],
+            stderr=sp.PIPE,
+        )
+    except Exception:
+        logger.exception("Interface %s Deletion failed", iface)
+

archive/uncloud_etcd_based/uncloud/common/parser.py

@@ -0,0 +1,13 @@
+import argparse
+
+
+class BaseParser:
+    def __init__(self, command):
+        self.arg_parser = argparse.ArgumentParser(command, add_help=False)
+        self.subparser = self.arg_parser.add_subparsers(dest='{}_subcommand'.format(command))
+        self.common_args = {'add_help': False}
+
+        methods = [attr for attr in dir(self) if not attr.startswith('__')
+                   and type(getattr(self, attr)).__name__ == 'method']
+        for method in methods:
+            getattr(self, method)(**self.common_args)

archive/uncloud_etcd_based/uncloud/common/request.py

@@ -2,9 +2,8 @@ import json
 from os.path import join
 from uuid import uuid4
 
-from etcd3_wrapper.etcd3_wrapper import PsuedoEtcdEntry
-
-from .classes import SpecificEtcdEntryBase
+from uncloud.common.etcd_wrapper import EtcdEntry
+from uncloud.common.classes import SpecificEtcdEntryBase
 
 
 class RequestType:
@@ -18,8 +17,9 @@ class RequestType:
 
 
 class RequestEntry(SpecificEtcdEntryBase):
-
     def __init__(self, e):
+        self.destination_sock_path = None
+        self.destination_host_key = None
         self.type = None  # type: str
         self.migration = None  # type: bool
         self.destination = None  # type: str
@@ -29,8 +29,8 @@ class RequestEntry(SpecificEtcdEntryBase):
 
     @classmethod
     def from_scratch(cls, request_prefix, **kwargs):
-        e = PsuedoEtcdEntry(join(request_prefix, uuid4().hex),
-                            value=json.dumps(kwargs).encode("utf-8"), value_in_json=True)
+        e = EtcdEntry(meta_or_key=join(request_prefix, uuid4().hex),
+                      value=json.dumps(kwargs).encode('utf-8'), value_in_json=True)
         return cls(e)
 
 

archive/uncloud_etcd_based/uncloud/common/schemas.py

@@ -0,0 +1,41 @@
+import bitmath
+
+from marshmallow import fields, Schema
+
+
+class StorageUnit(fields.Field):
+    def _serialize(self, value, attr, obj, **kwargs):
+        return str(value)
+
+    def _deserialize(self, value, attr, data, **kwargs):
+        return bitmath.parse_string_unsafe(value)
+
+
+class SpecsSchema(Schema):
+    cpu = fields.Int()
+    ram = StorageUnit()
+    os_ssd = StorageUnit(data_key="os-ssd", attribute="os-ssd")
+    hdd = fields.List(StorageUnit())
+
+
+class VMSchema(Schema):
+    name = fields.Str()
+    owner = fields.Str()
+    owner_realm = fields.Str()
+    specs = fields.Nested(SpecsSchema)
+    status = fields.Str()
+    log = fields.List(fields.Str())
+    vnc_socket = fields.Str()
+    image_uuid = fields.Str()
+    hostname = fields.Str()
+    metadata = fields.Dict()
+    network = fields.List(
+        fields.Tuple((fields.Str(), fields.Str(), fields.Int()))
+    )
+    in_migration = fields.Bool()
+
+
+class NetworkSchema(Schema):
+    _id = fields.Int(data_key="id", attribute="id")
+    _type = fields.Str(data_key="type", attribute="type")
+    ipv6 = fields.Str()

archive/uncloud_etcd_based/uncloud/common/settings.py

@@ -0,0 +1,136 @@
+import configparser
+import logging
+import sys
+import os
+
+from datetime import datetime
+from uncloud.common.etcd_wrapper import Etcd3Wrapper
+from os.path import join as join_path
+
+logger = logging.getLogger(__name__)
+settings = None
+
+
+class CustomConfigParser(configparser.RawConfigParser):
+    def __getitem__(self, key):
+        try:
+            result = super().__getitem__(key)
+        except KeyError as err:
+            raise KeyError(
+                'Key \'{}\' not found in configuration. Make sure you configure uncloud.'.format(
+                    key
+                )
+            ) from err
+        else:
+            return result
+
+
+class Settings(object):
+    def __init__(self, conf_dir, seed_value=None):
+        conf_name = 'uncloud.conf'
+        self.config_file = join_path(conf_dir, conf_name)
+
+        # this is used to cache config from etcd for 1 minutes. Without this we
+        # would make a lot of requests to etcd which slows down everything.
+        self.last_config_update = datetime.fromtimestamp(0)
+
+        self.config_parser = CustomConfigParser(allow_no_value=True)
+        self.config_parser.add_section('etcd')
+        self.config_parser.set('etcd', 'base_prefix', '/')
+
+        if os.access(self.config_file, os.R_OK):
+            self.config_parser.read(self.config_file)
+        else:
+            raise FileNotFoundError('Config file %s not found!', self.config_file)
+        self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
+
+        self.read_internal_values()
+
+        if seed_value is None:
+            seed_value = dict()
+
+        self.config_parser.read_dict(seed_value)
+
+    def get_etcd_client(self):
+        args = tuple()
+        try:
+            kwargs = {
+                'host': self.config_parser.get('etcd', 'url'),
+                'port': self.config_parser.get('etcd', 'port'),
+                'ca_cert': self.config_parser.get('etcd', 'ca_cert'),
+                'cert_cert': self.config_parser.get('etcd', 'cert_cert'),
+                'cert_key': self.config_parser.get('etcd', 'cert_key'),
+            }
+        except configparser.Error as err:
+            raise configparser.Error(
+                '{} in config file {}'.format(
+                    err.message, self.config_file
+                )
+            ) from err
+        else:
+            try:
+                wrapper = Etcd3Wrapper(*args, **kwargs)
+            except Exception as err:
+                logger.error(
+                    'etcd connection not successfull. Please check your config file.'
+                    '\nDetails: %s\netcd connection parameters: %s',
+                    err,
+                    kwargs,
+                )
+                sys.exit(1)
+            else:
+                return wrapper
+
+    def read_internal_values(self):
+        base_prefix = self['etcd']['base_prefix']
+        self.config_parser.read_dict(
+            {
+                'etcd': {
+                    'file_prefix': join_path(base_prefix, 'files/'),
+                    'host_prefix': join_path(base_prefix, 'hosts/'),
+                    'image_prefix': join_path(base_prefix, 'images/'),
+                    'image_store_prefix': join_path(base_prefix, 'imagestore/'),
+                    'network_prefix': join_path(base_prefix, 'networks/'),
+                    'request_prefix': join_path(base_prefix, 'requests/'),
+                    'user_prefix': join_path(base_prefix, 'users/'),
+                    'vm_prefix': join_path(base_prefix, 'vms/'),
+                    'vxlan_counter': join_path(base_prefix, 'counters/vxlan'),
+                    'tap_counter': join_path(base_prefix, 'counters/tap')
+                }
+            }
+        )
+
+    def read_config_file_values(self, config_file):
+        try:
+            # Trying to read configuration file
+            with open(config_file) as config_file_handle:
+                self.config_parser.read_file(config_file_handle)
+        except FileNotFoundError:
+            sys.exit('Configuration file {} not found!'.format(config_file))
+        except Exception as err:
+            logger.exception(err)
+            sys.exit('Error occurred while reading configuration file')
+
+    def read_values_from_etcd(self):
+        etcd_client = self.get_etcd_client()
+        if (datetime.utcnow() - self.last_config_update).total_seconds() > 60:
+            config_from_etcd = etcd_client.get(self.config_key, value_in_json=True)
+            if config_from_etcd:
+                self.config_parser.read_dict(config_from_etcd.value)
+                self.last_config_update = datetime.utcnow()
+            else:
+                raise KeyError('Key \'{}\' not found in etcd. Please configure uncloud.'.format(self.config_key))
+
+    def __getitem__(self, key):
+        # Allow failing to read from etcd if we have
+        # it locally
+        if key not in self.config_parser.sections():
+            try:
+                self.read_values_from_etcd()
+            except KeyError:
+                pass
+        return self.config_parser[key]
+
+
+def get_settings():
+    return settings

archive/uncloud_etcd_based/uncloud/common/shared.py

@@ -0,0 +1,34 @@
+from uncloud.common.settings import get_settings
+from uncloud.common.vm import VmPool
+from uncloud.common.host import HostPool
+from uncloud.common.request import RequestPool
+import uncloud.common.storage_handlers as storage_handlers
+
+
+class Shared:
+    @property
+    def settings(self):
+        return get_settings()
+
+    @property
+    def etcd_client(self):
+        return self.settings.get_etcd_client()
+
+    @property
+    def host_pool(self):
+        return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
+
+    @property
+    def vm_pool(self):
+        return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
+
+    @property
+    def request_pool(self):
+        return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
+
+    @property
+    def storage_handler(self):
+        return storage_handlers.get_storage_handler()
+
+
+shared = Shared()

archive/uncloud_etcd_based/uncloud/common/storage_handlers.py

@@ -6,17 +6,20 @@ import stat
 from abc import ABC
 from . import logger
 from os.path import join as join_path
+import uncloud.common.shared as shared
 
 
 class ImageStorageHandler(ABC):
+    handler_name = "base"
+
     def __init__(self, image_base, vm_base):
         self.image_base = image_base
         self.vm_base = vm_base
 
     def import_image(self, image_src, image_dest, protect=False):
         """Put an image at the destination
-        :param src: An Image file
-        :param dest: A path where :param src: is to be put.
+        :param image_src: An Image file
+        :param image_dest: A path where :param src: is to be put.
         :param protect: If protect is true then the dest is protect (readonly etc)
         The obj must exist on filesystem.
         """
@@ -26,8 +29,8 @@ class ImageStorageHandler(ABC):
     def make_vm_image(self, image_path, path):
         """Copy image from src to dest
 
-        :param src: A path
-        :param dest: A path
+        :param image_path: A path
+        :param path: A path
 
         src and destination must be on same storage system i.e both on file system or both on CEPH etc.
         """
@@ -43,14 +46,17 @@ class ImageStorageHandler(ABC):
     def delete_vm_image(self, path):
         raise NotImplementedError()
 
-    def execute_command(self, command, report=True):
+    def execute_command(self, command, report=True, error_origin=None):
+        if not error_origin:
+            error_origin = self.handler_name
+
         command = list(map(str, command))
         try:
-            output = sp.check_output(command, stderr=sp.PIPE)
-        except Exception as e:
+            sp.check_output(command, stderr=sp.PIPE)
+        except sp.CalledProcessError as e:
+            _stderr = e.stderr.decode("utf-8").strip()
             if report:
-                print(e)
-                logger.exception(e)
+                logger.exception("%s:- %s", error_origin, _stderr)
             return False
         return True
 
@@ -65,12 +71,16 @@ class ImageStorageHandler(ABC):
 
 
 class FileSystemBasedImageStorageHandler(ImageStorageHandler):
+    handler_name = "Filesystem"
+
     def import_image(self, src, dest, protect=False):
         dest = join_path(self.image_base, dest)
         try:
             shutil.copy(src, dest)
             if protect:
-                os.chmod(dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
+                os.chmod(
+                    dest, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH
+                )
         except Exception as e:
             logger.exception(e)
             return False
@@ -80,7 +90,7 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler):
         src = join_path(self.image_base, src)
         dest = join_path(self.vm_base, dest)
         try:
-            shutil.copy(src, dest)
+            shutil.copyfile(src, dest)
         except Exception as e:
             logger.exception(e)
             return False
@@ -88,7 +98,14 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler):
 
     def resize_vm_image(self, path, size):
         path = join_path(self.vm_base, path)
-        command = ["qemu-img", "resize", "-f", "raw", path, "{}M".format(size)]
+        command = [
+            "qemu-img",
+            "resize",
+            "-f",
+            "raw",
+            path,
+            "{}M".format(size),
+        ]
         if self.execute_command(command):
             return True
         else:
@@ -117,17 +134,33 @@ class FileSystemBasedImageStorageHandler(ImageStorageHandler):
 
 
 class CEPHBasedImageStorageHandler(ImageStorageHandler):
+    handler_name = "Ceph"
+
     def import_image(self, src, dest, protect=False):
         dest = join_path(self.image_base, dest)
-        command = ["rbd", "import", src, dest]
+        import_command = ["rbd", "import", src, dest]
+        commands = [import_command]
         if protect:
-            snap_create_command = ["rbd", "snap", "create", "{}@protected".format(dest)]
-            snap_protect_command = ["rbd", "snap", "protect", "{}@protected".format(dest)]
+            snap_create_command = [
+                "rbd",
+                "snap",
+                "create",
+                "{}@protected".format(dest),
+            ]
+            snap_protect_command = [
+                "rbd",
+                "snap",
+                "protect",
+                "{}@protected".format(dest),
+            ]
+            commands.append(snap_create_command)
+            commands.append(snap_protect_command)
 
-            return self.execute_command(command) and self.execute_command(snap_create_command) and\
-                self.execute_command(snap_protect_command)
+        result = True
+        for command in commands:
+            result = result and self.execute_command(command)
 
-        return self.execute_command(command)
+        return result
 
     def make_vm_image(self, src, dest):
         src = join_path(self.image_base, src)
@@ -156,3 +189,19 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
         path = join_path(self.vm_base, path)
         command = ["rbd", "info", path]
         return self.execute_command(command, report=False)
+
+
+def get_storage_handler():
+    __storage_backend = shared.shared.settings["storage"]["storage_backend"]
+    if __storage_backend == "filesystem":
+        return FileSystemBasedImageStorageHandler(
+            vm_base=shared.shared.settings["storage"]["vm_dir"],
+            image_base=shared.shared.settings["storage"]["image_dir"],
+        )
+    elif __storage_backend == "ceph":
+        return CEPHBasedImageStorageHandler(
+            vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
+            image_base=shared.shared.settings["storage"]["ceph_image_pool"],
+        )
+    else:
+        raise Exception("Unknown Image Storage Handler")

archive/uncloud_etcd_based/uncloud/common/vm.py

@@ -12,8 +12,13 @@ class VMStatus:
     error = "ERROR"  # An error occurred that cannot be resolved automatically
 
 
-class VMEntry(SpecificEtcdEntryBase):
+def declare_stopped(vm):
+    vm["hostname"] = ""
+    vm["in_migration"] = False
+    vm["status"] = VMStatus.stopped
 
+
+class VMEntry(SpecificEtcdEntryBase):
     def __init__(self, e):
         self.owner = None  # type: str
         self.specs = None  # type: dict
@@ -42,7 +47,9 @@ class VMEntry(SpecificEtcdEntryBase):
 
     def add_log(self, msg):
         self.log = self.log[:5]
-        self.log.append("{} - {}".format(datetime.now().isoformat(), msg))
+        self.log.append(
+            "{} - {}".format(datetime.now().isoformat(), msg)
+        )
 
 
 class VmPool:

archive/uncloud_etcd_based/uncloud/configure/main.py

@@ -0,0 +1,57 @@
+import os
+import argparse
+
+from uncloud.common.shared import shared
+
+arg_parser = argparse.ArgumentParser('configure', add_help=False)
+configure_subparsers = arg_parser.add_subparsers(dest='subcommand')
+
+otp_parser = configure_subparsers.add_parser('otp')
+otp_parser.add_argument('--verification-controller-url', required=True, metavar='URL')
+otp_parser.add_argument('--auth-name', required=True, metavar='OTP-NAME')
+otp_parser.add_argument('--auth-realm', required=True, metavar='OTP-REALM')
+otp_parser.add_argument('--auth-seed', required=True, metavar='OTP-SEED')
+
+network_parser = configure_subparsers.add_parser('network')
+network_parser.add_argument('--prefix-length', required=True, type=int)
+network_parser.add_argument('--prefix', required=True)
+network_parser.add_argument('--vxlan-phy-dev', required=True)
+
+netbox_parser = configure_subparsers.add_parser('netbox')
+netbox_parser.add_argument('--url', required=True)
+netbox_parser.add_argument('--token', required=True)
+
+ssh_parser = configure_subparsers.add_parser('ssh')
+ssh_parser.add_argument('--username', default='root')
+ssh_parser.add_argument('--private-key-path', default=os.path.expanduser('~/.ssh/id_rsa'),)
+
+storage_parser = configure_subparsers.add_parser('storage')
+storage_parser.add_argument('--file-dir', required=True)
+storage_parser_subparsers = storage_parser.add_subparsers(dest='storage_backend')
+
+filesystem_storage_parser = storage_parser_subparsers.add_parser('filesystem')
+filesystem_storage_parser.add_argument('--vm-dir', required=True)
+filesystem_storage_parser.add_argument('--image-dir', required=True)
+
+ceph_storage_parser = storage_parser_subparsers.add_parser('ceph')
+ceph_storage_parser.add_argument('--ceph-vm-pool', required=True)
+ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
+
+
+def update_config(section, kwargs):
+    uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
+    if not uncloud_config:
+        uncloud_config = {}
+    else:
+        uncloud_config = uncloud_config.value
+
+    uncloud_config[section] = kwargs
+    shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
+
+
+def main(arguments):
+    subcommand = arguments['subcommand']
+    if not subcommand:
+        arg_parser.print_help()
+    else:
+        update_config(subcommand, arguments)

archive/uncloud_etcd_based/uncloud/filescanner/main.py

@@ -0,0 +1,85 @@
+import glob
+import os
+import pathlib
+import subprocess as sp
+import time
+import argparse
+import bitmath
+
+from uuid import uuid4
+
+from . import logger
+from uncloud.common.shared import shared
+
+arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
+arg_parser.add_argument('--hostname', required=True)
+
+
+def sha512sum(file: str):
+    """Use sha512sum utility to compute sha512 sum of arg:file
+
+       IF arg:file does not exists:
+        raise FileNotFoundError exception
+       ELSE IF sum successfully computer:
+        return computed sha512 sum
+       ELSE:
+        return None
+    """
+    if not isinstance(file, str):
+        raise TypeError
+    try:
+        output = sp.check_output(['sha512sum', file], stderr=sp.PIPE)
+    except sp.CalledProcessError as e:
+        error = e.stderr.decode('utf-8')
+        if 'No such file or directory' in error:
+            raise FileNotFoundError from None
+    else:
+        output = output.decode('utf-8').strip()
+        output = output.split(' ')
+        return output[0]
+    return None
+
+
+def track_file(file, base_dir, host):
+    file_path = file.relative_to(base_dir)
+    file_str = str(file)
+    # Get Username
+    try:
+        owner = file_path.parts[0]
+    except IndexError:
+        pass
+    else:
+        file_path = file_path.relative_to(owner)
+        creation_date = time.ctime(os.stat(file_str).st_ctime)
+
+        entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
+        entry_value = {
+            'filename': str(file_path),
+            'owner': owner,
+            'sha512sum': sha512sum(file_str),
+            'creation_date': creation_date,
+            'size': str(bitmath.Byte(os.path.getsize(file_str)).to_MB()),
+            'host': host
+        }
+
+        logger.info('Tracking %s', file_str)
+
+        shared.etcd_client.put(entry_key, entry_value, value_in_json=True)
+
+
+def main(arguments):
+    hostname = arguments['hostname']
+    base_dir = shared.settings['storage']['file_dir']
+    # Recursively Get All Files and Folder below BASE_DIR
+    files = glob.glob('{}/**'.format(base_dir), recursive=True)
+    files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
+
+    # Files that are already tracked
+    tracked_files = [
+        pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
+        for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
+        if f.value['host'] == hostname
+    ]
+    untracked_files = set(files) - set(tracked_files)
+    for file in untracked_files:
+        track_file(file, base_dir, hostname)

archive/uncloud_etcd_based/uncloud/hack/__init__.py

@@ -0,0 +1 @@
+

archive/uncloud_etcd_based/uncloud/hack/config.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
+#
+# This file is part of uncloud.
+#
+# uncloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uncloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+class Config(object):
+    def __init__(self, arguments):
+        """ read arguments dicts as a base """
+
+        self.arguments = arguments
+
+        # Split them so *etcd_args can be used and we can
+        # iterate over etcd_hosts
+        self.etcd_hosts = [ arguments['etcd_host'] ]
+        self.etcd_args = {
+            'ca_cert': arguments['etcd_ca_cert'],
+            'cert_cert': arguments['etcd_cert_cert'],
+            'cert_key': arguments['etcd_cert_key'],
+#            'user': None,
+#            'password': None
+        }
+        self.etcd_prefix = '/nicohack/'

archive/uncloud_etcd_based/uncloud/hack/db.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
+#
+# This file is part of uncloud.
+#
+# uncloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uncloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import etcd3
+import json
+import logging
+import datetime
+import re
+
+from functools import wraps
+from uncloud import UncloudException
+
+log = logging.getLogger(__name__)
+
+def db_logentry(message):
+    timestamp = datetime.datetime.now()
+    return {
+        "timestamp": str(timestamp),
+        "message": message
+    }
+
+
+def readable_errors(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except etcd3.exceptions.ConnectionFailedError as e:
+            raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
+        except etcd3.exceptions.ConnectionTimeoutError as e:
+            raise UncloudException('etcd connection timeout. {}'.format(e))
+
+    return wrapper
+
+
+class DB(object):
+    def __init__(self, config, prefix="/"):
+        self.config = config
+
+        # Root for everything
+        self.base_prefix= '/nicohack'
+
+        # Can be set from outside
+        self.prefix = prefix
+
+        try:
+            self.connect()
+        except FileNotFoundError as e:
+            raise UncloudException("Is the path to the etcd certs correct? {}".format(e))
+
+    @readable_errors
+    def connect(self):
+        self._db_clients = []
+        for endpoint in self.config.etcd_hosts:
+            client = etcd3.client(host=endpoint, **self.config.etcd_args)
+            self._db_clients.append(client)
+
+    def realkey(self, key):
+        return "{}{}/{}".format(self.base_prefix,
+                                self.prefix,
+                                key)
+
+    @readable_errors
+    def get(self, key, as_json=False, **kwargs):
+        value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
+
+        if as_json:
+            value = json.loads(value)
+
+        return value
+
+    @readable_errors
+    def get_prefix(self, key, as_json=False, **kwargs):
+        for value, meta in self._db_clients[0].get_prefix(self.realkey(key), **kwargs):
+            k = meta.key.decode("utf-8")
+            value = value.decode("utf-8")
+            if as_json:
+                value = json.loads(value)
+
+            yield (k, value)
+
+
+    @readable_errors
+    def set(self, key, value, as_json=False, **kwargs):
+        if as_json:
+            value = json.dumps(value)
+
+        log.debug("Setting {} = {}".format(self.realkey(key), value))
+        # FIXME: iterate over clients in case of failure ?
+        return self._db_clients[0].put(self.realkey(key), value, **kwargs)
+
+
+    @readable_errors
+    def list_and_filter(self, key, filter_key=None, filter_regexp=None):
+        for k,v in self.get_prefix(key, as_json=True):
+
+            if filter_key and filter_regexp:
+                if filter_key in v:
+                    if re.match(filter_regexp, v[filter_key]):
+                        yield v
+            else:
+                yield v
+
+
+    @readable_errors
+    def increment(self, key, **kwargs):
+        print(self.realkey(key))
+
+
+        print("prelock")
+        lock = self._db_clients[0].lock('/nicohack/foo')
+        print("prelockacq")
+        lock.acquire()
+        print("prelockrelease")
+        lock.release()
+
+        with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock:
+            print("in lock")
+            pass
+
+#        with self._db_clients[0].lock(self.realkey(key)) as lock:#            value = int(self.get(self.realkey(key), **kwargs))
+#            self.set(self.realkey(key), str(value + 1), **kwargs)
+
+
+if __name__ == '__main__':
+    endpoints = [ "https://etcd1.ungleich.ch:2379",
+                  "https://etcd2.ungleich.ch:2379",
+                  "https://etcd3.ungleich.ch:2379" ]
+
+    db = DB(url=endpoints)

archive/uncloud_etcd_based/uncloud/hack/hackcloud/.gitignore

@@ -0,0 +1,3 @@
+*.iso
+radvdpid
+foo

archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py

@@ -0,0 +1 @@
+

archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+etcdctl  --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \
+         --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \
+         --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \
+         --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@"

archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo $@

archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifup.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+dev=$1; shift
+
+# bridge is setup from outside
+ip link set dev "$dev" master ${bridge}
+ip link set dev "$dev" up

archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last

@@ -0,0 +1 @@
+000000000252

archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix

@@ -0,0 +1 @@
+02:00