Merge branch 'master' of code.ungleich.ch:uncloud/uncloud

Signed-off-by: Nico Schottelius <nico@nico-notebook.schottelius.org>

.gitignore

@@ -1,6 +1,12 @@
-.idea
-.vscode
+.idea/
+.vscode/
+__pycache__/
 
+pay.conf
+log.txt
+test.py
+STRIPE
+venv/
 
 uncloud/docs/build
 logs.txt
@@ -16,3 +22,6 @@ uncloud/version.py
 build/
 venv/
 dist/
+
+*.iso
+*.sqlite3

.gitlab-ci.yml

@@ -0,0 +1,18 @@
+stages:
+  - lint
+  - test
+
+run-tests:
+  stage: test
+  image: code.ungleich.ch:5050/uncloud/uncloud/uncloud-ci:latest
+  services:
+    - postgres:latest
+  variables:
+    DATABASE_HOST: postgres
+    DATABASE_USER: postgres
+    POSTGRES_HOST_AUTH_METHOD: trust
+  coverage: /^TOTAL.+?(\d+\%)$/
+  script:
+    - pip install -r requirements.txt
+    - coverage run --source='.' ./manage.py test
+    - coverage report

README.md

@@ -1,3 +1,62 @@
-# ucloud
+# Uncloud
 
-Checkout https://ungleich.ch/ucloud/ for the documentation of ucloud.
+Cloud management platform, the ungleich way.
+
+
+[![pipeline status](https://code.ungleich.ch/uncloud/uncloud/badges/master/pipeline.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master)
+[![coverage report](https://code.ungleich.ch/uncloud/uncloud/badges/master/coverage.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master)
+
+## Useful commands
+
+* `./manage.py import-vat-rates path/to/csv`
+* `./manage.py make-admin username`
+
+## Development setup
+
+Install system dependencies:
+
+* On Fedora, you will need the following packages: `python3-virtualenv python3-devel openldap-devel gcc chromium`
+
+NOTE: you will need to configure a LDAP server and credentials for authentication. See `uncloud/settings.py`.
+
+```
+# Initialize virtualenv.
+» virtualenv .venv
+Using base prefix '/usr'
+New python executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python3
+Also creating executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python
+Installing setuptools, pip, wheel...
+done.
+
+# Enter virtualenv.
+» source .venv/bin/activate
+
+# Install dependencies.
+» pip install -r requirements.txt
+[...]
+
+# Run migrations.
+» ./manage.py migrate
+Operations to perform:
+  Apply all migrations: admin, auth, contenttypes, opennebula, sessions, uncloud_auth, uncloud_net, uncloud_pay, uncloud_service, uncloud_vm
+Running migrations:
+  [...]
+
+# Run webserver.
+» ./manage.py runserver
+Watching for file changes with StatReloader
+Performing system checks...
+
+System check identified no issues (0 silenced).
+May 07, 2020 - 10:17:08
+Django version 3.0.6, using settings 'uncloud.settings'
+Starting development server at http://127.0.0.1:8000/
+Quit the server with CONTROL-C.
+```
+
+### Note on PGSQL
+
+If you want to use Postgres:
+
+* Install on configure PGSQL on your base system.
+* OR use a container! `podman run --rm -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -it postgres:latest`

archive/issues.org

@@ -0,0 +1,6 @@
+* Intro
+  This file lists issues that should be handled, are small and likely
+  not yet high prio.
+* Issues
+** TODO Register prefered address in User model
+** TODO Allow to specify different recurring periods

archive/uncloud_django_based/hacks/abk-hacks.py

@@ -0,0 +1,55 @@
+"""
+investigate into a simple python function that maps an ldap user to a vat percentage. Basically you need to
+lookup the customer address, check if she is a business/registered tax number and if not apply the local
+vat
+"""
+
+import iso3166
+import datetime
+
+from csv import DictReader
+
+
+def get_vat(street_address, city, postal_code, country, vat_number=None):
+    vat = {
+        'Austria': [
+            {'period': '1984-01-01/', 'rate': 0.2},
+            {'period': '1976-01-01/1984-01-01', 'rate': 0.18},
+            {'period': '1973-01-01/1976-01-01', 'rate': 0.16},
+            ]
+    }
+    return iso3166.countries.get(country)
+
+    # return iso3166.countries_by_name[country]
+
+
+def main():
+    # vat = get_vat(
+    #     street_address='82 Nasheman-e-Iqbal near Wapda Town',
+    #     city='Lahore',
+    #     postal_code=53700,
+    #     country='Pakistan',
+    # )
+    # print(vat)
+    vat_rates = {}
+    with open('vat_rates.csv', newline='') as csvfile:
+        reader = DictReader(csvfile)
+        for row in reader:
+            territory_codes = row['territory_codes'].split('\n')
+            for code in territory_codes:
+                if code not in vat_rates:
+                    vat_rates[code] = {}
+
+                start_date = row['start_date']
+                stop_data = row['stop_date']
+                time_period = f'{start_date}|{stop_data}'
+                r = row.copy()
+                del r['start_date']
+                del r['stop_date']
+                del r['territory_codes']
+                vat_rates[code][time_period] = r
+    print(vat_rates)
+
+
+if __name__ == '__main__':
+    main()

archive/uncloud_django_based/hacks/abkhack/opennebula_hacks.py

@@ -0,0 +1,46 @@
+import importlib
+import sys
+import os
+
+from os.path import join as join_path
+from xmlrpc.client import ServerProxy as RPCClient
+
+root = os.path.dirname(os.getcwd())
+sys.path.append(join_path(root, 'uncloud'))
+secrets = importlib.import_module('uncloud.secrets')
+
+
+class OpenNebula:
+    def __init__(self, url, session_string):
+        self.session_string = session_string
+        self.client = RPCClient(secrets.OPENNEBULA_URL)
+
+    def create_user(self, username, password, authentication_driver='', group_id=None):
+        # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-user-allocate
+
+        if group_id is None:
+            group_id = []
+
+        return self.client.one.user.allocate(
+            self.session_string,
+            username,
+            password,
+            authentication_driver,
+            group_id
+        )
+
+    def chmod(self, vm_id, user_id=-1, group_id=-1):
+        # https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-vm-chown
+
+        return self.client.one.vm.chown(self.session_string, vm_id, user_id, group_id)
+
+
+one = OpenNebula(secrets.OPENNEBULA_URL, secrets.OPENNEBULA_USER_PASS)
+
+# Create User in OpenNebula
+# success, response, *_ = one.create_user(username='meow12345', password='hello_world')
+# print(success, response)
+
+# Change owner of a VM
+# success, response, *_ = one.chmod(vm_id=25589, user_id=706)
+# print(success, response)

archive/uncloud_django_based/hacks/command-wrapper.sh

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+dbhost=$1; shift
+
+ssh -L5432:localhost:5432 "$dbhost" &
+
+python manage.py "$@"
+
+
+
+# command only needs to be active while manage command is running
+
+# -T no pseudo terminal
+
+
+# alternatively: commands output shell code
+
+# ssh uncloud@dbhost "python manage.py --hostname xxx ..."

archive/uncloud_django_based/meow-payv1/README.md

@@ -0,0 +1,51 @@
+# uncloud-pay
+
+The generic product/payment system.
+
+## Installation
+
+```shell script
+pip3 install -r requirements.txt
+```
+
+## Getting Started
+
+```shell script
+python ucloud_pay.py
+```
+
+## Usage
+
+#### 1. Adding of products
+```shell script
+http --json http://[::]:5000/product/add username=your_username_here password=your_password_here specs:=@ipv6-only-vm.json
+```
+
+#### 2. Listing of products
+```shell script
+http --json http://[::]:5000/product/list
+```
+
+#### 3. Registering user's payment method (credit card for now using Stripe)
+
+```shell script
+http --json http://[::]:5000/user/register_payment card_number=4111111111111111 cvc=123 expiry_year=2020 expiry_month=8 card_holder_name="The test user" username=your_username_here password=your_password_here line1="your_billing_address" city="your_city" country="your_country"
+```
+
+#### 4. Ordering products
+
+First of all, user have to buy the membership first.
+
+```shell script
+http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=membership pay=True
+```
+
+```shell script
+http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=ipv6-only-vm cpu=1 ram=1 os-disk-space=10 os=alpine pay=True
+```
+
+#### 5. Listing users orders
+
+```shell script
+http --json POST http://[::]:5000/order/list username=your_username_here password=your_password_here
+```

archive/uncloud_django_based/meow-payv1/config.py

@@ -0,0 +1,21 @@
+import os
+
+from ungleich_common.ldap.ldap_manager import LdapManager
+from ungleich_common.std.configparser import StrictConfigParser
+from ungleich_common.etcd.etcd_wrapper import EtcdWrapper
+
+config_file = os.environ.get('meow-pay-config-file', default='pay.conf')
+
+config = StrictConfigParser(allow_no_value=True)
+config.read(config_file)
+
+etcd_client = EtcdWrapper(
+    host=config.get('etcd', 'host'), port=config.get('etcd', 'port'),
+    ca_cert=config.get('etcd', 'ca_cert'), cert_key=config.get('etcd', 'cert_key'),
+    cert_cert=config.get('etcd', 'cert_cert')
+)
+
+ldap_manager = LdapManager(
+    server=config.get('ldap', 'server'), admin_dn=config.get('ldap', 'admin_dn'),
+    admin_password=config.get('ldap', 'admin_password')
+)

archive/uncloud_django_based/meow-payv1/hack-a-vpn.py

@@ -0,0 +1,213 @@
+from flask import Flask, request
+from flask_restful import Resource, Api
+import etcd3
+import json
+import logging
+from functools import wraps
+
+from ldaptest import is_valid_ldap_user
+
+def authenticate(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        if not getattr(func, 'authenticated', True):
+            return func(*args, **kwargs)
+
+        # pass in username/password !
+        acct = basic_authentication()  # custom account lookup function
+
+        if acct:
+            return func(*args, **kwargs)
+
+        flask_restful.abort(401)
+    return wrapper
+
+def readable_errors(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except etcd3.exceptions.ConnectionFailedError as e:
+            raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
+        except etcd3.exceptions.ConnectionTimeoutError as e:
+            raise UncloudException('etcd connection timeout. {}'.format(e))
+
+    return wrapper
+
+
+class DB(object):
+    def __init__(self, config, prefix="/"):
+        self.config = config
+
+        # Root for everything
+        self.base_prefix= '/nicohack'
+
+        # Can be set from outside
+        self.prefix = prefix
+
+        self.connect()
+
+    @readable_errors
+    def connect(self):
+        self._db_clients = []
+        for endpoint in self.config.etcd_hosts:
+            client = etcd3.client(host=endpoint, **self.config.etcd_args)
+            self._db_clients.append(client)
+
+    def realkey(self, key):
+        return "{}{}/{}".format(self.base_prefix,
+                                self.prefix,
+                                key)
+
+    @readable_errors
+    def get(self, key, as_json=False, **kwargs):
+        value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
+
+        if as_json:
+            value = json.loads(value)
+
+        return value
+
+
+    @readable_errors
+    def set(self, key, value, as_json=False, **kwargs):
+        if as_json:
+            value = json.dumps(value)
+
+        # FIXME: iterate over clients in case of failure ?
+        return self._db_clients[0].put(self.realkey(key), value, **kwargs)
+
+
+class Membership(Resource):
+    def __init__(self, config):
+        self.config = config
+
+    def get(self):
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+        return {'message': 'Order successful' }, 200
+
+    def post(self):
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+        return {'message': 'Order 2x successful' }, 200
+
+
+class Order(Resource):
+    def __init__(self, config):
+        self.config = config
+
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+
+
+class Product(Resource):
+    def __init__(self, config):
+        self.config = config
+
+        self.products = []
+        self.products.append(
+            { "name": "membership-free",
+              "description": """
+This membership gives you access to the API and includes a VPN
+with 1 IPv6 address.
+See https://redmine.ungleich.ch/issues/7747?
+""",
+              "uuid": "a3883466-0012-4d01-80ff-cbf7469957af",
+              "recurring": True,
+              "recurring_time_frame": "per_year",
+              "features": [
+                  { "name": "membership",
+                    "price_one_time": 0,
+                    "price_recurring": 0
+                  }
+              ]
+            }
+        )
+        self.products.append(
+            { "name": "membership-standard",
+              "description": """
+This membership gives you access to the API and includes an IPv6-VPN with
+one IPv6 address ("Road warrior")
+See https://redmine.ungleich.ch/issues/7747?
+""",
+              "uuid": "1d85296b-0863-4dd6-a543-a6d5a4fbe4a6",
+              "recurring": True,
+              "recurring_time_frame": "per_month",
+              "features": [
+                  { "name": "membership",
+                    "price_one_time": 0,
+                    "price_recurring": 5
+                  }
+
+              ]
+            }
+        )
+        self.products.append(
+            { "name": "membership-premium",
+              "description": """
+This membership gives you access to the API and includes an
+IPv6-VPN with a /48 IPv6 network.
+See https://redmine.ungleich.ch/issues/7747?
+""",
+              "uuid": "bfd63fd2-d227-436f-a8b8-600de74dd6ce",
+              "recurring": True,
+              "recurring_time_frame": "per_month",
+              "features": [
+                  { "name": "membership",
+                    "price_one_time": 0,
+                    "price_recurring": 5
+                  }
+
+              ]
+            }
+        )
+        self.products.append(
+            { "name": "ipv6-vpn-with-/48",
+              "description": """
+An IPv6 VPN with a /48 network included.
+""",
+              "uuid": "fe5753f8-6fe1-4dc4-9b73-7b803de4c597",
+              "recurring": True,
+              "recurring_time_frame": "per_year",
+              "features": [
+                  { "name": "vpn",
+                    "price_one_time": 0,
+                    "price_recurring": 120
+                  }
+              ]
+            }
+        )
+
+
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+
+    def get(self):
+        data = request.get_json(silent=True) or {}
+        print("{} {}".format(data, config))
+
+        return self.products
+
+
+
+
+
+if __name__ == '__main__':
+    app = Flask(__name__)
+
+    config = {}
+
+    config['etcd_url']="https://etcd1.ungleich.ch"
+    config['ldap_url']="ldaps://ldap1.ungleich.ch"
+
+    api = Api(app)
+    api.add_resource(Order, '/orders', resource_class_args=( config, ))
+    api.add_resource(Product, '/products', resource_class_args=( config, ))
+    api.add_resource(Membership, '/membership',  resource_class_args=( config, ))
+
+    app.run(host='::', port=5000, debug=True)

archive/uncloud_django_based/meow-payv1/helper.py

@@ -0,0 +1,87 @@
+import logging
+
+import parsedatetime
+
+from datetime import datetime
+from stripe_utils import StripeUtils
+
+
+def get_plan_id_from_product(product):
+    plan_id = 'ucloud-v1-'
+    plan_id += product['name'].strip().replace(' ', '-')
+    return plan_id
+
+
+def get_pricing(price_in_chf_cents, product_type, recurring_period):
+    if product_type == 'recurring':
+        return 'CHF {}/{}'.format(price_in_chf_cents/100, recurring_period)
+    elif product_type == 'one-time':
+        return 'CHF {} (One time charge)'.format(price_in_chf_cents/100)
+
+
+def get_user_friendly_product(product_dict):
+    uf_product = {
+        'name': product_dict['name'],
+        'description': product_dict['description'],
+        'product_id': product_dict['usable-id'],
+        'pricing': get_pricing(
+            product_dict['price'], product_dict['type'], product_dict['recurring_period']
+        )
+    }
+    if product_dict['type'] == 'recurring':
+        uf_product['minimum_subscription_period'] = product_dict['minimum_subscription_period']
+    return uf_product
+
+
+def get_token(card_number, cvc, exp_month, exp_year):
+    stripe_utils = StripeUtils()
+    token_response = stripe_utils.get_token_from_card(
+        card_number, cvc, exp_month, exp_year
+    )
+    if token_response['response_object']:
+        return token_response['response_object'].id
+    else:
+        return None
+
+
+def resolve_product(usable_id, etcd_client):
+    products = etcd_client.get_prefix('/v1/products/', value_in_json=True)
+    for p in products:
+        if p.value['usable-id'] == usable_id:
+            return p.value
+    return None
+
+
+def calculate_charges(specification, data):
+    logging.debug('Calculating charges for specs:{} and data:{}'.format(specification, data))
+    one_time_charge = 0
+    recurring_charge = 0
+    for feature_name, feature_detail in specification['features'].items():
+        if feature_detail['constant']:
+            data[feature_name] = 1
+
+        if feature_detail['unit']['type'] != 'str':
+            one_time_charge += feature_detail['one_time_fee']
+            recurring_charge += (
+                    feature_detail['price_per_unit_per_period'] * data[feature_name] /
+                    feature_detail['unit']['value']
+            )
+    return one_time_charge, recurring_charge
+
+
+def is_order_valid(order_timestamp, renewal_period):
+    """
+    Sample Code Usage
+
+    >> current_datetime, status = cal.parse('Now')
+    >> current_datetime = datetime(*current_datetime[:6])
+
+    >> print('Is order valid: ', is_order_valid(current_datetime, '1 month'))
+    >> True
+    """
+    cal = parsedatetime.Calendar()
+
+    renewal_datetime, status = cal.parse(renewal_period)
+    renewal_datetime = datetime(*renewal_datetime[:6])
+
+    return order_timestamp <= renewal_datetime

archive/uncloud_django_based/meow-payv1/products/ipv6-only-django.json

@@ -0,0 +1,28 @@
+{
+  "usable-id": "ipv6-only-django-hosting",
+  "active": true,
+  "name": "IPv6 Only Django Hosting",
+  "description": "Host your Django application on our shiny IPv6 Only VM",
+  "recurring_period": "month",
+  "quantity": "inf",
+  "features": {
+    "cpu": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 3,
+      "one_time_fee": 0,
+      "constant": false
+    },
+    "ram": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 4,
+      "one_time_fee": 0,
+      "constant": false
+    },
+    "os-disk-space": {
+      "unit": {"value": 10, "type":"int"},
+      "one_time_fee": 0,
+      "price_per_unit_per_period": 3.5,
+      "constant": false
+    }
+  }
+}

archive/uncloud_django_based/meow-payv1/products/ipv6-only-vm.json

@@ -0,0 +1,34 @@
+{
+  "usable-id": "ipv6-only-vm",
+  "active": true,
+  "name": "IPv6 Only VM",
+  "description": "IPv6 Only VM are accessible to only those having IPv6 for themselves",
+  "recurring_period": "month",
+  "quantity": "inf",
+  "features": {
+    "cpu": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 3,
+      "one_time_fee": 0,
+      "constant": false
+    },
+    "ram": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 4,
+      "one_time_fee": 0,
+      "constant": false
+    },
+    "os-disk-space": {
+      "unit": {"value": 10, "type":"int"},
+      "one_time_fee": 0,
+      "price_per_unit_per_period": 4,
+      "constant": false
+    },
+    "os": {
+      "unit": {"value": 1, "type":"str"},
+      "one_time_fee": 0,
+      "price_per_unit_per_period": 0,
+      "constant": false
+    }
+  }
+}

archive/uncloud_django_based/meow-payv1/products/ipv6-only-vpn.json

@@ -0,0 +1,16 @@
+{
+  "usable-id": "ipv6-only-vpn",
+  "active": true,
+  "name": "IPv6 Only VPN",
+  "description": "IPv6 VPN enable you to access IPv6 only websites and more",
+  "recurring_period": "month",
+  "quantity": "inf",
+  "features": {
+    "vpn": {
+      "unit": {"value": 1, "type": "int"},
+      "price_per_unit_per_period": 10,
+      "one_time_fee": 0,
+      "constant": true
+    }
+  }
+}

archive/uncloud_django_based/meow-payv1/products/ipv6box.json

@@ -0,0 +1,16 @@
+{
+  "usable-id": "ipv6-box",
+  "active": true,
+  "name": "IPv6 Box",
+  "description": "A ready-to-go IPv6 Box: it creates a VPN to ungleich and distributes IPv6 addresses to all your computers.",
+  "recurring_period": "eternity",
+  "quantity": 4,
+  "features": {
+    "ipv6-box": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 0,
+      "one_time_fee": 250,
+      "constant": true
+    }
+  }
+}

archive/uncloud_django_based/meow-payv1/products/membership.json

@@ -0,0 +1,17 @@
+{
+  "usable-id": "membership",
+  "active": true,
+  "name": "Membership",
+  "description": "Membership to use uncloud-pay",
+  "recurring_period": "month",
+  "quantity": "inf",
+  "features": {
+    "membership": {
+      "unit": {"value": 1, "type":"int"},
+      "price_per_unit_per_period": 5,
+      "one_time_fee": 0,
+      "constant": true
+    }
+  },
+  "max_per_user": "1"
+}

archive/uncloud_django_based/meow-payv1/requirements.txt

@@ -0,0 +1,7 @@
+stripe
+flask
+Flask-RESTful
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-etcd&subdirectory=etcd
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-std&subdirectory=std
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-schemas&subdirectory=schemas

archive/uncloud_django_based/meow-payv1/sample-pay.conf

@@ -0,0 +1,17 @@
+[etcd]
+host = 127.0.0.1
+port = 2379
+ca_cert
+cert_cert
+cert_key
+
+[stripe]
+private_key=stripe_private_key
+
+[app]
+port = 5000
+
+[ldap]
+server = ldap_server_url
+admin_dn = ldap_admin_dn
+admin_password = ldap_admin_password

archive/uncloud_django_based/meow-payv1/schemas.py

@@ -0,0 +1,136 @@
+import logging
+import config
+import json
+import math
+
+from config import ldap_manager, etcd_client
+from helper import resolve_product
+from ungleich_common.schemas.schemas import BaseSchema, Field, ValidationException
+
+
+class AddProductSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.add_schema(UserCredentialSchema, data)
+        self.specs = Field('specs', dict, **self.get(data, 'specs'))
+        self.update = Field('update', bool, **self.get(data, 'update', return_default=True, default=False))
+
+    def validation(self):
+        user = self.objects['user']
+        user = json.loads(user.entry_to_json())
+        uid, ou, *dc = user['dn'].replace('ou=', '').replace('dc=', '').replace('uid=', '').split(',')
+        if ou != config.config.get('ldap', 'internal_user_ou', fallback='users'):
+            raise ValidationException('You do not have access to create product.')
+
+        product = resolve_product(self.specs.value['usable-id'], etcd_client)
+        if product:
+            self.objects['product'] = product
+
+
+class AddressSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.line1 = Field('line1', str, **self.get(data, 'line1'))
+        self.line2 = Field('line2', str, **self.get(data, 'line2', return_default=True))
+        self.city = Field('city', str, **self.get(data, 'city'))
+        self.country = Field('country', str, **self.get(data, 'country'))
+        self.state = Field('state', str, **self.get(data, 'state', return_default=True))
+        self.postal_code = Field('postal_code', str, **self.get(data, 'postal_code', return_default=True))
+
+
+class UserRegisterPaymentSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+
+        self.add_schema(UserCredentialSchema, data)
+        self.add_schema(AddressSchema, data, under_field_name='address')
+
+        self.card_number = Field('card_number', str, **self.get(data, 'card_number'))
+        self.cvc = Field('cvc', str, **self.get(data, 'cvc'))
+        self.expiry_year = Field('expiry_year', int, **self.get(data, 'expiry_year'))
+        self.expiry_month = Field('expiry_month', int, **self.get(data, 'expiry_month'))
+        self.card_holder_name = Field('card_holder_name', str, **self.get(data, 'card_holder_name'))
+
+
+class UserCredentialSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.username = Field('username', str, **self.get(data, 'username'))
+        self.password = Field('password', str, **self.get(data, 'password'))
+
+    def validation(self):
+        try:
+            entry = ldap_manager.is_password_valid(self.username.value, self.password.value, query_key='uid')
+        except ValueError:
+            raise ValidationException('No user with \'{}\' username found. You can create account at '
+                                      'https://account.ungleich.ch'.format(self.username.value))
+        except Exception:
+            raise ValidationException('Invalid username/password.')
+        else:
+            self.objects['user'] = entry
+
+
+class ProductOrderSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.product_id = Field(
+            'product_id', str, **self.get(data, 'product_id'), validators=[self.product_id_validation]
+        )
+        self.pay_consent = Field('pay', bool, **self.get(data, 'pay', return_default=True, default=False))
+        self.add_schema(UserCredentialSchema, data)
+
+    def product_id_validation(self):
+        product = resolve_product(self.product_id.value, etcd_client)
+        if product:
+            product['quantity'] = float(product['quantity'])
+            self.product_id.value = product['uuid']
+            self.objects['product'] = product
+            logging.debug('Got product {}'.format(product))
+
+            if not product['active']:
+                raise ValidationException('Product is not active at the moment.')
+
+            if product['quantity'] <= 0:
+                raise ValidationException('Out of stock.')
+        else:
+            raise ValidationException('No such product exists.')
+
+    def validation(self):
+        username = self.objects['user'].uid
+        customer_previous_orders = etcd_client.get_prefix('/v1/user/{}'.format(username), value_in_json=True)
+        customer_previous_orders = [o.value for o in customer_previous_orders]
+        membership = next(filter(lambda o: o['product'] == 'membership', customer_previous_orders), None)
+        if membership is None and self.objects['product']['usable-id'] != 'membership':
+            raise ValidationException('Please buy membership first to use this facility')
+        max_quantity_user_can_order = float(self.objects['product'].get('max_per_user', math.inf))
+        previous_order_of_same_product = [
+            o for o in customer_previous_orders if o['product'] == self.objects['product']['usable-id']
+        ]
+        if len(previous_order_of_same_product) >= max_quantity_user_can_order:
+            raise ValidationException(
+                'You cannot buy {} more than {} times'.format(
+                    self.objects['product']['name'], int(max_quantity_user_can_order)
+                )
+            )
+
+
+class OrderListSchema(BaseSchema):
+    def __init__(self, data):
+        super().__init__()
+        self.add_schema(UserCredentialSchema, data)
+
+
+def make_return_message(err, status_code=200):
+    logging.debug('message: {}'.format(str(err)))
+    return {'message': str(err)}, status_code
+
+
+def create_schema(specification, data):
+    fields = {}
+    for feature_name, feature_detail in specification['features'].items():
+        if not feature_detail['constant']:
+            fields[feature_name] = Field(
+                feature_name, eval(feature_detail['unit']['type']), **BaseSchema.get(data, feature_name)
+            )
+
+    return type('{}Schema'.format(specification['name']), (BaseSchema,), fields)

archive/uncloud_django_based/meow-payv1/stripe_hack.py

@@ -0,0 +1,7 @@
+import stripe_utils
+import os
+
+
+if __name__ == '__main__':
+    s = stripe_utils.StripeUtils(os.environ['STRIPE_PRIVATE_KEY'])
+    print(s.get_stripe_customer_from_email('coder.purple+2002@gmail.com'))

archive/uncloud_django_based/meow-payv1/stripe_utils.py

@@ -0,0 +1,491 @@
+import re
+import stripe
+import stripe.error
+import logging
+
+from config import etcd_client as client, config as config
+
+stripe.api_key = config.get('stripe', 'private_key')
+
+
+def handle_stripe_error(f):
+    def handle_problems(*args, **kwargs):
+        response = {
+            'paid': False,
+            'response_object': None,
+            'error': None
+        }
+
+        common_message = "Currently it's not possible to make payments."
+        try:
+            response_object = f(*args, **kwargs)
+            response = {
+                'response_object': response_object,
+                'error': None
+            }
+            return response
+        except stripe.error.CardError as e:
+            # Since it's a decline, stripe.error.CardError will be caught
+            body = e.json_body
+            err = body['error']
+            response.update({'error': err['message']})
+            logging.error(str(e))
+            return response
+        except stripe.error.RateLimitError:
+            response.update(
+                {'error': "Too many requests made to the API too quickly"})
+            return response
+        except stripe.error.InvalidRequestError as e:
+            logging.error(str(e))
+            response.update({'error': "Invalid parameters"})
+            return response
+        except stripe.error.AuthenticationError as e:
+            # Authentication with Stripe's API failed
+            # (maybe you changed API keys recently)
+            logging.error(str(e))
+            response.update({'error': common_message})
+            return response
+        except stripe.error.APIConnectionError as e:
+            logging.error(str(e))
+            response.update({'error': common_message})
+            return response
+        except stripe.error.StripeError as e:
+            # maybe send email
+            logging.error(str(e))
+            response.update({'error': common_message})
+            return response
+        except Exception as e:
+            # maybe send email
+            logging.error(str(e))
+            response.update({'error': common_message})
+            return response
+
+    return handle_problems
+
+
+class StripeUtils(object):
+    CURRENCY = 'chf'
+    INTERVAL = 'month'
+    SUCCEEDED_STATUS = 'succeeded'
+    STRIPE_PLAN_ALREADY_EXISTS = 'Plan already exists'
+    STRIPE_NO_SUCH_PLAN = 'No such plan'
+    PLAN_EXISTS_ERROR_MSG = 'Plan {} exists already.\nCreating a local StripePlan now.'
+    PLAN_DOES_NOT_EXIST_ERROR_MSG = 'Plan {} does not exist.'
+
+    def __init__(self, private_key):
+        self.stripe = stripe
+        stripe.api_key = private_key
+
+    @handle_stripe_error
+    def card_exists(self, customer, cc_number, exp_month, exp_year, cvc):
+        token_obj = stripe.Token.create(
+            card={
+                'number': cc_number,
+                'exp_month': exp_month,
+                'exp_year': exp_year,
+                'cvc': cvc,
+            },
+        )
+        cards = stripe.Customer.list_sources(
+            customer,
+            limit=20,
+            object='card'
+        )
+
+        for card in cards.data:
+            if (card.fingerprint == token_obj.card.fingerprint and
+                    int(card.exp_month) == int(exp_month) and int(card.exp_year) == int(exp_year)):
+                return True
+        return False
+
+    @staticmethod
+    def get_stripe_customer_from_email(email):
+        customer = stripe.Customer.list(limit=1, email=email)
+        return customer.data[0] if len(customer.data) == 1 else None
+
+    @staticmethod
+    def update_customer_token(customer, token):
+        customer.source = token
+        customer.save()
+
+    @handle_stripe_error
+    def get_token_from_card(self, cc_number, cvc, expiry_month, expiry_year):
+        token_obj = stripe.Token.create(
+            card={
+                'number': cc_number,
+                'exp_month': expiry_month,
+                'exp_year': expiry_year,
+                'cvc': cvc,
+            },
+        )
+        return token_obj
+
+    @handle_stripe_error
+    def associate_customer_card(self, stripe_customer_id, token,
+                                set_as_default=False):
+        customer = stripe.Customer.retrieve(stripe_customer_id)
+        card = customer.sources.create(source=token)
+        if set_as_default:
+            customer.default_source = card.id
+            customer.save()
+        return True
+
+    @handle_stripe_error
+    def dissociate_customer_card(self, stripe_customer_id, card_id):
+        customer = stripe.Customer.retrieve(stripe_customer_id)
+        card = customer.sources.retrieve(card_id)
+        card.delete()
+
+    @handle_stripe_error
+    def update_customer_card(self, customer_id, token):
+        customer = stripe.Customer.retrieve(customer_id)
+        current_card_token = customer.default_source
+        customer.sources.retrieve(current_card_token).delete()
+        customer.source = token
+        customer.save()
+        credit_card_raw_data = customer.sources.data.pop()
+        new_card_data = {
+            'last4': credit_card_raw_data.last4,
+            'brand': credit_card_raw_data.brand
+        }
+        return new_card_data
+
+    @handle_stripe_error
+    def get_card_details(self, customer_id):
+        customer = stripe.Customer.retrieve(customer_id)
+        credit_card_raw_data = customer.sources.data.pop()
+        card_details = {
+            'last4': credit_card_raw_data.last4,
+            'brand': credit_card_raw_data.brand,
+            'exp_month': credit_card_raw_data.exp_month,
+            'exp_year': credit_card_raw_data.exp_year,
+            'fingerprint': credit_card_raw_data.fingerprint,
+            'card_id': credit_card_raw_data.id
+        }
+        return card_details
+
+    @handle_stripe_error
+    def get_all_invoices(self, customer_id, created_gt):
+        return_list = []
+        has_more_invoices = True
+        starting_after = False
+        while has_more_invoices:
+            if starting_after:
+                invoices = stripe.Invoice.list(
+                    limit=10, customer=customer_id, created={'gt': created_gt},
+                    starting_after=starting_after
+                )
+            else:
+                invoices = stripe.Invoice.list(
+                    limit=10, customer=customer_id, created={'gt': created_gt}
+                )
+            has_more_invoices = invoices.has_more
+            for invoice in invoices.data:
+                sub_ids = []
+                for line in invoice.lines.data:
+                    if line.type == 'subscription':
+                        sub_ids.append(line.id)
+                    elif line.type == 'invoiceitem':
+                        sub_ids.append(line.subscription)
+                    else:
+                        sub_ids.append('')
+                invoice_details = {
+                    'created': invoice.created,
+                    'receipt_number': invoice.receipt_number,
+                    'invoice_number': invoice.number,
+                    'paid_at': invoice.status_transitions.paid_at if invoice.paid else 0,
+                    'period_start': invoice.period_start,
+                    'period_end': invoice.period_end,
+                    'billing_reason': invoice.billing_reason,
+                    'discount': invoice.discount.coupon.amount_off if invoice.discount else 0,
+                    'total': invoice.total,
+                    # to see how many line items we have in this invoice and
+                    # then later check if we have more than 1
+                    'lines_data_count': len(invoice.lines.data) if invoice.lines.data is not None else 0,
+                    'invoice_id': invoice.id,
+                    'lines_meta_data_csv': ','.join(
+                        [line.metadata.VM_ID if hasattr(line.metadata, 'VM_ID') else '' for line in invoice.lines.data]
+                    ),
+                    'subscription_ids_csv': ','.join(sub_ids),
+                    'line_items': invoice.lines.data
+                }
+                starting_after = invoice.id
+                return_list.append(invoice_details)
+        return return_list
+
+    @handle_stripe_error
+    def get_cards_details_from_token(self, token):
+        stripe_token = stripe.Token.retrieve(token)
+        card_details = {
+            'last4': stripe_token.card.last4,
+            'brand': stripe_token.card.brand,
+            'exp_month': stripe_token.card.exp_month,
+            'exp_year': stripe_token.card.exp_year,
+            'fingerprint': stripe_token.card.fingerprint,
+            'card_id': stripe_token.card.id
+        }
+        return card_details
+
+    def check_customer(self, stripe_cus_api_id, user, token):
+        try:
+            customer = stripe.Customer.retrieve(stripe_cus_api_id)
+        except stripe.error.InvalidRequestError:
+            customer = self.create_customer(token, user.email, user.name)
+            user.stripecustomer.stripe_id = customer.get(
+                'response_object').get('id')
+            user.stripecustomer.save()
+        if type(customer) is dict:
+            customer = customer['response_object']
+        return customer
+
+    @handle_stripe_error
+    def get_customer(self, stripe_api_cus_id):
+        customer = stripe.Customer.retrieve(stripe_api_cus_id)
+        # data = customer.get('response_object')
+        return customer
+
+    @handle_stripe_error
+    def create_customer(self, token, email, name=None, address=None):
+        if name is None or name.strip() == "":
+            name = email
+        customer = self.stripe.Customer.create(
+            source=token,
+            description=name,
+            email=email,
+            address=address
+        )
+        return customer
+
+    @handle_stripe_error
+    def make_charge(self, amount=None, customer=None):
+        _amount = float(amount)
+        amount = int(_amount * 100)  # stripe amount unit, in cents
+        charge = self.stripe.Charge.create(
+            amount=amount,  # in cents
+            currency=self.CURRENCY,
+            customer=customer
+        )
+        return charge
+
+    @staticmethod
+    def _get_all_stripe_plans():
+        all_stripe_plans = client.get("/v1/stripe_plans")
+        all_stripe_plans_set = set()
+        if all_stripe_plans:
+            all_stripe_plans_obj = all_stripe_plans.value
+            if all_stripe_plans_obj and len(all_stripe_plans_obj['plans']) > 0:
+                all_stripe_plans_set = set(all_stripe_plans_obj["plans"])
+        return all_stripe_plans_set
+
+    @staticmethod
+    def _save_all_stripe_plans(stripe_plans):
+        client.put("/v1/stripe_plans", {"plans": list(stripe_plans)})
+
+    @handle_stripe_error
+    def get_or_create_stripe_plan(self, product_name, amount, stripe_plan_id,
+                                  interval=INTERVAL):
+        """
+        This function checks if a StripePlan with the given
+        stripe_plan_id already exists. If it exists then the function
+        returns this object otherwise it creates a new StripePlan and
+        returns the new object.
+
+        :param amount: The amount in CHF cents
+        :param product_name: The name of the Stripe plan (product) to be created.
+        :param stripe_plan_id: The id of the Stripe plan to be
+               created. Use get_stripe_plan_id_string function to
+               obtain the name of the plan to be created
+        :param interval: The interval for subscription {month, year}. Defaults
+               to month if not provided
+        :return: The StripePlan object if it exists else creates a
+               Plan object in Stripe and a local StripePlan and
+               returns it. Returns None in case of Stripe error
+        """
+        _amount = float(amount)
+        amount = int(_amount * 100)  # stripe amount unit, in cents
+        all_stripe_plans = self._get_all_stripe_plans()
+        if stripe_plan_id in all_stripe_plans:
+            logging.debug("{} plan exists in db.".format(stripe_plan_id))
+        else:
+            logging.debug(("{} plan DOES NOT exist in db. "
+                          "Creating").format(stripe_plan_id))
+            try:
+                plan_obj = self.stripe.Plan.retrieve(id=stripe_plan_id)
+                logging.debug("{} plan exists in Stripe".format(stripe_plan_id))
+                all_stripe_plans.add(stripe_plan_id)
+            except stripe.error.InvalidRequestError as e:
+                if "No such plan" in str(e):
+                    logging.debug("Plan {} does not exist in Stripe, Creating")
+                    plan_obj = self.stripe.Plan.create(
+                        amount=amount,
+                        product={'name': product_name},
+                        interval=interval,
+                        currency=self.CURRENCY,
+                        id=stripe_plan_id)
+                    logging.debug(self.PLAN_EXISTS_ERROR_MSG.format(stripe_plan_id))
+                    all_stripe_plans.add(stripe_plan_id)
+            self._save_all_stripe_plans(all_stripe_plans)
+        return stripe_plan_id
+
+    @handle_stripe_error
+    def delete_stripe_plan(self, stripe_plan_id):
+        """
+        Deletes the Plan in Stripe and also deletes the local db copy
+        of the plan if it exists
+
+        :param stripe_plan_id: The stripe plan id that needs to be
+               deleted
+        :return: True if the plan was deleted successfully from
+               Stripe, False otherwise.
+        """
+        return_value = False
+        try:
+            plan = self.stripe.Plan.retrieve(stripe_plan_id)
+            plan.delete()
+            return_value = True
+            all_stripe_plans = self._get_all_stripe_plans()
+            all_stripe_plans.remove(stripe_plan_id)
+            self._save_all_stripe_plans(all_stripe_plans)
+        except stripe.error.InvalidRequestError as e:
+            if self.STRIPE_NO_SUCH_PLAN in str(e):
+                logging.debug(
+                    self.PLAN_DOES_NOT_EXIST_ERROR_MSG.format(stripe_plan_id))
+        return return_value
+
+    @handle_stripe_error
+    def subscribe_customer_to_plan(self, customer, plans, trial_end=None):
+        """
+        Subscribes the given customer to the list of given plans
+
+        :param customer: The stripe customer identifier
+        :param plans: A list of stripe plans.
+        :param trial_end: An integer representing when the Stripe subscription
+               is supposed to end
+        Ref: https://stripe.com/docs/api/python#create_subscription-items
+              e.g.
+                    plans = [
+                                {
+                                  "plan": "dcl-v1-cpu-2-ram-5gb-ssd-10gb",
+                                },
+                            ]
+        :return: The subscription StripeObject
+        """
+
+        subscription_result = self.stripe.Subscription.create(
+            customer=customer, items=plans, trial_end=trial_end
+        )
+        return subscription_result
+
+    @handle_stripe_error
+    def set_subscription_metadata(self, subscription_id, metadata):
+        subscription = stripe.Subscription.retrieve(subscription_id)
+        subscription.metadata = metadata
+        subscription.save()
+
+    @handle_stripe_error
+    def unsubscribe_customer(self, subscription_id):
+        """
+        Cancels a given subscription
+
+        :param subscription_id: The Stripe subscription id string
+        :return:
+        """
+        sub = stripe.Subscription.retrieve(subscription_id)
+        return sub.delete()
+
+    @handle_stripe_error
+    def make_payment(self, customer, amount, token):
+        charge = self.stripe.Charge.create(
+            amount=amount,  # in cents
+            currency=self.CURRENCY,
+            customer=customer
+        )
+        return charge
+
+    @staticmethod
+    def get_stripe_plan_id(cpu, ram, ssd, version, app='dcl', hdd=None,
+                           price=None):
+        """
+        Returns the Stripe plan id string of the form
+        `dcl-v1-cpu-2-ram-5gb-ssd-10gb` based on the input parameters
+
+        :param cpu: The number of cores
+        :param ram: The size of the RAM in GB
+        :param ssd: The size of ssd storage in GB
+        :param hdd: The size of hdd storage in GB
+        :param version: The version of the Stripe plans
+        :param app: The application to which the stripe plan belongs
+        to. By default it is 'dcl'
+        :param price: The price for this plan
+        :return: A string of the form `dcl-v1-cpu-2-ram-5gb-ssd-10gb`
+        """
+        dcl_plan_string = 'cpu-{cpu}-ram-{ram}gb-ssd-{ssd}gb'.format(cpu=cpu,
+                                                                     ram=ram,
+                                                                     ssd=ssd)
+        if hdd is not None:
+            dcl_plan_string = '{dcl_plan_string}-hdd-{hdd}gb'.format(
+                dcl_plan_string=dcl_plan_string, hdd=hdd)
+        stripe_plan_id_string = '{app}-v{version}-{plan}'.format(
+            app=app,
+            version=version,
+            plan=dcl_plan_string
+        )
+        if price is not None:
+            stripe_plan_id_string_with_price = '{}-{}chf'.format(
+                stripe_plan_id_string,
+                round(price, 2)
+            )
+            return stripe_plan_id_string_with_price
+        else:
+            return stripe_plan_id_string
+
+    @staticmethod
+    def get_vm_config_from_stripe_id(stripe_id):
+        """
+        Given a string like "dcl-v1-cpu-2-ram-5gb-ssd-10gb" return different
+        configuration params as a dict
+
+        :param stripe_id|str
+        :return: dict
+        """
+        pattern = re.compile(r'^dcl-v(\d+)-cpu-(\d+)-ram-(\d+\.?\d*)gb-ssd-(\d+)gb-?(\d*\.?\d*)(chf)?$')
+        match_res = pattern.match(stripe_id)
+        if match_res is not None:
+            price = None
+            try:
+                price = match_res.group(5)
+            except IndexError:
+                logging.debug("Did not find price in {}".format(stripe_id))
+            return {
+                'version': match_res.group(1),
+                'cores': match_res.group(2),
+                'ram': match_res.group(3),
+                'ssd': match_res.group(4),
+                'price': price
+            }
+
+    @staticmethod
+    def get_stripe_plan_name(cpu, memory, disk_size, price):
+        """
+        Returns the Stripe plan name
+        :return:
+        """
+        return "{cpu} Cores, {memory} GB RAM, {disk_size} GB SSD, " \
+               "{price} CHF".format(
+                    cpu=cpu,
+                    memory=memory,
+                    disk_size=disk_size,
+                    price=round(price, 2)
+                )
+
+    @handle_stripe_error
+    def set_subscription_meta_data(self, subscription_id, meta_data):
+        """
+        Adds VM metadata to a subscription
+        :param subscription_id: Stripe identifier for the subscription
+        :param meta_data: A dict of meta data to be added
+        :return:
+        """
+        subscription = stripe.Subscription.retrieve(subscription_id)
+        subscription.metadata = meta_data
+        subscription.save()

archive/uncloud_django_based/meow-payv1/ucloud_pay.py

@@ -0,0 +1,338 @@
+import logging
+
+from datetime import datetime
+from uuid import uuid4
+
+from flask import Flask, request
+from flask_restful import Resource, Api
+from werkzeug.exceptions import HTTPException
+from config import etcd_client as client, config as config
+from stripe_utils import StripeUtils
+from schemas import (
+    make_return_message, ValidationException, UserRegisterPaymentSchema,
+    AddProductSchema, ProductOrderSchema, OrderListSchema, create_schema
+)
+from helper import get_plan_id_from_product, calculate_charges
+
+
+class ListProducts(Resource):
+    @staticmethod
+    def get():
+        products = client.get_prefix('/v1/products/')
+        products = [
+            product
+            for product in [p.value for p in products]
+            if product['active']
+        ]
+        prod_dict = {}
+        for p in products:
+            prod_dict[p['usable-id']] = {
+                'name': p['name'],
+                'description': p['description'],
+            }
+        logger.debug('Products = {}'.format(prod_dict))
+        return prod_dict, 200
+
+class AddProduct(Resource):
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+
+        try:
+            logger.debug('Got data: {}'.format(str(data)))
+            validator = AddProductSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            cleaned_values = validator.get_cleaned_values()
+            previous_product = cleaned_values.get('product', None)
+            if previous_product:
+                if not cleaned_values['update']:
+                    return make_return_message('Product already exists. Pass --update to update the product.')
+                else:
+                    product_uuid = previous_product.pop('uuid')
+            else:
+                product_uuid = uuid4().hex
+
+            product_value = cleaned_values['specs']
+
+            product_key = '/v1/products/{}'.format(product_uuid)
+            product_value['uuid'] = product_uuid
+
+            logger.debug('Adding product data: {}'.format(str(product_value)))
+            client.put(product_key, product_value)
+            if not previous_product:
+                return make_return_message('Product created.')
+            else:
+                return make_return_message('Product updated.')
+
+################################################################################
+# Nico-ok-marker
+
+
+class UserRegisterPayment(Resource):
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+
+        try:
+            logger.debug('Got data: {}'.format(str(data)))
+            validator = UserRegisterPaymentSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            cleaned_values = validator.get_cleaned_values()
+            last4 = data['card_number'].strip()[-4:]
+
+            stripe_utils = StripeUtils()
+
+            # Does customer already exist ?
+            stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail'])
+
+            # Does customer already exist ?
+            if stripe_customer is not None:
+                logger.debug('Customer {}-{} exists already'.format(
+                    cleaned_values['username'], cleaned_values['user']['mail'])
+                )
+
+                # Check if the card already exists
+                ce_response = stripe_utils.card_exists(
+                        stripe_customer.id, cc_number=data['card_number'],
+                        exp_month=int(data['expiry_month']),
+                        exp_year=int(data['expiry_year']),
+                        cvc=data['cvc'])
+
+                if ce_response['response_object']:
+                    message = 'The given card ending in {} exists already.'.format(last4)
+                    return make_return_message(message, 400)
+
+                elif ce_response['response_object'] is False:
+                    # Associate card with user
+                    logger.debug('Adding card ending in {}'.format(last4))
+                    token_response = stripe_utils.get_token_from_card(
+                        data['card_number'], data['cvc'], data['expiry_month'],
+                        data['expiry_year']
+                    )
+                    if token_response['response_object']:
+                        logger.debug('Token {}'.format(token_response['response_object'].id))
+                        resp = stripe_utils.associate_customer_card(
+                            stripe_customer.id, token_response['response_object'].id
+                        )
+                        if resp['response_object']:
+                            return make_return_message(
+                                'Card ending in {} registered as your payment source'.format(last4)
+                            )
+                    else:
+                        return make_return_message('Error with payment gateway. Contact support', 400)
+                else:
+                    return make_return_message('Error: {}'.format(ce_response['error']), 400)
+            else:
+                # Stripe customer does not exist, create a new one
+                logger.debug(
+                    'Customer {} does not exist, creating new'.format(cleaned_values['user']['mail'])
+                )
+                token_response = stripe_utils.get_token_from_card(
+                    cleaned_values['card_number'], cleaned_values['cvc'],
+                    cleaned_values['expiry_month'], cleaned_values['expiry_year']
+                )
+                if token_response['response_object']:
+                    logger.debug('Token {}'.format(token_response['response_object'].id))
+
+                    # Create stripe customer
+                    stripe_customer_resp = stripe_utils.create_customer(
+                        name=cleaned_values['card_holder_name'],
+                        token=token_response['response_object'].id,
+                        email=cleaned_values['user']['mail'],
+                        address=cleaned_values['address']
+                    )
+                    stripe_customer = stripe_customer_resp['response_object']
+
+                    if stripe_customer:
+                        logger.debug('Created stripe customer {}'.format(stripe_customer.id))
+                        return make_return_message(
+                            'Card ending in {} registered as your payment source'.format(last4)
+                        )
+                    else:
+                        return make_return_message('Error with card. Contact support', 400)
+                else:
+                    return make_return_message('Error with payment gateway. Contact support', 400)
+
+
+class ProductOrder(Resource):
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+
+        try:
+            validator = ProductOrderSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            cleaned_values = validator.get_cleaned_values()
+            stripe_utils = StripeUtils()
+
+            product = cleaned_values['product']
+
+            # Check the user has a payment source added
+            stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail'])
+
+            if not stripe_customer or len(stripe_customer.sources) == 0:
+                return make_return_message('Please register your payment method first.', 400)
+
+            try:
+                product_schema = create_schema(product, data)
+                product_schema = product_schema()
+                product_schema.is_valid()
+            except ValidationException as err:
+                return make_return_message(err, 400)
+            else:
+                transformed_data = product_schema.get_cleaned_values()
+                logger.debug('Tranformed data: {}'.format(transformed_data))
+                one_time_charge, recurring_charge = calculate_charges(product, transformed_data)
+                recurring_charge = int(recurring_charge)
+
+                if not cleaned_values['pay']:
+                    return make_return_message(
+                        'You would be charged {} CHF one time and {} CHF every {}. '
+                        'Add --pay to command to order.'.format(
+                            one_time_charge, recurring_charge, product['recurring_period']
+                        )
+                    )
+
+                with client.client.lock('product-order') as _:
+                    # Initiate a one-time/subscription based on product type
+                    if recurring_charge > 0:
+                        logger.debug('Product {} is recurring payment'.format(product['name']))
+                        plan_id = get_plan_id_from_product(product)
+                        res = stripe_utils.get_or_create_stripe_plan(
+                            product_name=product['name'],
+                            stripe_plan_id=plan_id, amount=recurring_charge,
+                            interval=product['recurring_period'],
+                        )
+                        if res['response_object']:
+                            logger.debug('Obtained plan {}'.format(plan_id))
+                            subscription_res = stripe_utils.subscribe_customer_to_plan(
+                                stripe_customer.id,
+                                [{'plan': plan_id}]
+                            )
+                            subscription_obj = subscription_res['response_object']
+                            if subscription_obj is None or subscription_obj.status != 'active':
+                                return make_return_message(
+                                    'Error subscribing to plan. Detail: {}'.format(subscription_res['error']), 400
+                                )
+                            else:
+                                order_obj = {
+                                    'order-id': uuid4().hex,
+                                    'ordered-at': datetime.now().isoformat(),
+                                    'product': product['usable-id'],
+                                    'one-time-price': one_time_charge,
+                                    'recurring-price': recurring_charge,
+                                    'recurring-period': product['recurring_period']
+                                }
+                                client.put(
+                                    '/v1/user/{}/orders/{}'.format(
+                                        cleaned_values['username'], order_obj['order-id']
+                                    ), order_obj
+                                )
+                                product['quantity'] -= 1
+                                client.put('/v1/products/{}'.format(product['uuid']), product)
+
+                                return {
+                                    'message': 'Order Successful.',
+                                    **order_obj
+                                }
+                        else:
+                            logger.error('Could not create plan {}'.format(plan_id))
+                            return make_return_message('Something wrong happened. Contact administrator', 400)
+
+                    elif recurring_charge == 0 and one_time_charge > 0:
+                        logger.debug('Product {} is one-time payment'.format(product['name']))
+                        charge_response = stripe_utils.make_charge(
+                            amount=one_time_charge,
+                            customer=stripe_customer.id
+                        )
+                        stripe_onetime_charge = charge_response.get('response_object')
+
+                        # Check if the payment was approved
+                        if not stripe_onetime_charge:
+                            msg = charge_response.get('error')
+                            return make_return_message('Error subscribing to plan. Details: {}'.format(msg), 400)
+
+                        order_obj = {
+                            'order-id': uuid4().hex,
+                            'ordered-at': datetime.now().isoformat(),
+                            'product': product['usable-id'],
+                            'one-time-price': one_time_charge,
+                        }
+                        client.put(
+                            '/v1/user/{}/orders/{}'.format(cleaned_values['username'], order_obj['order-id']),
+                            order_obj
+                        )
+                        product['quantity'] -= 1
+                        client.put('/v1/products/{}'.format(product['uuid']), product)
+
+                        return {'message': 'Order successful', **order_obj}, 200
+
+
+class OrderList(Resource):
+    @staticmethod
+    def post():
+        data = request.get_json(silent=True) or {}
+
+        try:
+            validator = OrderListSchema(data)
+            validator.is_valid()
+        except ValidationException as err:
+            return make_return_message(err, 400)
+        else:
+            cleaned_values = validator.get_cleaned_values()
+            orders = client.get_prefix('/v1/user/{}/orders'.format(cleaned_values['username']))
+            orders_dict = {
+                order.value['order-id']: {
+                    **order.value
+                }
+                for order in orders
+            }
+            logger.debug('Orders = {}'.format(orders_dict))
+            return {'orders': orders_dict}, 200
+
+
+if __name__ == '__main__':
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    log_formater = logging.Formatter('[%(filename)s:%(lineno)d] %(message)s')
+
+    stream_logger = logging.StreamHandler()
+    stream_logger.setFormatter(log_formater)
+
+    # file_logger = logging.FileHandler('log.txt')
+    # file_logger.setLevel(logging.DEBUG)
+    # file_logger.setFormatter(log_formater)
+
+    logger.addHandler(stream_logger)
+    # logger.addHandler(file_logger)
+
+    app = Flask(__name__)
+
+    api = Api(app)
+    api.add_resource(ListProducts, '/product/list')
+    api.add_resource(AddProduct, '/product/add')
+    api.add_resource(ProductOrder, '/product/order')
+    api.add_resource(UserRegisterPayment, '/user/register_payment')
+    api.add_resource(OrderList, '/order/list')
+
+    app.run(host='::', port=config.get('app', 'port', fallback=5000), debug=True)
+
+
+    @app.errorhandler(Exception)
+    def handle_exception(e):
+        app.logger.error(e)
+        # pass through HTTP errors
+        if isinstance(e, HTTPException):
+            return e
+
+        # now you're handling non-HTTP exceptions only
+        return {'message': 'Server Error'}, 500

archive/uncloud_django_based/notes-abk.md

@@ -0,0 +1,11 @@
+## TODO 2020-02-22
+
+* ~~move the current rest api to /opennebula~~
+* ~~make the /opennebula api only accessible by an admin account~~
+* ~~create a new filtered api on /vm/list that~~
+    * ~~a) requires authentication~~
+    * ~~b) only shows the VMs of the current user~~
+* ~~the new api should not contain all details, but: cpus (as read by the vcpu field), ram, ips, disks~~
+* ~~also make a (random) uuid the primary key for VMs - everything in this uncloud hack will use uuids as the id~~
+* ~~still expose the opennebula id as opennebula_id~~
+* ~~note put all secrets/configs into uncloud.secrets - I added a sample file into the repo~~

archive/uncloud_django_based/notes-nico.org

@@ -0,0 +1,102 @@
+* snapshot feature
+** product: vm-snapshot
+** flow
+*** list all my VMs
+**** get the uuid of the VM I want to take a snapshot of
+*** request a snapshot
+```
+vmuuid=$(http nicocustomer
+http -a nicocustomer:xxx http://uncloud.ch/vm/create_snapshot uuid=
+password=...
+```
+** backend realisation
+*** list snapshots
+    - have them in the DB
+    - create an entry on create
+*** creating snapshots
+    - vm sync / fsync?
+    - rbd snapshot
+      - host/cluster mapping?
+      - need image(s)
+
+* steps
+** DONE authenticate via ldap
+   CLOSED: [2020-02-20 Thu 19:05]
+** DONE Make classes / views require authentication
+   CLOSED: [2020-02-20 Thu 19:05]
+** TODO register credit card
+*** TODO find out what saving with us
+*** Info
+**** should not be fully saved in the DB
+**** model needs to be a bit different
+* Decide where to save sensitive data
+** stripe access key, etc.
+* python requirements (nicohack202002)
+  django djangorestframework django-auth-ldap stripe
+* os package requirements (alpine)
+  openldap-dev
+* VPN case
+** put on /orders with uuid
+** register cc
+* CC
+** TODO check whether we can register or not at stripe
+* membership
+** required for "smaller" / "shorter" products
+
+* TODO Membership missing
+* Flows to be implemented - see https://redmine.ungleich.ch/issues/7609
+** Membership
+*** 5 CHF
+** Django Hosting
+*** One time payment 35 CHF
+*** Monthly payment depends on VM size
+*** Parameters: same as IPv6 only VM
+** IPv6 VPN
+*** Parameters: none
+*** Is for free if the customer has an active VM
+** IPv6 only VM
+*** Parameters: cores, ram, os_disk_size, OS
+* Django rest framework
+** viewset: .list and .create
+** view: .get .post
+* TODO register CC
+* DONE list products
+  CLOSED: [2020-02-24 Mon 20:15]
+* An ungleich account - can be registered for free on
+  https://account.ungleich.ch
+* httpie installed (provides the http command)
+
+## Get a membership
+
+
+## Registering a payment method
+
+To be able to pay for the membership, you will need to register a
+credit card or apply for payment on bill (TO BE IMPLEMENTED).
+
+### Register credit card
+
+```
+http POST https://api.ungleich.ch/membership \
+    username=nico password=yourpassword \
+    cc_number=.. \
+    cc_
+
+```
+
+
+
+### Request payment via bill
+
+
+
+
+## Create the membership
+
+
+```
+http POST https://api.ungleich.ch/membership username=nico password=yourpassword
+
+```
+
+## List available products

archive/uncloud_django_based/plan.org

@@ -0,0 +1,6 @@
+* TODO register CC
+* TODO list products
+* ahmed
+** schemas
+*** field: is_valid? - used by schemas
+*** definition of a "schema"

archive/uncloud_django_based/uncloud/.gitignore

@@ -0,0 +1,4 @@
+db.sqlite3
+uncloud/secrets.py
+debug.log
+uncloud/local_settings.py

archive/uncloud_etcd_based/bin/gen-version

@@ -1,22 +1,22 @@
 #!/bin/sh
 # -*- coding: utf-8 -*-
 #
-# 2019 Nico Schottelius (nico-ucloud at schottelius.org)
+# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org)
 #
-# This file is part of ucloud.
+# This file is part of uncloud.
 #
-# ucloud is free software: you can redistribute it and/or modify
+# uncloud is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
 #
-# ucloud is distributed in the hope that it will be useful,
+# uncloud is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
 #
 # You should have received a copy of the GNU General Public License
-# along with ucloud. If not, see <http://www.gnu.org/licenses/>.
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
 #
 #
 
@@ -26,4 +26,4 @@ dir=${0%/*}
 
 # Ensure version is present - the bundled/shipped version contains a static version,
 # the git version contains a dynamic version
-printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py
+printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py

archive/uncloud_etcd_based/bin/uncloud-run-reinstall

@@ -24,6 +24,6 @@
 dir=${0%/*}
 
 ${dir}/gen-version;
-pip uninstall -y uncloud
-python setup.py install
+pip uninstall -y uncloud >/dev/null
+python setup.py install  >/dev/null
 ${dir}/uncloud "$@"

archive/uncloud_etcd_based/docs/Makefile

@@ -7,7 +7,7 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source/
 BUILDDIR      = build/
-DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/
+DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/
 
 .PHONY: all build clean
 

archive/uncloud_etcd_based/docs/source/admin-guide.rst

@@ -56,40 +56,13 @@ To start host we created earlier, execute the following command
 
     ucloud host ungleich.ch
 
-Create OS Image
----------------
+File & image scanners
+--------------------------
 
-Create ucloud-init ready OS image (Optional)
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-This step is optional if you just want to test ucloud. However, sooner or later
-you want to create OS images with ucloud-init to properly
-contexualize VMs.
-
-1. Start a VM with OS image on which you want to install ucloud-init
-2. Execute the following command on the started VM
-
-   .. code-block:: sh
-
-    apk add git
-    git clone https://code.ungleich.ch/ucloud/ucloud-init.git
-    cd ucloud-init
-    sh ./install.sh
-3. Congratulations. Your image is now ucloud-init ready.
-
-
-Upload Sample OS Image
-~~~~~~~~~~~~~~~~~~~~~~
-Execute the following to get the sample OS image file.
-
-.. code-block:: sh
-
-    mkdir /var/www/admin
-    (cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download)
-
-Run File Scanner and Image Scanner
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make
-images from tracked files. So, we need to track the file by running File Scanner
+Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our
+uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by
+ucloud. We can only make images from tracked files. So, we need to track the
+file by running File Scanner
 
 .. code-block:: sh
 

archive/uncloud_etcd_based/docs/source/hacking.rst

@@ -0,0 +1,36 @@
+Hacking
+=======
+Using uncloud in hacking (aka development) mode.
+
+
+Get the code
+------------
+.. code-block:: sh
+    :linenos:
+
+     git clone https://code.ungleich.ch/uncloud/uncloud.git
+
+
+
+Install python requirements
+---------------------------
+You need to have python3 installed.
+
+.. code-block:: sh
+    :linenos:
+
+    cd uncloud!
+    python -m venv venv
+    . ./venv/bin/activate
+    ./bin/uncloud-run-reinstall
+
+
+
+Install os requirements
+-----------------------
+Install the following software packages: **dnsmasq**.
+
+If you already have a working IPv6 SLAAC and DNS setup,
+this step can be skipped.
+
+Note that you need at least one /64 IPv6 network to run uncloud.

archive/uncloud_etcd_based/docs/source/index.rst

@@ -11,14 +11,13 @@ Welcome to ucloud's documentation!
    :caption: Contents:
 
    introduction
-   user-guide
    setup-install
+   vm-images
+   user-guide
    admin-guide
-   user-guide/how-to-create-an-os-image-for-ucloud
    troubleshooting
    hacking
 
-
 Indices and tables
 ==================
 

archive/uncloud_etcd_based/docs/source/vm-images.rst

@@ -0,0 +1,66 @@
+VM images
+==================================
+
+Overview
+---------
+
+ucloud tries to be least invasise towards VMs and only require
+strictly necessary changes for running in a virtualised
+environment. This includes configurations for:
+
+* Configuring the network
+* Managing access via ssh keys
+* Resizing the attached disk(s)
+
+Upstream images
+---------------
+
+The 'official' uncloud images are defined in the `uncloud/images
+<https://code.ungleich.ch/uncloud/images>`_ repository.
+
+How to make you own Uncloud images
+----------------------------------
+
+.. note::
+  It is fairly easy to create your own images for uncloud, as the common
+  operations (which are detailed below) can be automatically handled by the
+  `uncloud/uncloud-init <https://code.ungleich.ch/uncloud/uncloud-init>`_ tool.
+
+Network configuration
+~~~~~~~~~~~~~~~~~~~~~
+All VMs in ucloud are required to support IPv6. The primary network
+configuration is always done using SLAAC. A VM thus needs only to be
+configured to
+
+* accept router advertisements on all network interfaces
+* use the router advertisements to configure the network interfaces
+* accept the DNS entries from the router advertisements
+
+
+Configuring SSH keys
+~~~~~~~~~~~~~~~~~~~~
+
+To be able to access the VM, ucloud support provisioning SSH keys.
+
+To accept ssh keys in your VM, request the URL
+*http://metadata/ssh_keys*. Add the content to the appropriate user's
+**authorized_keys** file. Below you find sample code to accomplish
+this task:
+
+.. code-block:: sh
+
+    tmp=$(mktemp)
+    curl -s http://metadata/ssk_keys > "$tmp"
+    touch ~/.ssh/authorized_keys # ensure it exists
+    cat ~/.ssh/authorized_keys >> "$tmp"
+    sort "$tmp" | uniq > ~/.ssh/authorized_keys
+
+
+Disk resize
+~~~~~~~~~~~
+In virtualised environments, the disk sizes might grow. The operating
+system should detect disks that are bigger than the existing partition
+table and resize accordingly. This task is os specific.
+
+ucloud does not support shrinking disks due to the complexity and
+intra OS dependencies.

archive/uncloud_etcd_based/scripts/uncloud

@@ -0,0 +1,89 @@
+#!/usr/bin/env python3
+import logging
+import sys
+import importlib
+import argparse
+import os
+
+from etcd3.exceptions import ConnectionFailedError
+
+from uncloud.common import settings
+from uncloud import UncloudException
+from uncloud.common.cli import resolve_otp_credentials
+
+# Components that use etcd
+ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner',
+                   'imagescanner', 'metadata', 'configure', 'hack']
+
+ALL_COMPONENTS = ETCD_COMPONENTS.copy()
+ALL_COMPONENTS.append('oneshot')
+#ALL_COMPONENTS.append('cli')
+
+
+if __name__ == '__main__':
+    arg_parser = argparse.ArgumentParser()
+    subparsers = arg_parser.add_subparsers(dest='command')
+
+    parent_parser = argparse.ArgumentParser(add_help=False)
+    parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
+                               help='More verbose logging')
+    parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
+                               default=os.path.expanduser('~/uncloud'))
+
+    etcd_parser = argparse.ArgumentParser(add_help=False)
+    etcd_parser.add_argument('--etcd-host')
+    etcd_parser.add_argument('--etcd-port')
+    etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
+    etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
+    etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
+
+    for component in ALL_COMPONENTS:
+        mod = importlib.import_module('uncloud.{}.main'.format(component))
+        parser = getattr(mod, 'arg_parser')
+
+        if component in ETCD_COMPONENTS:
+            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser])
+        else:
+            subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
+
+    arguments = vars(arg_parser.parse_args())
+    etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
+    etcd_arguments = {
+        'etcd': {
+            key.replace('etcd_', ''): arguments[key]
+            for key in etcd_arguments
+        }
+    }
+    if not arguments['command']:
+        arg_parser.print_help()
+    else:
+        # Initializing Settings and resolving otp_credentials
+        # It is neccessary to resolve_otp_credentials after argument parsing is done because
+        # previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
+        # providing the default values for --name, --realm and --seed arguments from the values
+        # we read from file. But, now we are asking user about where the config file lives. So,
+        # to providing default value is not possible before parsing arguments. So, we are doing
+        # it after..
+#        settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
+#        resolve_otp_credentials(arguments)
+
+        name = arguments.pop('command')
+        mod = importlib.import_module('uncloud.{}.main'.format(name))
+        main = getattr(mod, 'main')
+
+        if arguments['debug']:
+            logging.basicConfig(level=logging.DEBUG)
+        else:
+            logging.basicConfig(level=logging.INFO)
+
+        log = logging.getLogger()
+
+        try:
+            main(arguments)
+        except UncloudException as err:
+            log.error(err)
+            sys.exit(1)
+#        except ConnectionFailedError as err:
+#            log.error('Cannot connect to etcd: {}'.format(err))
+        except Exception as err:
+            log.exception(err)

archive/uncloud_etcd_based/setup.py

@@ -40,7 +40,8 @@ setup(
         "pynetbox",
         "colorama",
         "etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3",
-        "marshmallow"
+        "marshmallow",
+        "ldap3"
     ],
     scripts=["scripts/uncloud"],
     data_files=[

archive/uncloud_etcd_based/test/test_mac_local.py

@@ -0,0 +1,37 @@
+import unittest
+from unittest.mock import Mock
+
+from uncloud.hack.mac import MAC
+from uncloud import UncloudException
+
+class TestMacLocal(unittest.TestCase):
+    def setUp(self):
+        self.config = Mock()
+        self.config.arguments = {"no_db":True}
+        self.mac = MAC(self.config)
+        self.mac.create()
+
+    def testMacInt(self):
+        self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong first MAC index")
+
+    def testMacRepr(self):
+        self.assertEqual(self.mac.__repr__(), '420000000001', "wrong first MAC index")
+
+    def testMacStr(self):
+        self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong first MAC index")
+
+    def testValidationRaise(self):
+        with self.assertRaises(UncloudException):
+            self.mac.validate_mac("2")
+
+    def testValidation(self):
+        self.assertTrue(self.mac.validate_mac("42:00:00:00:00:01"), "Validation of a given MAC not working properly")
+
+    def testNextMAC(self):
+        self.mac.create()
+        self.assertEqual(self.mac.__repr__(), '420000000001', "wrong second MAC index")
+        self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong second MAC index")
+        self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong second MAC index")
+
+if __name__ == '__main__':
+    unittest.main()

archive/uncloud_etcd_based/uncloud/__init__.py

@@ -0,0 +1,2 @@
+class UncloudException(Exception):
+    pass

archive/uncloud_etcd_based/uncloud/api/common_fields.py

@@ -1,7 +1,6 @@
 import os
 
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 
 
 class Optional:
@@ -54,9 +53,7 @@ class VmUUIDField(Field):
 
     def vm_uuid_validation(self):
         r = shared.etcd_client.get(
-            os.path.join(settings["etcd"]["vm_prefix"], self.uuid)
+            os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
         )
         if not r:
-            self.add_error(
-                "VM with uuid {} does not exists".format(self.uuid)
-            )
+            self.add_error("VM with uuid {} does not exists".format(self.uuid))

archive/uncloud_etcd_based/uncloud/api/create_image_store.py

@@ -4,7 +4,6 @@ import os
 from uuid import uuid4
 
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 
 data = {
     'is_public': True,
@@ -15,6 +14,6 @@ data = {
 }
 
 shared.etcd_client.put(
-    os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex),
+    os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
     json.dumps(data),
 )

archive/uncloud_etcd_based/uncloud/api/helper.py

@@ -7,7 +7,6 @@ import requests
 from pyotp import TOTP
 
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 
 logger = logging.getLogger(__name__)
 
@@ -15,9 +14,9 @@ logger = logging.getLogger(__name__)
 def check_otp(name, realm, token):
     try:
         data = {
-            "auth_name": settings["otp"]["auth_name"],
-            "auth_token": TOTP(settings["otp"]["auth_seed"]).now(),
-            "auth_realm": settings["otp"]["auth_realm"],
+            "auth_name": shared.settings["otp"]["auth_name"],
+            "auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
+            "auth_realm": shared.settings["otp"]["auth_realm"],
             "name": name,
             "realm": realm,
             "token": token,
@@ -25,13 +24,13 @@ def check_otp(name, realm, token):
     except binascii.Error as err:
         logger.error(
             "Cannot compute OTP for seed: {}".format(
-                settings["otp"]["auth_seed"]
+                shared.settings["otp"]["auth_seed"]
             )
         )
         return 400
 
     response = requests.post(
-        settings["otp"]["verification_controller_url"], json=data
+        shared.settings["otp"]["verification_controller_url"], json=data
     )
     return response.status_code
 
@@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client):
         )
 
     images = etcd_client.get_prefix(
-        settings["etcd"]["image_prefix"], value_in_json=True
+        shared.settings["etcd"]["image_prefix"], value_in_json=True
     )
 
     # Try to find image with name == image_name and store_name == store_name
@@ -111,9 +110,7 @@ def random_bytes(num=6):
     return [random.randrange(256) for _ in range(num)]
 
 
-def generate_mac(
-    uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
-):
+def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
     mac = random_bytes()
     if oui:
         if type(oui) == str:
@@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix):
     lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
     prefix = ipaddress.IPv6Address(prefix)
     return str(prefix + int(lower_part))
+

archive/uncloud_etcd_based/uncloud/api/main.py

@@ -15,9 +15,8 @@ from uncloud.common.shared import shared
 from uncloud.common import counters
 from uncloud.common.vm import VMStatus
 from uncloud.common.request import RequestEntry, RequestType
-from uncloud.common.settings import settings
-from . import schemas
-from .helper import generate_mac, mac2ipv6
+from uncloud.api import schemas
+from uncloud.api.helper import generate_mac, mac2ipv6
 from uncloud import UncloudException
 
 logger = logging.getLogger(__name__)
@@ -50,7 +49,7 @@ class CreateVM(Resource):
         validator = schemas.CreateVMSchema(data)
         if validator.is_valid():
             vm_uuid = uuid4().hex
-            vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid)
+            vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
             specs = {
                 'cpu': validator.specs['cpu'],
                 'ram': validator.specs['ram'],
@@ -60,7 +59,7 @@ class CreateVM(Resource):
             macs = [generate_mac() for _ in range(len(data['network']))]
             tap_ids = [
                 counters.increment_etcd_counter(
-                    shared.etcd_client, settings['etcd']['tap_counter']
+                    shared.etcd_client, shared.settings['etcd']['tap_counter']
                 )
                 for _ in range(len(data['network']))
             ]
@@ -84,7 +83,7 @@ class CreateVM(Resource):
             r = RequestEntry.from_scratch(
                 type=RequestType.ScheduleVM,
                 uuid=vm_uuid,
-                request_prefix=settings['etcd']['request_prefix'],
+                request_prefix=shared.settings['etcd']['request_prefix'],
             )
             shared.request_pool.put(r)
 
@@ -99,7 +98,7 @@ class VmStatus(Resource):
         validator = schemas.VMStatusSchema(data)
         if validator.is_valid():
             vm = shared.vm_pool.get(
-                join_path(settings['etcd']['vm_prefix'], data['uuid'])
+                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
             )
             vm_value = vm.value.copy()
             vm_value['ip'] = []
@@ -107,7 +106,7 @@ class VmStatus(Resource):
                 network_name, mac, tap = network_mac_and_tap
                 network = shared.etcd_client.get(
                     join_path(
-                        settings['etcd']['network_prefix'],
+                        shared.settings['etcd']['network_prefix'],
                         data['name'],
                         network_name,
                     ),
@@ -130,7 +129,7 @@ class CreateImage(Resource):
         validator = schemas.CreateImageSchema(data)
         if validator.is_valid():
             file_entry = shared.etcd_client.get(
-                join_path(settings['etcd']['file_prefix'], data['uuid'])
+                join_path(shared.settings['etcd']['file_prefix'], data['uuid'])
             )
             file_entry_value = json.loads(file_entry.value)
 
@@ -144,7 +143,7 @@ class CreateImage(Resource):
             }
             shared.etcd_client.put(
                 join_path(
-                    settings['etcd']['image_prefix'], data['uuid']
+                    shared.settings['etcd']['image_prefix'], data['uuid']
                 ),
                 json.dumps(image_entry_json),
             )
@@ -157,7 +156,7 @@ class ListPublicImages(Resource):
     @staticmethod
     def get():
         images = shared.etcd_client.get_prefix(
-            settings['etcd']['image_prefix'], value_in_json=True
+            shared.settings['etcd']['image_prefix'], value_in_json=True
         )
         r = {'images': []}
         for image in images:
@@ -178,7 +177,7 @@ class VMAction(Resource):
 
         if validator.is_valid():
             vm_entry = shared.vm_pool.get(
-                join_path(settings['etcd']['vm_prefix'], data['uuid'])
+                join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
             )
             action = data['action']
 
@@ -208,7 +207,7 @@ class VMAction(Resource):
                 type='{}VM'.format(action.title()),
                 uuid=data['uuid'],
                 hostname=vm_entry.hostname,
-                request_prefix=settings['etcd']['request_prefix'],
+                request_prefix=shared.settings['etcd']['request_prefix'],
             )
             shared.request_pool.put(r)
             return (
@@ -231,10 +230,10 @@ class VMMigration(Resource):
                 type=RequestType.InitVMMigration,
                 uuid=vm.uuid,
                 hostname=join_path(
-                    settings['etcd']['host_prefix'],
+                    shared.settings['etcd']['host_prefix'],
                     validator.destination.value,
                 ),
-                request_prefix=settings['etcd']['request_prefix'],
+                request_prefix=shared.settings['etcd']['request_prefix'],
             )
 
             shared.request_pool.put(r)
@@ -254,7 +253,7 @@ class ListUserVM(Resource):
 
         if validator.is_valid():
             vms = shared.etcd_client.get_prefix(
-                settings['etcd']['vm_prefix'], value_in_json=True
+                shared.settings['etcd']['vm_prefix'], value_in_json=True
             )
             return_vms = []
             user_vms = filter(
@@ -287,7 +286,7 @@ class ListUserFiles(Resource):
 
         if validator.is_valid():
             files = shared.etcd_client.get_prefix(
-                settings['etcd']['file_prefix'], value_in_json=True
+                shared.settings['etcd']['file_prefix'], value_in_json=True
             )
             return_files = []
             user_files = [f for f in files if f.value['owner'] == data['name']]
@@ -312,7 +311,7 @@ class CreateHost(Resource):
         validator = schemas.CreateHostSchema(data)
         if validator.is_valid():
             host_key = join_path(
-                settings['etcd']['host_prefix'], uuid4().hex
+                shared.settings['etcd']['host_prefix'], uuid4().hex
             )
             host_entry = {
                 'specs': data['specs'],
@@ -354,7 +353,7 @@ class GetSSHKeys(Resource):
 
                 # {user_prefix}/{realm}/{name}/key/
                 etcd_key = join_path(
-                    settings['etcd']['user_prefix'],
+                    shared.settings['etcd']['user_prefix'],
                     data['realm'],
                     data['name'],
                     'key',
@@ -372,7 +371,7 @@ class GetSSHKeys(Resource):
 
                 # {user_prefix}/{realm}/{name}/key/{key_name}
                 etcd_key = join_path(
-                    settings['etcd']['user_prefix'],
+                    shared.settings['etcd']['user_prefix'],
                     data['realm'],
                     data['name'],
                     'key',
@@ -405,7 +404,7 @@ class AddSSHKey(Resource):
 
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                settings['etcd']['user_prefix'],
+                shared.settings['etcd']['user_prefix'],
                 data['realm'],
                 data['name'],
                 'key',
@@ -439,7 +438,7 @@ class RemoveSSHKey(Resource):
 
             # {user_prefix}/{realm}/{name}/key/{key_name}
             etcd_key = join_path(
-                settings['etcd']['user_prefix'],
+                shared.settings['etcd']['user_prefix'],
                 data['realm'],
                 data['name'],
                 'key',
@@ -471,23 +470,23 @@ class CreateNetwork(Resource):
 
             network_entry = {
                 'id': counters.increment_etcd_counter(
-                    shared.etcd_client, settings['etcd']['vxlan_counter']
+                    shared.etcd_client, shared.settings['etcd']['vxlan_counter']
                 ),
                 'type': data['type'],
             }
             if validator.user.value:
                 try:
                     nb = pynetbox.api(
-                        url=settings['netbox']['url'],
-                        token=settings['netbox']['token'],
+                        url=shared.settings['netbox']['url'],
+                        token=shared.settings['netbox']['token'],
                     )
                     nb_prefix = nb.ipam.prefixes.get(
-                        prefix=settings['network']['prefix']
+                        prefix=shared.settings['network']['prefix']
                     )
                     prefix = nb_prefix.available_prefixes.create(
                         data={
                             'prefix_length': int(
-                                settings['network']['prefix_length']
+                                shared.settings['network']['prefix_length']
                             ),
                             'description': '{}\'s network "{}"'.format(
                                 data['name'], data['network_name']
@@ -506,7 +505,7 @@ class CreateNetwork(Resource):
                 network_entry['ipv6'] = 'fd00::/64'
 
             network_key = join_path(
-                settings['etcd']['network_prefix'],
+                shared.settings['etcd']['network_prefix'],
                 data['name'],
                 data['network_name'],
             )
@@ -526,7 +525,7 @@ class ListUserNetwork(Resource):
 
         if validator.is_valid():
             prefix = join_path(
-                settings['etcd']['network_prefix'], data['name']
+                shared.settings['etcd']['network_prefix'], data['name']
             )
             networks = shared.etcd_client.get_prefix(
                 prefix, value_in_json=True
@@ -570,7 +569,7 @@ def main(arguments):
     try:
         image_stores = list(
             shared.etcd_client.get_prefix(
-                settings['etcd']['image_store_prefix'], value_in_json=True
+                shared.settings['etcd']['image_store_prefix'], value_in_json=True
             )
         )
     except KeyError:
@@ -590,7 +589,7 @@ def main(arguments):
 
     #     shared.etcd_client.put(
     #         join_path(
-    #             settings['etcd']['image_store_prefix'], uuid4().hex
+    #             shared.settings['etcd']['image_store_prefix'], uuid4().hex
     #         ),
     #         json.dumps(data),
     #     )

archive/uncloud_etcd_based/uncloud/api/schemas.py

@@ -22,7 +22,6 @@ import bitmath
 from uncloud.common.host import HostStatus
 from uncloud.common.vm import VMStatus
 from uncloud.common.shared import shared
-from uncloud.common.settings import settings
 from . import helper, logger
 from .common_fields import Field, VmUUIDField
 from .helper import check_otp, resolve_vm_name
@@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema):
     def file_uuid_validation(self):
         file_entry = shared.etcd_client.get(
             os.path.join(
-                settings["etcd"]["file_prefix"], self.uuid.value
+                shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
             )
         )
         if file_entry is None:
@@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema):
     def image_store_name_validation(self):
         image_stores = list(
             shared.etcd_client.get_prefix(
-                settings["etcd"]["image_store_prefix"]
+                shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
             )
         )
 
@@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema):
             for net in _network:
                 network = shared.etcd_client.get(
                     os.path.join(
-                        settings["etcd"]["network_prefix"],
+                        shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
                         self.name.value,
                         net,
                     ),
@@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema):
             self.add_error("Can't migrate non-running VM")
 
         if vm.hostname == os.path.join(
-            settings["etcd"]["host_prefix"], self.destination.value
+            shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
         ):
             self.add_error(
                 "Destination host couldn't be same as Source Host"
@@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema):
         super().__init__(data, fields=fields)
 
     def network_name_validation(self):
-        print(self.name.value, self.network_name.value)
-        key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
-        print(key)
+        key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
         network = shared.etcd_client.get(key, value_in_json=True)
         if network:
             self.add_error(

archive/uncloud_etcd_based/uncloud/cli/helper.py

@@ -0,0 +1,46 @@
+import requests
+import json
+import argparse
+import binascii
+
+from pyotp import TOTP
+from os.path import join as join_path
+from uncloud.common.shared import shared
+
+
+def get_otp_parser():
+    otp_parser = argparse.ArgumentParser('otp')
+    otp_parser.add_argument('--name')
+    otp_parser.add_argument('--realm')
+    otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
+
+    return otp_parser
+
+
+def load_dump_pretty(content):
+    if isinstance(content, bytes):
+        content = content.decode('utf-8')
+    parsed = json.loads(content)
+    return json.dumps(parsed, indent=4, sort_keys=True)
+
+
+def make_request(*args, data=None, request_method=requests.post):
+    try:
+        r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
+    except requests.exceptions.RequestException:
+        print('Error occurred while connecting to API server.')
+    else:
+        try:
+            print(load_dump_pretty(r.content))
+        except Exception:
+            print('Error occurred while getting output from api server.')
+
+
+def get_token(seed):
+    if seed is not None:
+        try:
+            token = TOTP(seed).now()
+        except binascii.Error:
+            raise argparse.ArgumentTypeError('Invalid seed')
+        else:
+            return token

archive/uncloud_etcd_based/uncloud/client/main.py

@@ -0,0 +1,23 @@
+import argparse
+import etcd3
+from uncloud.common.etcd_wrapper import Etcd3Wrapper
+
+arg_parser = argparse.ArgumentParser('client', add_help=False)
+arg_parser.add_argument('--dump-etcd-contents-prefix', help="Dump contents below the given prefix")
+
+def dump_etcd_contents(prefix):
+    etcd = Etcd3Wrapper()
+    for k,v in etcd.get_prefix_raw(prefix):
+        k = k.decode('utf-8')
+        v = v.decode('utf-8')
+        print("{} = {}".format(k,v))
+#    print("{} = {}".format(k,v))
+
+#    for k,v in etcd.get_prefix(prefix):
+#
+    print("done")
+
+
+def main(arguments):
+    if 'dump_etcd_contents_prefix' in arguments:
+        dump_etcd_contents(prefix=arguments['dump_etcd_contents_prefix'])

archive/uncloud_etcd_based/uncloud/common/cli.py

@@ -0,0 +1,26 @@
+from uncloud.common.shared import shared
+from pyotp import TOTP
+
+
+def get_token(seed):
+    if seed is not None:
+        try:
+            token = TOTP(seed).now()
+        except Exception:
+            raise Exception('Invalid seed')
+        else:
+            return token
+
+
+def resolve_otp_credentials(kwargs):
+    d = {
+        'name': shared.settings['client']['name'],
+        'realm': shared.settings['client']['realm'],
+        'token': get_token(shared.settings['client']['seed'])
+    }
+
+    for k, v in d.items():
+        if k in kwargs and kwargs[k] is None:
+            kwargs.update({k: v})
+
+    return d

archive/uncloud_etcd_based/uncloud/common/settings.py

@@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper
 from os.path import join as join_path
 
 logger = logging.getLogger(__name__)
+settings = None
 
 
 class CustomConfigParser(configparser.RawConfigParser):
@@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser):
 
 
 class Settings(object):
-    def __init__(self):
+    def __init__(self, conf_dir, seed_value=None):
         conf_name = 'uncloud.conf'
-        conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
         self.config_file = join_path(conf_dir, conf_name)
 
         # this is used to cache config from etcd for 1 minutes. Without this we
@@ -38,15 +38,19 @@ class Settings(object):
         self.config_parser.add_section('etcd')
         self.config_parser.set('etcd', 'base_prefix', '/')
 
-        try:
+        if os.access(self.config_file, os.R_OK):
             self.config_parser.read(self.config_file)
-        except Exception as err:
-            logger.error('%s', err)
-
+        else:
+            raise FileNotFoundError('Config file %s not found!', self.config_file)
         self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
 
         self.read_internal_values()
 
+        if seed_value is None:
+            seed_value = dict()
+
+        self.config_parser.read_dict(seed_value)
+
     def get_etcd_client(self):
         args = tuple()
         try:
@@ -128,4 +132,5 @@ class Settings(object):
         return self.config_parser[key]
 
 
-settings = Settings()
+def get_settings():
+    return settings

archive/uncloud_etcd_based/uncloud/common/shared.py

@@ -0,0 +1,34 @@
+from uncloud.common.settings import get_settings
+from uncloud.common.vm import VmPool
+from uncloud.common.host import HostPool
+from uncloud.common.request import RequestPool
+import uncloud.common.storage_handlers as storage_handlers
+
+
+class Shared:
+    @property
+    def settings(self):
+        return get_settings()
+
+    @property
+    def etcd_client(self):
+        return self.settings.get_etcd_client()
+
+    @property
+    def host_pool(self):
+        return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
+
+    @property
+    def vm_pool(self):
+        return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
+
+    @property
+    def request_pool(self):
+        return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
+
+    @property
+    def storage_handler(self):
+        return storage_handlers.get_storage_handler()
+
+
+shared = Shared()

archive/uncloud_etcd_based/uncloud/common/storage_handlers.py

@@ -6,8 +6,7 @@ import stat
 from abc import ABC
 from . import logger
 from os.path import join as join_path
-
-from uncloud.common.settings import settings as config
+import uncloud.common.shared as shared
 
 
 class ImageStorageHandler(ABC):
@@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
 
 
 def get_storage_handler():
-    __storage_backend = config["storage"]["storage_backend"]
+    __storage_backend = shared.shared.settings["storage"]["storage_backend"]
     if __storage_backend == "filesystem":
         return FileSystemBasedImageStorageHandler(
-            vm_base=config["storage"]["vm_dir"],
-            image_base=config["storage"]["image_dir"],
+            vm_base=shared.shared.settings["storage"]["vm_dir"],
+            image_base=shared.shared.settings["storage"]["image_dir"],
         )
     elif __storage_backend == "ceph":
         return CEPHBasedImageStorageHandler(
-            vm_base=config["storage"]["ceph_vm_pool"],
-            image_base=config["storage"]["ceph_image_pool"],
+            vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
+            image_base=shared.shared.settings["storage"]["ceph_image_pool"],
         )
     else:
         raise Exception("Unknown Image Storage Handler")

archive/uncloud_etcd_based/uncloud/configure/main.py

@@ -1,7 +1,6 @@
 import os
 import argparse
 
-from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 
 arg_parser = argparse.ArgumentParser('configure', add_help=False)
@@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
 
 
 def update_config(section, kwargs):
-    uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
+    uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
     if not uncloud_config:
         uncloud_config = {}
     else:
         uncloud_config = uncloud_config.value
 
     uncloud_config[section] = kwargs
-    shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
+    shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
 
 
-def main(**kwargs):
-    subcommand = kwargs.pop('subcommand')
+def main(arguments):
+    subcommand = arguments['subcommand']
     if not subcommand:
         arg_parser.print_help()
     else:
-        update_config(subcommand, kwargs)
+        update_config(subcommand, arguments)

archive/uncloud_etcd_based/uncloud/filescanner/main.py

@@ -9,7 +9,6 @@ import bitmath
 from uuid import uuid4
 
 from . import logger
-from uncloud.common.settings import settings
 from uncloud.common.shared import shared
 
 arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
@@ -53,7 +52,7 @@ def track_file(file, base_dir, host):
         file_path = file_path.relative_to(owner)
         creation_date = time.ctime(os.stat(file_str).st_ctime)
 
-        entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
+        entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
         entry_value = {
             'filename': str(file_path),
             'owner': owner,
@@ -70,7 +69,7 @@ def track_file(file, base_dir, host):
 
 def main(arguments):
     hostname = arguments['hostname']
-    base_dir = settings['storage']['file_dir']
+    base_dir = shared.settings['storage']['file_dir']
     # Recursively Get All Files and Folder below BASE_DIR
     files = glob.glob('{}/**'.format(base_dir), recursive=True)
     files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
@@ -78,7 +77,7 @@ def main(arguments):
     # Files that are already tracked
     tracked_files = [
         pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
-        for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
+        for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
         if f.value['host'] == hostname
     ]
     untracked_files = set(files) - set(tracked_files)

archive/uncloud_etcd_based/uncloud/hack/__init__.py

@@ -0,0 +1 @@
+

archive/uncloud_etcd_based/uncloud/hack/config.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
+#
+# This file is part of uncloud.
+#
+# uncloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uncloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+class Config(object):
+    def __init__(self, arguments):
+        """ read arguments dicts as a base """
+
+        self.arguments = arguments
+
+        # Split them so *etcd_args can be used and we can
+        # iterate over etcd_hosts
+        self.etcd_hosts = [ arguments['etcd_host'] ]
+        self.etcd_args = {
+            'ca_cert': arguments['etcd_ca_cert'],
+            'cert_cert': arguments['etcd_cert_cert'],
+            'cert_key': arguments['etcd_cert_key'],
+#            'user': None,
+#            'password': None
+        }
+        self.etcd_prefix = '/nicohack/'

archive/uncloud_etcd_based/uncloud/hack/db.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
+#
+# This file is part of uncloud.
+#
+# uncloud is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# uncloud is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import etcd3
+import json
+import logging
+import datetime
+import re
+
+from functools import wraps
+from uncloud import UncloudException
+
+log = logging.getLogger(__name__)
+
+def db_logentry(message):
+    timestamp = datetime.datetime.now()
+    return {
+        "timestamp": str(timestamp),
+        "message": message
+    }
+
+
+def readable_errors(func):
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except etcd3.exceptions.ConnectionFailedError as e:
+            raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
+        except etcd3.exceptions.ConnectionTimeoutError as e:
+            raise UncloudException('etcd connection timeout. {}'.format(e))
+
+    return wrapper
+
+
+class DB(object):
+    def __init__(self, config, prefix="/"):
+        self.config = config
+
+        # Root for everything
+        self.base_prefix= '/nicohack'
+
+        # Can be set from outside
+        self.prefix = prefix
+
+        try:
+            self.connect()
+        except FileNotFoundError as e:
+            raise UncloudException("Is the path to the etcd certs correct? {}".format(e))
+
+    @readable_errors
+    def connect(self):
+        self._db_clients = []
+        for endpoint in self.config.etcd_hosts:
+            client = etcd3.client(host=endpoint, **self.config.etcd_args)
+            self._db_clients.append(client)
+
+    def realkey(self, key):
+        return "{}{}/{}".format(self.base_prefix,
+                                self.prefix,
+                                key)
+
+    @readable_errors
+    def get(self, key, as_json=False, **kwargs):
+        value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
+
+        if as_json:
+            value = json.loads(value)
+
+        return value
+
+    @readable_errors
+    def get_prefix(self, key, as_json=False, **kwargs):
+        for value, meta in self._db_clients[0].get_prefix(self.realkey(key), **kwargs):
+            k = meta.key.decode("utf-8")
+            value = value.decode("utf-8")
+            if as_json:
+                value = json.loads(value)
+
+            yield (k, value)
+
+
+    @readable_errors
+    def set(self, key, value, as_json=False, **kwargs):
+        if as_json:
+            value = json.dumps(value)
+
+        log.debug("Setting {} = {}".format(self.realkey(key), value))
+        # FIXME: iterate over clients in case of failure ?
+        return self._db_clients[0].put(self.realkey(key), value, **kwargs)
+
+
+    @readable_errors
+    def list_and_filter(self, key, filter_key=None, filter_regexp=None):
+        for k,v in self.get_prefix(key, as_json=True):
+
+            if filter_key and filter_regexp:
+                if filter_key in v:
+                    if re.match(filter_regexp, v[filter_key]):
+                        yield v
+            else:
+                yield v
+
+
+    @readable_errors
+    def increment(self, key, **kwargs):
+        print(self.realkey(key))
+
+
+        print("prelock")
+        lock = self._db_clients[0].lock('/nicohack/foo')
+        print("prelockacq")
+        lock.acquire()
+        print("prelockrelease")
+        lock.release()
+
+        with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock:
+            print("in lock")
+            pass
+
+#        with self._db_clients[0].lock(self.realkey(key)) as lock:#            value = int(self.get(self.realkey(key), **kwargs))
+#            self.set(self.realkey(key), str(value + 1), **kwargs)
+
+
+if __name__ == '__main__':
+    endpoints = [ "https://etcd1.ungleich.ch:2379",
+                  "https://etcd2.ungleich.ch:2379",
+                  "https://etcd3.ungleich.ch:2379" ]
+
+    db = DB(url=endpoints)

archive/uncloud_etcd_based/uncloud/hack/hackcloud/__init__.py

@@ -0,0 +1 @@
+

archive/uncloud_etcd_based/uncloud/hack/hackcloud/etcd-client.sh

@@ -0,0 +1,6 @@
+#!/bin/sh
+
+etcdctl  --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \
+         --key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \
+         --cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \
+         --endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@"

archive/uncloud_etcd_based/uncloud/hack/hackcloud/ifdown.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo $@

archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-last

@@ -0,0 +1 @@
+000000000252

archive/uncloud_etcd_based/uncloud/hack/hackcloud/mac-prefix

@@ -0,0 +1 @@
+02:00