Merge branch 'master' of code.ungleich.ch:uncloud/uncloud

Signed-off-by: Nico Schottelius <nico@nico-notebook.schottelius.org>
This commit is contained in:
Nico Schottelius 2020-05-17 22:38:49 +02:00
commit 04fac71a85
284 changed files with 13011 additions and 345 deletions

13
.gitignore vendored
View File

@ -1,6 +1,12 @@
.idea
.vscode
.idea/
.vscode/
__pycache__/
pay.conf
log.txt
test.py
STRIPE
venv/
uncloud/docs/build
logs.txt
@ -16,3 +22,6 @@ uncloud/version.py
build/
venv/
dist/
*.iso
*.sqlite3

18
.gitlab-ci.yml Normal file
View File

@ -0,0 +1,18 @@
stages:
- lint
- test
run-tests:
stage: test
image: code.ungleich.ch:5050/uncloud/uncloud/uncloud-ci:latest
services:
- postgres:latest
variables:
DATABASE_HOST: postgres
DATABASE_USER: postgres
POSTGRES_HOST_AUTH_METHOD: trust
coverage: /^TOTAL.+?(\d+\%)$/
script:
- pip install -r requirements.txt
- coverage run --source='.' ./manage.py test
- coverage report

View File

@ -1,3 +1,62 @@
# ucloud
# Uncloud
Checkout https://ungleich.ch/ucloud/ for the documentation of ucloud.
Cloud management platform, the ungleich way.
[![pipeline status](https://code.ungleich.ch/uncloud/uncloud/badges/master/pipeline.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master)
[![coverage report](https://code.ungleich.ch/uncloud/uncloud/badges/master/coverage.svg)](https://code.ungleich.ch/uncloud/uncloud/commits/master)
## Useful commands
* `./manage.py import-vat-rates path/to/csv`
* `./manage.py make-admin username`
## Development setup
Install system dependencies:
* On Fedora, you will need the following packages: `python3-virtualenv python3-devel openldap-devel gcc chromium`
NOTE: you will need to configure a LDAP server and credentials for authentication. See `uncloud/settings.py`.
```
# Initialize virtualenv.
» virtualenv .venv
Using base prefix '/usr'
New python executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python3
Also creating executable in /home/fnux/Workspace/ungleich/uncloud/uncloud/.venv/bin/python
Installing setuptools, pip, wheel...
done.
# Enter virtualenv.
» source .venv/bin/activate
# Install dependencies.
» pip install -r requirements.txt
[...]
# Run migrations.
» ./manage.py migrate
Operations to perform:
Apply all migrations: admin, auth, contenttypes, opennebula, sessions, uncloud_auth, uncloud_net, uncloud_pay, uncloud_service, uncloud_vm
Running migrations:
[...]
# Run webserver.
» ./manage.py runserver
Watching for file changes with StatReloader
Performing system checks...
System check identified no issues (0 silenced).
May 07, 2020 - 10:17:08
Django version 3.0.6, using settings 'uncloud.settings'
Starting development server at http://127.0.0.1:8000/
Quit the server with CONTROL-C.
```
### Note on PGSQL
If you want to use Postgres:
* Install on configure PGSQL on your base system.
* OR use a container! `podman run --rm -p 5432:5432 -e POSTGRES_HOST_AUTH_METHOD=trust -it postgres:latest`

6
archive/issues.org Normal file
View File

@ -0,0 +1,6 @@
* Intro
This file lists issues that should be handled, are small and likely
not yet high prio.
* Issues
** TODO Register prefered address in User model
** TODO Allow to specify different recurring periods

View File

@ -0,0 +1,55 @@
"""
investigate into a simple python function that maps an ldap user to a vat percentage. Basically you need to
lookup the customer address, check if she is a business/registered tax number and if not apply the local
vat
"""
import iso3166
import datetime
from csv import DictReader
def get_vat(street_address, city, postal_code, country, vat_number=None):
vat = {
'Austria': [
{'period': '1984-01-01/', 'rate': 0.2},
{'period': '1976-01-01/1984-01-01', 'rate': 0.18},
{'period': '1973-01-01/1976-01-01', 'rate': 0.16},
]
}
return iso3166.countries.get(country)
# return iso3166.countries_by_name[country]
def main():
# vat = get_vat(
# street_address='82 Nasheman-e-Iqbal near Wapda Town',
# city='Lahore',
# postal_code=53700,
# country='Pakistan',
# )
# print(vat)
vat_rates = {}
with open('vat_rates.csv', newline='') as csvfile:
reader = DictReader(csvfile)
for row in reader:
territory_codes = row['territory_codes'].split('\n')
for code in territory_codes:
if code not in vat_rates:
vat_rates[code] = {}
start_date = row['start_date']
stop_data = row['stop_date']
time_period = f'{start_date}|{stop_data}'
r = row.copy()
del r['start_date']
del r['stop_date']
del r['territory_codes']
vat_rates[code][time_period] = r
print(vat_rates)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,46 @@
import importlib
import sys
import os
from os.path import join as join_path
from xmlrpc.client import ServerProxy as RPCClient
root = os.path.dirname(os.getcwd())
sys.path.append(join_path(root, 'uncloud'))
secrets = importlib.import_module('uncloud.secrets')
class OpenNebula:
def __init__(self, url, session_string):
self.session_string = session_string
self.client = RPCClient(secrets.OPENNEBULA_URL)
def create_user(self, username, password, authentication_driver='', group_id=None):
# https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-user-allocate
if group_id is None:
group_id = []
return self.client.one.user.allocate(
self.session_string,
username,
password,
authentication_driver,
group_id
)
def chmod(self, vm_id, user_id=-1, group_id=-1):
# https://docs.opennebula.org/5.10/integration/system_interfaces/api.html#one-vm-chown
return self.client.one.vm.chown(self.session_string, vm_id, user_id, group_id)
one = OpenNebula(secrets.OPENNEBULA_URL, secrets.OPENNEBULA_USER_PASS)
# Create User in OpenNebula
# success, response, *_ = one.create_user(username='meow12345', password='hello_world')
# print(success, response)
# Change owner of a VM
# success, response, *_ = one.chmod(vm_id=25589, user_id=706)
# print(success, response)

View File

@ -0,0 +1,18 @@
#!/bin/sh
dbhost=$1; shift
ssh -L5432:localhost:5432 "$dbhost" &
python manage.py "$@"
# command only needs to be active while manage command is running
# -T no pseudo terminal
# alternatively: commands output shell code
# ssh uncloud@dbhost "python manage.py --hostname xxx ..."

View File

@ -0,0 +1,51 @@
# uncloud-pay
The generic product/payment system.
## Installation
```shell script
pip3 install -r requirements.txt
```
## Getting Started
```shell script
python ucloud_pay.py
```
## Usage
#### 1. Adding of products
```shell script
http --json http://[::]:5000/product/add username=your_username_here password=your_password_here specs:=@ipv6-only-vm.json
```
#### 2. Listing of products
```shell script
http --json http://[::]:5000/product/list
```
#### 3. Registering user's payment method (credit card for now using Stripe)
```shell script
http --json http://[::]:5000/user/register_payment card_number=4111111111111111 cvc=123 expiry_year=2020 expiry_month=8 card_holder_name="The test user" username=your_username_here password=your_password_here line1="your_billing_address" city="your_city" country="your_country"
```
#### 4. Ordering products
First of all, user have to buy the membership first.
```shell script
http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=membership pay=True
```
```shell script
http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=ipv6-only-vm cpu=1 ram=1 os-disk-space=10 os=alpine pay=True
```
#### 5. Listing users orders
```shell script
http --json POST http://[::]:5000/order/list username=your_username_here password=your_password_here
```

View File

@ -0,0 +1,21 @@
import os
from ungleich_common.ldap.ldap_manager import LdapManager
from ungleich_common.std.configparser import StrictConfigParser
from ungleich_common.etcd.etcd_wrapper import EtcdWrapper
config_file = os.environ.get('meow-pay-config-file', default='pay.conf')
config = StrictConfigParser(allow_no_value=True)
config.read(config_file)
etcd_client = EtcdWrapper(
host=config.get('etcd', 'host'), port=config.get('etcd', 'port'),
ca_cert=config.get('etcd', 'ca_cert'), cert_key=config.get('etcd', 'cert_key'),
cert_cert=config.get('etcd', 'cert_cert')
)
ldap_manager = LdapManager(
server=config.get('ldap', 'server'), admin_dn=config.get('ldap', 'admin_dn'),
admin_password=config.get('ldap', 'admin_password')
)

View File

@ -0,0 +1,213 @@
from flask import Flask, request
from flask_restful import Resource, Api
import etcd3
import json
import logging
from functools import wraps
from ldaptest import is_valid_ldap_user
def authenticate(func):
@wraps(func)
def wrapper(*args, **kwargs):
if not getattr(func, 'authenticated', True):
return func(*args, **kwargs)
# pass in username/password !
acct = basic_authentication() # custom account lookup function
if acct:
return func(*args, **kwargs)
flask_restful.abort(401)
return wrapper
def readable_errors(func):
@wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except etcd3.exceptions.ConnectionFailedError as e:
raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
except etcd3.exceptions.ConnectionTimeoutError as e:
raise UncloudException('etcd connection timeout. {}'.format(e))
return wrapper
class DB(object):
def __init__(self, config, prefix="/"):
self.config = config
# Root for everything
self.base_prefix= '/nicohack'
# Can be set from outside
self.prefix = prefix
self.connect()
@readable_errors
def connect(self):
self._db_clients = []
for endpoint in self.config.etcd_hosts:
client = etcd3.client(host=endpoint, **self.config.etcd_args)
self._db_clients.append(client)
def realkey(self, key):
return "{}{}/{}".format(self.base_prefix,
self.prefix,
key)
@readable_errors
def get(self, key, as_json=False, **kwargs):
value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
if as_json:
value = json.loads(value)
return value
@readable_errors
def set(self, key, value, as_json=False, **kwargs):
if as_json:
value = json.dumps(value)
# FIXME: iterate over clients in case of failure ?
return self._db_clients[0].put(self.realkey(key), value, **kwargs)
class Membership(Resource):
def __init__(self, config):
self.config = config
def get(self):
data = request.get_json(silent=True) or {}
print("{} {}".format(data, config))
return {'message': 'Order successful' }, 200
def post(self):
data = request.get_json(silent=True) or {}
print("{} {}".format(data, config))
return {'message': 'Order 2x successful' }, 200
class Order(Resource):
def __init__(self, config):
self.config = config
@staticmethod
def post():
data = request.get_json(silent=True) or {}
print("{} {}".format(data, config))
class Product(Resource):
def __init__(self, config):
self.config = config
self.products = []
self.products.append(
{ "name": "membership-free",
"description": """
This membership gives you access to the API and includes a VPN
with 1 IPv6 address.
See https://redmine.ungleich.ch/issues/7747?
""",
"uuid": "a3883466-0012-4d01-80ff-cbf7469957af",
"recurring": True,
"recurring_time_frame": "per_year",
"features": [
{ "name": "membership",
"price_one_time": 0,
"price_recurring": 0
}
]
}
)
self.products.append(
{ "name": "membership-standard",
"description": """
This membership gives you access to the API and includes an IPv6-VPN with
one IPv6 address ("Road warrior")
See https://redmine.ungleich.ch/issues/7747?
""",
"uuid": "1d85296b-0863-4dd6-a543-a6d5a4fbe4a6",
"recurring": True,
"recurring_time_frame": "per_month",
"features": [
{ "name": "membership",
"price_one_time": 0,
"price_recurring": 5
}
]
}
)
self.products.append(
{ "name": "membership-premium",
"description": """
This membership gives you access to the API and includes an
IPv6-VPN with a /48 IPv6 network.
See https://redmine.ungleich.ch/issues/7747?
""",
"uuid": "bfd63fd2-d227-436f-a8b8-600de74dd6ce",
"recurring": True,
"recurring_time_frame": "per_month",
"features": [
{ "name": "membership",
"price_one_time": 0,
"price_recurring": 5
}
]
}
)
self.products.append(
{ "name": "ipv6-vpn-with-/48",
"description": """
An IPv6 VPN with a /48 network included.
""",
"uuid": "fe5753f8-6fe1-4dc4-9b73-7b803de4c597",
"recurring": True,
"recurring_time_frame": "per_year",
"features": [
{ "name": "vpn",
"price_one_time": 0,
"price_recurring": 120
}
]
}
)
@staticmethod
def post():
data = request.get_json(silent=True) or {}
print("{} {}".format(data, config))
def get(self):
data = request.get_json(silent=True) or {}
print("{} {}".format(data, config))
return self.products
if __name__ == '__main__':
app = Flask(__name__)
config = {}
config['etcd_url']="https://etcd1.ungleich.ch"
config['ldap_url']="ldaps://ldap1.ungleich.ch"
api = Api(app)
api.add_resource(Order, '/orders', resource_class_args=( config, ))
api.add_resource(Product, '/products', resource_class_args=( config, ))
api.add_resource(Membership, '/membership', resource_class_args=( config, ))
app.run(host='::', port=5000, debug=True)

View File

@ -0,0 +1,87 @@
import logging
import parsedatetime
from datetime import datetime
from stripe_utils import StripeUtils
def get_plan_id_from_product(product):
plan_id = 'ucloud-v1-'
plan_id += product['name'].strip().replace(' ', '-')
return plan_id
def get_pricing(price_in_chf_cents, product_type, recurring_period):
if product_type == 'recurring':
return 'CHF {}/{}'.format(price_in_chf_cents/100, recurring_period)
elif product_type == 'one-time':
return 'CHF {} (One time charge)'.format(price_in_chf_cents/100)
def get_user_friendly_product(product_dict):
uf_product = {
'name': product_dict['name'],
'description': product_dict['description'],
'product_id': product_dict['usable-id'],
'pricing': get_pricing(
product_dict['price'], product_dict['type'], product_dict['recurring_period']
)
}
if product_dict['type'] == 'recurring':
uf_product['minimum_subscription_period'] = product_dict['minimum_subscription_period']
return uf_product
def get_token(card_number, cvc, exp_month, exp_year):
stripe_utils = StripeUtils()
token_response = stripe_utils.get_token_from_card(
card_number, cvc, exp_month, exp_year
)
if token_response['response_object']:
return token_response['response_object'].id
else:
return None
def resolve_product(usable_id, etcd_client):
products = etcd_client.get_prefix('/v1/products/', value_in_json=True)
for p in products:
if p.value['usable-id'] == usable_id:
return p.value
return None
def calculate_charges(specification, data):
logging.debug('Calculating charges for specs:{} and data:{}'.format(specification, data))
one_time_charge = 0
recurring_charge = 0
for feature_name, feature_detail in specification['features'].items():
if feature_detail['constant']:
data[feature_name] = 1
if feature_detail['unit']['type'] != 'str':
one_time_charge += feature_detail['one_time_fee']
recurring_charge += (
feature_detail['price_per_unit_per_period'] * data[feature_name] /
feature_detail['unit']['value']
)
return one_time_charge, recurring_charge
def is_order_valid(order_timestamp, renewal_period):
"""
Sample Code Usage
>> current_datetime, status = cal.parse('Now')
>> current_datetime = datetime(*current_datetime[:6])
>> print('Is order valid: ', is_order_valid(current_datetime, '1 month'))
>> True
"""
cal = parsedatetime.Calendar()
renewal_datetime, status = cal.parse(renewal_period)
renewal_datetime = datetime(*renewal_datetime[:6])
return order_timestamp <= renewal_datetime

View File

@ -0,0 +1,28 @@
{
"usable-id": "ipv6-only-django-hosting",
"active": true,
"name": "IPv6 Only Django Hosting",
"description": "Host your Django application on our shiny IPv6 Only VM",
"recurring_period": "month",
"quantity": "inf",
"features": {
"cpu": {
"unit": {"value": 1, "type":"int"},
"price_per_unit_per_period": 3,
"one_time_fee": 0,
"constant": false
},
"ram": {
"unit": {"value": 1, "type":"int"},
"price_per_unit_per_period": 4,
"one_time_fee": 0,
"constant": false
},
"os-disk-space": {
"unit": {"value": 10, "type":"int"},
"one_time_fee": 0,
"price_per_unit_per_period": 3.5,
"constant": false
}
}
}

View File

@ -0,0 +1,34 @@
{
"usable-id": "ipv6-only-vm",
"active": true,
"name": "IPv6 Only VM",
"description": "IPv6 Only VM are accessible to only those having IPv6 for themselves",
"recurring_period": "month",
"quantity": "inf",
"features": {
"cpu": {
"unit": {"value": 1, "type":"int"},
"price_per_unit_per_period": 3,
"one_time_fee": 0,
"constant": false
},
"ram": {
"unit": {"value": 1, "type":"int"},
"price_per_unit_per_period": 4,
"one_time_fee": 0,
"constant": false
},
"os-disk-space": {
"unit": {"value": 10, "type":"int"},
"one_time_fee": 0,
"price_per_unit_per_period": 4,
"constant": false
},
"os": {
"unit": {"value": 1, "type":"str"},
"one_time_fee": 0,
"price_per_unit_per_period": 0,
"constant": false
}
}
}

View File

@ -0,0 +1,16 @@
{
"usable-id": "ipv6-only-vpn",
"active": true,
"name": "IPv6 Only VPN",
"description": "IPv6 VPN enable you to access IPv6 only websites and more",
"recurring_period": "month",
"quantity": "inf",
"features": {
"vpn": {
"unit": {"value": 1, "type": "int"},
"price_per_unit_per_period": 10,
"one_time_fee": 0,
"constant": true
}
}
}

View File

@ -0,0 +1,16 @@
{
"usable-id": "ipv6-box",
"active": true,
"name": "IPv6 Box",
"description": "A ready-to-go IPv6 Box: it creates a VPN to ungleich and distributes IPv6 addresses to all your computers.",
"recurring_period": "eternity",
"quantity": 4,
"features": {
"ipv6-box": {
"unit": {"value": 1, "type":"int"},
"price_per_unit_per_period": 0,
"one_time_fee": 250,
"constant": true
}
}
}

View File

@ -0,0 +1,17 @@
{
"usable-id": "membership",
"active": true,
"name": "Membership",
"description": "Membership to use uncloud-pay",
"recurring_period": "month",
"quantity": "inf",
"features": {
"membership": {
"unit": {"value": 1, "type":"int"},
"price_per_unit_per_period": 5,
"one_time_fee": 0,
"constant": true
}
},
"max_per_user": "1"
}

View File

@ -0,0 +1,7 @@
stripe
flask
Flask-RESTful
git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-etcd&subdirectory=etcd
git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap
git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-std&subdirectory=std
git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-schemas&subdirectory=schemas

View File

@ -0,0 +1,17 @@
[etcd]
host = 127.0.0.1
port = 2379
ca_cert
cert_cert
cert_key
[stripe]
private_key=stripe_private_key
[app]
port = 5000
[ldap]
server = ldap_server_url
admin_dn = ldap_admin_dn
admin_password = ldap_admin_password

View File

@ -0,0 +1,136 @@
import logging
import config
import json
import math
from config import ldap_manager, etcd_client
from helper import resolve_product
from ungleich_common.schemas.schemas import BaseSchema, Field, ValidationException
class AddProductSchema(BaseSchema):
def __init__(self, data):
super().__init__()
self.add_schema(UserCredentialSchema, data)
self.specs = Field('specs', dict, **self.get(data, 'specs'))
self.update = Field('update', bool, **self.get(data, 'update', return_default=True, default=False))
def validation(self):
user = self.objects['user']
user = json.loads(user.entry_to_json())
uid, ou, *dc = user['dn'].replace('ou=', '').replace('dc=', '').replace('uid=', '').split(',')
if ou != config.config.get('ldap', 'internal_user_ou', fallback='users'):
raise ValidationException('You do not have access to create product.')
product = resolve_product(self.specs.value['usable-id'], etcd_client)
if product:
self.objects['product'] = product
class AddressSchema(BaseSchema):
def __init__(self, data):
super().__init__()
self.line1 = Field('line1', str, **self.get(data, 'line1'))
self.line2 = Field('line2', str, **self.get(data, 'line2', return_default=True))
self.city = Field('city', str, **self.get(data, 'city'))
self.country = Field('country', str, **self.get(data, 'country'))
self.state = Field('state', str, **self.get(data, 'state', return_default=True))
self.postal_code = Field('postal_code', str, **self.get(data, 'postal_code', return_default=True))
class UserRegisterPaymentSchema(BaseSchema):
def __init__(self, data):
super().__init__()
self.add_schema(UserCredentialSchema, data)
self.add_schema(AddressSchema, data, under_field_name='address')
self.card_number = Field('card_number', str, **self.get(data, 'card_number'))
self.cvc = Field('cvc', str, **self.get(data, 'cvc'))
self.expiry_year = Field('expiry_year', int, **self.get(data, 'expiry_year'))
self.expiry_month = Field('expiry_month', int, **self.get(data, 'expiry_month'))
self.card_holder_name = Field('card_holder_name', str, **self.get(data, 'card_holder_name'))
class UserCredentialSchema(BaseSchema):
def __init__(self, data):
super().__init__()
self.username = Field('username', str, **self.get(data, 'username'))
self.password = Field('password', str, **self.get(data, 'password'))
def validation(self):
try:
entry = ldap_manager.is_password_valid(self.username.value, self.password.value, query_key='uid')
except ValueError:
raise ValidationException('No user with \'{}\' username found. You can create account at '
'https://account.ungleich.ch'.format(self.username.value))
except Exception:
raise ValidationException('Invalid username/password.')
else:
self.objects['user'] = entry
class ProductOrderSchema(BaseSchema):
def __init__(self, data):
super().__init__()
self.product_id = Field(
'product_id', str, **self.get(data, 'product_id'), validators=[self.product_id_validation]
)
self.pay_consent = Field('pay', bool, **self.get(data, 'pay', return_default=True, default=False))
self.add_schema(UserCredentialSchema, data)
def product_id_validation(self):
product = resolve_product(self.product_id.value, etcd_client)
if product:
product['quantity'] = float(product['quantity'])
self.product_id.value = product['uuid']
self.objects['product'] = product
logging.debug('Got product {}'.format(product))
if not product['active']:
raise ValidationException('Product is not active at the moment.')
if product['quantity'] <= 0:
raise ValidationException('Out of stock.')
else:
raise ValidationException('No such product exists.')
def validation(self):
username = self.objects['user'].uid
customer_previous_orders = etcd_client.get_prefix('/v1/user/{}'.format(username), value_in_json=True)
customer_previous_orders = [o.value for o in customer_previous_orders]
membership = next(filter(lambda o: o['product'] == 'membership', customer_previous_orders), None)
if membership is None and self.objects['product']['usable-id'] != 'membership':
raise ValidationException('Please buy membership first to use this facility')
max_quantity_user_can_order = float(self.objects['product'].get('max_per_user', math.inf))
previous_order_of_same_product = [
o for o in customer_previous_orders if o['product'] == self.objects['product']['usable-id']
]
if len(previous_order_of_same_product) >= max_quantity_user_can_order:
raise ValidationException(
'You cannot buy {} more than {} times'.format(
self.objects['product']['name'], int(max_quantity_user_can_order)
)
)
class OrderListSchema(BaseSchema):
def __init__(self, data):
super().__init__()
self.add_schema(UserCredentialSchema, data)
def make_return_message(err, status_code=200):
logging.debug('message: {}'.format(str(err)))
return {'message': str(err)}, status_code
def create_schema(specification, data):
fields = {}
for feature_name, feature_detail in specification['features'].items():
if not feature_detail['constant']:
fields[feature_name] = Field(
feature_name, eval(feature_detail['unit']['type']), **BaseSchema.get(data, feature_name)
)
return type('{}Schema'.format(specification['name']), (BaseSchema,), fields)

View File

@ -0,0 +1,7 @@
import stripe_utils
import os
if __name__ == '__main__':
s = stripe_utils.StripeUtils(os.environ['STRIPE_PRIVATE_KEY'])
print(s.get_stripe_customer_from_email('coder.purple+2002@gmail.com'))

View File

@ -0,0 +1,491 @@
import re
import stripe
import stripe.error
import logging
from config import etcd_client as client, config as config
stripe.api_key = config.get('stripe', 'private_key')
def handle_stripe_error(f):
def handle_problems(*args, **kwargs):
response = {
'paid': False,
'response_object': None,
'error': None
}
common_message = "Currently it's not possible to make payments."
try:
response_object = f(*args, **kwargs)
response = {
'response_object': response_object,
'error': None
}
return response
except stripe.error.CardError as e:
# Since it's a decline, stripe.error.CardError will be caught
body = e.json_body
err = body['error']
response.update({'error': err['message']})
logging.error(str(e))
return response
except stripe.error.RateLimitError:
response.update(
{'error': "Too many requests made to the API too quickly"})
return response
except stripe.error.InvalidRequestError as e:
logging.error(str(e))
response.update({'error': "Invalid parameters"})
return response
except stripe.error.AuthenticationError as e:
# Authentication with Stripe's API failed
# (maybe you changed API keys recently)
logging.error(str(e))
response.update({'error': common_message})
return response
except stripe.error.APIConnectionError as e:
logging.error(str(e))
response.update({'error': common_message})
return response
except stripe.error.StripeError as e:
# maybe send email
logging.error(str(e))
response.update({'error': common_message})
return response
except Exception as e:
# maybe send email
logging.error(str(e))
response.update({'error': common_message})
return response
return handle_problems
class StripeUtils(object):
CURRENCY = 'chf'
INTERVAL = 'month'
SUCCEEDED_STATUS = 'succeeded'
STRIPE_PLAN_ALREADY_EXISTS = 'Plan already exists'
STRIPE_NO_SUCH_PLAN = 'No such plan'
PLAN_EXISTS_ERROR_MSG = 'Plan {} exists already.\nCreating a local StripePlan now.'
PLAN_DOES_NOT_EXIST_ERROR_MSG = 'Plan {} does not exist.'
def __init__(self, private_key):
self.stripe = stripe
stripe.api_key = private_key
@handle_stripe_error
def card_exists(self, customer, cc_number, exp_month, exp_year, cvc):
token_obj = stripe.Token.create(
card={
'number': cc_number,
'exp_month': exp_month,
'exp_year': exp_year,
'cvc': cvc,
},
)
cards = stripe.Customer.list_sources(
customer,
limit=20,
object='card'
)
for card in cards.data:
if (card.fingerprint == token_obj.card.fingerprint and
int(card.exp_month) == int(exp_month) and int(card.exp_year) == int(exp_year)):
return True
return False
@staticmethod
def get_stripe_customer_from_email(email):
customer = stripe.Customer.list(limit=1, email=email)
return customer.data[0] if len(customer.data) == 1 else None
@staticmethod
def update_customer_token(customer, token):
customer.source = token
customer.save()
@handle_stripe_error
def get_token_from_card(self, cc_number, cvc, expiry_month, expiry_year):
token_obj = stripe.Token.create(
card={
'number': cc_number,
'exp_month': expiry_month,
'exp_year': expiry_year,
'cvc': cvc,
},
)
return token_obj
@handle_stripe_error
def associate_customer_card(self, stripe_customer_id, token,
set_as_default=False):
customer = stripe.Customer.retrieve(stripe_customer_id)
card = customer.sources.create(source=token)
if set_as_default:
customer.default_source = card.id
customer.save()
return True
@handle_stripe_error
def dissociate_customer_card(self, stripe_customer_id, card_id):
customer = stripe.Customer.retrieve(stripe_customer_id)
card = customer.sources.retrieve(card_id)
card.delete()
@handle_stripe_error
def update_customer_card(self, customer_id, token):
customer = stripe.Customer.retrieve(customer_id)
current_card_token = customer.default_source
customer.sources.retrieve(current_card_token).delete()
customer.source = token
customer.save()
credit_card_raw_data = customer.sources.data.pop()
new_card_data = {
'last4': credit_card_raw_data.last4,
'brand': credit_card_raw_data.brand
}
return new_card_data
@handle_stripe_error
def get_card_details(self, customer_id):
customer = stripe.Customer.retrieve(customer_id)
credit_card_raw_data = customer.sources.data.pop()
card_details = {
'last4': credit_card_raw_data.last4,
'brand': credit_card_raw_data.brand,
'exp_month': credit_card_raw_data.exp_month,
'exp_year': credit_card_raw_data.exp_year,
'fingerprint': credit_card_raw_data.fingerprint,
'card_id': credit_card_raw_data.id
}
return card_details
@handle_stripe_error
def get_all_invoices(self, customer_id, created_gt):
return_list = []
has_more_invoices = True
starting_after = False
while has_more_invoices:
if starting_after:
invoices = stripe.Invoice.list(
limit=10, customer=customer_id, created={'gt': created_gt},
starting_after=starting_after
)
else:
invoices = stripe.Invoice.list(
limit=10, customer=customer_id, created={'gt': created_gt}
)
has_more_invoices = invoices.has_more
for invoice in invoices.data:
sub_ids = []
for line in invoice.lines.data:
if line.type == 'subscription':
sub_ids.append(line.id)
elif line.type == 'invoiceitem':
sub_ids.append(line.subscription)
else:
sub_ids.append('')
invoice_details = {
'created': invoice.created,
'receipt_number': invoice.receipt_number,
'invoice_number': invoice.number,
'paid_at': invoice.status_transitions.paid_at if invoice.paid else 0,
'period_start': invoice.period_start,
'period_end': invoice.period_end,
'billing_reason': invoice.billing_reason,
'discount': invoice.discount.coupon.amount_off if invoice.discount else 0,
'total': invoice.total,
# to see how many line items we have in this invoice and
# then later check if we have more than 1
'lines_data_count': len(invoice.lines.data) if invoice.lines.data is not None else 0,
'invoice_id': invoice.id,
'lines_meta_data_csv': ','.join(
[line.metadata.VM_ID if hasattr(line.metadata, 'VM_ID') else '' for line in invoice.lines.data]
),
'subscription_ids_csv': ','.join(sub_ids),
'line_items': invoice.lines.data
}
starting_after = invoice.id
return_list.append(invoice_details)
return return_list
@handle_stripe_error
def get_cards_details_from_token(self, token):
stripe_token = stripe.Token.retrieve(token)
card_details = {
'last4': stripe_token.card.last4,
'brand': stripe_token.card.brand,
'exp_month': stripe_token.card.exp_month,
'exp_year': stripe_token.card.exp_year,
'fingerprint': stripe_token.card.fingerprint,
'card_id': stripe_token.card.id
}
return card_details
def check_customer(self, stripe_cus_api_id, user, token):
try:
customer = stripe.Customer.retrieve(stripe_cus_api_id)
except stripe.error.InvalidRequestError:
customer = self.create_customer(token, user.email, user.name)
user.stripecustomer.stripe_id = customer.get(
'response_object').get('id')
user.stripecustomer.save()
if type(customer) is dict:
customer = customer['response_object']
return customer
@handle_stripe_error
def get_customer(self, stripe_api_cus_id):
customer = stripe.Customer.retrieve(stripe_api_cus_id)
# data = customer.get('response_object')
return customer
@handle_stripe_error
def create_customer(self, token, email, name=None, address=None):
if name is None or name.strip() == "":
name = email
customer = self.stripe.Customer.create(
source=token,
description=name,
email=email,
address=address
)
return customer
@handle_stripe_error
def make_charge(self, amount=None, customer=None):
_amount = float(amount)
amount = int(_amount * 100) # stripe amount unit, in cents
charge = self.stripe.Charge.create(
amount=amount, # in cents
currency=self.CURRENCY,
customer=customer
)
return charge
@staticmethod
def _get_all_stripe_plans():
all_stripe_plans = client.get("/v1/stripe_plans")
all_stripe_plans_set = set()
if all_stripe_plans:
all_stripe_plans_obj = all_stripe_plans.value
if all_stripe_plans_obj and len(all_stripe_plans_obj['plans']) > 0:
all_stripe_plans_set = set(all_stripe_plans_obj["plans"])
return all_stripe_plans_set
@staticmethod
def _save_all_stripe_plans(stripe_plans):
client.put("/v1/stripe_plans", {"plans": list(stripe_plans)})
@handle_stripe_error
def get_or_create_stripe_plan(self, product_name, amount, stripe_plan_id,
interval=INTERVAL):
"""
This function checks if a StripePlan with the given
stripe_plan_id already exists. If it exists then the function
returns this object otherwise it creates a new StripePlan and
returns the new object.
:param amount: The amount in CHF cents
:param product_name: The name of the Stripe plan (product) to be created.
:param stripe_plan_id: The id of the Stripe plan to be
created. Use get_stripe_plan_id_string function to
obtain the name of the plan to be created
:param interval: The interval for subscription {month, year}. Defaults
to month if not provided
:return: The StripePlan object if it exists else creates a
Plan object in Stripe and a local StripePlan and
returns it. Returns None in case of Stripe error
"""
_amount = float(amount)
amount = int(_amount * 100) # stripe amount unit, in cents
all_stripe_plans = self._get_all_stripe_plans()
if stripe_plan_id in all_stripe_plans:
logging.debug("{} plan exists in db.".format(stripe_plan_id))
else:
logging.debug(("{} plan DOES NOT exist in db. "
"Creating").format(stripe_plan_id))
try:
plan_obj = self.stripe.Plan.retrieve(id=stripe_plan_id)
logging.debug("{} plan exists in Stripe".format(stripe_plan_id))
all_stripe_plans.add(stripe_plan_id)
except stripe.error.InvalidRequestError as e:
if "No such plan" in str(e):
logging.debug("Plan {} does not exist in Stripe, Creating")
plan_obj = self.stripe.Plan.create(
amount=amount,
product={'name': product_name},
interval=interval,
currency=self.CURRENCY,
id=stripe_plan_id)
logging.debug(self.PLAN_EXISTS_ERROR_MSG.format(stripe_plan_id))
all_stripe_plans.add(stripe_plan_id)
self._save_all_stripe_plans(all_stripe_plans)
return stripe_plan_id
@handle_stripe_error
def delete_stripe_plan(self, stripe_plan_id):
"""
Deletes the Plan in Stripe and also deletes the local db copy
of the plan if it exists
:param stripe_plan_id: The stripe plan id that needs to be
deleted
:return: True if the plan was deleted successfully from
Stripe, False otherwise.
"""
return_value = False
try:
plan = self.stripe.Plan.retrieve(stripe_plan_id)
plan.delete()
return_value = True
all_stripe_plans = self._get_all_stripe_plans()
all_stripe_plans.remove(stripe_plan_id)
self._save_all_stripe_plans(all_stripe_plans)
except stripe.error.InvalidRequestError as e:
if self.STRIPE_NO_SUCH_PLAN in str(e):
logging.debug(
self.PLAN_DOES_NOT_EXIST_ERROR_MSG.format(stripe_plan_id))
return return_value
@handle_stripe_error
def subscribe_customer_to_plan(self, customer, plans, trial_end=None):
"""
Subscribes the given customer to the list of given plans
:param customer: The stripe customer identifier
:param plans: A list of stripe plans.
:param trial_end: An integer representing when the Stripe subscription
is supposed to end
Ref: https://stripe.com/docs/api/python#create_subscription-items
e.g.
plans = [
{
"plan": "dcl-v1-cpu-2-ram-5gb-ssd-10gb",
},
]
:return: The subscription StripeObject
"""
subscription_result = self.stripe.Subscription.create(
customer=customer, items=plans, trial_end=trial_end
)
return subscription_result
@handle_stripe_error
def set_subscription_metadata(self, subscription_id, metadata):
subscription = stripe.Subscription.retrieve(subscription_id)
subscription.metadata = metadata
subscription.save()
@handle_stripe_error
def unsubscribe_customer(self, subscription_id):
"""
Cancels a given subscription
:param subscription_id: The Stripe subscription id string
:return:
"""
sub = stripe.Subscription.retrieve(subscription_id)
return sub.delete()
@handle_stripe_error
def make_payment(self, customer, amount, token):
charge = self.stripe.Charge.create(
amount=amount, # in cents
currency=self.CURRENCY,
customer=customer
)
return charge
@staticmethod
def get_stripe_plan_id(cpu, ram, ssd, version, app='dcl', hdd=None,
price=None):
"""
Returns the Stripe plan id string of the form
`dcl-v1-cpu-2-ram-5gb-ssd-10gb` based on the input parameters
:param cpu: The number of cores
:param ram: The size of the RAM in GB
:param ssd: The size of ssd storage in GB
:param hdd: The size of hdd storage in GB
:param version: The version of the Stripe plans
:param app: The application to which the stripe plan belongs
to. By default it is 'dcl'
:param price: The price for this plan
:return: A string of the form `dcl-v1-cpu-2-ram-5gb-ssd-10gb`
"""
dcl_plan_string = 'cpu-{cpu}-ram-{ram}gb-ssd-{ssd}gb'.format(cpu=cpu,
ram=ram,
ssd=ssd)
if hdd is not None:
dcl_plan_string = '{dcl_plan_string}-hdd-{hdd}gb'.format(
dcl_plan_string=dcl_plan_string, hdd=hdd)
stripe_plan_id_string = '{app}-v{version}-{plan}'.format(
app=app,
version=version,
plan=dcl_plan_string
)
if price is not None:
stripe_plan_id_string_with_price = '{}-{}chf'.format(
stripe_plan_id_string,
round(price, 2)
)
return stripe_plan_id_string_with_price
else:
return stripe_plan_id_string
@staticmethod
def get_vm_config_from_stripe_id(stripe_id):
"""
Given a string like "dcl-v1-cpu-2-ram-5gb-ssd-10gb" return different
configuration params as a dict
:param stripe_id|str
:return: dict
"""
pattern = re.compile(r'^dcl-v(\d+)-cpu-(\d+)-ram-(\d+\.?\d*)gb-ssd-(\d+)gb-?(\d*\.?\d*)(chf)?$')
match_res = pattern.match(stripe_id)
if match_res is not None:
price = None
try:
price = match_res.group(5)
except IndexError:
logging.debug("Did not find price in {}".format(stripe_id))
return {
'version': match_res.group(1),
'cores': match_res.group(2),
'ram': match_res.group(3),
'ssd': match_res.group(4),
'price': price
}
@staticmethod
def get_stripe_plan_name(cpu, memory, disk_size, price):
"""
Returns the Stripe plan name
:return:
"""
return "{cpu} Cores, {memory} GB RAM, {disk_size} GB SSD, " \
"{price} CHF".format(
cpu=cpu,
memory=memory,
disk_size=disk_size,
price=round(price, 2)
)
@handle_stripe_error
def set_subscription_meta_data(self, subscription_id, meta_data):
"""
Adds VM metadata to a subscription
:param subscription_id: Stripe identifier for the subscription
:param meta_data: A dict of meta data to be added
:return:
"""
subscription = stripe.Subscription.retrieve(subscription_id)
subscription.metadata = meta_data
subscription.save()

View File

@ -0,0 +1,338 @@
import logging
from datetime import datetime
from uuid import uuid4
from flask import Flask, request
from flask_restful import Resource, Api
from werkzeug.exceptions import HTTPException
from config import etcd_client as client, config as config
from stripe_utils import StripeUtils
from schemas import (
make_return_message, ValidationException, UserRegisterPaymentSchema,
AddProductSchema, ProductOrderSchema, OrderListSchema, create_schema
)
from helper import get_plan_id_from_product, calculate_charges
class ListProducts(Resource):
@staticmethod
def get():
products = client.get_prefix('/v1/products/')
products = [
product
for product in [p.value for p in products]
if product['active']
]
prod_dict = {}
for p in products:
prod_dict[p['usable-id']] = {
'name': p['name'],
'description': p['description'],
}
logger.debug('Products = {}'.format(prod_dict))
return prod_dict, 200
class AddProduct(Resource):
@staticmethod
def post():
data = request.get_json(silent=True) or {}
try:
logger.debug('Got data: {}'.format(str(data)))
validator = AddProductSchema(data)
validator.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
cleaned_values = validator.get_cleaned_values()
previous_product = cleaned_values.get('product', None)
if previous_product:
if not cleaned_values['update']:
return make_return_message('Product already exists. Pass --update to update the product.')
else:
product_uuid = previous_product.pop('uuid')
else:
product_uuid = uuid4().hex
product_value = cleaned_values['specs']
product_key = '/v1/products/{}'.format(product_uuid)
product_value['uuid'] = product_uuid
logger.debug('Adding product data: {}'.format(str(product_value)))
client.put(product_key, product_value)
if not previous_product:
return make_return_message('Product created.')
else:
return make_return_message('Product updated.')
################################################################################
# Nico-ok-marker
class UserRegisterPayment(Resource):
@staticmethod
def post():
data = request.get_json(silent=True) or {}
try:
logger.debug('Got data: {}'.format(str(data)))
validator = UserRegisterPaymentSchema(data)
validator.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
cleaned_values = validator.get_cleaned_values()
last4 = data['card_number'].strip()[-4:]
stripe_utils = StripeUtils()
# Does customer already exist ?
stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail'])
# Does customer already exist ?
if stripe_customer is not None:
logger.debug('Customer {}-{} exists already'.format(
cleaned_values['username'], cleaned_values['user']['mail'])
)
# Check if the card already exists
ce_response = stripe_utils.card_exists(
stripe_customer.id, cc_number=data['card_number'],
exp_month=int(data['expiry_month']),
exp_year=int(data['expiry_year']),
cvc=data['cvc'])
if ce_response['response_object']:
message = 'The given card ending in {} exists already.'.format(last4)
return make_return_message(message, 400)
elif ce_response['response_object'] is False:
# Associate card with user
logger.debug('Adding card ending in {}'.format(last4))
token_response = stripe_utils.get_token_from_card(
data['card_number'], data['cvc'], data['expiry_month'],
data['expiry_year']
)
if token_response['response_object']:
logger.debug('Token {}'.format(token_response['response_object'].id))
resp = stripe_utils.associate_customer_card(
stripe_customer.id, token_response['response_object'].id
)
if resp['response_object']:
return make_return_message(
'Card ending in {} registered as your payment source'.format(last4)
)
else:
return make_return_message('Error with payment gateway. Contact support', 400)
else:
return make_return_message('Error: {}'.format(ce_response['error']), 400)
else:
# Stripe customer does not exist, create a new one
logger.debug(
'Customer {} does not exist, creating new'.format(cleaned_values['user']['mail'])
)
token_response = stripe_utils.get_token_from_card(
cleaned_values['card_number'], cleaned_values['cvc'],
cleaned_values['expiry_month'], cleaned_values['expiry_year']
)
if token_response['response_object']:
logger.debug('Token {}'.format(token_response['response_object'].id))
# Create stripe customer
stripe_customer_resp = stripe_utils.create_customer(
name=cleaned_values['card_holder_name'],
token=token_response['response_object'].id,
email=cleaned_values['user']['mail'],
address=cleaned_values['address']
)
stripe_customer = stripe_customer_resp['response_object']
if stripe_customer:
logger.debug('Created stripe customer {}'.format(stripe_customer.id))
return make_return_message(
'Card ending in {} registered as your payment source'.format(last4)
)
else:
return make_return_message('Error with card. Contact support', 400)
else:
return make_return_message('Error with payment gateway. Contact support', 400)
class ProductOrder(Resource):
@staticmethod
def post():
data = request.get_json(silent=True) or {}
try:
validator = ProductOrderSchema(data)
validator.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
cleaned_values = validator.get_cleaned_values()
stripe_utils = StripeUtils()
product = cleaned_values['product']
# Check the user has a payment source added
stripe_customer = stripe_utils.get_stripe_customer_from_email(cleaned_values['user']['mail'])
if not stripe_customer or len(stripe_customer.sources) == 0:
return make_return_message('Please register your payment method first.', 400)
try:
product_schema = create_schema(product, data)
product_schema = product_schema()
product_schema.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
transformed_data = product_schema.get_cleaned_values()
logger.debug('Tranformed data: {}'.format(transformed_data))
one_time_charge, recurring_charge = calculate_charges(product, transformed_data)
recurring_charge = int(recurring_charge)
if not cleaned_values['pay']:
return make_return_message(
'You would be charged {} CHF one time and {} CHF every {}. '
'Add --pay to command to order.'.format(
one_time_charge, recurring_charge, product['recurring_period']
)
)
with client.client.lock('product-order') as _:
# Initiate a one-time/subscription based on product type
if recurring_charge > 0:
logger.debug('Product {} is recurring payment'.format(product['name']))
plan_id = get_plan_id_from_product(product)
res = stripe_utils.get_or_create_stripe_plan(
product_name=product['name'],
stripe_plan_id=plan_id, amount=recurring_charge,
interval=product['recurring_period'],
)
if res['response_object']:
logger.debug('Obtained plan {}'.format(plan_id))
subscription_res = stripe_utils.subscribe_customer_to_plan(
stripe_customer.id,
[{'plan': plan_id}]
)
subscription_obj = subscription_res['response_object']
if subscription_obj is None or subscription_obj.status != 'active':
return make_return_message(
'Error subscribing to plan. Detail: {}'.format(subscription_res['error']), 400
)
else:
order_obj = {
'order-id': uuid4().hex,
'ordered-at': datetime.now().isoformat(),
'product': product['usable-id'],
'one-time-price': one_time_charge,
'recurring-price': recurring_charge,
'recurring-period': product['recurring_period']
}
client.put(
'/v1/user/{}/orders/{}'.format(
cleaned_values['username'], order_obj['order-id']
), order_obj
)
product['quantity'] -= 1
client.put('/v1/products/{}'.format(product['uuid']), product)
return {
'message': 'Order Successful.',
**order_obj
}
else:
logger.error('Could not create plan {}'.format(plan_id))
return make_return_message('Something wrong happened. Contact administrator', 400)
elif recurring_charge == 0 and one_time_charge > 0:
logger.debug('Product {} is one-time payment'.format(product['name']))
charge_response = stripe_utils.make_charge(
amount=one_time_charge,
customer=stripe_customer.id
)
stripe_onetime_charge = charge_response.get('response_object')
# Check if the payment was approved
if not stripe_onetime_charge:
msg = charge_response.get('error')
return make_return_message('Error subscribing to plan. Details: {}'.format(msg), 400)
order_obj = {
'order-id': uuid4().hex,
'ordered-at': datetime.now().isoformat(),
'product': product['usable-id'],
'one-time-price': one_time_charge,
}
client.put(
'/v1/user/{}/orders/{}'.format(cleaned_values['username'], order_obj['order-id']),
order_obj
)
product['quantity'] -= 1
client.put('/v1/products/{}'.format(product['uuid']), product)
return {'message': 'Order successful', **order_obj}, 200
class OrderList(Resource):
@staticmethod
def post():
data = request.get_json(silent=True) or {}
try:
validator = OrderListSchema(data)
validator.is_valid()
except ValidationException as err:
return make_return_message(err, 400)
else:
cleaned_values = validator.get_cleaned_values()
orders = client.get_prefix('/v1/user/{}/orders'.format(cleaned_values['username']))
orders_dict = {
order.value['order-id']: {
**order.value
}
for order in orders
}
logger.debug('Orders = {}'.format(orders_dict))
return {'orders': orders_dict}, 200
if __name__ == '__main__':
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
log_formater = logging.Formatter('[%(filename)s:%(lineno)d] %(message)s')
stream_logger = logging.StreamHandler()
stream_logger.setFormatter(log_formater)
# file_logger = logging.FileHandler('log.txt')
# file_logger.setLevel(logging.DEBUG)
# file_logger.setFormatter(log_formater)
logger.addHandler(stream_logger)
# logger.addHandler(file_logger)
app = Flask(__name__)
api = Api(app)
api.add_resource(ListProducts, '/product/list')
api.add_resource(AddProduct, '/product/add')
api.add_resource(ProductOrder, '/product/order')
api.add_resource(UserRegisterPayment, '/user/register_payment')
api.add_resource(OrderList, '/order/list')
app.run(host='::', port=config.get('app', 'port', fallback=5000), debug=True)
@app.errorhandler(Exception)
def handle_exception(e):
app.logger.error(e)
# pass through HTTP errors
if isinstance(e, HTTPException):
return e
# now you're handling non-HTTP exceptions only
return {'message': 'Server Error'}, 500

View File

@ -0,0 +1,11 @@
## TODO 2020-02-22
* ~~move the current rest api to /opennebula~~
* ~~make the /opennebula api only accessible by an admin account~~
* ~~create a new filtered api on /vm/list that~~
* ~~a) requires authentication~~
* ~~b) only shows the VMs of the current user~~
* ~~the new api should not contain all details, but: cpus (as read by the vcpu field), ram, ips, disks~~
* ~~also make a (random) uuid the primary key for VMs - everything in this uncloud hack will use uuids as the id~~
* ~~still expose the opennebula id as opennebula_id~~
* ~~note put all secrets/configs into uncloud.secrets - I added a sample file into the repo~~

View File

@ -0,0 +1,102 @@
* snapshot feature
** product: vm-snapshot
** flow
*** list all my VMs
**** get the uuid of the VM I want to take a snapshot of
*** request a snapshot
```
vmuuid=$(http nicocustomer
http -a nicocustomer:xxx http://uncloud.ch/vm/create_snapshot uuid=
password=...
```
** backend realisation
*** list snapshots
- have them in the DB
- create an entry on create
*** creating snapshots
- vm sync / fsync?
- rbd snapshot
- host/cluster mapping?
- need image(s)
* steps
** DONE authenticate via ldap
CLOSED: [2020-02-20 Thu 19:05]
** DONE Make classes / views require authentication
CLOSED: [2020-02-20 Thu 19:05]
** TODO register credit card
*** TODO find out what saving with us
*** Info
**** should not be fully saved in the DB
**** model needs to be a bit different
* Decide where to save sensitive data
** stripe access key, etc.
* python requirements (nicohack202002)
django djangorestframework django-auth-ldap stripe
* os package requirements (alpine)
openldap-dev
* VPN case
** put on /orders with uuid
** register cc
* CC
** TODO check whether we can register or not at stripe
* membership
** required for "smaller" / "shorter" products
* TODO Membership missing
* Flows to be implemented - see https://redmine.ungleich.ch/issues/7609
** Membership
*** 5 CHF
** Django Hosting
*** One time payment 35 CHF
*** Monthly payment depends on VM size
*** Parameters: same as IPv6 only VM
** IPv6 VPN
*** Parameters: none
*** Is for free if the customer has an active VM
** IPv6 only VM
*** Parameters: cores, ram, os_disk_size, OS
* Django rest framework
** viewset: .list and .create
** view: .get .post
* TODO register CC
* DONE list products
CLOSED: [2020-02-24 Mon 20:15]
* An ungleich account - can be registered for free on
https://account.ungleich.ch
* httpie installed (provides the http command)
## Get a membership
## Registering a payment method
To be able to pay for the membership, you will need to register a
credit card or apply for payment on bill (TO BE IMPLEMENTED).
### Register credit card
```
http POST https://api.ungleich.ch/membership \
username=nico password=yourpassword \
cc_number=.. \
cc_
```
### Request payment via bill
## Create the membership
```
http POST https://api.ungleich.ch/membership username=nico password=yourpassword
```
## List available products

View File

@ -0,0 +1,6 @@
* TODO register CC
* TODO list products
* ahmed
** schemas
*** field: is_valid? - used by schemas
*** definition of a "schema"

View File

@ -0,0 +1,4 @@
db.sqlite3
uncloud/secrets.py
debug.log
uncloud/local_settings.py

View File

@ -1,22 +1,22 @@
#!/bin/sh
# -*- coding: utf-8 -*-
#
# 2019 Nico Schottelius (nico-ucloud at schottelius.org)
# 2019-2020 Nico Schottelius (nico-uncloud at schottelius.org)
#
# This file is part of ucloud.
# This file is part of uncloud.
#
# ucloud is free software: you can redistribute it and/or modify
# uncloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# ucloud is distributed in the hope that it will be useful,
# uncloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with ucloud. If not, see <http://www.gnu.org/licenses/>.
# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
#
#
@ -26,4 +26,4 @@ dir=${0%/*}
# Ensure version is present - the bundled/shipped version contains a static version,
# the git version contains a dynamic version
printf "VERSION = \"%s\"\n" "$(git describe)" > ${dir}/../uncloud/version.py
printf "VERSION = \"%s\"\n" "$(git describe --tags --abbrev=0)" > ${dir}/../uncloud/version.py

View File

@ -24,6 +24,6 @@
dir=${0%/*}
${dir}/gen-version;
pip uninstall -y uncloud
python setup.py install
pip uninstall -y uncloud >/dev/null
python setup.py install >/dev/null
${dir}/uncloud "$@"

View File

@ -7,7 +7,7 @@ SPHINXOPTS ?=
SPHINXBUILD ?= sphinx-build
SOURCEDIR = source/
BUILDDIR = build/
DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/
DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/uncloud/
.PHONY: all build clean

View File

@ -56,40 +56,13 @@ To start host we created earlier, execute the following command
ucloud host ungleich.ch
Create OS Image
---------------
File & image scanners
--------------------------
Create ucloud-init ready OS image (Optional)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
This step is optional if you just want to test ucloud. However, sooner or later
you want to create OS images with ucloud-init to properly
contexualize VMs.
1. Start a VM with OS image on which you want to install ucloud-init
2. Execute the following command on the started VM
.. code-block:: sh
apk add git
git clone https://code.ungleich.ch/ucloud/ucloud-init.git
cd ucloud-init
sh ./install.sh
3. Congratulations. Your image is now ucloud-init ready.
Upload Sample OS Image
~~~~~~~~~~~~~~~~~~~~~~
Execute the following to get the sample OS image file.
.. code-block:: sh
mkdir /var/www/admin
(cd /var/www/admin && wget https://cloud.ungleich.ch/s/qTb5dFYW5ii8KsD/download)
Run File Scanner and Image Scanner
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
Currently, our uploaded file *alpine-untouched.qcow2* is not tracked by ucloud. We can only make
images from tracked files. So, we need to track the file by running File Scanner
Let's assume we have uploaded an *alpine-uploaded.qcow2* disk images to our
uncloud server. Currently, our *alpine-untouched.qcow2* is not tracked by
ucloud. We can only make images from tracked files. So, we need to track the
file by running File Scanner
.. code-block:: sh

View File

@ -0,0 +1,36 @@
Hacking
=======
Using uncloud in hacking (aka development) mode.
Get the code
------------
.. code-block:: sh
:linenos:
git clone https://code.ungleich.ch/uncloud/uncloud.git
Install python requirements
---------------------------
You need to have python3 installed.
.. code-block:: sh
:linenos:
cd uncloud!
python -m venv venv
. ./venv/bin/activate
./bin/uncloud-run-reinstall
Install os requirements
-----------------------
Install the following software packages: **dnsmasq**.
If you already have a working IPv6 SLAAC and DNS setup,
this step can be skipped.
Note that you need at least one /64 IPv6 network to run uncloud.

View File

Before

Width:  |  Height:  |  Size: 37 KiB

After

Width:  |  Height:  |  Size: 37 KiB

View File

@ -11,14 +11,13 @@ Welcome to ucloud's documentation!
:caption: Contents:
introduction
user-guide
setup-install
vm-images
user-guide
admin-guide
user-guide/how-to-create-an-os-image-for-ucloud
troubleshooting
hacking
Indices and tables
==================

View File

@ -0,0 +1,66 @@
VM images
==================================
Overview
---------
ucloud tries to be least invasise towards VMs and only require
strictly necessary changes for running in a virtualised
environment. This includes configurations for:
* Configuring the network
* Managing access via ssh keys
* Resizing the attached disk(s)
Upstream images
---------------
The 'official' uncloud images are defined in the `uncloud/images
<https://code.ungleich.ch/uncloud/images>`_ repository.
How to make you own Uncloud images
----------------------------------
.. note::
It is fairly easy to create your own images for uncloud, as the common
operations (which are detailed below) can be automatically handled by the
`uncloud/uncloud-init <https://code.ungleich.ch/uncloud/uncloud-init>`_ tool.
Network configuration
~~~~~~~~~~~~~~~~~~~~~
All VMs in ucloud are required to support IPv6. The primary network
configuration is always done using SLAAC. A VM thus needs only to be
configured to
* accept router advertisements on all network interfaces
* use the router advertisements to configure the network interfaces
* accept the DNS entries from the router advertisements
Configuring SSH keys
~~~~~~~~~~~~~~~~~~~~
To be able to access the VM, ucloud support provisioning SSH keys.
To accept ssh keys in your VM, request the URL
*http://metadata/ssh_keys*. Add the content to the appropriate user's
**authorized_keys** file. Below you find sample code to accomplish
this task:
.. code-block:: sh
tmp=$(mktemp)
curl -s http://metadata/ssk_keys > "$tmp"
touch ~/.ssh/authorized_keys # ensure it exists
cat ~/.ssh/authorized_keys >> "$tmp"
sort "$tmp" | uniq > ~/.ssh/authorized_keys
Disk resize
~~~~~~~~~~~
In virtualised environments, the disk sizes might grow. The operating
system should detect disks that are bigger than the existing partition
table and resize accordingly. This task is os specific.
ucloud does not support shrinking disks due to the complexity and
intra OS dependencies.

View File

@ -0,0 +1,89 @@
#!/usr/bin/env python3
import logging
import sys
import importlib
import argparse
import os
from etcd3.exceptions import ConnectionFailedError
from uncloud.common import settings
from uncloud import UncloudException
from uncloud.common.cli import resolve_otp_credentials
# Components that use etcd
ETCD_COMPONENTS = ['api', 'scheduler', 'host', 'filescanner',
'imagescanner', 'metadata', 'configure', 'hack']
ALL_COMPONENTS = ETCD_COMPONENTS.copy()
ALL_COMPONENTS.append('oneshot')
#ALL_COMPONENTS.append('cli')
if __name__ == '__main__':
arg_parser = argparse.ArgumentParser()
subparsers = arg_parser.add_subparsers(dest='command')
parent_parser = argparse.ArgumentParser(add_help=False)
parent_parser.add_argument('--debug', '-d', action='store_true', default=False,
help='More verbose logging')
parent_parser.add_argument('--conf-dir', '-c', help='Configuration directory',
default=os.path.expanduser('~/uncloud'))
etcd_parser = argparse.ArgumentParser(add_help=False)
etcd_parser.add_argument('--etcd-host')
etcd_parser.add_argument('--etcd-port')
etcd_parser.add_argument('--etcd-ca-cert', help='CA that signed the etcd certificate')
etcd_parser.add_argument('--etcd-cert-cert', help='Path to client certificate')
etcd_parser.add_argument('--etcd-cert-key', help='Path to client certificate key')
for component in ALL_COMPONENTS:
mod = importlib.import_module('uncloud.{}.main'.format(component))
parser = getattr(mod, 'arg_parser')
if component in ETCD_COMPONENTS:
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser, etcd_parser])
else:
subparsers.add_parser(name=parser.prog, parents=[parser, parent_parser])
arguments = vars(arg_parser.parse_args())
etcd_arguments = [key for key, value in arguments.items() if key.startswith('etcd_') and value]
etcd_arguments = {
'etcd': {
key.replace('etcd_', ''): arguments[key]
for key in etcd_arguments
}
}
if not arguments['command']:
arg_parser.print_help()
else:
# Initializing Settings and resolving otp_credentials
# It is neccessary to resolve_otp_credentials after argument parsing is done because
# previously we were reading config file which was fixed to ~/uncloud/uncloud.conf and
# providing the default values for --name, --realm and --seed arguments from the values
# we read from file. But, now we are asking user about where the config file lives. So,
# to providing default value is not possible before parsing arguments. So, we are doing
# it after..
# settings.settings = settings.Settings(arguments['conf_dir'], seed_value=etcd_arguments)
# resolve_otp_credentials(arguments)
name = arguments.pop('command')
mod = importlib.import_module('uncloud.{}.main'.format(name))
main = getattr(mod, 'main')
if arguments['debug']:
logging.basicConfig(level=logging.DEBUG)
else:
logging.basicConfig(level=logging.INFO)
log = logging.getLogger()
try:
main(arguments)
except UncloudException as err:
log.error(err)
sys.exit(1)
# except ConnectionFailedError as err:
# log.error('Cannot connect to etcd: {}'.format(err))
except Exception as err:
log.exception(err)

View File

@ -40,7 +40,8 @@ setup(
"pynetbox",
"colorama",
"etcd3 @ https://github.com/kragniz/python-etcd3/tarball/master#egg=etcd3",
"marshmallow"
"marshmallow",
"ldap3"
],
scripts=["scripts/uncloud"],
data_files=[

View File

@ -0,0 +1,37 @@
import unittest
from unittest.mock import Mock
from uncloud.hack.mac import MAC
from uncloud import UncloudException
class TestMacLocal(unittest.TestCase):
def setUp(self):
self.config = Mock()
self.config.arguments = {"no_db":True}
self.mac = MAC(self.config)
self.mac.create()
def testMacInt(self):
self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong first MAC index")
def testMacRepr(self):
self.assertEqual(self.mac.__repr__(), '420000000001', "wrong first MAC index")
def testMacStr(self):
self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong first MAC index")
def testValidationRaise(self):
with self.assertRaises(UncloudException):
self.mac.validate_mac("2")
def testValidation(self):
self.assertTrue(self.mac.validate_mac("42:00:00:00:00:01"), "Validation of a given MAC not working properly")
def testNextMAC(self):
self.mac.create()
self.assertEqual(self.mac.__repr__(), '420000000001', "wrong second MAC index")
self.assertEqual(self.mac.__int__(), int("0x420000000001",0), "wrong second MAC index")
self.assertEqual(self.mac.__str__(), '42:00:00:00:00:01', "wrong second MAC index")
if __name__ == '__main__':
unittest.main()

View File

@ -0,0 +1,2 @@
class UncloudException(Exception):
pass

View File

@ -1,7 +1,6 @@
import os
from uncloud.common.shared import shared
from uncloud.common.settings import settings
class Optional:
@ -54,9 +53,7 @@ class VmUUIDField(Field):
def vm_uuid_validation(self):
r = shared.etcd_client.get(
os.path.join(settings["etcd"]["vm_prefix"], self.uuid)
os.path.join(shared.settings["etcd"]["vm_prefix"], self.uuid)
)
if not r:
self.add_error(
"VM with uuid {} does not exists".format(self.uuid)
)
self.add_error("VM with uuid {} does not exists".format(self.uuid))

View File

@ -4,7 +4,6 @@ import os
from uuid import uuid4
from uncloud.common.shared import shared
from uncloud.common.settings import settings
data = {
'is_public': True,
@ -15,6 +14,6 @@ data = {
}
shared.etcd_client.put(
os.path.join(settings['etcd']['image_store_prefix'], uuid4().hex),
os.path.join(shared.settings['etcd']['image_store_prefix'], uuid4().hex),
json.dumps(data),
)

View File

@ -7,7 +7,6 @@ import requests
from pyotp import TOTP
from uncloud.common.shared import shared
from uncloud.common.settings import settings
logger = logging.getLogger(__name__)
@ -15,9 +14,9 @@ logger = logging.getLogger(__name__)
def check_otp(name, realm, token):
try:
data = {
"auth_name": settings["otp"]["auth_name"],
"auth_token": TOTP(settings["otp"]["auth_seed"]).now(),
"auth_realm": settings["otp"]["auth_realm"],
"auth_name": shared.settings["otp"]["auth_name"],
"auth_token": TOTP(shared.settings["otp"]["auth_seed"]).now(),
"auth_realm": shared.settings["otp"]["auth_realm"],
"name": name,
"realm": realm,
"token": token,
@ -25,13 +24,13 @@ def check_otp(name, realm, token):
except binascii.Error as err:
logger.error(
"Cannot compute OTP for seed: {}".format(
settings["otp"]["auth_seed"]
shared.settings["otp"]["auth_seed"]
)
)
return 400
response = requests.post(
settings["otp"]["verification_controller_url"], json=data
shared.settings["otp"]["verification_controller_url"], json=data
)
return response.status_code
@ -87,7 +86,7 @@ def resolve_image_name(name, etcd_client):
)
images = etcd_client.get_prefix(
settings["etcd"]["image_prefix"], value_in_json=True
shared.settings["etcd"]["image_prefix"], value_in_json=True
)
# Try to find image with name == image_name and store_name == store_name
@ -111,9 +110,7 @@ def random_bytes(num=6):
return [random.randrange(256) for _ in range(num)]
def generate_mac(
uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"
):
def generate_mac(uaa=False, multicast=False, oui=None, separator=":", byte_fmt="%02x"):
mac = random_bytes()
if oui:
if type(oui) == str:
@ -148,3 +145,4 @@ def mac2ipv6(mac, prefix):
lower_part = ipaddress.IPv6Address(":".join(ipv6_parts))
prefix = ipaddress.IPv6Address(prefix)
return str(prefix + int(lower_part))

View File

@ -15,9 +15,8 @@ from uncloud.common.shared import shared
from uncloud.common import counters
from uncloud.common.vm import VMStatus
from uncloud.common.request import RequestEntry, RequestType
from uncloud.common.settings import settings
from . import schemas
from .helper import generate_mac, mac2ipv6
from uncloud.api import schemas
from uncloud.api.helper import generate_mac, mac2ipv6
from uncloud import UncloudException
logger = logging.getLogger(__name__)
@ -50,7 +49,7 @@ class CreateVM(Resource):
validator = schemas.CreateVMSchema(data)
if validator.is_valid():
vm_uuid = uuid4().hex
vm_key = join_path(settings['etcd']['vm_prefix'], vm_uuid)
vm_key = join_path(shared.settings['etcd']['vm_prefix'], vm_uuid)
specs = {
'cpu': validator.specs['cpu'],
'ram': validator.specs['ram'],
@ -60,7 +59,7 @@ class CreateVM(Resource):
macs = [generate_mac() for _ in range(len(data['network']))]
tap_ids = [
counters.increment_etcd_counter(
shared.etcd_client, settings['etcd']['tap_counter']
shared.etcd_client, shared.settings['etcd']['tap_counter']
)
for _ in range(len(data['network']))
]
@ -84,7 +83,7 @@ class CreateVM(Resource):
r = RequestEntry.from_scratch(
type=RequestType.ScheduleVM,
uuid=vm_uuid,
request_prefix=settings['etcd']['request_prefix'],
request_prefix=shared.settings['etcd']['request_prefix'],
)
shared.request_pool.put(r)
@ -99,7 +98,7 @@ class VmStatus(Resource):
validator = schemas.VMStatusSchema(data)
if validator.is_valid():
vm = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], data['uuid'])
join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
)
vm_value = vm.value.copy()
vm_value['ip'] = []
@ -107,7 +106,7 @@ class VmStatus(Resource):
network_name, mac, tap = network_mac_and_tap
network = shared.etcd_client.get(
join_path(
settings['etcd']['network_prefix'],
shared.settings['etcd']['network_prefix'],
data['name'],
network_name,
),
@ -130,7 +129,7 @@ class CreateImage(Resource):
validator = schemas.CreateImageSchema(data)
if validator.is_valid():
file_entry = shared.etcd_client.get(
join_path(settings['etcd']['file_prefix'], data['uuid'])
join_path(shared.settings['etcd']['file_prefix'], data['uuid'])
)
file_entry_value = json.loads(file_entry.value)
@ -144,7 +143,7 @@ class CreateImage(Resource):
}
shared.etcd_client.put(
join_path(
settings['etcd']['image_prefix'], data['uuid']
shared.settings['etcd']['image_prefix'], data['uuid']
),
json.dumps(image_entry_json),
)
@ -157,7 +156,7 @@ class ListPublicImages(Resource):
@staticmethod
def get():
images = shared.etcd_client.get_prefix(
settings['etcd']['image_prefix'], value_in_json=True
shared.settings['etcd']['image_prefix'], value_in_json=True
)
r = {'images': []}
for image in images:
@ -178,7 +177,7 @@ class VMAction(Resource):
if validator.is_valid():
vm_entry = shared.vm_pool.get(
join_path(settings['etcd']['vm_prefix'], data['uuid'])
join_path(shared.settings['etcd']['vm_prefix'], data['uuid'])
)
action = data['action']
@ -208,7 +207,7 @@ class VMAction(Resource):
type='{}VM'.format(action.title()),
uuid=data['uuid'],
hostname=vm_entry.hostname,
request_prefix=settings['etcd']['request_prefix'],
request_prefix=shared.settings['etcd']['request_prefix'],
)
shared.request_pool.put(r)
return (
@ -231,10 +230,10 @@ class VMMigration(Resource):
type=RequestType.InitVMMigration,
uuid=vm.uuid,
hostname=join_path(
settings['etcd']['host_prefix'],
shared.settings['etcd']['host_prefix'],
validator.destination.value,
),
request_prefix=settings['etcd']['request_prefix'],
request_prefix=shared.settings['etcd']['request_prefix'],
)
shared.request_pool.put(r)
@ -254,7 +253,7 @@ class ListUserVM(Resource):
if validator.is_valid():
vms = shared.etcd_client.get_prefix(
settings['etcd']['vm_prefix'], value_in_json=True
shared.settings['etcd']['vm_prefix'], value_in_json=True
)
return_vms = []
user_vms = filter(
@ -287,7 +286,7 @@ class ListUserFiles(Resource):
if validator.is_valid():
files = shared.etcd_client.get_prefix(
settings['etcd']['file_prefix'], value_in_json=True
shared.settings['etcd']['file_prefix'], value_in_json=True
)
return_files = []
user_files = [f for f in files if f.value['owner'] == data['name']]
@ -312,7 +311,7 @@ class CreateHost(Resource):
validator = schemas.CreateHostSchema(data)
if validator.is_valid():
host_key = join_path(
settings['etcd']['host_prefix'], uuid4().hex
shared.settings['etcd']['host_prefix'], uuid4().hex
)
host_entry = {
'specs': data['specs'],
@ -354,7 +353,7 @@ class GetSSHKeys(Resource):
# {user_prefix}/{realm}/{name}/key/
etcd_key = join_path(
settings['etcd']['user_prefix'],
shared.settings['etcd']['user_prefix'],
data['realm'],
data['name'],
'key',
@ -372,7 +371,7 @@ class GetSSHKeys(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path(
settings['etcd']['user_prefix'],
shared.settings['etcd']['user_prefix'],
data['realm'],
data['name'],
'key',
@ -405,7 +404,7 @@ class AddSSHKey(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path(
settings['etcd']['user_prefix'],
shared.settings['etcd']['user_prefix'],
data['realm'],
data['name'],
'key',
@ -439,7 +438,7 @@ class RemoveSSHKey(Resource):
# {user_prefix}/{realm}/{name}/key/{key_name}
etcd_key = join_path(
settings['etcd']['user_prefix'],
shared.settings['etcd']['user_prefix'],
data['realm'],
data['name'],
'key',
@ -471,23 +470,23 @@ class CreateNetwork(Resource):
network_entry = {
'id': counters.increment_etcd_counter(
shared.etcd_client, settings['etcd']['vxlan_counter']
shared.etcd_client, shared.settings['etcd']['vxlan_counter']
),
'type': data['type'],
}
if validator.user.value:
try:
nb = pynetbox.api(
url=settings['netbox']['url'],
token=settings['netbox']['token'],
url=shared.settings['netbox']['url'],
token=shared.settings['netbox']['token'],
)
nb_prefix = nb.ipam.prefixes.get(
prefix=settings['network']['prefix']
prefix=shared.settings['network']['prefix']
)
prefix = nb_prefix.available_prefixes.create(
data={
'prefix_length': int(
settings['network']['prefix_length']
shared.settings['network']['prefix_length']
),
'description': '{}\'s network "{}"'.format(
data['name'], data['network_name']
@ -506,7 +505,7 @@ class CreateNetwork(Resource):
network_entry['ipv6'] = 'fd00::/64'
network_key = join_path(
settings['etcd']['network_prefix'],
shared.settings['etcd']['network_prefix'],
data['name'],
data['network_name'],
)
@ -526,7 +525,7 @@ class ListUserNetwork(Resource):
if validator.is_valid():
prefix = join_path(
settings['etcd']['network_prefix'], data['name']
shared.settings['etcd']['network_prefix'], data['name']
)
networks = shared.etcd_client.get_prefix(
prefix, value_in_json=True
@ -570,7 +569,7 @@ def main(arguments):
try:
image_stores = list(
shared.etcd_client.get_prefix(
settings['etcd']['image_store_prefix'], value_in_json=True
shared.settings['etcd']['image_store_prefix'], value_in_json=True
)
)
except KeyError:
@ -590,7 +589,7 @@ def main(arguments):
# shared.etcd_client.put(
# join_path(
# settings['etcd']['image_store_prefix'], uuid4().hex
# shared.settings['etcd']['image_store_prefix'], uuid4().hex
# ),
# json.dumps(data),
# )

View File

@ -22,7 +22,6 @@ import bitmath
from uncloud.common.host import HostStatus
from uncloud.common.vm import VMStatus
from uncloud.common.shared import shared
from uncloud.common.settings import settings
from . import helper, logger
from .common_fields import Field, VmUUIDField
from .helper import check_otp, resolve_vm_name
@ -112,7 +111,7 @@ class CreateImageSchema(BaseSchema):
def file_uuid_validation(self):
file_entry = shared.etcd_client.get(
os.path.join(
settings["etcd"]["file_prefix"], self.uuid.value
shared.shared.shared.shared.shared.settings["etcd"]["file_prefix"], self.uuid.value
)
)
if file_entry is None:
@ -125,7 +124,7 @@ class CreateImageSchema(BaseSchema):
def image_store_name_validation(self):
image_stores = list(
shared.etcd_client.get_prefix(
settings["etcd"]["image_store_prefix"]
shared.shared.shared.shared.shared.settings["etcd"]["image_store_prefix"]
)
)
@ -283,7 +282,7 @@ class CreateVMSchema(OTPSchema):
for net in _network:
network = shared.etcd_client.get(
os.path.join(
settings["etcd"]["network_prefix"],
shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"],
self.name.value,
net,
),
@ -488,7 +487,7 @@ class VmMigrationSchema(OTPSchema):
self.add_error("Can't migrate non-running VM")
if vm.hostname == os.path.join(
settings["etcd"]["host_prefix"], self.destination.value
shared.shared.shared.shared.shared.settings["etcd"]["host_prefix"], self.destination.value
):
self.add_error(
"Destination host couldn't be same as Source Host"
@ -539,9 +538,7 @@ class CreateNetwork(OTPSchema):
super().__init__(data, fields=fields)
def network_name_validation(self):
print(self.name.value, self.network_name.value)
key = os.path.join(settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
print(key)
key = os.path.join(shared.shared.shared.shared.shared.settings["etcd"]["network_prefix"], self.name.value, self.network_name.value)
network = shared.etcd_client.get(key, value_in_json=True)
if network:
self.add_error(

View File

@ -0,0 +1,46 @@
import requests
import json
import argparse
import binascii
from pyotp import TOTP
from os.path import join as join_path
from uncloud.common.shared import shared
def get_otp_parser():
otp_parser = argparse.ArgumentParser('otp')
otp_parser.add_argument('--name')
otp_parser.add_argument('--realm')
otp_parser.add_argument('--seed', type=get_token, dest='token', metavar='SEED')
return otp_parser
def load_dump_pretty(content):
if isinstance(content, bytes):
content = content.decode('utf-8')
parsed = json.loads(content)
return json.dumps(parsed, indent=4, sort_keys=True)
def make_request(*args, data=None, request_method=requests.post):
try:
r = request_method(join_path(shared.settings['client']['api_server'], *args), json=data)
except requests.exceptions.RequestException:
print('Error occurred while connecting to API server.')
else:
try:
print(load_dump_pretty(r.content))
except Exception:
print('Error occurred while getting output from api server.')
def get_token(seed):
if seed is not None:
try:
token = TOTP(seed).now()
except binascii.Error:
raise argparse.ArgumentTypeError('Invalid seed')
else:
return token

View File

@ -0,0 +1,23 @@
import argparse
import etcd3
from uncloud.common.etcd_wrapper import Etcd3Wrapper
arg_parser = argparse.ArgumentParser('client', add_help=False)
arg_parser.add_argument('--dump-etcd-contents-prefix', help="Dump contents below the given prefix")
def dump_etcd_contents(prefix):
etcd = Etcd3Wrapper()
for k,v in etcd.get_prefix_raw(prefix):
k = k.decode('utf-8')
v = v.decode('utf-8')
print("{} = {}".format(k,v))
# print("{} = {}".format(k,v))
# for k,v in etcd.get_prefix(prefix):
#
print("done")
def main(arguments):
if 'dump_etcd_contents_prefix' in arguments:
dump_etcd_contents(prefix=arguments['dump_etcd_contents_prefix'])

View File

@ -0,0 +1,26 @@
from uncloud.common.shared import shared
from pyotp import TOTP
def get_token(seed):
if seed is not None:
try:
token = TOTP(seed).now()
except Exception:
raise Exception('Invalid seed')
else:
return token
def resolve_otp_credentials(kwargs):
d = {
'name': shared.settings['client']['name'],
'realm': shared.settings['client']['realm'],
'token': get_token(shared.settings['client']['seed'])
}
for k, v in d.items():
if k in kwargs and kwargs[k] is None:
kwargs.update({k: v})
return d

View File

@ -8,6 +8,7 @@ from uncloud.common.etcd_wrapper import Etcd3Wrapper
from os.path import join as join_path
logger = logging.getLogger(__name__)
settings = None
class CustomConfigParser(configparser.RawConfigParser):
@ -25,9 +26,8 @@ class CustomConfigParser(configparser.RawConfigParser):
class Settings(object):
def __init__(self):
def __init__(self, conf_dir, seed_value=None):
conf_name = 'uncloud.conf'
conf_dir = os.environ.get('UCLOUD_CONF_DIR', os.path.expanduser('~/uncloud/'))
self.config_file = join_path(conf_dir, conf_name)
# this is used to cache config from etcd for 1 minutes. Without this we
@ -38,15 +38,19 @@ class Settings(object):
self.config_parser.add_section('etcd')
self.config_parser.set('etcd', 'base_prefix', '/')
try:
if os.access(self.config_file, os.R_OK):
self.config_parser.read(self.config_file)
except Exception as err:
logger.error('%s', err)
else:
raise FileNotFoundError('Config file %s not found!', self.config_file)
self.config_key = join_path(self['etcd']['base_prefix'] + 'uncloud/config/')
self.read_internal_values()
if seed_value is None:
seed_value = dict()
self.config_parser.read_dict(seed_value)
def get_etcd_client(self):
args = tuple()
try:
@ -128,4 +132,5 @@ class Settings(object):
return self.config_parser[key]
settings = Settings()
def get_settings():
return settings

View File

@ -0,0 +1,34 @@
from uncloud.common.settings import get_settings
from uncloud.common.vm import VmPool
from uncloud.common.host import HostPool
from uncloud.common.request import RequestPool
import uncloud.common.storage_handlers as storage_handlers
class Shared:
@property
def settings(self):
return get_settings()
@property
def etcd_client(self):
return self.settings.get_etcd_client()
@property
def host_pool(self):
return HostPool(self.etcd_client, self.settings["etcd"]["host_prefix"])
@property
def vm_pool(self):
return VmPool(self.etcd_client, self.settings["etcd"]["vm_prefix"])
@property
def request_pool(self):
return RequestPool(self.etcd_client, self.settings["etcd"]["request_prefix"])
@property
def storage_handler(self):
return storage_handlers.get_storage_handler()
shared = Shared()

View File

@ -6,8 +6,7 @@ import stat
from abc import ABC
from . import logger
from os.path import join as join_path
from uncloud.common.settings import settings as config
import uncloud.common.shared as shared
class ImageStorageHandler(ABC):
@ -193,16 +192,16 @@ class CEPHBasedImageStorageHandler(ImageStorageHandler):
def get_storage_handler():
__storage_backend = config["storage"]["storage_backend"]
__storage_backend = shared.shared.settings["storage"]["storage_backend"]
if __storage_backend == "filesystem":
return FileSystemBasedImageStorageHandler(
vm_base=config["storage"]["vm_dir"],
image_base=config["storage"]["image_dir"],
vm_base=shared.shared.settings["storage"]["vm_dir"],
image_base=shared.shared.settings["storage"]["image_dir"],
)
elif __storage_backend == "ceph":
return CEPHBasedImageStorageHandler(
vm_base=config["storage"]["ceph_vm_pool"],
image_base=config["storage"]["ceph_image_pool"],
vm_base=shared.shared.settings["storage"]["ceph_vm_pool"],
image_base=shared.shared.settings["storage"]["ceph_image_pool"],
)
else:
raise Exception("Unknown Image Storage Handler")
raise Exception("Unknown Image Storage Handler")

View File

@ -1,7 +1,6 @@
import os
import argparse
from uncloud.common.settings import settings
from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('configure', add_help=False)
@ -40,19 +39,19 @@ ceph_storage_parser.add_argument('--ceph-image-pool', required=True)
def update_config(section, kwargs):
uncloud_config = shared.etcd_client.get(settings.config_key, value_in_json=True)
uncloud_config = shared.etcd_client.get(shared.settings.config_key, value_in_json=True)
if not uncloud_config:
uncloud_config = {}
else:
uncloud_config = uncloud_config.value
uncloud_config[section] = kwargs
shared.etcd_client.put(settings.config_key, uncloud_config, value_in_json=True)
shared.etcd_client.put(shared.settings.config_key, uncloud_config, value_in_json=True)
def main(**kwargs):
subcommand = kwargs.pop('subcommand')
def main(arguments):
subcommand = arguments['subcommand']
if not subcommand:
arg_parser.print_help()
else:
update_config(subcommand, kwargs)
update_config(subcommand, arguments)

View File

@ -9,7 +9,6 @@ import bitmath
from uuid import uuid4
from . import logger
from uncloud.common.settings import settings
from uncloud.common.shared import shared
arg_parser = argparse.ArgumentParser('filescanner', add_help=False)
@ -53,7 +52,7 @@ def track_file(file, base_dir, host):
file_path = file_path.relative_to(owner)
creation_date = time.ctime(os.stat(file_str).st_ctime)
entry_key = os.path.join(settings['etcd']['file_prefix'], str(uuid4()))
entry_key = os.path.join(shared.settings['etcd']['file_prefix'], str(uuid4()))
entry_value = {
'filename': str(file_path),
'owner': owner,
@ -70,7 +69,7 @@ def track_file(file, base_dir, host):
def main(arguments):
hostname = arguments['hostname']
base_dir = settings['storage']['file_dir']
base_dir = shared.settings['storage']['file_dir']
# Recursively Get All Files and Folder below BASE_DIR
files = glob.glob('{}/**'.format(base_dir), recursive=True)
files = [pathlib.Path(f) for f in files if pathlib.Path(f).is_file()]
@ -78,7 +77,7 @@ def main(arguments):
# Files that are already tracked
tracked_files = [
pathlib.Path(os.path.join(base_dir, f.value['owner'], f.value['filename']))
for f in shared.etcd_client.get_prefix(settings['etcd']['file_prefix'], value_in_json=True)
for f in shared.etcd_client.get_prefix(shared.settings['etcd']['file_prefix'], value_in_json=True)
if f.value['host'] == hostname
]
untracked_files = set(files) - set(tracked_files)

View File

@ -0,0 +1 @@

View File

@ -0,0 +1,39 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
#
# This file is part of uncloud.
#
# uncloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# uncloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
#
#
class Config(object):
def __init__(self, arguments):
""" read arguments dicts as a base """
self.arguments = arguments
# Split them so *etcd_args can be used and we can
# iterate over etcd_hosts
self.etcd_hosts = [ arguments['etcd_host'] ]
self.etcd_args = {
'ca_cert': arguments['etcd_ca_cert'],
'cert_cert': arguments['etcd_cert_cert'],
'cert_key': arguments['etcd_cert_key'],
# 'user': None,
# 'password': None
}
self.etcd_prefix = '/nicohack/'

View File

@ -0,0 +1,149 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2020 Nico Schottelius (nico.schottelius at ungleich.ch)
#
# This file is part of uncloud.
#
# uncloud is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# uncloud is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with uncloud. If not, see <http://www.gnu.org/licenses/>.
#
#
import etcd3
import json
import logging
import datetime
import re
from functools import wraps
from uncloud import UncloudException
log = logging.getLogger(__name__)
def db_logentry(message):
timestamp = datetime.datetime.now()
return {
"timestamp": str(timestamp),
"message": message
}
def readable_errors(func):
@wraps(func)
def wrapper(*args, **kwargs):
try:
return func(*args, **kwargs)
except etcd3.exceptions.ConnectionFailedError as e:
raise UncloudException('Cannot connect to etcd: is etcd running and reachable? {}'.format(e))
except etcd3.exceptions.ConnectionTimeoutError as e:
raise UncloudException('etcd connection timeout. {}'.format(e))
return wrapper
class DB(object):
def __init__(self, config, prefix="/"):
self.config = config
# Root for everything
self.base_prefix= '/nicohack'
# Can be set from outside
self.prefix = prefix
try:
self.connect()
except FileNotFoundError as e:
raise UncloudException("Is the path to the etcd certs correct? {}".format(e))
@readable_errors
def connect(self):
self._db_clients = []
for endpoint in self.config.etcd_hosts:
client = etcd3.client(host=endpoint, **self.config.etcd_args)
self._db_clients.append(client)
def realkey(self, key):
return "{}{}/{}".format(self.base_prefix,
self.prefix,
key)
@readable_errors
def get(self, key, as_json=False, **kwargs):
value, _ = self._db_clients[0].get(self.realkey(key), **kwargs)
if as_json:
value = json.loads(value)
return value
@readable_errors
def get_prefix(self, key, as_json=False, **kwargs):
for value, meta in self._db_clients[0].get_prefix(self.realkey(key), **kwargs):
k = meta.key.decode("utf-8")
value = value.decode("utf-8")
if as_json:
value = json.loads(value)
yield (k, value)
@readable_errors
def set(self, key, value, as_json=False, **kwargs):
if as_json:
value = json.dumps(value)
log.debug("Setting {} = {}".format(self.realkey(key), value))
# FIXME: iterate over clients in case of failure ?
return self._db_clients[0].put(self.realkey(key), value, **kwargs)
@readable_errors
def list_and_filter(self, key, filter_key=None, filter_regexp=None):
for k,v in self.get_prefix(key, as_json=True):
if filter_key and filter_regexp:
if filter_key in v:
if re.match(filter_regexp, v[filter_key]):
yield v
else:
yield v
@readable_errors
def increment(self, key, **kwargs):
print(self.realkey(key))
print("prelock")
lock = self._db_clients[0].lock('/nicohack/foo')
print("prelockacq")
lock.acquire()
print("prelockrelease")
lock.release()
with self._db_clients[0].lock("/nicohack/mac/last_used_index") as lock:
print("in lock")
pass
# with self._db_clients[0].lock(self.realkey(key)) as lock:# value = int(self.get(self.realkey(key), **kwargs))
# self.set(self.realkey(key), str(value + 1), **kwargs)
if __name__ == '__main__':
endpoints = [ "https://etcd1.ungleich.ch:2379",
"https://etcd2.ungleich.ch:2379",
"https://etcd3.ungleich.ch:2379" ]
db = DB(url=endpoints)

View File

@ -0,0 +1,6 @@
#!/bin/sh
etcdctl --cert=$HOME/vcs/ungleich-dot-cdist/files/etcd/nico.pem \
--key=/home/nico/vcs/ungleich-dot-cdist/files/etcd/nico-key.pem \
--cacert=$HOME/vcs/ungleich-dot-cdist/files/etcd/ca.pem \
--endpoints https://etcd1.ungleich.ch:2379,https://etcd2.ungleich.ch:2379,https://etcd3.ungleich.ch:2379 "$@"

View File

@ -0,0 +1,3 @@
#!/bin/sh
echo $@

View File

@ -0,0 +1 @@
000000000252

View File

@ -0,0 +1 @@
02:00

Some files were not shown because too many files have changed in this diff Show More