Merge remote-tracking branch 'ahmed/master'

README.md

@@ -1,25 +1,24 @@
 # uncloud-pay
 
-The pay module for the uncloud
+The generic product/payment system.
 
-- uses [etcd3](https://coreos.com/blog/etcd3-a-new-etcd.html) for storage.
-- uses [Stripe](https://stripe.com/docs/api) as the payment gateway.
-- uses [ldap3](https://github.com/cannatag/ldap3) for ldap authentication.
+## Installation
 
+```shell script
+pip3 install -r requirements.txt
+```
 
-## Getting started as a user
+## Getting Started
 
-
-
-**TODO**
+```shell script
+python ucloud_pay.py
+```
 
 ## Usage
 
-Currently handles very basic features, such as:
-
 #### 1. Adding of products
 ```shell script
-http --json http://[::]:5000/product/add email=your_email_here password=your_password_here specs:=@ipv6-only-vm.json
+http --json http://[::]:5000/product/add username=your_username_here password=your_password_here specs:=@ipv6-only-vm.json
 ```
 
 #### 2. Listing of products
@@ -27,20 +26,26 @@ http --json http://[::]:5000/product/add email=your_email_here password=your_pas
 http --json http://[::]:5000/product/list
 ```
 
-#### 3. Ordering products
-```shell script
-http --json http://[::]:5000/product/order email=your_email_here password=your_password_here product_id=5332cb89453d495381e2b2167f32c842 cpu=1 ram=1gb os-disk-space=10gb os=alpine
-```
-
-#### 4. Listing users orders
+#### 3. Registering user's payment method (credit card for now using Stripe)
 
 ```shell script
-http --json GET http://[::]:5000/order/list email=your_email_here password=your_password_here
+http --json http://[::]:5000/user/register_payment card_number=4111111111111111 cvc=123 expiry_year=2020 expiry_month=8 card_holder_name="The test user" username=your_username_here password=your_password_here line1="your_billing_address" city="your_city" country="your_country"
 ```
 
+#### 4. Ordering products
 
-#### 5. Registering user's payment method (credit card for now using Stripe)
+First of all, user have to buy the membership first.
 
 ```shell script
-http --json http://[::]:5000/user/register_payment card_number=4111111111111111 cvc=123 expiry_year=2020 expiry_month=8 card_holder_name="The test user" email=your_email_here password=your_password_here
+http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=membership pay=True
+```
+
+```shell script
+http --json http://[::]:5000/product/order username=your_username_here password=your_password_here product_id=ipv6-only-vm cpu=1 ram=1 os-disk-space=10 os=alpine pay=True
+```
+
+#### 5. Listing users orders
+
+```shell script
+http --json POST http://[::]:5000/order/list username=your_username_here password=your_password_here
 ```

config.py

@@ -1,14 +1,21 @@
-import configparser
-from etcd_wrapper import EtcdWrapper
-from ldap_manager import LdapManager
+import os
 
-config = configparser.ConfigParser()
-config.read('pay.conf')
+from ungleich_common.ldap.ldap_manager import LdapManager
+from ungleich_common.std.configparser import StrictConfigParser
+from ungleich_common.etcd.etcd_wrapper import EtcdWrapper
 
-# Note 2020-02-15: this stuff clearly does not belong here,
-# if config.py is used everywhere.
+config_file = os.environ.get('meow-pay-config-file', default='pay.conf')
 
-etcd_client = EtcdWrapper(host=config['etcd']['host'], port=config['etcd']['port'])
+config = StrictConfigParser(allow_no_value=True)
+config.read(config_file)
 
-ldap_manager = LdapManager(server=config['ldap']['server'], admin_dn=config['ldap']['admin_dn'],
-                           admin_password=config['ldap']['admin_password'])
+etcd_client = EtcdWrapper(
+    host=config.get('etcd', 'host'), port=config.get('etcd', 'port'),
+    ca_cert=config.get('etcd', 'ca_cert'), cert_key=config.get('etcd', 'cert_key'),
+    cert_cert=config.get('etcd', 'cert_cert')
+)
+
+ldap_manager = LdapManager(
+    server=config.get('ldap', 'server'), admin_dn=config.get('ldap', 'admin_dn'),
+    admin_password=config.get('ldap', 'admin_password')
+)

etcd_wrapper.py

@@ -1,75 +0,0 @@
-import etcd3
-import json
-
-from functools import wraps
-
-from uncloud import UncloudException
-from uncloud.common import logger
-
-
-class EtcdEntry:
-    def __init__(self, meta_or_key, value, value_in_json=True):
-        if hasattr(meta_or_key, 'key'):
-            # if meta has attr 'key' then get it
-            self.key = meta_or_key.key.decode('utf-8')
-        else:
-            # otherwise meta is the 'key'
-            self.key = meta_or_key
-        self.value = value.decode('utf-8')
-
-        if value_in_json:
-            self.value = json.loads(self.value)
-
-
-def readable_errors(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except etcd3.exceptions.ConnectionFailedError:
-            raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?')
-        except etcd3.exceptions.ConnectionTimeoutError as err:
-            raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err
-        except Exception as err:
-            logger.exception('Some etcd error occured. See syslog for details.', err)
-
-    return wrapper
-
-
-class EtcdWrapper:
-    @readable_errors
-    def __init__(self, *args, **kwargs):
-        self.client = etcd3.client(*args, **kwargs)
-
-    @readable_errors
-    def get(self, *args, value_in_json=True, **kwargs):
-        _value, _key = self.client.get(*args, **kwargs)
-        if _key is None or _value is None:
-            return None
-        return EtcdEntry(_key, _value, value_in_json=value_in_json)
-
-    @readable_errors
-    def put(self, *args, value_in_json=True, **kwargs):
-        _key, _value = args
-        if value_in_json:
-            _value = json.dumps(_value)
-
-        if not isinstance(_key, str):
-            _key = _key.decode('utf-8')
-
-        return self.client.put(_key, _value, **kwargs)
-
-    @readable_errors
-    def get_prefix(self, *args, value_in_json=True, **kwargs):
-        event_iterator = self.client.get_prefix(*args, **kwargs)
-        for e in event_iterator:
-            yield EtcdEntry(*e[::-1], value_in_json=value_in_json)
-
-    @readable_errors
-    def watch_prefix(self, key, value_in_json=True):
-        event_iterator, cancel = self.client.watch_prefix(key)
-        for e in event_iterator:
-            if hasattr(e, '_event'):
-                e = getattr('e', '_event')
-                if e.type == e.PUT:
-                    yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json)

helper.py

@@ -1,5 +1,8 @@
 import logging
 
+import parsedatetime
+
+from datetime import datetime
 from stripe_utils import StripeUtils
 
 
@@ -64,3 +67,21 @@ def calculate_charges(specification, data):
                     feature_detail['unit']['value']
             )
     return one_time_charge, recurring_charge
+
+
+def is_order_valid(order_timestamp, renewal_period):
+    """
+    Sample Code Usage
+
+    >> current_datetime, status = cal.parse('Now')
+    >> current_datetime = datetime(*current_datetime[:6])
+
+    >> print('Is order valid: ', is_order_valid(current_datetime, '1 month'))
+    >> True
+    """
+    cal = parsedatetime.Calendar()
+
+    renewal_datetime, status = cal.parse(renewal_period)
+    renewal_datetime = datetime(*renewal_datetime[:6])
+
+    return order_timestamp <= renewal_datetime

ldap_manager.py

@@ -1,69 +0,0 @@
-import hashlib
-import random
-import base64
-
-from ldap3 import Server, Connection, ObjectDef, Reader, ALL
-
-
-class LdapManager:
-    def __init__(self, server, admin_dn, admin_password):
-        self.server = Server(server, get_info=ALL)
-        self.conn = Connection(server, admin_dn, admin_password, auto_bind=True)
-        self.person_obj_def = ObjectDef('inetOrgPerson', self.conn)
-
-    def get(self, query=None, search_base='dc=ungleich,dc=ch'):
-        kwargs = {
-            'connection': self.conn,
-            'object_def': self.person_obj_def,
-            'base': search_base,
-        }
-        if query:
-            kwargs['query'] = query
-        r = Reader(**kwargs)
-        return r.search()
-
-    def is_password_valid(self, query_value, password, query_key='mail', **kwargs):
-        entries = self.get(query='({}={})'.format(query_key, query_value), **kwargs)
-        if entries:
-            password_in_ldap = entries[0].userPassword.value
-            found = self._check_password(password_in_ldap, password)
-            if not found:
-                raise Exception('Invalid Password')
-            else:
-                return entries[0]
-        else:
-            raise ValueError('Such {}={} not found'.format(query_key, query_value))
-
-    @staticmethod
-    def _check_password(tagged_digest_salt, password):
-        digest_salt_b64 = tagged_digest_salt[6:]
-        digest_salt = base64.decodebytes(digest_salt_b64)
-        digest = digest_salt[:20]
-        salt = digest_salt[20:]
-
-        sha = hashlib.sha1(password.encode('utf-8'))
-        sha.update(salt)
-
-        return digest == sha.digest()
-
-    @staticmethod
-    def ssha_password(password):
-        """
-        Apply the SSHA password hashing scheme to the given *password*.
-        *password* must be a :class:`bytes` object, containing the utf-8
-        encoded password.
-
-        Return a :class:`bytes` object containing ``ascii``-compatible data
-        which can be used as LDAP value, e.g. after armoring it once more using
-        base64 or decoding it to unicode from ``ascii``.
-        """
-        SALT_BYTES = 15
-
-        sha1 = hashlib.sha1()
-        salt = random.SystemRandom().getrandbits(SALT_BYTES * 8).to_bytes(SALT_BYTES, 'little')
-        sha1.update(password)
-        sha1.update(salt)
-
-        digest = sha1.digest()
-        passwd = b'{SSHA}' + base64.b64encode(digest + salt)
-        return passwd

nicohack202002/uncloud/opennebula/admin.py

@@ -0,0 +1,3 @@
+from django.contrib import admin
+
+# Register your models here.

nicohack202002/uncloud/opennebula/apps.py

@@ -0,0 +1,5 @@
+from django.apps import AppConfig
+
+
+class OpennebulaConfig(AppConfig):
+    name = 'opennebula'

nicohack202002/uncloud/opennebula/management/commands/syncvm.py

@@ -0,0 +1,40 @@
+import os
+import json
+
+from django.core.management.base import BaseCommand
+from django.contrib.auth import get_user_model
+from xmlrpc.client import ServerProxy as RPCClient
+
+from xmltodict import parse
+
+from opennebula.models import VM as VMModel
+
+OCA_SESSION_STRING = os.environ.get('OCASECRETS', '')
+
+
+class Command(BaseCommand):
+    help = 'Syncronize VM information from OpenNebula'
+
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **options):
+        with RPCClient('https://opennebula.ungleich.ch:2634/RPC2') as rpc_client:
+            success, response, *_ = rpc_client.one.vmpool.infoextended(
+                OCA_SESSION_STRING, -2, -1, -1, -1
+            )
+            if success:
+                vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM']
+                for i, vm in enumerate(vms):
+                    vm_id = vm['ID']
+                    vm_owner = vm['UNAME']
+                    try:
+                        user = get_user_model().objects.get(username=vm_owner)
+                    except get_user_model().DoesNotExist:
+                        user = get_user_model().objects.create_user(username=vm_owner)
+
+                    vm_object = VMModel.objects.create(vmid=vm_id, owner=user, data=vm)
+                    vm_object.save()
+            else:
+                print(response)
+

nicohack202002/uncloud/opennebula/migrations/0001_initial.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.3 on 2020-02-21 10:22
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='VM',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('vmid', models.IntegerField()),
+                ('owner', models.CharField(max_length=128)),
+                ('data', models.CharField(max_length=65536)),
+            ],
+        ),
+    ]

nicohack202002/uncloud/opennebula/migrations/0002_auto_20200221_1024.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.3 on 2020-02-21 10:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('opennebula', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='vm',
+            name='data',
+            field=models.CharField(max_length=65536, null=True),
+        ),
+        migrations.AlterField(
+            model_name='vm',
+            name='owner',
+            field=models.CharField(max_length=128, null=True),
+        ),
+    ]

nicohack202002/uncloud/opennebula/migrations/0003_auto_20200221_1113.py

@@ -0,0 +1,21 @@
+# Generated by Django 3.0.3 on 2020-02-21 11:13
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('opennebula', '0002_auto_20200221_1024'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='vm',
+            name='owner',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+        ),
+    ]

nicohack202002/uncloud/opennebula/models.py

@@ -0,0 +1,8 @@
+from django.db import models
+from django.contrib.auth import get_user_model
+
+
+class VM(models.Model):
+    vmid = models.IntegerField()
+    owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE)
+    data = models.CharField(max_length=65536, null=True)

nicohack202002/uncloud/opennebula/serializers.py

@@ -0,0 +1,8 @@
+from rest_framework import serializers
+from opennebula.models import VM
+
+
+class VMSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = VM
+        fields = ['vmid', 'owner', 'data']

nicohack202002/uncloud/opennebula/tests.py

@@ -0,0 +1,3 @@
+from django.test import TestCase
+
+# Create your tests here.

nicohack202002/uncloud/opennebula/views.py

@@ -0,0 +1,14 @@
+from rest_framework import viewsets, generics
+from .models import VM
+from .serializers import VMSerializer
+
+
+class VMList(generics.ListAPIView):
+    queryset = VM.objects.all()
+    serializer_class = VMSerializer
+
+
+class VMDetail(generics.RetrieveAPIView):
+    lookup_field = 'vmid'
+    queryset = VM.objects.all()
+    serializer_class = VMSerializer

nicohack202002/uncloud/uncloud/settings.py

@@ -39,7 +39,8 @@ INSTALLED_APPS = [
     'django.contrib.staticfiles',
     'rest_framework',
     'uncloud_api',
-    'uncloud_auth'
+    'uncloud_auth',
+    'opennebula'
 ]
 
 MIDDLEWARE = [
@@ -159,7 +160,7 @@ STATIC_URL = '/static/'
 
 
 # Uncommitted file
-import uncloud.secrets
-
-import stripe
-stripe.api_key = uncloud.secrets.STRIPE_KEY
+# import uncloud.secrets
+#
+# import stripe
+# stripe.api_key = uncloud.secrets.STRIPE_KEY

nicohack202002/uncloud/uncloud/urls.py

@@ -19,6 +19,8 @@ from django.urls import path, include
 from rest_framework import routers
 from uncloud_api import views
 
+from opennebula import views as oneviews
+
 router = routers.DefaultRouter()
 router.register(r'users', views.UserViewSet)
 router.register(r'groups', views.GroupViewSet)
@@ -28,10 +30,7 @@ router.register(r'groups', views.GroupViewSet)
 urlpatterns = [
     path('', include(router.urls)),
     path('admin/', admin.site.urls),
-    path('api-auth/', include('rest_framework.urls', namespace='rest_framework'))
+    path('api-auth/', include('rest_framework.urls', namespace='rest_framework')),
+    path('vm/list/', oneviews.VMList.as_view(), name='vm_list'),
+    path('vm/detail/<int:vmid>/', oneviews.VMDetail.as_view(), name='vm_detail'),
 ]
-
-#urlpatterns = [
-#    path('admin/', admin.site.urls),
-#    path('api/', include('api.urls')),
-#]

nicohack202002/uncloud/uncloud_auth/models.py

@@ -1,4 +1,5 @@
 from django.contrib.auth.models import AbstractUser
 
+
 class User(AbstractUser):
     pass

requirements.txt

@@ -1,4 +1,11 @@
-flask-restful
-ldap3
-etcd3
+xmltodict
+djangorestframework
+django
+done
 stripe
+flask
+Flask-RESTful
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-etcd&subdirectory=etcd
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-std&subdirectory=std
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-schemas&subdirectory=schemas

sample-pay.conf

@@ -0,0 +1,17 @@
+[etcd]
+host = 127.0.0.1
+port = 2379
+ca_cert
+cert_cert
+cert_key
+
+[stripe]
+private_key=stripe_private_key
+
+[app]
+port = 5000
+
+[ldap]
+server = ldap_server_url
+admin_dn = ldap_admin_dn
+admin_password = ldap_admin_password

schemas.py

@@ -3,95 +3,9 @@ import config
 import json
 import math
 
-from config import ldap_manager
+from config import ldap_manager, etcd_client
 from helper import resolve_product
-
-etcd_client = config.etcd_client
-
-
-class ValidationException(Exception):
-    """Validation Error"""
-
-
-class Field:
-    def __init__(self, _name, _type, _value=None, validators=None, disable_validation=False):
-        self.validation_disabled = disable_validation
-        self.name = _name
-        self.value = _value
-        self.type = _type
-        self.validators = validators or []
-
-    def is_valid(self):
-        if not self.validation_disabled:
-            if not isinstance(self.value, self.type):
-                try:
-                    self.value = self.type(self.value)
-                except Exception:
-                    raise ValidationException("Incorrect Type for '{}' field".format(self.name))
-
-            for validator in self.validators:
-                validator()
-
-    def __repr__(self):
-        return self.name
-
-
-class BaseSchema:
-    def __init__(self):
-        self.objects = {}
-
-    def validation(self):
-        # custom validation is optional
-        return True
-
-    def get_fields(self):
-        return [getattr(self, field) for field in dir(self) if isinstance(getattr(self, field), Field)]
-
-    def is_valid(self):
-        for field in self.get_fields():
-            field.is_valid()
-        self.validation()
-
-    def get_cleaned_values(self):
-        field_kv_dict = {
-            field.name: field.value
-            for field in self.get_fields()
-        }
-        cleaned_values = field_kv_dict
-        cleaned_values.update(self.objects)
-
-        return cleaned_values
-
-    def add_schema(self, schema, data, under_field_name=None):
-        s = schema(data)
-        s.is_valid()
-
-        base = self
-        if under_field_name:
-            # Create a field in self
-            setattr(self, under_field_name, Field(under_field_name, dict, _value={}, disable_validation=True))
-            base = getattr(self, under_field_name)
-
-        for field in s.get_fields():
-            if under_field_name:
-                getattr(base, 'value')[field.name] = field.value
-            else:
-                setattr(base, field.name, field)
-
-        self.objects.update(s.objects)
-
-    @staticmethod
-    def get(dictionary: dict, key: str, return_default=False, default=None):
-        if dictionary is None:
-            raise ValidationException('No data provided at all.')
-        try:
-            value = dictionary[key]
-        except KeyError:
-            if return_default:
-                return {'_value': default, 'disable_validation': True}
-            raise ValidationException("Missing data for '{}' field.".format(key))
-        else:
-            return {'_value': value, 'disable_validation': False}
+from ungleich_common.schemas.schemas import BaseSchema, Field, ValidationException
 
 
 class AddProductSchema(BaseSchema):
@@ -105,7 +19,7 @@ class AddProductSchema(BaseSchema):
         user = self.objects['user']
         user = json.loads(user.entry_to_json())
         uid, ou, *dc = user['dn'].replace('ou=', '').replace('dc=', '').replace('uid=', '').split(',')
-        if ou != config.config['ldap']['internal_user_ou']:
+        if ou != config.config.get('ldap', 'internal_user_ou', fallback='users'):
             raise ValidationException('You do not have access to create product.')
 
         product = resolve_product(self.specs.value['usable-id'], etcd_client)

stripe_utils.py

@@ -1,12 +1,11 @@
-import json
 import re
 import stripe
 import stripe.error
 import logging
 
-# FIXME: way too many dependencies in this import
-# Most of them are not needed for stripe
-#from config import etcd_client as client, config as config
+from config import etcd_client as client, config as config
+
+stripe.api_key = config.get('stripe', 'private_key')
 
 
 def handle_stripe_error(f):
@@ -292,7 +291,7 @@ class StripeUtils(object):
         returns the new object.
 
         :param amount: The amount in CHF cents
-        :param name: The name of the Stripe plan to be created.
+        :param product_name: The name of the Stripe plan (product) to be created.
         :param stripe_plan_id: The id of the Stripe plan to be
                created. Use get_stripe_plan_id_string function to
                obtain the name of the plan to be created

ucloud_pay.py

@@ -5,7 +5,7 @@ from uuid import uuid4
 
 from flask import Flask, request
 from flask_restful import Resource, Api
-
+from werkzeug.exceptions import HTTPException
 from config import etcd_client as client, config as config
 from stripe_utils import StripeUtils
 from schemas import (
@@ -324,4 +324,15 @@ if __name__ == '__main__':
     api.add_resource(UserRegisterPayment, '/user/register_payment')
     api.add_resource(OrderList, '/order/list')
 
-    app.run(host='::', port=config['app']['port'], debug=True)
+    app.run(host='::', port=config.get('app', 'port', fallback=5000), debug=True)
+
+
+    @app.errorhandler(Exception)
+    def handle_exception(e):
+        app.logger.error(e)
+        # pass through HTTP errors
+        if isinstance(e, HTTPException):
+            return e
+
+        # now you're handling non-HTTP exceptions only
+        return {'message': 'Server Error'}, 500