From 446c13b77c12461f3436b33ec993eb9daadf5979 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 25 Feb 2020 19:23:39 +0100 Subject: [PATCH] fix/simplify syncvm --- nicohack202002/uncloud/opennebula/models.py | 11 -- nicohack202002/uncloud/opennebula/views.py | 59 --------- nicohack202002/uncloud/uncloud_api/models.py | 125 ------------------ nicohack202002/uncloud/uncloud_api/views.py | 83 ------------ .../opennebula/management/commands/syncvm.py | 37 ++---- .../migrations/0004_auto_20200225_1816.py | 23 ++++ uncloud/opennebula/models.py | 3 +- uncloud/requirements.txt | 1 - uncloud/uncloud/settings.py | 14 +- .../0002_vmsnapshotproduct_vm_uuid.py | 19 +++ uncloud/uncloud_api/models.py | 2 +- 11 files changed, 57 insertions(+), 320 deletions(-) delete mode 100644 nicohack202002/uncloud/opennebula/models.py delete mode 100644 nicohack202002/uncloud/opennebula/views.py delete mode 100644 nicohack202002/uncloud/uncloud_api/models.py delete mode 100644 nicohack202002/uncloud/uncloud_api/views.py create mode 100644 uncloud/opennebula/migrations/0004_auto_20200225_1816.py create mode 100644 uncloud/uncloud_api/migrations/0002_vmsnapshotproduct_vm_uuid.py diff --git a/nicohack202002/uncloud/opennebula/models.py b/nicohack202002/uncloud/opennebula/models.py deleted file mode 100644 index 915862a..0000000 --- a/nicohack202002/uncloud/opennebula/models.py +++ /dev/null @@ -1,11 +0,0 @@ -import uuid - -from django.db import models -from django.contrib.auth import get_user_model - - -class VM(models.Model): - uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - vmid = models.IntegerField() - owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) - data = models.CharField(max_length=65536, null=True) diff --git a/nicohack202002/uncloud/opennebula/views.py b/nicohack202002/uncloud/opennebula/views.py deleted file mode 100644 index 1030101..0000000 --- a/nicohack202002/uncloud/opennebula/views.py +++ /dev/null @@ -1,59 +0,0 @@ -import json - -from rest_framework import generics -from rest_framework.authentication import SessionAuthentication, BasicAuthentication -from rest_framework.permissions import IsAuthenticated, IsAdminUser - -from .models import VM -from .serializers import VMSerializer - -class VMList(generics.ListAPIView): - authentication_classes = [SessionAuthentication, BasicAuthentication] - permission_classes = [IsAuthenticated, IsAdminUser] - queryset = VM.objects.all() - serializer_class = VMSerializer - - -class VMDetail(generics.RetrieveAPIView): - authentication_classes = [SessionAuthentication, BasicAuthentication] - permission_classes = [IsAuthenticated, IsAdminUser] - lookup_field = 'uuid' - queryset = VM.objects.all() - serializer_class = VMSerializer - - -class UserVMList(generics.ListAPIView): - authentication_classes = [SessionAuthentication, BasicAuthentication] - permission_classes = [IsAuthenticated] - serializer_class = VMSerializer - - def get_queryset(self): - user_email = self.request.user.ldap_user.attrs.data['mail'] - vms = [] - for mail in user_email: - vms += VM.objects.filter(owner__username=mail) - - for vm in vms: - data = json.loads(vm.data) - vm_template = data['TEMPLATE'] - vm.data = { - 'cpu': vm_template['VCPU'], - 'ram': vm_template['MEMORY'], - 'nic': vm_template['NIC'], - 'disks': vm_template['DISK'] - } - - return vms - -####################################### -# Following for quick experimentation # -####################################### - -# from django.http import HttpResponse -# -# def test(request): -# user_email = request.user.ldap_user.attrs.data['mail'] -# vms = [] -# for mail in user_email: -# vms += VM.objects.filter(owner__username=mail) -# return HttpResponse("Hello World") diff --git a/nicohack202002/uncloud/uncloud_api/models.py b/nicohack202002/uncloud/uncloud_api/models.py deleted file mode 100644 index 7eaec7b..0000000 --- a/nicohack202002/uncloud/uncloud_api/models.py +++ /dev/null @@ -1,125 +0,0 @@ -import uuid - -from django.db import models -from django.contrib.auth import get_user_model - -# Product in DB vs. product in code -# DB: -# - need to define params (+param types) in db -> messy? -# - get /products/ is easy / automatic -# -# code -# - can have serializer/verification of fields easily in DRF -# - can have per product side effects / extra code running -# - might (??) make features easier?? -# - how to setup / query the recurring period (?) -# - could get products list via getattr() + re ...Product() classes -# -> this could include the url for ordering => /order/vm_snapshot (params) -# ---> this would work with urlpatterns - -# Combination: create specific product in DB (?) -# - a table per product (?) with 1 entry? - -# Orders -# define state in DB -# select a price from a product => product might change, order stays -# params: -# - the product uuid or name (?) => productuuid -# - the product parameters => for each feature -# - -# logs -# Should have a log = ... => 1:n field for most models! - - -class Product(models.Model): - # override these fields by default - description = "" - recurring_period = "not_recurring" - uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - status = models.CharField( - max_length=256, choices=( - ('pending', 'Pending'), - ('being_created', 'Being created'), - ('created_active', 'Created'), - ('deleted', 'Deleted') - ), - default='pending' - ) - - def __str__(self): - return "{}".format(self.name) - - -class VMSnapshotProduct(Product): - price_per_gb_ssd = 0.35 - price_per_gb_hdd = 1.5/100 - - sample_ssd = 10 - sample_hdd = 100 - - def recurring_price(self): - return 0 - - def one_time_price(self): - return 0 - - @classmethod - def sample_price(cls): - return cls.sample_ssd * cls.price_per_gb_ssd + cls.sample_hdd * cls.price_per_gb_hdd - - description = "Create snapshot of a VM" - recurring_period = "monthly" - - @classmethod - def pricing_model(cls): - return """ -Pricing is on monthly basis and storage prices are equivalent to the storage -price in the VM. - -Price per GB SSD is: {} -Price per GB HDD is: {} - - -Sample price for a VM with {} GB SSD and {} GB HDD VM is: {}. -""".format(cls.price_per_gb_ssd, cls.price_per_gb_hdd, - cls.sample_ssd, cls.sample_hdd, cls.sample_price()) - - gb_ssd = models.FloatField() - gb_hdd = models.FloatField() - - -class Feature(models.Model): - uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - name = models.CharField(max_length=256) - - recurring_price = models.FloatField(default=0) - one_time_price = models.FloatField() - - product = models.ForeignKey(Product, on_delete=models.CASCADE) - - # params for "cpu": cpu_count -> int - # each feature can only have one parameters - # could call this "value" and set whether it is user usable - # has_value = True/False - # value = string -> int (?) - # value_int - # value_str - # value_float - - def __str__(self): - return "'{}' - '{}'".format(self.product, self.name) - - -class Order(models.Model): - uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) - - owner = models.ForeignKey(get_user_model(), - on_delete=models.CASCADE) - - product = models.ForeignKey(Product, - on_delete=models.CASCADE) - - -class VMSnapshotOrder(Order): - pass diff --git a/nicohack202002/uncloud/uncloud_api/views.py b/nicohack202002/uncloud/uncloud_api/views.py deleted file mode 100644 index 68963ff..0000000 --- a/nicohack202002/uncloud/uncloud_api/views.py +++ /dev/null @@ -1,83 +0,0 @@ -from django.shortcuts import render -from django.contrib.auth import get_user_model -from django.contrib.auth.models import Group - -from rest_framework import viewsets, permissions, generics -from .serializers import UserSerializer, GroupSerializer -from rest_framework.views import APIView -from rest_framework.response import Response - - -class CreditCardViewSet(viewsets.ModelViewSet): - - """ - API endpoint that allows credit cards to be listed - """ - queryset = get_user_model().objects.all().order_by('-date_joined') - serializer_class = UserSerializer - - permission_classes = [permissions.IsAuthenticated] - - -class UserViewSet(viewsets.ModelViewSet): - - """ - API endpoint that allows users to be viewed or edited. - """ - queryset = get_user_model().objects.all().order_by('-date_joined') - serializer_class = UserSerializer - - permission_classes = [permissions.IsAuthenticated] - -class GroupViewSet(viewsets.ModelViewSet): - """ - API endpoint that allows groups to be viewed or edited. - """ - queryset = Group.objects.all() - serializer_class = GroupSerializer - - permission_classes = [permissions.IsAuthenticated] - -class GroupViewSet(viewsets.ModelViewSet): - """ - API endpoint that allows groups to be viewed or edited. - """ - queryset = Group.objects.all() - serializer_class = GroupSerializer - - permission_classes = [permissions.IsAuthenticated] - - -# POST /vm/snapshot/ vmuuid=... => create snapshot, returns snapshot uuid -# GET /vm/snapshot => list -# DEL /vm/snapshot/ => delete -# create-list -> get, post => ListCreateAPIView -# del on other! -class VMSnapshotView(generics.ListCreateAPIView): - #lookup_field = 'uuid' - permission_classes = [permissions.IsAuthenticated] - -import inspect -import sys -import re - -# Next: create /order/ urls -# Next: strip off "Product" at the end -class ProductsView(APIView): - def get(self, request, format=None): - clsmembers = inspect.getmembers(sys.modules['uncloud_api.models'], inspect.isclass) - products = [] - for name, c in clsmembers: - # Include everything that ends in Product, but not Product itself - m = re.match(r'(?P.+)Product$', name) - if m: - products.append({ - 'name': m.group('pname'), - 'description': c.description, - 'recurring_period': c.recurring_period, - 'pricing_model': c.pricing_model() - } - ) - - - return Response(products) diff --git a/uncloud/opennebula/management/commands/syncvm.py b/uncloud/opennebula/management/commands/syncvm.py index 55844e3..779db61 100644 --- a/uncloud/opennebula/management/commands/syncvm.py +++ b/uncloud/opennebula/management/commands/syncvm.py @@ -8,15 +8,10 @@ from xmlrpc.client import ServerProxy as RPCClient from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model from xmltodict import parse -from ungleich_common.ldap.ldap_manager import LdapManager from opennebula.models import VM as VMModel - -def find_user_based_on_email(users, email): - for user in users: - if email in user.mail.values: - return user +from django_auth_ldap.backend import LDAPBackend class Command(BaseCommand): @@ -26,39 +21,29 @@ class Command(BaseCommand): pass def handle(self, *args, **options): - ldap_server_uri = secrets.LDAP_SERVER_URI.split(',')[0] - ldap_manager = LdapManager( - server=ldap_server_uri, - admin_dn=secrets.LDAP_ADMIN_DN, - admin_password=secrets.LDAP_ADMIN_PASSWORD, - ) - users = ldap_manager.get('') # Get all users - with RPCClient(secrets.OPENNEBULA_URL) as rpc_client: success, response, *_ = rpc_client.one.vmpool.infoextended( secrets.OPENNEBULA_USER_PASS, -2, -1, -1, -1 ) if success: vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] - unknown_user_with_email = set() + unknown_user = set() + + backend = LDAPBackend() for vm in vms: vm_id = vm['ID'] - vm_owner_email = vm['UNAME'] + vm_owner = vm['UNAME'] + + user = backend.populate_user(username=vm_owner) - user = find_user_based_on_email(users, vm_owner_email) if not user: - unknown_user_with_email.add(vm_owner_email) + unknown_user.add(vm_owner) else: - try: - user_in_db = get_user_model().objects.get(email=vm_owner_email) - except get_user_model().DoesNotExist: - user_in_db = get_user_model().objects.create_user(username=user.uid, email=vm_owner_email) VMModel.objects.update_or_create( - id=f'opennebula{vm_id}', - defaults={'data': vm, 'owner': user_in_db} + vmid=vm_id, + defaults={'data': vm, 'owner': user} ) - print('User with email but not found in ldap:', unknown_user_with_email) + print('User not found in ldap:', unknown_user) else: print(response) - print(secrets.OPENNEBULA_USER_PASS) diff --git a/uncloud/opennebula/migrations/0004_auto_20200225_1816.py b/uncloud/opennebula/migrations/0004_auto_20200225_1816.py new file mode 100644 index 0000000..5b39f26 --- /dev/null +++ b/uncloud/opennebula/migrations/0004_auto_20200225_1816.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-02-25 18:16 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('opennebula', '0003_auto_20200225_1428'), + ] + + operations = [ + migrations.RemoveField( + model_name='vm', + name='id', + ), + migrations.AddField( + model_name='vm', + name='vmid', + field=models.IntegerField(default=42, primary_key=True, serialize=False), + preserve_default=False, + ), + ] diff --git a/uncloud/opennebula/models.py b/uncloud/opennebula/models.py index 904699d..fff811b 100644 --- a/uncloud/opennebula/models.py +++ b/uncloud/opennebula/models.py @@ -5,7 +5,7 @@ from django.contrib.postgres.fields import JSONField class VM(models.Model): - id = models.CharField(primary_key=True, editable=True, default=uuid.uuid4, unique=True, max_length=64) + vmid = models.IntegerField(primary_key=True) owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) data = JSONField() @@ -34,7 +34,6 @@ class VM(models.Model): disks = [] if 'DISK' in self.data['TEMPLATE']: - if type(self.data['TEMPLATE']['DISK']) is dict: disks = [ self.data['TEMPLATE']['DISK'] ] else: diff --git a/uncloud/requirements.txt b/uncloud/requirements.txt index c7efd69..1b4e05b 100644 --- a/uncloud/requirements.txt +++ b/uncloud/requirements.txt @@ -3,5 +3,4 @@ djangorestframework django-auth-ldap stripe xmltodict -git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap psycopg2 diff --git a/uncloud/uncloud/settings.py b/uncloud/uncloud/settings.py index b32b89a..624c9bb 100644 --- a/uncloud/uncloud/settings.py +++ b/uncloud/uncloud/settings.py @@ -18,7 +18,7 @@ import ldap # Uncommitted file with secrets import uncloud.secrets -from django_auth_ldap.config import LDAPSearch +from django_auth_ldap.config import LDAPSearch, LDAPSearchUnion # Uncommitted file with local settings i.e logging try: @@ -129,9 +129,7 @@ AUTH_LDAP_USER_ATTR_MAP = { AUTH_LDAP_BIND_DN = uncloud.secrets.LDAP_ADMIN_DN AUTH_LDAP_BIND_PASSWORD = uncloud.secrets.LDAP_ADMIN_PASSWORD -AUTH_LDAP_USER_SEARCH = LDAPSearch( - "dc=ungleich,dc=ch", ldap.SCOPE_SUBTREE, "(uid=%(user)s)" -) +AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=ungleich,dc=ch", ldap.SCOPE_SUBTREE, "(uid=%(user)s)") ################################################################################ @@ -174,12 +172,4 @@ USE_TZ = True STATIC_URL = '/static/' -# https://docs.djangoproject.com/en/3.0/ref/settings/#databases -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'NAME': uncloud.secrets.POSTGRESQL_DB_NAME, - } -} - stripe.api_key = uncloud.secrets.STRIPE_KEY diff --git a/uncloud/uncloud_api/migrations/0002_vmsnapshotproduct_vm_uuid.py b/uncloud/uncloud_api/migrations/0002_vmsnapshotproduct_vm_uuid.py new file mode 100644 index 0000000..b35317e --- /dev/null +++ b/uncloud/uncloud_api/migrations/0002_vmsnapshotproduct_vm_uuid.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-02-25 18:16 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_api', '0001_initial'), + ] + + operations = [ + migrations.AddField( + model_name='vmsnapshotproduct', + name='vm_uuid', + field=models.UUIDField(default=uuid.uuid4, editable=False), + ), + ] diff --git a/uncloud/uncloud_api/models.py b/uncloud/uncloud_api/models.py index 50857fb..6affaa3 100644 --- a/uncloud/uncloud_api/models.py +++ b/uncloud/uncloud_api/models.py @@ -72,7 +72,7 @@ class VMSnapshotProduct(Product): gb_ssd = models.FloatField() gb_hdd = models.FloatField() - vm_uuid = models.UUIDField() + vm_uuid = models.UUIDField(default=uuid.uuid4, editable=False) # Need to setup recurring_price and one_time_price and recurring period