Merge remote-tracking branch 'ahmed/master' into ahmed_merge

Signed-off-by: Nico Schottelius <nico@nico-notebook.schottelius.org>

uncloud/.gitignore

@@ -1,3 +1,4 @@
 db.sqlite3
 uncloud/secrets.py
-debug.log
+debug.log
+uncloud/local_settings.py

uncloud/opennebula/management/commands/syncvm.py

@@ -1,15 +1,23 @@
-import os
 import json
 
+import uncloud.secrets as secrets
+
+
+from xmlrpc.client import ServerProxy as RPCClient
+
 from django.core.management.base import BaseCommand
 from django.contrib.auth import get_user_model
-from xmlrpc.client import ServerProxy as RPCClient
-
 from xmltodict import parse
+from ungleich_common.ldap.ldap_manager import LdapManager
 
 from opennebula.models import VM as VMModel
 
-import uncloud.secrets
+
+def find_user_based_on_email(users, email):
+    for user in users:
+        if email in user.mail.values:
+            return user
+
 
 class Command(BaseCommand):
     help = 'Syncronize VM information from OpenNebula'
@@ -18,29 +26,39 @@ class Command(BaseCommand):
         pass
 
     def handle(self, *args, **options):
-        with RPCClient(uncloud.secrets.OPENNEBULA_URL) as rpc_client:
+        ldap_server_uri = secrets.LDAP_SERVER_URI.split(',')[0]
+        ldap_manager = LdapManager(
+            server=ldap_server_uri,
+            admin_dn=secrets.LDAP_ADMIN_DN,
+            admin_password=secrets.LDAP_ADMIN_PASSWORD,
+        )
+        users = ldap_manager.get('')  # Get all users
+
+        with RPCClient(secrets.OPENNEBULA_URL) as rpc_client:
             success, response, *_ = rpc_client.one.vmpool.infoextended(
-                uncloud.secrets.OPENNEBULA_USER_PASS, -2, -1, -1, -1
+                secrets.OPENNEBULA_USER_PASS, -2, -1, -1, -1
             )
             if success:
                 vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM']
-                for i, vm in enumerate(vms):
+                unknown_user_with_email = set()
+
+                for vm in vms:
                     vm_id = vm['ID']
                     vm_owner_email = vm['UNAME']
 
-                    try:
-                        user = get_user_model().objects.get(email=vm_owner_email)
-                    except get_user_model().DoesNotExist:
-                        print("Skipping VM import for unknown user with email: {}".format(vm_owner_email))
-                        continue
-                        # user = get_user_model().objects.create_user(username=vm_owner)
-
-                    VMModel.objects.update_or_create(
-                        defaults= { 'data': vm,
-                                    'owner': user },
-                        vmid=vm_id
-                    )
-
+                    user = find_user_based_on_email(users, vm_owner_email)
+                    if not user:
+                        unknown_user_with_email.add(vm_owner_email)
+                    else:
+                        try:
+                            user_in_db = get_user_model().objects.get(email=vm_owner_email)
+                        except get_user_model().DoesNotExist:
+                            user_in_db = get_user_model().objects.create_user(username=user.uid, email=vm_owner_email)
+                        VMModel.objects.update_or_create(
+                            id=f'opennebula{vm_id}',
+                            defaults={'data': vm, 'owner': user_in_db}
+                        )
+                print('User with email but not found in ldap:', unknown_user_with_email)
             else:
                 print(response)
-                print(uncloud.secrets.OPENNEBULA_USER_PASS)
+                print(secrets.OPENNEBULA_USER_PASS)

uncloud/opennebula/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.0.3 on 2020-02-23 17:08
+# Generated by Django 3.0.3 on 2020-02-23 17:12
 
 from django.conf import settings
 import django.contrib.postgres.fields.jsonb

uncloud/opennebula/migrations/0002_auto_20200225_1335.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.0.3 on 2020-02-25 13:35
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('opennebula', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RemoveField(
+            model_name='vm',
+            name='uuid',
+        ),
+        migrations.RemoveField(
+            model_name='vm',
+            name='vmid',
+        ),
+        migrations.AddField(
+            model_name='vm',
+            name='id',
+            field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True),
+        ),
+    ]

uncloud/opennebula/migrations/0003_auto_20200225_1428.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.3 on 2020-02-25 14:28
+
+from django.db import migrations, models
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('opennebula', '0002_auto_20200225_1335'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='vm',
+            name='id',
+            field=models.CharField(default=uuid.uuid4, max_length=64, primary_key=True, serialize=False, unique=True),
+        ),
+    ]

uncloud/opennebula/models.py

@@ -1,15 +1,17 @@
 import uuid
 from django.db import models
 from django.contrib.auth import get_user_model
-
 from django.contrib.postgres.fields import JSONField
 
+
 class VM(models.Model):
-    vmid = models.IntegerField(primary_key=True)
-    uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True)
+    id = models.CharField(primary_key=True, editable=True, default=uuid.uuid4, unique=True, max_length=64)
     owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE)
     data = JSONField()
 
+    def save(self, *args, **kwargs):
+        self.id = 'opennebula' + str(self.data.get("ID"))
+        super().save(*args, **kwargs)
 
     @property
     def cores(self):
@@ -48,3 +50,11 @@ class VM(models.Model):
         ]
 
         return disks
+
+    @property
+    def last_host(self):
+        return ((self.data.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None)
+
+    @property
+    def graphics(self):
+        return self.data.get('TEMPLATE', {}).get('GRAPHICS', {})

uncloud/opennebula/serializers.py

@@ -5,10 +5,10 @@ from opennebula.models import VM
 class VMSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = VM
-        fields = ['vmid', 'owner', 'data']
+        fields = ['id', 'owner', 'data']
 
 
 class OpenNebulaVMSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = VM
-        fields = ['vmid', 'owner', 'cores', 'ram_in_gb', 'disks' ]
+        fields = ['id', 'owner', 'cores', 'ram_in_gb', 'disks', 'last_host', 'graphics']

uncloud/opennebula/views.py

@@ -1,18 +1,18 @@
-from rest_framework import viewsets, generics, permissions
+from rest_framework import viewsets, permissions
 from rest_framework.response import Response
-
-from django.contrib.auth import get_user_model
+from django.shortcuts import get_object_or_404
 
 from .models import VM
 from .serializers import VMSerializer, OpenNebulaVMSerializer
 
+
 class RawVMViewSet(viewsets.ModelViewSet):
     queryset = VM.objects.all()
     serializer_class = VMSerializer
     permission_classes = [permissions.IsAdminUser]
 
 
-class VMViewSet(viewsets.ModelViewSet):
+class VMViewSet(viewsets.ViewSet):
     permission_classes = [permissions.IsAuthenticated]
 
     def list(self, request):
@@ -22,6 +22,6 @@ class VMViewSet(viewsets.ModelViewSet):
 
     def retrieve(self, request, pk=None):
         queryset = VM.objects.filter(owner=request.user)
-        user = get_object_or_404(queryset, pk=pk)
-        serializer = OpenNebulaVMSerializer(queryset)
+        vm = get_object_or_404(queryset, pk=pk)
+        serializer = OpenNebulaVMSerializer(vm, context={'request': request})
         return Response(serializer.data)

uncloud/requirements.txt

@@ -3,3 +3,4 @@ djangorestframework
 django-auth-ldap
 stripe
 xmltodict
+git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap

uncloud/uncloud/settings.py

@@ -12,18 +12,26 @@ https://docs.djangoproject.com/en/3.0/ref/settings/
 
 import os
 
+import stripe
+import ldap
 
 # Uncommitted file with secrets
 import uncloud.secrets
 
-import stripe
-import ldap
-
-import uncloud.secrets as secrets
-
 from django_auth_ldap.config import LDAPSearch
 
-
+# Uncommitted file with local settings i.e logging
+try:
+    from uncloud.local_settings import LOGGING, DATABASES
+except ModuleNotFoundError:
+    LOGGING = {}
+    # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
+    DATABASES = {
+        'default': {
+            'ENGINE': 'django.db.backends.postgresql',
+            'NAME': uncloud.secrets.POSTGRESQL_DB_NAME,
+        }
+    }
 
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
@@ -88,8 +96,6 @@ TEMPLATES = [
 WSGI_APPLICATION = 'uncloud.wsgi.application'
 
 
-
-
 # Password validation
 # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators
 
@@ -168,33 +174,6 @@ USE_TZ = True
 
 STATIC_URL = '/static/'
 
-stripe.api_key = secrets.STRIPE_KEY
-
-# FIXME: not sure if we really need this
-# LOGGING = {
-#     'version': 1,
-#     'disable_existing_loggers': False,
-#     'handlers': {
-#         'file': {
-#             'level': 'DEBUG',
-#             'class': 'logging.FileHandler',
-#             'filename': 'debug.log',
-#         },
-#     },
-#     'loggers': {
-#         'django': {
-#             'handlers': ['file'],
-#             'level': 'DEBUG',
-#             'propagate': True,
-#         },
-#         'django_auth_ldap': {
-#             'handlers': ['file'],
-#             'level': 'DEBUG',
-#             'propagate': True
-#         }
-#     },
-# }
-
 # https://docs.djangoproject.com/en/3.0/ref/settings/#databases
 DATABASES = {
     'default': {
@@ -202,3 +181,5 @@ DATABASES = {
         'NAME': uncloud.secrets.POSTGRESQL_DB_NAME,
     }
 }
+
+stripe.api_key = uncloud.secrets.STRIPE_KEY

uncloud/uncloud_api/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.0.3 on 2020-02-23 17:09
+# Generated by Django 3.0.3 on 2020-02-23 17:12
 
 from django.conf import settings
 from django.db import migrations, models

uncloud/uncloud_auth/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 3.0.3 on 2020-02-23 17:08
+# Generated by Django 3.0.3 on 2020-02-23 17:11
 
 import django.contrib.auth.models
 import django.contrib.auth.validators