diff --git a/uncloud/.gitignore b/uncloud/.gitignore index 4ade18f..71202e1 100644 --- a/uncloud/.gitignore +++ b/uncloud/.gitignore @@ -1,3 +1,4 @@ db.sqlite3 uncloud/secrets.py -debug.log \ No newline at end of file +debug.log +uncloud/local_settings.py \ No newline at end of file diff --git a/uncloud/opennebula/management/commands/syncvm.py b/uncloud/opennebula/management/commands/syncvm.py index 795d53a..55844e3 100644 --- a/uncloud/opennebula/management/commands/syncvm.py +++ b/uncloud/opennebula/management/commands/syncvm.py @@ -1,15 +1,23 @@ -import os import json +import uncloud.secrets as secrets + + +from xmlrpc.client import ServerProxy as RPCClient + from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model -from xmlrpc.client import ServerProxy as RPCClient - from xmltodict import parse +from ungleich_common.ldap.ldap_manager import LdapManager from opennebula.models import VM as VMModel -import uncloud.secrets + +def find_user_based_on_email(users, email): + for user in users: + if email in user.mail.values: + return user + class Command(BaseCommand): help = 'Syncronize VM information from OpenNebula' @@ -18,29 +26,39 @@ class Command(BaseCommand): pass def handle(self, *args, **options): - with RPCClient(uncloud.secrets.OPENNEBULA_URL) as rpc_client: + ldap_server_uri = secrets.LDAP_SERVER_URI.split(',')[0] + ldap_manager = LdapManager( + server=ldap_server_uri, + admin_dn=secrets.LDAP_ADMIN_DN, + admin_password=secrets.LDAP_ADMIN_PASSWORD, + ) + users = ldap_manager.get('') # Get all users + + with RPCClient(secrets.OPENNEBULA_URL) as rpc_client: success, response, *_ = rpc_client.one.vmpool.infoextended( - uncloud.secrets.OPENNEBULA_USER_PASS, -2, -1, -1, -1 + secrets.OPENNEBULA_USER_PASS, -2, -1, -1, -1 ) if success: vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] - for i, vm in enumerate(vms): + unknown_user_with_email = set() + + for vm in vms: vm_id = vm['ID'] vm_owner_email = vm['UNAME'] - try: - user = get_user_model().objects.get(email=vm_owner_email) - except get_user_model().DoesNotExist: - print("Skipping VM import for unknown user with email: {}".format(vm_owner_email)) - continue - # user = get_user_model().objects.create_user(username=vm_owner) - - VMModel.objects.update_or_create( - defaults= { 'data': vm, - 'owner': user }, - vmid=vm_id - ) - + user = find_user_based_on_email(users, vm_owner_email) + if not user: + unknown_user_with_email.add(vm_owner_email) + else: + try: + user_in_db = get_user_model().objects.get(email=vm_owner_email) + except get_user_model().DoesNotExist: + user_in_db = get_user_model().objects.create_user(username=user.uid, email=vm_owner_email) + VMModel.objects.update_or_create( + id=f'opennebula{vm_id}', + defaults={'data': vm, 'owner': user_in_db} + ) + print('User with email but not found in ldap:', unknown_user_with_email) else: print(response) - print(uncloud.secrets.OPENNEBULA_USER_PASS) + print(secrets.OPENNEBULA_USER_PASS) diff --git a/uncloud/opennebula/migrations/0001_initial.py b/uncloud/opennebula/migrations/0001_initial.py index 7fa9154..4c0527a 100644 --- a/uncloud/opennebula/migrations/0001_initial.py +++ b/uncloud/opennebula/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.0.3 on 2020-02-23 17:08 +# Generated by Django 3.0.3 on 2020-02-23 17:12 from django.conf import settings import django.contrib.postgres.fields.jsonb diff --git a/uncloud/opennebula/migrations/0002_auto_20200225_1335.py b/uncloud/opennebula/migrations/0002_auto_20200225_1335.py new file mode 100644 index 0000000..1554aa6 --- /dev/null +++ b/uncloud/opennebula/migrations/0002_auto_20200225_1335.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.3 on 2020-02-25 13:35 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('opennebula', '0001_initial'), + ] + + operations = [ + migrations.RemoveField( + model_name='vm', + name='uuid', + ), + migrations.RemoveField( + model_name='vm', + name='vmid', + ), + migrations.AddField( + model_name='vm', + name='id', + field=models.UUIDField(default=uuid.uuid4, primary_key=True, serialize=False, unique=True), + ), + ] diff --git a/uncloud/opennebula/migrations/0003_auto_20200225_1428.py b/uncloud/opennebula/migrations/0003_auto_20200225_1428.py new file mode 100644 index 0000000..8bb3d8d --- /dev/null +++ b/uncloud/opennebula/migrations/0003_auto_20200225_1428.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-02-25 14:28 + +from django.db import migrations, models +import uuid + + +class Migration(migrations.Migration): + + dependencies = [ + ('opennebula', '0002_auto_20200225_1335'), + ] + + operations = [ + migrations.AlterField( + model_name='vm', + name='id', + field=models.CharField(default=uuid.uuid4, max_length=64, primary_key=True, serialize=False, unique=True), + ), + ] diff --git a/uncloud/opennebula/models.py b/uncloud/opennebula/models.py index 0b0f307..904699d 100644 --- a/uncloud/opennebula/models.py +++ b/uncloud/opennebula/models.py @@ -1,15 +1,17 @@ import uuid from django.db import models from django.contrib.auth import get_user_model - from django.contrib.postgres.fields import JSONField + class VM(models.Model): - vmid = models.IntegerField(primary_key=True) - uuid = models.UUIDField(default=uuid.uuid4, editable=False, unique=True) + id = models.CharField(primary_key=True, editable=True, default=uuid.uuid4, unique=True, max_length=64) owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE) data = JSONField() + def save(self, *args, **kwargs): + self.id = 'opennebula' + str(self.data.get("ID")) + super().save(*args, **kwargs) @property def cores(self): @@ -48,3 +50,11 @@ class VM(models.Model): ] return disks + + @property + def last_host(self): + return ((self.data.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None) + + @property + def graphics(self): + return self.data.get('TEMPLATE', {}).get('GRAPHICS', {}) diff --git a/uncloud/opennebula/serializers.py b/uncloud/opennebula/serializers.py index 30bd20a..6bfaf56 100644 --- a/uncloud/opennebula/serializers.py +++ b/uncloud/opennebula/serializers.py @@ -5,10 +5,10 @@ from opennebula.models import VM class VMSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = VM - fields = ['vmid', 'owner', 'data'] + fields = ['id', 'owner', 'data'] class OpenNebulaVMSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = VM - fields = ['vmid', 'owner', 'cores', 'ram_in_gb', 'disks' ] + fields = ['id', 'owner', 'cores', 'ram_in_gb', 'disks', 'last_host', 'graphics'] diff --git a/uncloud/opennebula/views.py b/uncloud/opennebula/views.py index 29fdb64..61ed5a4 100644 --- a/uncloud/opennebula/views.py +++ b/uncloud/opennebula/views.py @@ -1,18 +1,18 @@ -from rest_framework import viewsets, generics, permissions +from rest_framework import viewsets, permissions from rest_framework.response import Response - -from django.contrib.auth import get_user_model +from django.shortcuts import get_object_or_404 from .models import VM from .serializers import VMSerializer, OpenNebulaVMSerializer + class RawVMViewSet(viewsets.ModelViewSet): queryset = VM.objects.all() serializer_class = VMSerializer permission_classes = [permissions.IsAdminUser] -class VMViewSet(viewsets.ModelViewSet): +class VMViewSet(viewsets.ViewSet): permission_classes = [permissions.IsAuthenticated] def list(self, request): @@ -22,6 +22,6 @@ class VMViewSet(viewsets.ModelViewSet): def retrieve(self, request, pk=None): queryset = VM.objects.filter(owner=request.user) - user = get_object_or_404(queryset, pk=pk) - serializer = OpenNebulaVMSerializer(queryset) + vm = get_object_or_404(queryset, pk=pk) + serializer = OpenNebulaVMSerializer(vm, context={'request': request}) return Response(serializer.data) diff --git a/uncloud/requirements.txt b/uncloud/requirements.txt index 11ab309..e79f479 100644 --- a/uncloud/requirements.txt +++ b/uncloud/requirements.txt @@ -3,3 +3,4 @@ djangorestframework django-auth-ldap stripe xmltodict +git+https://code.ungleich.ch/ahmedbilal/ungleich-common/#egg=ungleich-common-ldap&subdirectory=ldap diff --git a/uncloud/uncloud/settings.py b/uncloud/uncloud/settings.py index bdef1df..b32b89a 100644 --- a/uncloud/uncloud/settings.py +++ b/uncloud/uncloud/settings.py @@ -12,18 +12,26 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ import os +import stripe +import ldap # Uncommitted file with secrets import uncloud.secrets -import stripe -import ldap - -import uncloud.secrets as secrets - from django_auth_ldap.config import LDAPSearch - +# Uncommitted file with local settings i.e logging +try: + from uncloud.local_settings import LOGGING, DATABASES +except ModuleNotFoundError: + LOGGING = {} + # https://docs.djangoproject.com/en/3.0/ref/settings/#databases + DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.postgresql', + 'NAME': uncloud.secrets.POSTGRESQL_DB_NAME, + } + } # Build paths inside the project like this: os.path.join(BASE_DIR, ...) @@ -88,8 +96,6 @@ TEMPLATES = [ WSGI_APPLICATION = 'uncloud.wsgi.application' - - # Password validation # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators @@ -168,33 +174,6 @@ USE_TZ = True STATIC_URL = '/static/' -stripe.api_key = secrets.STRIPE_KEY - -# FIXME: not sure if we really need this -# LOGGING = { -# 'version': 1, -# 'disable_existing_loggers': False, -# 'handlers': { -# 'file': { -# 'level': 'DEBUG', -# 'class': 'logging.FileHandler', -# 'filename': 'debug.log', -# }, -# }, -# 'loggers': { -# 'django': { -# 'handlers': ['file'], -# 'level': 'DEBUG', -# 'propagate': True, -# }, -# 'django_auth_ldap': { -# 'handlers': ['file'], -# 'level': 'DEBUG', -# 'propagate': True -# } -# }, -# } - # https://docs.djangoproject.com/en/3.0/ref/settings/#databases DATABASES = { 'default': { @@ -202,3 +181,5 @@ DATABASES = { 'NAME': uncloud.secrets.POSTGRESQL_DB_NAME, } } + +stripe.api_key = uncloud.secrets.STRIPE_KEY diff --git a/uncloud/uncloud_api/migrations/0001_initial.py b/uncloud/uncloud_api/migrations/0001_initial.py index cc3944c..67bdd2e 100644 --- a/uncloud/uncloud_api/migrations/0001_initial.py +++ b/uncloud/uncloud_api/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.0.3 on 2020-02-23 17:09 +# Generated by Django 3.0.3 on 2020-02-23 17:12 from django.conf import settings from django.db import migrations, models diff --git a/uncloud/uncloud_auth/migrations/0001_initial.py b/uncloud/uncloud_auth/migrations/0001_initial.py index 73072a5..63885c4 100644 --- a/uncloud/uncloud_auth/migrations/0001_initial.py +++ b/uncloud/uncloud_auth/migrations/0001_initial.py @@ -1,4 +1,4 @@ -# Generated by Django 3.0.3 on 2020-02-23 17:08 +# Generated by Django 3.0.3 on 2020-02-23 17:11 import django.contrib.auth.models import django.contrib.auth.validators