Add debug to opennebula, create VM disks from opennebula correctly

uncloud/opennebula/management/commands/opennebula-to-uncloud.py

@@ -1,13 +1,18 @@
+import sys
 from datetime import datetime
 
 from django.core.management.base import BaseCommand
 from django.utils import timezone
+from django.contrib.auth import get_user_model
 
 from opennebula.models import VM as VMModel
 from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct
 
 from uncloud_pay.models import Order
 
+import logging
+
+log = logging.getLogger(__name__)
 
 def convert_mac_to_int(mac_address: str):
     # Remove octet connecting characters
@@ -41,24 +46,35 @@ def create_nics(one_vm, vm_product):
         )
 
 
-def create_disk_and_image(one_vm, vm_product):
+def sync_disk_and_image(one_vm, vm_product, disk_owner):
-    for disk in one_vm.disks:
+    """
-        owner = one_vm.owner
+    a) Check all opennebula disk if they are in the uncloud VM, if not add
-        name = disk.get('image')
+    b) Check all uncloud disks and remove them if they are not in the opennebula VM
+    """
 
-        # TODO: Fix the following hard coded values
+    vmdisknum = 0
-        is_os_image, is_public, status = True, True, 'active'
+
+    one_disks_extra_data = []
+
+    for disk in one_vm.disks:
+        vmowner = one_vm.owner
+        name = disk.get('image')
+        vmdisknum += 1
+
+        log.info("Checking disk {} for VM {}".format(name, one_vm))
+
+        is_os_image, is_public, status = True, False, 'active'
 
         image_size_in_gb = disk.get('image_size_in_gb')
         disk_size_in_gb = disk.get('size_in_gb')
-        storage_class = disk.get('pool_name')
+        storage_class = disk.get('storage_class')
         image_source = disk.get('source')
         image_source_type = disk.get('source_type')
 
         image, _ = VMDiskImageProduct.objects.update_or_create(
             name=name,
             defaults={
-                'owner': owner,
+                'owner': disk_owner,
                 'is_os_image': is_os_image,
                 'is_public': is_public,
                 'size_in_gb': image_size_in_gb,
@@ -68,29 +84,59 @@ def create_disk_and_image(one_vm, vm_product):
                 'status': status
             }
         )
-        VMDiskProduct.objects.update_or_create(
-            owner=owner, vm=vm_product,
-            defaults={
-                'image': image,
-                'size_in_gb': disk_size_in_gb
-            }
-        )
 
+        # identify vmdisk from opennebula - primary mapping key
+        extra_data = {
+            'opennebula_vm': one_vm.vmid,
+            'opennebula_size_in_gb': disk_size_in_gb,
+            'opennebula_source': disk.get('opennebula_source'),
+            'opennebula_disk_num': vmdisknum
+        }
+        # Save for comparing later
+        one_disks_extra_data.append(extra_data)
+
+        try:
+            vm_disk = VMDiskProduct.objects.get(extra_data=extra_data)
+        except VMDiskProduct.DoesNotExist:
+            vm_disk = VMDiskProduct.objects.create(
+                owner=vmowner,
+                vm=vm_product,
+                image=image,
+                size_in_gb=disk_size_in_gb,
+                extra_data=extra_data
+            )
+
+    # Now remove all disks that are not in above extra_data list
+    for disk in VMDiskProduct.objects.filter(vm=vm_product):
+        extra_data = disk.extra_data
+        if not extra_data in one_disks_extra_data:
+            log.info("Removing disk {} from VM {}".format(disk, vm_product))
+            disk.delete()
+
+    disks = [ disk.extra_data for disk in VMDiskProduct.objects.filter(vm=vm_product) ]
+    log.info("VM {} has disks: {}".format(vm_product, disks))
 
 class Command(BaseCommand):
     help = 'Migrate Opennebula VM to regular (uncloud) vm'
 
+    def add_arguments(self, parser):
+        parser.add_argument('--disk-owner', required=True, help="The user who owns the the opennebula disks")
+
     def handle(self, *args, **options):
+        log.debug("{} {}".format(args, options))
+
+        disk_owner = get_user_model().objects.get(username=options['disk_owner'])
+
         for one_vm in VMModel.objects.all():
 
             if not one_vm.last_host:
-                print("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid))
+                log.warning("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid))
                 continue
 
             try:
                 vmhost = VMHost.objects.get(hostname=one_vm.last_host)
             except VMHost.DoesNotExist:
-                print("VMHost {} does not exist, aborting".format(one_vm.last_host))
+                log.error("VMHost {} does not exist, aborting".format(one_vm.last_host))
                 raise
 
             cores = one_vm.cores
@@ -98,9 +144,6 @@ class Command(BaseCommand):
             owner = one_vm.owner
             status = 'active'
 
-            # Total Amount of SSD Storage
-            # TODO: What would happen if the attached storage is not SSD but HDD?
-
             ssd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['ssd', 'one'] ])
             hdd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['hdd'] ])
 
@@ -119,30 +162,32 @@ class Command(BaseCommand):
                                            len(ipv4), len(ipv6))
 
             try:
-                vm_product = VMProduct.objects.get(name=one_vm.uncloud_name)
+                vm_product = VMProduct.objects.get(extra_data__opennebula_id=one_vm.vmid)
             except VMProduct.DoesNotExist:
                 order = Order.objects.create(
                     owner=owner,
                     creation_date=creation_date,
                     starting_date=starting_date
-#                    one_time_price=one_time_price,
-#                    recurring_price=recurring_price,
-#                    recurring_period=recurring_period
                 )
-                vm_product, _ = VMProduct.objects.update_or_create(
+                vm_product = VMProduct(
+                    extra_data={ 'opennebula_id': one_vm.vmid },
                     name=one_vm.uncloud_name,
-                    defaults={
+                    order=order
-                        'cores': cores,
-                        'ram_in_gb': ram_in_gb,
-                        'owner': owner,
-                        'vmhost': vmhost,
-                        'order': order,
-                        'status': status
-                    }
                 )
 
+            # we don't use update_or_create, as filtering by json AND setting json
+            # at the same time does not work
+
+            vm_product.vmhost = vmhost
+            vm_product.owner = owner
+            vm_product.cores = cores
+            vm_product.ram_in_gb = ram_in_gb
+            vm_product.status = status
+
+            vm_product.save()
+
             # Create VMNetworkCards
             create_nics(one_vm, vm_product)
 
             # Create VMDiskImageProduct and VMDiskProduct
-            create_disk_and_image(one_vm, vm_product)
+            sync_disk_and_image(one_vm, vm_product, disk_owner=disk_owner)

uncloud/opennebula/models.py

@@ -3,6 +3,12 @@ from django.db import models
 from django.contrib.auth import get_user_model
 from django.contrib.postgres.fields import JSONField
 
+# ungleich specific
+storage_class_mapping = {
+    'one': 'ssd',
+    'ssd': 'ssd',
+    'hdd': 'hdd'
+}
 
 class VM(models.Model):
     vmid = models.IntegerField(primary_key=True)
@@ -48,7 +54,8 @@ class VM(models.Model):
                 'pool_name': d['POOL_NAME'],
                 'image': d['IMAGE'],
                 'source': d['SOURCE'],
-                'source_type': d['TM_MAD']
+                'source_type': d['TM_MAD'],
+                'storage_class': storage_class_mapping[d['POOL_NAME']]
 
             }
             for d in disks

uncloud/opennebula/serializers.py

@@ -5,4 +5,6 @@ from opennebula.models import VM
 class OpenNebulaVMSerializer(serializers.HyperlinkedModelSerializer):
     class Meta:
         model = VM
-        fields = '__all__'
+        fields = [ 'vmid', 'owner', 'data',
+                   'uncloud_name', 'cores', 'ram_in_gb',
+                   'disks', 'nics', 'ips' ]

uncloud/uncloud/urls.py

@@ -28,9 +28,9 @@ router = routers.DefaultRouter()
 
 # VM
 router.register(r'vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct')
+router.register(r'vm/diskimage', vmviews.VMDiskImageProductViewSet, basename='vmdiskimageproduct')
 router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct')
-router.register(r'vm/image/mine', vmviews.VMDiskImageProductMineViewSet, basename='vmdiskimagemineproduct')
+
-router.register(r'vm/image/public', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct')
 
 # images the provider provides :-)
 # router.register(r'vm/image/official', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct')

uncloud/uncloud_auth/serializers.py

@@ -9,5 +9,7 @@ class UserSerializer(serializers.ModelSerializer):
         model = get_user_model()
         fields = ['username', 'email', 'balance', 'maximum_credit' ]
 
+
+
     balance = serializers.DecimalField(max_digits=AMOUNT_MAX_DIGITS,
                                        decimal_places=AMOUNT_DECIMALS)

uncloud/uncloud_vm/models.py

@@ -119,7 +119,7 @@ class VMDiskImageProduct(UncloudModel):
 
     name = models.CharField(max_length=256)
     is_os_image = models.BooleanField(default=False)
-    is_public = models.BooleanField(default=False)
+    is_public = models.BooleanField(default=False, editable=False) # only allow admins to set this
 
     size_in_gb = models.FloatField(null=True, blank=True)
     import_url = models.URLField(null=True, blank=True)

uncloud/uncloud_vm/serializers.py

@@ -84,8 +84,9 @@ class VMProductSerializer(serializers.ModelSerializer):
         model = VMProduct
         fields = ['uuid', 'order', 'owner', 'status', 'name',
                   'cores', 'ram_in_gb', 'recurring_period',
-                  'snapshots', 'disks' ]
+                  'snapshots', 'disks',
-        read_only_fields = ['uuid', 'order', 'owner', 'status']
+                  'extra_data' ]
+        read_only_fields = ['uuid', 'order', 'owner', 'status' ]
 
     # Custom field used at creation (= ordering) only.
     recurring_period = serializers.ChoiceField(

uncloud/uncloud_vm/views.py

@@ -24,7 +24,7 @@ class VMHostViewSet(viewsets.ModelViewSet):
     queryset = VMHost.objects.all()
     permission_classes = [permissions.IsAdminUser]
 
-class VMDiskImageProductMineViewSet(viewsets.ModelViewSet):
+class VMDiskImageProductViewSet(viewsets.ModelViewSet):
     permission_classes = [permissions.IsAuthenticated]
     serializer_class = VMDiskImageProductSerializer
 
@@ -32,7 +32,7 @@ class VMDiskImageProductMineViewSet(viewsets.ModelViewSet):
         if self.request.user.is_superuser:
             obj = VMDiskImageProduct.objects.all()
         else:
-            obj = VMDiskImageProduct.objects.filter(owner=self.request.user)
+            obj = VMDiskImageProduct.objects.filter(owner=self.request.user) | VMDiskImageProduct.objects.filter(is_public=True)
 
         return obj