From 08fe3e689ef6acc11e66f721fc26cf1cb601039e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 22 Mar 2020 17:30:55 +0100 Subject: [PATCH] Add debug to opennebula, create VM disks from opennebula correctly --- .../commands/opennebula-to-uncloud.py | 113 ++++++++++++------ uncloud/opennebula/models.py | 9 +- uncloud/opennebula/serializers.py | 4 +- uncloud/uncloud/urls.py | 4 +- uncloud/uncloud_auth/serializers.py | 2 + uncloud/uncloud_vm/models.py | 2 +- uncloud/uncloud_vm/serializers.py | 5 +- uncloud/uncloud_vm/views.py | 4 +- 8 files changed, 100 insertions(+), 43 deletions(-) diff --git a/uncloud/opennebula/management/commands/opennebula-to-uncloud.py b/uncloud/opennebula/management/commands/opennebula-to-uncloud.py index dc7cb45..230159a 100644 --- a/uncloud/opennebula/management/commands/opennebula-to-uncloud.py +++ b/uncloud/opennebula/management/commands/opennebula-to-uncloud.py @@ -1,13 +1,18 @@ +import sys from datetime import datetime from django.core.management.base import BaseCommand from django.utils import timezone +from django.contrib.auth import get_user_model from opennebula.models import VM as VMModel from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct from uncloud_pay.models import Order +import logging + +log = logging.getLogger(__name__) def convert_mac_to_int(mac_address: str): # Remove octet connecting characters @@ -41,24 +46,35 @@ def create_nics(one_vm, vm_product): ) -def create_disk_and_image(one_vm, vm_product): - for disk in one_vm.disks: - owner = one_vm.owner - name = disk.get('image') +def sync_disk_and_image(one_vm, vm_product, disk_owner): + """ + a) Check all opennebula disk if they are in the uncloud VM, if not add + b) Check all uncloud disks and remove them if they are not in the opennebula VM + """ - # TODO: Fix the following hard coded values - is_os_image, is_public, status = True, True, 'active' + vmdisknum = 0 + + one_disks_extra_data = [] + + for disk in one_vm.disks: + vmowner = one_vm.owner + name = disk.get('image') + vmdisknum += 1 + + log.info("Checking disk {} for VM {}".format(name, one_vm)) + + is_os_image, is_public, status = True, False, 'active' image_size_in_gb = disk.get('image_size_in_gb') disk_size_in_gb = disk.get('size_in_gb') - storage_class = disk.get('pool_name') + storage_class = disk.get('storage_class') image_source = disk.get('source') image_source_type = disk.get('source_type') image, _ = VMDiskImageProduct.objects.update_or_create( name=name, defaults={ - 'owner': owner, + 'owner': disk_owner, 'is_os_image': is_os_image, 'is_public': is_public, 'size_in_gb': image_size_in_gb, @@ -68,29 +84,59 @@ def create_disk_and_image(one_vm, vm_product): 'status': status } ) - VMDiskProduct.objects.update_or_create( - owner=owner, vm=vm_product, - defaults={ - 'image': image, - 'size_in_gb': disk_size_in_gb - } - ) + # identify vmdisk from opennebula - primary mapping key + extra_data = { + 'opennebula_vm': one_vm.vmid, + 'opennebula_size_in_gb': disk_size_in_gb, + 'opennebula_source': disk.get('opennebula_source'), + 'opennebula_disk_num': vmdisknum + } + # Save for comparing later + one_disks_extra_data.append(extra_data) + + try: + vm_disk = VMDiskProduct.objects.get(extra_data=extra_data) + except VMDiskProduct.DoesNotExist: + vm_disk = VMDiskProduct.objects.create( + owner=vmowner, + vm=vm_product, + image=image, + size_in_gb=disk_size_in_gb, + extra_data=extra_data + ) + + # Now remove all disks that are not in above extra_data list + for disk in VMDiskProduct.objects.filter(vm=vm_product): + extra_data = disk.extra_data + if not extra_data in one_disks_extra_data: + log.info("Removing disk {} from VM {}".format(disk, vm_product)) + disk.delete() + + disks = [ disk.extra_data for disk in VMDiskProduct.objects.filter(vm=vm_product) ] + log.info("VM {} has disks: {}".format(vm_product, disks)) class Command(BaseCommand): help = 'Migrate Opennebula VM to regular (uncloud) vm' + def add_arguments(self, parser): + parser.add_argument('--disk-owner', required=True, help="The user who owns the the opennebula disks") + def handle(self, *args, **options): + log.debug("{} {}".format(args, options)) + + disk_owner = get_user_model().objects.get(username=options['disk_owner']) + for one_vm in VMModel.objects.all(): if not one_vm.last_host: - print("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid)) + log.warning("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid)) continue try: vmhost = VMHost.objects.get(hostname=one_vm.last_host) except VMHost.DoesNotExist: - print("VMHost {} does not exist, aborting".format(one_vm.last_host)) + log.error("VMHost {} does not exist, aborting".format(one_vm.last_host)) raise cores = one_vm.cores @@ -98,9 +144,6 @@ class Command(BaseCommand): owner = one_vm.owner status = 'active' - # Total Amount of SSD Storage - # TODO: What would happen if the attached storage is not SSD but HDD? - ssd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['ssd', 'one'] ]) hdd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['hdd'] ]) @@ -119,30 +162,32 @@ class Command(BaseCommand): len(ipv4), len(ipv6)) try: - vm_product = VMProduct.objects.get(name=one_vm.uncloud_name) + vm_product = VMProduct.objects.get(extra_data__opennebula_id=one_vm.vmid) except VMProduct.DoesNotExist: order = Order.objects.create( owner=owner, creation_date=creation_date, starting_date=starting_date -# one_time_price=one_time_price, -# recurring_price=recurring_price, -# recurring_period=recurring_period ) - vm_product, _ = VMProduct.objects.update_or_create( + vm_product = VMProduct( + extra_data={ 'opennebula_id': one_vm.vmid }, name=one_vm.uncloud_name, - defaults={ - 'cores': cores, - 'ram_in_gb': ram_in_gb, - 'owner': owner, - 'vmhost': vmhost, - 'order': order, - 'status': status - } + order=order ) + # we don't use update_or_create, as filtering by json AND setting json + # at the same time does not work + + vm_product.vmhost = vmhost + vm_product.owner = owner + vm_product.cores = cores + vm_product.ram_in_gb = ram_in_gb + vm_product.status = status + + vm_product.save() + # Create VMNetworkCards create_nics(one_vm, vm_product) # Create VMDiskImageProduct and VMDiskProduct - create_disk_and_image(one_vm, vm_product) + sync_disk_and_image(one_vm, vm_product, disk_owner=disk_owner) diff --git a/uncloud/opennebula/models.py b/uncloud/opennebula/models.py index f5faeb5..826b615 100644 --- a/uncloud/opennebula/models.py +++ b/uncloud/opennebula/models.py @@ -3,6 +3,12 @@ from django.db import models from django.contrib.auth import get_user_model from django.contrib.postgres.fields import JSONField +# ungleich specific +storage_class_mapping = { + 'one': 'ssd', + 'ssd': 'ssd', + 'hdd': 'hdd' +} class VM(models.Model): vmid = models.IntegerField(primary_key=True) @@ -48,7 +54,8 @@ class VM(models.Model): 'pool_name': d['POOL_NAME'], 'image': d['IMAGE'], 'source': d['SOURCE'], - 'source_type': d['TM_MAD'] + 'source_type': d['TM_MAD'], + 'storage_class': storage_class_mapping[d['POOL_NAME']] } for d in disks diff --git a/uncloud/opennebula/serializers.py b/uncloud/opennebula/serializers.py index 8e0c513..cd00622 100644 --- a/uncloud/opennebula/serializers.py +++ b/uncloud/opennebula/serializers.py @@ -5,4 +5,6 @@ from opennebula.models import VM class OpenNebulaVMSerializer(serializers.HyperlinkedModelSerializer): class Meta: model = VM - fields = '__all__' + fields = [ 'vmid', 'owner', 'data', + 'uncloud_name', 'cores', 'ram_in_gb', + 'disks', 'nics', 'ips' ] diff --git a/uncloud/uncloud/urls.py b/uncloud/uncloud/urls.py index 856e59c..50d59c3 100644 --- a/uncloud/uncloud/urls.py +++ b/uncloud/uncloud/urls.py @@ -28,9 +28,9 @@ router = routers.DefaultRouter() # VM router.register(r'vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct') +router.register(r'vm/diskimage', vmviews.VMDiskImageProductViewSet, basename='vmdiskimageproduct') router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct') -router.register(r'vm/image/mine', vmviews.VMDiskImageProductMineViewSet, basename='vmdiskimagemineproduct') -router.register(r'vm/image/public', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct') + # images the provider provides :-) # router.register(r'vm/image/official', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct') diff --git a/uncloud/uncloud_auth/serializers.py b/uncloud/uncloud_auth/serializers.py index 3627149..de369c3 100644 --- a/uncloud/uncloud_auth/serializers.py +++ b/uncloud/uncloud_auth/serializers.py @@ -9,5 +9,7 @@ class UserSerializer(serializers.ModelSerializer): model = get_user_model() fields = ['username', 'email', 'balance', 'maximum_credit' ] + + balance = serializers.DecimalField(max_digits=AMOUNT_MAX_DIGITS, decimal_places=AMOUNT_DECIMALS) diff --git a/uncloud/uncloud_vm/models.py b/uncloud/uncloud_vm/models.py index bdd3a43..a4b7f2a 100644 --- a/uncloud/uncloud_vm/models.py +++ b/uncloud/uncloud_vm/models.py @@ -119,7 +119,7 @@ class VMDiskImageProduct(UncloudModel): name = models.CharField(max_length=256) is_os_image = models.BooleanField(default=False) - is_public = models.BooleanField(default=False) + is_public = models.BooleanField(default=False, editable=False) # only allow admins to set this size_in_gb = models.FloatField(null=True, blank=True) import_url = models.URLField(null=True, blank=True) diff --git a/uncloud/uncloud_vm/serializers.py b/uncloud/uncloud_vm/serializers.py index 96454f7..6d26cbe 100644 --- a/uncloud/uncloud_vm/serializers.py +++ b/uncloud/uncloud_vm/serializers.py @@ -84,8 +84,9 @@ class VMProductSerializer(serializers.ModelSerializer): model = VMProduct fields = ['uuid', 'order', 'owner', 'status', 'name', 'cores', 'ram_in_gb', 'recurring_period', - 'snapshots', 'disks' ] - read_only_fields = ['uuid', 'order', 'owner', 'status'] + 'snapshots', 'disks', + 'extra_data' ] + read_only_fields = ['uuid', 'order', 'owner', 'status' ] # Custom field used at creation (= ordering) only. recurring_period = serializers.ChoiceField( diff --git a/uncloud/uncloud_vm/views.py b/uncloud/uncloud_vm/views.py index 1ef4974..6d4e5a9 100644 --- a/uncloud/uncloud_vm/views.py +++ b/uncloud/uncloud_vm/views.py @@ -24,7 +24,7 @@ class VMHostViewSet(viewsets.ModelViewSet): queryset = VMHost.objects.all() permission_classes = [permissions.IsAdminUser] -class VMDiskImageProductMineViewSet(viewsets.ModelViewSet): +class VMDiskImageProductViewSet(viewsets.ModelViewSet): permission_classes = [permissions.IsAuthenticated] serializer_class = VMDiskImageProductSerializer @@ -32,7 +32,7 @@ class VMDiskImageProductMineViewSet(viewsets.ModelViewSet): if self.request.user.is_superuser: obj = VMDiskImageProduct.objects.all() else: - obj = VMDiskImageProduct.objects.filter(owner=self.request.user) + obj = VMDiskImageProduct.objects.filter(owner=self.request.user) | VMDiskImageProduct.objects.filter(is_public=True) return obj