ahmedbilal/uncloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Add debug to opennebula, create VM disks from opennebula correctly

Browse source
This commit is contained in:
Nico Schottelius 2020-03-22 17:30:55 +01:00
commit 08fe3e689e
8 changed files with 100 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

109
uncloud/opennebula/management/commands/opennebula-to-uncloud.py
View file

@ -1,13 +1,18 @@
import sys
from datetime import datetime
from django.core.management.base import BaseCommand
from django.utils import timezone
from django.contrib.auth import get_user_model
from opennebula.models import VM as VMModel
from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct
from uncloud_pay.models import Order
import logging
log = logging.getLogger(__name__)
def convert_mac_to_int(mac_address: str):
# Remove octet connecting characters
@ -41,24 +46,35 @@ def create_nics(one_vm, vm_product):
)
def create_disk_and_image(one_vm, vm_product):
for disk in one_vm.disks:
owner = one_vm.owner
name = disk.get('image')
def sync_disk_and_image(one_vm, vm_product, disk_owner):
"""
a) Check all opennebula disk if they are in the uncloud VM, if not add
b) Check all uncloud disks and remove them if they are not in the opennebula VM
"""
# TODO: Fix the following hard coded values
is_os_image, is_public, status = True, True, 'active'
vmdisknum = 0
one_disks_extra_data = []
for disk in one_vm.disks:
vmowner = one_vm.owner
name = disk.get('image')
vmdisknum += 1
log.info("Checking disk {} for VM {}".format(name, one_vm))
is_os_image, is_public, status = True, False, 'active'
image_size_in_gb = disk.get('image_size_in_gb')
disk_size_in_gb = disk.get('size_in_gb')
storage_class = disk.get('pool_name')
storage_class = disk.get('storage_class')
image_source = disk.get('source')
image_source_type = disk.get('source_type')
image, _ = VMDiskImageProduct.objects.update_or_create(
name=name,
defaults={
'owner': owner,
'owner': disk_owner,
'is_os_image': is_os_image,
'is_public': is_public,
'size_in_gb': image_size_in_gb,
@ -68,29 +84,59 @@ def create_disk_and_image(one_vm, vm_product):
'status': status
}
)
VMDiskProduct.objects.update_or_create(
owner=owner, vm=vm_product,
defaults={
'image': image,
'size_in_gb': disk_size_in_gb
# identify vmdisk from opennebula - primary mapping key
extra_data = {
'opennebula_vm': one_vm.vmid,
'opennebula_size_in_gb': disk_size_in_gb,
'opennebula_source': disk.get('opennebula_source'),
'opennebula_disk_num': vmdisknum
}
# Save for comparing later
one_disks_extra_data.append(extra_data)
try:
vm_disk = VMDiskProduct.objects.get(extra_data=extra_data)
except VMDiskProduct.DoesNotExist:
vm_disk = VMDiskProduct.objects.create(
owner=vmowner,
vm=vm_product,
image=image,
size_in_gb=disk_size_in_gb,
extra_data=extra_data
)
# Now remove all disks that are not in above extra_data list
for disk in VMDiskProduct.objects.filter(vm=vm_product):
extra_data = disk.extra_data
if not extra_data in one_disks_extra_data:
log.info("Removing disk {} from VM {}".format(disk, vm_product))
disk.delete()
disks = [ disk.extra_data for disk in VMDiskProduct.objects.filter(vm=vm_product) ]
log.info("VM {} has disks: {}".format(vm_product, disks))
class Command(BaseCommand):
help = 'Migrate Opennebula VM to regular (uncloud) vm'
def add_arguments(self, parser):
parser.add_argument('--disk-owner', required=True, help="The user who owns the the opennebula disks")
def handle(self, *args, **options):
log.debug("{} {}".format(args, options))
disk_owner = get_user_model().objects.get(username=options['disk_owner'])
for one_vm in VMModel.objects.all():
if not one_vm.last_host:
print("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid))
log.warning("No VMHost for VM {} - VM might be on hold - skipping".format(one_vm.vmid))
continue
try:
vmhost = VMHost.objects.get(hostname=one_vm.last_host)
except VMHost.DoesNotExist:
print("VMHost {} does not exist, aborting".format(one_vm.last_host))
log.error("VMHost {} does not exist, aborting".format(one_vm.last_host))
raise
cores = one_vm.cores
@ -98,9 +144,6 @@ class Command(BaseCommand):
owner = one_vm.owner
status = 'active'
# Total Amount of SSD Storage
# TODO: What would happen if the attached storage is not SSD but HDD?
ssd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['ssd', 'one'] ])
hdd_size = sum([ disk['size_in_gb'] for disk in one_vm.disks if disk['pool_name'] in ['hdd'] ])
@ -119,30 +162,32 @@ class Command(BaseCommand):
len(ipv4), len(ipv6))
try:
vm_product = VMProduct.objects.get(name=one_vm.uncloud_name)
vm_product = VMProduct.objects.get(extra_data__opennebula_id=one_vm.vmid)
except VMProduct.DoesNotExist:
order = Order.objects.create(
owner=owner,
creation_date=creation_date,
starting_date=starting_date
# one_time_price=one_time_price,
# recurring_price=recurring_price,
# recurring_period=recurring_period
)
vm_product, _ = VMProduct.objects.update_or_create(
vm_product = VMProduct(
extra_data={ 'opennebula_id': one_vm.vmid },
name=one_vm.uncloud_name,
defaults={
'cores': cores,
'ram_in_gb': ram_in_gb,
'owner': owner,
'vmhost': vmhost,
'order': order,
'status': status
}
order=order
)
# we don't use update_or_create, as filtering by json AND setting json
# at the same time does not work
vm_product.vmhost = vmhost
vm_product.owner = owner
vm_product.cores = cores
vm_product.ram_in_gb = ram_in_gb
vm_product.status = status
vm_product.save()
# Create VMNetworkCards
create_nics(one_vm, vm_product)
# Create VMDiskImageProduct and VMDiskProduct
create_disk_and_image(one_vm, vm_product)
sync_disk_and_image(one_vm, vm_product, disk_owner=disk_owner)

9
uncloud/opennebula/models.py
View file

@ -3,6 +3,12 @@ from django.db import models
from django.contrib.auth import get_user_model
from django.contrib.postgres.fields import JSONField
# ungleich specific
storage_class_mapping = {
'one': 'ssd',
'ssd': 'ssd',
'hdd': 'hdd'
}
class VM(models.Model):
vmid = models.IntegerField(primary_key=True)
@ -48,7 +54,8 @@ class VM(models.Model):
'pool_name': d['POOL_NAME'],
'image': d['IMAGE'],
'source': d['SOURCE'],
'source_type': d['TM_MAD']
'source_type': d['TM_MAD'],
'storage_class': storage_class_mapping[d['POOL_NAME']]
}
for d in disks

4
uncloud/opennebula/serializers.py
View file

@ -5,4 +5,6 @@ from opennebula.models import VM
class OpenNebulaVMSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = VM
fields = '__all__'
fields = [ 'vmid', 'owner', 'data',
'uncloud_name', 'cores', 'ram_in_gb',
'disks', 'nics', 'ips' ]

4
uncloud/uncloud/urls.py
View file

@ -28,9 +28,9 @@ router = routers.DefaultRouter()
# VM
router.register(r'vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct')
router.register(r'vm/diskimage', vmviews.VMDiskImageProductViewSet, basename='vmdiskimageproduct')
router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct')
router.register(r'vm/image/mine', vmviews.VMDiskImageProductMineViewSet, basename='vmdiskimagemineproduct')
router.register(r'vm/image/public', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct')
# images the provider provides :-)
# router.register(r'vm/image/official', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct')

2
uncloud/uncloud_auth/serializers.py
View file

@ -9,5 +9,7 @@ class UserSerializer(serializers.ModelSerializer):
model = get_user_model()
fields = ['username', 'email', 'balance', 'maximum_credit' ]
balance = serializers.DecimalField(max_digits=AMOUNT_MAX_DIGITS,
decimal_places=AMOUNT_DECIMALS)

2
uncloud/uncloud_vm/models.py
View file

@ -119,7 +119,7 @@ class VMDiskImageProduct(UncloudModel):
name = models.CharField(max_length=256)
is_os_image = models.BooleanField(default=False)
is_public = models.BooleanField(default=False)
is_public = models.BooleanField(default=False, editable=False) # only allow admins to set this
size_in_gb = models.FloatField(null=True, blank=True)
import_url = models.URLField(null=True, blank=True)

5
uncloud/uncloud_vm/serializers.py
View file

@ -84,8 +84,9 @@ class VMProductSerializer(serializers.ModelSerializer):
model = VMProduct
fields = ['uuid', 'order', 'owner', 'status', 'name',
'cores', 'ram_in_gb', 'recurring_period',
'snapshots', 'disks' ]
read_only_fields = ['uuid', 'order', 'owner', 'status']
'snapshots', 'disks',
'extra_data' ]
read_only_fields = ['uuid', 'order', 'owner', 'status' ]
# Custom field used at creation (= ordering) only.
recurring_period = serializers.ChoiceField(

4
uncloud/uncloud_vm/views.py
View file

@ -24,7 +24,7 @@ class VMHostViewSet(viewsets.ModelViewSet):
queryset = VMHost.objects.all()
permission_classes = [permissions.IsAdminUser]
class VMDiskImageProductMineViewSet(viewsets.ModelViewSet):
class VMDiskImageProductViewSet(viewsets.ModelViewSet):
permission_classes = [permissions.IsAuthenticated]
serializer_class = VMDiskImageProductSerializer
@ -32,7 +32,7 @@ class VMDiskImageProductMineViewSet(viewsets.ModelViewSet):
if self.request.user.is_superuser:
obj = VMDiskImageProduct.objects.all()
else:
obj = VMDiskImageProduct.objects.filter(owner=self.request.user)
obj = VMDiskImageProduct.objects.filter(owner=self.request.user) | VMDiskImageProduct.objects.filter(is_public=True)
return obj