Merge remote-tracking branch 'ahmed/migrate-one-to-regular-vm'

uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py

@@ -0,0 +1,139 @@
+from datetime import datetime
+
+from django.core.management.base import BaseCommand
+from django.utils import timezone
+
+from opennebula.models import VM as VMModel
+from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct
+from uncloud_pay.models import Order
+
+
+def convert_mac_to_int(mac_address: str):
+    # Remove octet connecting characters
+    mac_address = mac_address.replace(':', '')
+    mac_address = mac_address.replace('.', '')
+    mac_address = mac_address.replace('-', '')
+    mac_address = mac_address.replace(' ', '')
+
+    # Parse the resulting number as hexadecimal
+    mac_address = int(mac_address, base=16)
+
+    return mac_address
+
+
+def get_vm_price(core, ram, storage, n_of_ipv4, n_of_ipv6):
+    storage = storage / 10  # Division by 10 because our base storage unit is 10 GB
+    total = 3 * core + 4 * ram + 3.5 * storage + 8 * n_of_ipv4 + 0 * n_of_ipv6
+
+    # TODO: Find some reason about the following magical subtraction.
+    total -= 8
+
+    return total
+
+
+def create_nics(one_vm, vm_product):
+    for nic in one_vm.nics:
+        mac_address = convert_mac_to_int(nic.get('MAC'))
+        ip_address = nic.get('IP', None) or nic.get('IP6_GLOBAL', None)
+
+        VMNetworkCard.objects.update_or_create(
+            mac_address=mac_address, vm=vm_product, defaults={'ip_address': ip_address}
+        )
+
+
+def create_disk_and_image(one_vm, vm_product):
+    for disk in one_vm.disks:
+        owner = one_vm.owner
+        name = disk.get('image')
+
+        # TODO: Fix the following hard coded values
+        is_os_image, is_public, status = True, True, 'active'
+
+        image_size_in_gb = disk.get('image_size_in_gb')
+        disk_size_in_gb = disk.get('size_in_gb')
+        storage_class = disk.get('pool_name')
+        image_source = disk.get('source')
+        image_source_type = disk.get('source_type')
+
+        image, _ = VMDiskImageProduct.objects.update_or_create(
+            name=name,
+            defaults={
+                'owner': owner,
+                'is_os_image': is_os_image,
+                'is_public': is_public,
+                'size_in_gb': image_size_in_gb,
+                'storage_class': storage_class,
+                'image_source': image_source,
+                'image_source_type': image_source_type,
+                'status': status
+            }
+        )
+        VMDiskProduct.objects.update_or_create(
+            owner=owner, vm=vm_product,
+            defaults={
+                'image': image,
+                'size_in_gb': disk_size_in_gb
+            }
+        )
+
+
+class Command(BaseCommand):
+    help = 'Migrate Opennebula VM to regular (uncloud) vm'
+
+    def handle(self, *args, **options):
+        for one_vm in VMModel.objects.all():
+            # Host on which the VM is currently residing
+            host = VMHost.objects.filter(vms__icontains=one_vm.vmid).first()
+
+            # VCPU, RAM, Owner, Status
+            # TODO: Set actual status instead of hard coded 'active'
+            vm_id, cores, ram_in_gb = one_vm.vmid, one_vm.cores, one_vm.ram_in_gb
+            owner, status = one_vm.owner, 'active'
+
+            # Total Amount of SSD Storage
+            # TODO: What would happen if the attached storage is not SSD but HDD?
+            total_storage_in_gb = sum([disk['size_in_gb'] for disk in one_vm.disks])
+
+            # List of IPv4 addresses and Global IPv6 addresses
+            ipv4, ipv6 = one_vm.ips
+
+            # TODO: Insert actual/real creation_date, starting_date, ending_date
+            #       instead of pseudo one we are putting currently
+            creation_date = starting_date = ending_date = datetime.now(tz=timezone.utc)
+
+            # Price calculation
+
+            # TODO: Make the following non-hardcoded
+            one_time_price = 0
+            recurring_period = 'per_month'
+
+            recurring_price = get_vm_price(cores, ram_in_gb, total_storage_in_gb, len(ipv4), len(ipv6))
+            try:
+                vm_product = VMProduct.objects.get(vmid=vm_id)
+            except VMProduct.DoesNotExist:
+                order = Order.objects.create(
+                    owner=one_vm.owner,
+                    creation_date=creation_date,
+                    starting_date=starting_date,
+                    ending_date=ending_date,
+                    one_time_price=one_time_price,
+                    recurring_price=recurring_price,
+                    recurring_period=recurring_period
+                )
+                vm_product, _ = VMProduct.objects.update_or_create(
+                    vmid=vm_id,
+                    defaults={
+                        'cores': cores,
+                        'ram_in_gb': ram_in_gb,
+                        'owner': owner,
+                        'vmhost': host,
+                        'order': order,
+                        'status': status
+                    }
+                )
+
+            # Create VMNetworkCards
+            create_nics(one_vm, vm_product)
+
+            # Create VMDiskImageProduct and VMDiskProduct
+            create_disk_and_image(one_vm, vm_product)

uncloud/opennebula/management/commands/synchost.py

@@ -0,0 +1,74 @@
+import json
+
+import uncloud.secrets as secrets
+
+from xmlrpc.client import ServerProxy as RPCClient
+
+from django.core.management.base import BaseCommand
+from xmltodict import parse
+from enum import IntEnum
+from opennebula.models import VM as VMModel
+from uncloud_vm.models import VMHost
+from django_auth_ldap.backend import LDAPBackend
+
+
+class HostStates(IntEnum):
+    """
+    The following flags are copied from
+    https://docs.opennebula.org/5.8/integration/system_interfaces/api.html#schemas-for-host
+    """
+    INIT = 0  # Initial state for enabled hosts
+    MONITORING_MONITORED = 1  # Monitoring the host (from monitored)
+    MONITORED = 2  # The host has been successfully monitored
+    ERROR = 3  # An error ocurrer while monitoring the host
+    DISABLED = 4  # The host is disabled
+    MONITORING_ERROR = 5  # Monitoring the host (from error)
+    MONITORING_INIT = 6  # Monitoring the host (from init)
+    MONITORING_DISABLED = 7  # Monitoring the host (from disabled)
+    OFFLINE = 8  # The host is totally offline
+
+
+class Command(BaseCommand):
+    help = 'Syncronize Host information from OpenNebula'
+
+    def add_arguments(self, parser):
+        pass
+
+    def handle(self, *args, **options):
+        with RPCClient(secrets.OPENNEBULA_URL) as rpc_client:
+            success, response, *_ = rpc_client.one.hostpool.info(secrets.OPENNEBULA_USER_PASS)
+            if success:
+                response = json.loads(json.dumps(parse(response)))
+                host_pool = response.get('HOST_POOL', {}).get('HOST', {})
+                for host in host_pool:
+                    host_share = host.get('HOST_SHARE', {})
+
+                    host_name = host.get('NAME')
+                    state = int(host.get('STATE', HostStates.OFFLINE.value))
+
+                    if state == HostStates.MONITORED:
+                        status = 'active'
+                    elif state == HostStates.DISABLED:
+                        status = 'disabled'
+                    else:
+                        status = 'unusable'
+
+                    usable_cores = host_share.get('TOTAL_CPU')
+                    usable_ram_in_kb = int(host_share.get('TOTAL_MEM', 0))
+                    usable_ram_in_gb = int(usable_ram_in_kb / 2 ** 20)
+
+                    vms = host.get('VMS', {}) or {}
+                    vms = vms.get('ID', []) or []
+                    vms = ','.join(vms)
+
+                    VMHost.objects.update_or_create(
+                        hostname=host_name,
+                        defaults={
+                            'usable_cores': usable_cores,
+                            'usable_ram_in_gb': usable_ram_in_gb,
+                            'status': status,
+                            'vms': vms
+                        }
+                    )
+            else:
+                print(response)

uncloud/opennebula/management/commands/syncvm.py

@@ -5,14 +5,12 @@ import uncloud.secrets as secrets
 
 from xmlrpc.client import ServerProxy as RPCClient
 
+from django_auth_ldap.backend import LDAPBackend
 from django.core.management.base import BaseCommand
-from django.contrib.auth import get_user_model
 from xmltodict import parse
 
 from opennebula.models import VM as VMModel
 
-from django_auth_ldap.backend import LDAPBackend
-
 
 class Command(BaseCommand):
     help = 'Syncronize VM information from OpenNebula'

uncloud/opennebula/models.py

@@ -19,7 +19,7 @@ class VM(models.Model):
 
     @property
     def ram_in_gb(self):
-        return (int(self.data['TEMPLATE']['MEMORY'])/1024.)
+        return int(self.data['TEMPLATE']['MEMORY'])/1024
 
     @property
     def disks(self):
@@ -35,15 +35,21 @@ class VM(models.Model):
 
         if 'DISK' in self.data['TEMPLATE']:
             if type(self.data['TEMPLATE']['DISK']) is dict:
-                disks = [ self.data['TEMPLATE']['DISK'] ]
+                disks = [self.data['TEMPLATE']['DISK']]
             else:
                 disks = self.data['TEMPLATE']['DISK']
 
         disks = [
             {
-                'size_in_gb': int(d['SIZE'])/1024. ,
+                'size_in_gb': int(d['SIZE'])/1024,
                 'opennebula_source': d['SOURCE'],
                 'opennebula_name': d['IMAGE'],
+                'image_size_in_gb': int(d['ORIGINAL_SIZE'])/1024,
+                'pool_name': d['POOL_NAME'],
+                'image': d['IMAGE'],
+                'source': d['SOURCE'],
+                'source_type': d['TM_MAD']
+
             }
             for d in disks
         ]
@@ -57,3 +63,22 @@ class VM(models.Model):
     @property
     def graphics(self):
         return self.data.get('TEMPLATE', {}).get('GRAPHICS', {})
+
+    @property
+    def nics(self):
+        _nics = self.data.get('TEMPLATE', {}).get('NIC', {})
+        if isinstance(_nics, dict):
+            _nics = [_nics]
+        return _nics
+
+    @property
+    def ips(self):
+        ipv4, ipv6 = [], []
+        for nic in self.nics:
+            ip = nic.get('IP')
+            ip6 = nic.get('IP6_GLOBAL')
+            if ip:
+                ipv4.append(ip)
+            if ip6:
+                ipv6.append(ip6)
+        return ipv4, ipv6

uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.3 on 2020-03-03 09:27
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0008_auto_20200229_1611'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmhost',
+            name='vms',
+            field=models.TextField(default=''),
+        ),
+        migrations.AlterField(
+            model_name='vmdiskproduct',
+            name='size_in_gb',
+            field=models.FloatField(blank=True),
+        ),
+    ]

uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.3 on 2020-03-03 12:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0009_auto_20200303_0927'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='vmnetworkcard',
+            name='mac_address',
+            field=models.BigIntegerField(),
+        ),
+    ]

uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.3 on 2020-03-03 18:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0010_auto_20200303_1208'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmdiskimageproduct',
+            name='source_type',
+            field=models.CharField(max_length=128, null=True),
+        ),
+    ]

uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.3 on 2020-03-03 18:35
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0011_vmdiskimageproduct_source_type'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmdiskimageproduct',
+            name='source',
+            field=models.CharField(max_length=128, null=True),
+        ),
+    ]

uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.3 on 2020-03-03 18:45
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0012_vmdiskimageproduct_source'),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name='vmdiskimageproduct',
+            old_name='source',
+            new_name='image_source',
+        ),
+        migrations.RenameField(
+            model_name='vmdiskimageproduct',
+            old_name='source_type',
+            new_name='image_source_type',
+        ),
+    ]

uncloud/uncloud_vm/migrations/0014_vmproduct_vmid.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.3 on 2020-03-04 07:52
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0013_auto_20200303_1845'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmproduct',
+            name='vmid',
+            field=models.IntegerField(null=True),
+        ),
+    ]

uncloud/uncloud_vm/models.py

@@ -40,6 +40,12 @@ class VMHost(models.Model):
         max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT
     )
 
+    # List of VMs running on this host
+    vms = models.TextField(default='')
+
+    def get_vms(self):
+        return self.vms.split(',')
+
 
 class VMProduct(Product):
     vmhost = models.ForeignKey(
@@ -51,6 +57,7 @@ class VMProduct(Product):
     name = models.CharField(max_length=32)
     cores = models.IntegerField()
     ram_in_gb = models.FloatField()
+    vmid = models.IntegerField(null=True)
 
     def recurring_price(self, recurring_period=RecurringPeriod.PER_MONTH):
         # TODO: move magic numbers in variables
@@ -97,7 +104,8 @@ class VMDiskImageProduct(models.Model):
 
     size_in_gb = models.FloatField(null=True, blank=True)
     import_url = models.URLField(null=True, blank=True)
-
+    image_source = models.CharField(max_length=128, null=True)
+    image_source_type = models.CharField(max_length=128, null=True)
     storage_class = models.CharField(
         max_length=32,
         choices=(
@@ -150,7 +158,7 @@ class VMDiskProduct(models.Model):
 class VMNetworkCard(models.Model):
     vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE)
 
-    mac_address = models.IntegerField()
+    mac_address = models.BigIntegerField()
 
     ip_address = models.GenericIPAddressField(blank=True,
                                               null=True)