From fea0568bb96be25d62c2fa27fae9094def750f3f Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 3 Mar 2020 23:46:39 +0500 Subject: [PATCH] init commit --- .../commands/migrate-one-vm-to-regular.py | 107 ++++++++++++++++++ .../management/commands/synchost.py | 74 ++++++++++++ .../opennebula/management/commands/syncvm.py | 3 +- uncloud/opennebula/models.py | 29 ++++- .../migrations/0009_auto_20200303_0927.py | 23 ++++ .../migrations/0010_auto_20200303_1208.py | 18 +++ .../0011_vmdiskimageproduct_source_type.py | 18 +++ .../0012_vmdiskimageproduct_source.py | 18 +++ .../migrations/0013_auto_20200303_1845.py | 23 ++++ uncloud/uncloud_vm/models.py | 11 +- 10 files changed, 318 insertions(+), 6 deletions(-) create mode 100644 uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py create mode 100644 uncloud/opennebula/management/commands/synchost.py create mode 100644 uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py create mode 100644 uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py create mode 100644 uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py create mode 100644 uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py create mode 100644 uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py diff --git a/uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py b/uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py new file mode 100644 index 0000000..16a6449 --- /dev/null +++ b/uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py @@ -0,0 +1,107 @@ +from datetime import datetime + +from django.core.management.base import BaseCommand +from django.utils import timezone + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct +from uncloud_pay.models import Order + + +def get_vm_price(core, ram, storage, n_of_ipv4, n_of_ipv6): + storage = storage / 10 + total = 3 * core + 4 * ram + 3.5 * storage + 8 * n_of_ipv4 + 0 * n_of_ipv6 + + # TODO: Find some reason about the following magical subtraction. + total -= 8 + + return total + + +def create_nics(one_vm, vm_product): + for nic in one_vm.nics: + mac_address = nic.get('MAC') + ip_address = nic.get('IP', None) or nic.get('IP6_GLOBAL', None) + + mac_address = mac_address.replace(':', '') + mac_address = mac_address.replace('.', '') + mac_address = mac_address.replace('-', '') + mac_address = mac_address.replace(' ', '') + mac_address = int(mac_address, base=16) + + VMNetworkCard.objects.create( + mac_address=mac_address, vm=vm_product, ip_address=ip_address + ) + + +def create_disk_and_image(one_vm, vm_product): + for disk in one_vm.disks: + owner = one_vm.owner + name = disk.get('image') + + # TODO: Fix the following hard coded values + is_os_image = True + is_public = True + status = 'active' + + image_size_in_gb = disk.get('image_size_in_gb') + disk_size_in_gb = disk.get('size_in_gb') + storage_class = disk.get('pool_name') + image_source = disk.get('source') + image_source_type = disk.get('source_type') + + image = VMDiskImageProduct.objects.create( + owner=owner, name=name, is_os_image=is_os_image, is_public=is_public, + size_in_gb=image_size_in_gb, storage_class=storage_class, + image_source=image_source, image_source_type=image_source_type, status=status + ) + vm_disk = VMDiskProduct.objects.create( + owner=owner, vm=vm_product, image=image, size_in_gb=disk_size_in_gb + ) + + +class Command(BaseCommand): + help = 'Migrate Opennebula VM to regular (uncloud) vm' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + for one_vm in VMModel.objects.all(): + # Host on which the VM is currently residing + host = VMHost.objects.filter(vms__icontains=one_vm.vmid).first() + + # VCPU, RAM, Owner, Status + # TODO: Set actual status instead of hard coded 'active' + cores, ram_in_gb, owner, status = one_vm.cores, one_vm.ram_in_gb, one_vm.owner, 'active' + + # Total Amount of SSD Storage + # TODO: What would happen if the attached storage is not SSD but HDD? + total_storage_in_gb = sum([disk['size_in_gb'] for disk in one_vm.disks]) + + # List of IPv4 addresses and Global IPv6 addresses + ipv4, ipv6 = one_vm.ips + + # TODO: Insert actual/real creation_date, starting_date, ending_date + # instead of pseudo one we are putting currently + order = Order.objects.create( + owner=one_vm.owner, + creation_date=datetime.now(tz=timezone.utc), + starting_date=datetime.now(tz=timezone.utc), + ending_date=datetime.now(tz=timezone.utc), + one_time_price=0, + recurring_price=get_vm_price(cores, ram_in_gb, total_storage_in_gb, len(ipv4), len(ipv6)), + recurring_period='per_month' + ) + + vm_product = VMProduct.objects.create( + cores=cores, ram_in_gb=ram_in_gb, + owner=one_vm.owner, vmhost=host, + order=order, status=status + ) + + # Create VMNetworkCards + create_nics(one_vm, vm_product) + + # Create VMDiskImageProduct and VMDiskProduct + create_disk_and_image(one_vm, vm_product) diff --git a/uncloud/opennebula/management/commands/synchost.py b/uncloud/opennebula/management/commands/synchost.py new file mode 100644 index 0000000..6e4ea0f --- /dev/null +++ b/uncloud/opennebula/management/commands/synchost.py @@ -0,0 +1,74 @@ +import json + +import uncloud.secrets as secrets + +from xmlrpc.client import ServerProxy as RPCClient + +from django.core.management.base import BaseCommand +from xmltodict import parse +from enum import IntEnum +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost +from django_auth_ldap.backend import LDAPBackend + + +class HostStates(IntEnum): + """ + The following flags are copied from + https://docs.opennebula.org/5.8/integration/system_interfaces/api.html#schemas-for-host + """ + INIT = 0 # Initial state for enabled hosts + MONITORING_MONITORED = 1 # Monitoring the host (from monitored) + MONITORED = 2 # The host has been successfully monitored + ERROR = 3 # An error ocurrer while monitoring the host + DISABLED = 4 # The host is disabled + MONITORING_ERROR = 5 # Monitoring the host (from error) + MONITORING_INIT = 6 # Monitoring the host (from init) + MONITORING_DISABLED = 7 # Monitoring the host (from disabled) + OFFLINE = 8 # The host is totally offline + + +class Command(BaseCommand): + help = 'Syncronize Host information from OpenNebula' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + with RPCClient(secrets.OPENNEBULA_URL) as rpc_client: + success, response, *_ = rpc_client.one.hostpool.info(secrets.OPENNEBULA_USER_PASS) + if success: + response = json.loads(json.dumps(parse(response))) + host_pool = response.get('HOST_POOL', {}).get('HOST', {}) + for host in host_pool: + host_share = host.get('HOST_SHARE', {}) + + host_name = host.get('NAME') + state = int(host.get('STATE', HostStates.OFFLINE.value)) + + if state == HostStates.MONITORED: + status = 'active' + elif state == HostStates.DISABLED: + status = 'disabled' + else: + status = 'unusable' + + usable_cores = host_share.get('TOTAL_CPU') + usable_ram_in_kb = int(host_share.get('TOTAL_MEM', 0)) + usable_ram_in_gb = int(usable_ram_in_kb / 2 ** 20) + + vms = host.get('VMS', {}) or {} + vms = vms.get('ID', []) or [] + vms = ','.join(vms) + + VMHost.objects.update_or_create( + hostname=host_name, + defaults={ + 'usable_cores': usable_cores, + 'usable_ram_in_gb': usable_ram_in_gb, + 'status': status, + 'vms': vms + } + ) + else: + print(response) diff --git a/uncloud/opennebula/management/commands/syncvm.py b/uncloud/opennebula/management/commands/syncvm.py index 779db61..ff620f7 100644 --- a/uncloud/opennebula/management/commands/syncvm.py +++ b/uncloud/opennebula/management/commands/syncvm.py @@ -6,7 +6,6 @@ import uncloud.secrets as secrets from xmlrpc.client import ServerProxy as RPCClient from django.core.management.base import BaseCommand -from django.contrib.auth import get_user_model from xmltodict import parse from opennebula.models import VM as VMModel @@ -31,7 +30,7 @@ class Command(BaseCommand): backend = LDAPBackend() - for vm in vms: + for i, vm in enumerate(vms): vm_id = vm['ID'] vm_owner = vm['UNAME'] diff --git a/uncloud/opennebula/models.py b/uncloud/opennebula/models.py index fff811b..059ba5a 100644 --- a/uncloud/opennebula/models.py +++ b/uncloud/opennebula/models.py @@ -35,15 +35,21 @@ class VM(models.Model): if 'DISK' in self.data['TEMPLATE']: if type(self.data['TEMPLATE']['DISK']) is dict: - disks = [ self.data['TEMPLATE']['DISK'] ] + disks = [self.data['TEMPLATE']['DISK']] else: disks = self.data['TEMPLATE']['DISK'] disks = [ { - 'size_in_gb': int(d['SIZE'])/1024. , + 'size_in_gb': int(d['SIZE'])/1024.0, 'opennebula_source': d['SOURCE'], 'opennebula_name': d['IMAGE'], + 'image_size_in_gb': int(d['ORIGINAL_SIZE'])/1024.0, + 'pool_name': d['POOL_NAME'], + 'image': d['IMAGE'], + 'source': d['SOURCE'], + 'source_type': d['TM_MAD'] + } for d in disks ] @@ -57,3 +63,22 @@ class VM(models.Model): @property def graphics(self): return self.data.get('TEMPLATE', {}).get('GRAPHICS', {}) + + @property + def nics(self): + _nics = self.data.get('TEMPLATE', {}).get('NIC', {}) + if isinstance(_nics, dict): + _nics = [_nics] + return _nics + + @property + def ips(self): + ipv4, ipv6 = [], [] + for nic in self.nics: + ip = nic.get('IP') + ip6 = nic.get('IP6_GLOBAL') + if ip: + ipv4.append(ip) + if ip6: + ipv6.append(ip6) + return ipv4, ipv6 diff --git a/uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py b/uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py new file mode 100644 index 0000000..7815f46 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-03-03 09:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0008_auto_20200229_1611'), + ] + + operations = [ + migrations.AddField( + model_name='vmhost', + name='vms', + field=models.TextField(default=''), + ), + migrations.AlterField( + model_name='vmdiskproduct', + name='size_in_gb', + field=models.FloatField(blank=True), + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py b/uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py new file mode 100644 index 0000000..39a20e3 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-03 12:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0009_auto_20200303_0927'), + ] + + operations = [ + migrations.AlterField( + model_name='vmnetworkcard', + name='mac_address', + field=models.BigIntegerField(), + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py b/uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py new file mode 100644 index 0000000..3d445cf --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-03 18:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0010_auto_20200303_1208'), + ] + + operations = [ + migrations.AddField( + model_name='vmdiskimageproduct', + name='source_type', + field=models.CharField(max_length=128, null=True), + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py b/uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py new file mode 100644 index 0000000..4072d82 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-03 18:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0011_vmdiskimageproduct_source_type'), + ] + + operations = [ + migrations.AddField( + model_name='vmdiskimageproduct', + name='source', + field=models.CharField(max_length=128, null=True), + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py b/uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py new file mode 100644 index 0000000..55aed73 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-03-03 18:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0012_vmdiskimageproduct_source'), + ] + + operations = [ + migrations.RenameField( + model_name='vmdiskimageproduct', + old_name='source', + new_name='image_source', + ), + migrations.RenameField( + model_name='vmdiskimageproduct', + old_name='source_type', + new_name='image_source_type', + ), + ] diff --git a/uncloud/uncloud_vm/models.py b/uncloud/uncloud_vm/models.py index 4b0d511..72317be 100644 --- a/uncloud/uncloud_vm/models.py +++ b/uncloud/uncloud_vm/models.py @@ -39,6 +39,12 @@ class VMHost(models.Model): max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT ) + # List of VMs running on this host + vms = models.TextField(default='') + + def get_vms(self): + return self.vms.split(',') + class VMProduct(Product): vmhost = models.ForeignKey( @@ -74,7 +80,8 @@ class VMDiskImageProduct(models.Model): size_in_gb = models.FloatField(null=True, blank=True) import_url = models.URLField(null=True, blank=True) - + image_source = models.CharField(max_length=128, null=True) + image_source_type = models.CharField(max_length=128, null=True) storage_class = models.CharField( max_length=32, choices=( @@ -127,7 +134,7 @@ class VMDiskProduct(models.Model): class VMNetworkCard(models.Model): vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE) - mac_address = models.IntegerField() + mac_address = models.BigIntegerField() ip_address = models.GenericIPAddressField(blank=True, null=True)