diff --git a/uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py b/uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py new file mode 100644 index 0000000..68cf1f2 --- /dev/null +++ b/uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py @@ -0,0 +1,139 @@ +from datetime import datetime + +from django.core.management.base import BaseCommand +from django.utils import timezone + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct +from uncloud_pay.models import Order + + +def convert_mac_to_int(mac_address: str): + # Remove octet connecting characters + mac_address = mac_address.replace(':', '') + mac_address = mac_address.replace('.', '') + mac_address = mac_address.replace('-', '') + mac_address = mac_address.replace(' ', '') + + # Parse the resulting number as hexadecimal + mac_address = int(mac_address, base=16) + + return mac_address + + +def get_vm_price(core, ram, storage, n_of_ipv4, n_of_ipv6): + storage = storage / 10 # Division by 10 because our base storage unit is 10 GB + total = 3 * core + 4 * ram + 3.5 * storage + 8 * n_of_ipv4 + 0 * n_of_ipv6 + + # TODO: Find some reason about the following magical subtraction. + total -= 8 + + return total + + +def create_nics(one_vm, vm_product): + for nic in one_vm.nics: + mac_address = convert_mac_to_int(nic.get('MAC')) + ip_address = nic.get('IP', None) or nic.get('IP6_GLOBAL', None) + + VMNetworkCard.objects.update_or_create( + mac_address=mac_address, vm=vm_product, defaults={'ip_address': ip_address} + ) + + +def create_disk_and_image(one_vm, vm_product): + for disk in one_vm.disks: + owner = one_vm.owner + name = disk.get('image') + + # TODO: Fix the following hard coded values + is_os_image, is_public, status = True, True, 'active' + + image_size_in_gb = disk.get('image_size_in_gb') + disk_size_in_gb = disk.get('size_in_gb') + storage_class = disk.get('pool_name') + image_source = disk.get('source') + image_source_type = disk.get('source_type') + + image, _ = VMDiskImageProduct.objects.update_or_create( + name=name, + defaults={ + 'owner': owner, + 'is_os_image': is_os_image, + 'is_public': is_public, + 'size_in_gb': image_size_in_gb, + 'storage_class': storage_class, + 'image_source': image_source, + 'image_source_type': image_source_type, + 'status': status + } + ) + VMDiskProduct.objects.update_or_create( + owner=owner, vm=vm_product, + defaults={ + 'image': image, + 'size_in_gb': disk_size_in_gb + } + ) + + +class Command(BaseCommand): + help = 'Migrate Opennebula VM to regular (uncloud) vm' + + def handle(self, *args, **options): + for one_vm in VMModel.objects.all(): + # Host on which the VM is currently residing + host = VMHost.objects.filter(vms__icontains=one_vm.vmid).first() + + # VCPU, RAM, Owner, Status + # TODO: Set actual status instead of hard coded 'active' + vm_id, cores, ram_in_gb = one_vm.vmid, one_vm.cores, one_vm.ram_in_gb + owner, status = one_vm.owner, 'active' + + # Total Amount of SSD Storage + # TODO: What would happen if the attached storage is not SSD but HDD? + total_storage_in_gb = sum([disk['size_in_gb'] for disk in one_vm.disks]) + + # List of IPv4 addresses and Global IPv6 addresses + ipv4, ipv6 = one_vm.ips + + # TODO: Insert actual/real creation_date, starting_date, ending_date + # instead of pseudo one we are putting currently + creation_date = starting_date = ending_date = datetime.now(tz=timezone.utc) + + # Price calculation + + # TODO: Make the following non-hardcoded + one_time_price = 0 + recurring_period = 'per_month' + + recurring_price = get_vm_price(cores, ram_in_gb, total_storage_in_gb, len(ipv4), len(ipv6)) + try: + vm_product = VMProduct.objects.get(vmid=vm_id) + except VMProduct.DoesNotExist: + order = Order.objects.create( + owner=one_vm.owner, + creation_date=creation_date, + starting_date=starting_date, + ending_date=ending_date, + one_time_price=one_time_price, + recurring_price=recurring_price, + recurring_period=recurring_period + ) + vm_product, _ = VMProduct.objects.update_or_create( + vmid=vm_id, + defaults={ + 'cores': cores, + 'ram_in_gb': ram_in_gb, + 'owner': owner, + 'vmhost': host, + 'order': order, + 'status': status + } + ) + + # Create VMNetworkCards + create_nics(one_vm, vm_product) + + # Create VMDiskImageProduct and VMDiskProduct + create_disk_and_image(one_vm, vm_product) diff --git a/uncloud/opennebula/management/commands/synchost.py b/uncloud/opennebula/management/commands/synchost.py new file mode 100644 index 0000000..6e4ea0f --- /dev/null +++ b/uncloud/opennebula/management/commands/synchost.py @@ -0,0 +1,74 @@ +import json + +import uncloud.secrets as secrets + +from xmlrpc.client import ServerProxy as RPCClient + +from django.core.management.base import BaseCommand +from xmltodict import parse +from enum import IntEnum +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost +from django_auth_ldap.backend import LDAPBackend + + +class HostStates(IntEnum): + """ + The following flags are copied from + https://docs.opennebula.org/5.8/integration/system_interfaces/api.html#schemas-for-host + """ + INIT = 0 # Initial state for enabled hosts + MONITORING_MONITORED = 1 # Monitoring the host (from monitored) + MONITORED = 2 # The host has been successfully monitored + ERROR = 3 # An error ocurrer while monitoring the host + DISABLED = 4 # The host is disabled + MONITORING_ERROR = 5 # Monitoring the host (from error) + MONITORING_INIT = 6 # Monitoring the host (from init) + MONITORING_DISABLED = 7 # Monitoring the host (from disabled) + OFFLINE = 8 # The host is totally offline + + +class Command(BaseCommand): + help = 'Syncronize Host information from OpenNebula' + + def add_arguments(self, parser): + pass + + def handle(self, *args, **options): + with RPCClient(secrets.OPENNEBULA_URL) as rpc_client: + success, response, *_ = rpc_client.one.hostpool.info(secrets.OPENNEBULA_USER_PASS) + if success: + response = json.loads(json.dumps(parse(response))) + host_pool = response.get('HOST_POOL', {}).get('HOST', {}) + for host in host_pool: + host_share = host.get('HOST_SHARE', {}) + + host_name = host.get('NAME') + state = int(host.get('STATE', HostStates.OFFLINE.value)) + + if state == HostStates.MONITORED: + status = 'active' + elif state == HostStates.DISABLED: + status = 'disabled' + else: + status = 'unusable' + + usable_cores = host_share.get('TOTAL_CPU') + usable_ram_in_kb = int(host_share.get('TOTAL_MEM', 0)) + usable_ram_in_gb = int(usable_ram_in_kb / 2 ** 20) + + vms = host.get('VMS', {}) or {} + vms = vms.get('ID', []) or [] + vms = ','.join(vms) + + VMHost.objects.update_or_create( + hostname=host_name, + defaults={ + 'usable_cores': usable_cores, + 'usable_ram_in_gb': usable_ram_in_gb, + 'status': status, + 'vms': vms + } + ) + else: + print(response) diff --git a/uncloud/opennebula/management/commands/syncvm.py b/uncloud/opennebula/management/commands/syncvm.py index 779db61..458528b 100644 --- a/uncloud/opennebula/management/commands/syncvm.py +++ b/uncloud/opennebula/management/commands/syncvm.py @@ -5,14 +5,12 @@ import uncloud.secrets as secrets from xmlrpc.client import ServerProxy as RPCClient +from django_auth_ldap.backend import LDAPBackend from django.core.management.base import BaseCommand -from django.contrib.auth import get_user_model from xmltodict import parse from opennebula.models import VM as VMModel -from django_auth_ldap.backend import LDAPBackend - class Command(BaseCommand): help = 'Syncronize VM information from OpenNebula' diff --git a/uncloud/opennebula/models.py b/uncloud/opennebula/models.py index fff811b..0748ff5 100644 --- a/uncloud/opennebula/models.py +++ b/uncloud/opennebula/models.py @@ -19,7 +19,7 @@ class VM(models.Model): @property def ram_in_gb(self): - return (int(self.data['TEMPLATE']['MEMORY'])/1024.) + return int(self.data['TEMPLATE']['MEMORY'])/1024 @property def disks(self): @@ -35,15 +35,21 @@ class VM(models.Model): if 'DISK' in self.data['TEMPLATE']: if type(self.data['TEMPLATE']['DISK']) is dict: - disks = [ self.data['TEMPLATE']['DISK'] ] + disks = [self.data['TEMPLATE']['DISK']] else: disks = self.data['TEMPLATE']['DISK'] disks = [ { - 'size_in_gb': int(d['SIZE'])/1024. , + 'size_in_gb': int(d['SIZE'])/1024, 'opennebula_source': d['SOURCE'], 'opennebula_name': d['IMAGE'], + 'image_size_in_gb': int(d['ORIGINAL_SIZE'])/1024, + 'pool_name': d['POOL_NAME'], + 'image': d['IMAGE'], + 'source': d['SOURCE'], + 'source_type': d['TM_MAD'] + } for d in disks ] @@ -57,3 +63,22 @@ class VM(models.Model): @property def graphics(self): return self.data.get('TEMPLATE', {}).get('GRAPHICS', {}) + + @property + def nics(self): + _nics = self.data.get('TEMPLATE', {}).get('NIC', {}) + if isinstance(_nics, dict): + _nics = [_nics] + return _nics + + @property + def ips(self): + ipv4, ipv6 = [], [] + for nic in self.nics: + ip = nic.get('IP') + ip6 = nic.get('IP6_GLOBAL') + if ip: + ipv4.append(ip) + if ip6: + ipv6.append(ip6) + return ipv4, ipv6 diff --git a/uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py b/uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py new file mode 100644 index 0000000..7815f46 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-03-03 09:27 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0008_auto_20200229_1611'), + ] + + operations = [ + migrations.AddField( + model_name='vmhost', + name='vms', + field=models.TextField(default=''), + ), + migrations.AlterField( + model_name='vmdiskproduct', + name='size_in_gb', + field=models.FloatField(blank=True), + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py b/uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py new file mode 100644 index 0000000..39a20e3 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-03 12:08 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0009_auto_20200303_0927'), + ] + + operations = [ + migrations.AlterField( + model_name='vmnetworkcard', + name='mac_address', + field=models.BigIntegerField(), + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py b/uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py new file mode 100644 index 0000000..3d445cf --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-03 18:33 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0010_auto_20200303_1208'), + ] + + operations = [ + migrations.AddField( + model_name='vmdiskimageproduct', + name='source_type', + field=models.CharField(max_length=128, null=True), + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py b/uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py new file mode 100644 index 0000000..4072d82 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-03 18:35 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0011_vmdiskimageproduct_source_type'), + ] + + operations = [ + migrations.AddField( + model_name='vmdiskimageproduct', + name='source', + field=models.CharField(max_length=128, null=True), + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py b/uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py new file mode 100644 index 0000000..55aed73 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-03-03 18:45 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0012_vmdiskimageproduct_source'), + ] + + operations = [ + migrations.RenameField( + model_name='vmdiskimageproduct', + old_name='source', + new_name='image_source', + ), + migrations.RenameField( + model_name='vmdiskimageproduct', + old_name='source_type', + new_name='image_source_type', + ), + ] diff --git a/uncloud/uncloud_vm/migrations/0014_vmproduct_vmid.py b/uncloud/uncloud_vm/migrations/0014_vmproduct_vmid.py new file mode 100644 index 0000000..4f43f77 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0014_vmproduct_vmid.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-03-04 07:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0013_auto_20200303_1845'), + ] + + operations = [ + migrations.AddField( + model_name='vmproduct', + name='vmid', + field=models.IntegerField(null=True), + ), + ] diff --git a/uncloud/uncloud_vm/models.py b/uncloud/uncloud_vm/models.py index 02fb13e..7612d86 100644 --- a/uncloud/uncloud_vm/models.py +++ b/uncloud/uncloud_vm/models.py @@ -40,6 +40,12 @@ class VMHost(models.Model): max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT ) + # List of VMs running on this host + vms = models.TextField(default='') + + def get_vms(self): + return self.vms.split(',') + class VMProduct(Product): vmhost = models.ForeignKey( @@ -51,6 +57,7 @@ class VMProduct(Product): name = models.CharField(max_length=32) cores = models.IntegerField() ram_in_gb = models.FloatField() + vmid = models.IntegerField(null=True) def recurring_price(self, recurring_period=RecurringPeriod.PER_MONTH): # TODO: move magic numbers in variables @@ -97,7 +104,8 @@ class VMDiskImageProduct(models.Model): size_in_gb = models.FloatField(null=True, blank=True) import_url = models.URLField(null=True, blank=True) - + image_source = models.CharField(max_length=128, null=True) + image_source_type = models.CharField(max_length=128, null=True) storage_class = models.CharField( max_length=32, choices=( @@ -150,7 +158,7 @@ class VMDiskProduct(models.Model): class VMNetworkCard(models.Model): vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE) - mac_address = models.IntegerField() + mac_address = models.BigIntegerField() ip_address = models.GenericIPAddressField(blank=True, null=True)