amal/uncloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge remote-tracking branch 'ahmed/migrate-one-to-regular-vm'

Browse source
This commit is contained in:
Nico Schottelius 2020-03-05 11:34:15 +01:00
commit 14a4fa8cc1
11 changed files with 370 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

139
uncloud/opennebula/management/commands/migrate-one-vm-to-regular.py Normal file
View file

@ -0,0 +1,139 @@
from datetime import datetime
from django.core.management.base import BaseCommand
from django.utils import timezone
from opennebula.models import VM as VMModel
from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct
from uncloud_pay.models import Order
def convert_mac_to_int(mac_address: str):
# Remove octet connecting characters
mac_address = mac_address.replace(':', '')
mac_address = mac_address.replace('.', '')
mac_address = mac_address.replace('-', '')
mac_address = mac_address.replace(' ', '')
# Parse the resulting number as hexadecimal
mac_address = int(mac_address, base=16)
return mac_address
def get_vm_price(core, ram, storage, n_of_ipv4, n_of_ipv6):
storage = storage / 10 # Division by 10 because our base storage unit is 10 GB
total = 3 * core + 4 * ram + 3.5 * storage + 8 * n_of_ipv4 + 0 * n_of_ipv6
# TODO: Find some reason about the following magical subtraction.
total -= 8
return total
def create_nics(one_vm, vm_product):
for nic in one_vm.nics:
mac_address = convert_mac_to_int(nic.get('MAC'))
ip_address = nic.get('IP', None) or nic.get('IP6_GLOBAL', None)
VMNetworkCard.objects.update_or_create(
mac_address=mac_address, vm=vm_product, defaults={'ip_address': ip_address}
)
def create_disk_and_image(one_vm, vm_product):
for disk in one_vm.disks:
owner = one_vm.owner
name = disk.get('image')
# TODO: Fix the following hard coded values
is_os_image, is_public, status = True, True, 'active'
image_size_in_gb = disk.get('image_size_in_gb')
disk_size_in_gb = disk.get('size_in_gb')
storage_class = disk.get('pool_name')
image_source = disk.get('source')
image_source_type = disk.get('source_type')
image, _ = VMDiskImageProduct.objects.update_or_create(
name=name,
defaults={
'owner': owner,
'is_os_image': is_os_image,
'is_public': is_public,
'size_in_gb': image_size_in_gb,
'storage_class': storage_class,
'image_source': image_source,
'image_source_type': image_source_type,
'status': status
}
)
VMDiskProduct.objects.update_or_create(
owner=owner, vm=vm_product,
defaults={
'image': image,
'size_in_gb': disk_size_in_gb
}
)
class Command(BaseCommand):
help = 'Migrate Opennebula VM to regular (uncloud) vm'
def handle(self, *args, **options):
for one_vm in VMModel.objects.all():
# Host on which the VM is currently residing
host = VMHost.objects.filter(vms__icontains=one_vm.vmid).first()
# VCPU, RAM, Owner, Status
# TODO: Set actual status instead of hard coded 'active'
vm_id, cores, ram_in_gb = one_vm.vmid, one_vm.cores, one_vm.ram_in_gb
owner, status = one_vm.owner, 'active'
# Total Amount of SSD Storage
# TODO: What would happen if the attached storage is not SSD but HDD?
total_storage_in_gb = sum([disk['size_in_gb'] for disk in one_vm.disks])
# List of IPv4 addresses and Global IPv6 addresses
ipv4, ipv6 = one_vm.ips
# TODO: Insert actual/real creation_date, starting_date, ending_date
# instead of pseudo one we are putting currently
creation_date = starting_date = ending_date = datetime.now(tz=timezone.utc)
# Price calculation
# TODO: Make the following non-hardcoded
one_time_price = 0
recurring_period = 'per_month'
recurring_price = get_vm_price(cores, ram_in_gb, total_storage_in_gb, len(ipv4), len(ipv6))
try:
vm_product = VMProduct.objects.get(vmid=vm_id)
except VMProduct.DoesNotExist:
order = Order.objects.create(
owner=one_vm.owner,
creation_date=creation_date,
starting_date=starting_date,
ending_date=ending_date,
one_time_price=one_time_price,
recurring_price=recurring_price,
recurring_period=recurring_period
)
vm_product, _ = VMProduct.objects.update_or_create(
vmid=vm_id,
defaults={
'cores': cores,
'ram_in_gb': ram_in_gb,
'owner': owner,
'vmhost': host,
'order': order,
'status': status
}
)
# Create VMNetworkCards
create_nics(one_vm, vm_product)
# Create VMDiskImageProduct and VMDiskProduct
create_disk_and_image(one_vm, vm_product)

74
uncloud/opennebula/management/commands/synchost.py Normal file
View file

@ -0,0 +1,74 @@
import json
import uncloud.secrets as secrets
from xmlrpc.client import ServerProxy as RPCClient
from django.core.management.base import BaseCommand
from xmltodict import parse
from enum import IntEnum
from opennebula.models import VM as VMModel
from uncloud_vm.models import VMHost
from django_auth_ldap.backend import LDAPBackend
class HostStates(IntEnum):
"""
The following flags are copied from
https://docs.opennebula.org/5.8/integration/system_interfaces/api.html#schemas-for-host
"""
INIT = 0 # Initial state for enabled hosts
MONITORING_MONITORED = 1 # Monitoring the host (from monitored)
MONITORED = 2 # The host has been successfully monitored
ERROR = 3 # An error ocurrer while monitoring the host
DISABLED = 4 # The host is disabled
MONITORING_ERROR = 5 # Monitoring the host (from error)
MONITORING_INIT = 6 # Monitoring the host (from init)
MONITORING_DISABLED = 7 # Monitoring the host (from disabled)
OFFLINE = 8 # The host is totally offline
class Command(BaseCommand):
help = 'Syncronize Host information from OpenNebula'
def add_arguments(self, parser):
pass
def handle(self, *args, **options):
with RPCClient(secrets.OPENNEBULA_URL) as rpc_client:
success, response, *_ = rpc_client.one.hostpool.info(secrets.OPENNEBULA_USER_PASS)
if success:
response = json.loads(json.dumps(parse(response)))
host_pool = response.get('HOST_POOL', {}).get('HOST', {})
for host in host_pool:
host_share = host.get('HOST_SHARE', {})
host_name = host.get('NAME')
state = int(host.get('STATE', HostStates.OFFLINE.value))
if state == HostStates.MONITORED:
status = 'active'
elif state == HostStates.DISABLED:
status = 'disabled'
else:
status = 'unusable'
usable_cores = host_share.get('TOTAL_CPU')
usable_ram_in_kb = int(host_share.get('TOTAL_MEM', 0))
usable_ram_in_gb = int(usable_ram_in_kb / 2 ** 20)
vms = host.get('VMS', {}) or {}
vms = vms.get('ID', []) or []
vms = ','.join(vms)
VMHost.objects.update_or_create(
hostname=host_name,
defaults={
'usable_cores': usable_cores,
'usable_ram_in_gb': usable_ram_in_gb,
'status': status,
'vms': vms
}
)
else:
print(response)

4
uncloud/opennebula/management/commands/syncvm.py
View file

@ -5,14 +5,12 @@ import uncloud.secrets as secrets
from xmlrpc.client import ServerProxy as RPCClient
from django_auth_ldap.backend import LDAPBackend
from django.core.management.base import BaseCommand
from django.contrib.auth import get_user_model
from xmltodict import parse
from opennebula.models import VM as VMModel
from django_auth_ldap.backend import LDAPBackend
class Command(BaseCommand):
help = 'Syncronize VM information from OpenNebula'

31
uncloud/opennebula/models.py
View file

@ -19,7 +19,7 @@ class VM(models.Model):
@property
def ram_in_gb(self):
return (int(self.data['TEMPLATE']['MEMORY'])/1024.)
return int(self.data['TEMPLATE']['MEMORY'])/1024
@property
def disks(self):
@ -35,15 +35,21 @@ class VM(models.Model):
if 'DISK' in self.data['TEMPLATE']:
if type(self.data['TEMPLATE']['DISK']) is dict:
disks = [ self.data['TEMPLATE']['DISK'] ]
disks = [self.data['TEMPLATE']['DISK']]
else:
disks = self.data['TEMPLATE']['DISK']
disks = [
{
'size_in_gb': int(d['SIZE'])/1024. ,
'size_in_gb': int(d['SIZE'])/1024,
'opennebula_source': d['SOURCE'],
'opennebula_name': d['IMAGE'],
'image_size_in_gb': int(d['ORIGINAL_SIZE'])/1024,
'pool_name': d['POOL_NAME'],
'image': d['IMAGE'],
'source': d['SOURCE'],
'source_type': d['TM_MAD']
}
for d in disks
]
@ -57,3 +63,22 @@ class VM(models.Model):
@property
def graphics(self):
return self.data.get('TEMPLATE', {}).get('GRAPHICS', {})
@property
def nics(self):
_nics = self.data.get('TEMPLATE', {}).get('NIC', {})
if isinstance(_nics, dict):
_nics = [_nics]
return _nics
@property
def ips(self):
ipv4, ipv6 = [], []
for nic in self.nics:
ip = nic.get('IP')
ip6 = nic.get('IP6_GLOBAL')
if ip:
ipv4.append(ip)
if ip6:
ipv6.append(ip6)
return ipv4, ipv6

23
uncloud/uncloud_vm/migrations/0009_auto_20200303_0927.py Normal file
View file

@ -0,0 +1,23 @@
# Generated by Django 3.0.3 on 2020-03-03 09:27
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('uncloud_vm', '0008_auto_20200229_1611'),
]
operations = [
migrations.AddField(
model_name='vmhost',
name='vms',
field=models.TextField(default=''),
),
migrations.AlterField(
model_name='vmdiskproduct',
name='size_in_gb',
field=models.FloatField(blank=True),
),
]

18
uncloud/uncloud_vm/migrations/0010_auto_20200303_1208.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.0.3 on 2020-03-03 12:08
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('uncloud_vm', '0009_auto_20200303_0927'),
]
operations = [
migrations.AlterField(
model_name='vmnetworkcard',
name='mac_address',
field=models.BigIntegerField(),
),
]

18
uncloud/uncloud_vm/migrations/0011_vmdiskimageproduct_source_type.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.0.3 on 2020-03-03 18:33
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('uncloud_vm', '0010_auto_20200303_1208'),
]
operations = [
migrations.AddField(
model_name='vmdiskimageproduct',
name='source_type',
field=models.CharField(max_length=128, null=True),
),
]

18
uncloud/uncloud_vm/migrations/0012_vmdiskimageproduct_source.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.0.3 on 2020-03-03 18:35
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('uncloud_vm', '0011_vmdiskimageproduct_source_type'),
]
operations = [
migrations.AddField(
model_name='vmdiskimageproduct',
name='source',
field=models.CharField(max_length=128, null=True),
),
]

23
uncloud/uncloud_vm/migrations/0013_auto_20200303_1845.py Normal file
View file

@ -0,0 +1,23 @@
# Generated by Django 3.0.3 on 2020-03-03 18:45
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('uncloud_vm', '0012_vmdiskimageproduct_source'),
]
operations = [
migrations.RenameField(
model_name='vmdiskimageproduct',
old_name='source',
new_name='image_source',
),
migrations.RenameField(
model_name='vmdiskimageproduct',
old_name='source_type',
new_name='image_source_type',
),
]

18
uncloud/uncloud_vm/migrations/0014_vmproduct_vmid.py Normal file
View file

@ -0,0 +1,18 @@
# Generated by Django 3.0.3 on 2020-03-04 07:52
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('uncloud_vm', '0013_auto_20200303_1845'),
]
operations = [
migrations.AddField(
model_name='vmproduct',
name='vmid',
field=models.IntegerField(null=True),
),
]

12
uncloud/uncloud_vm/models.py
View file

@ -40,6 +40,12 @@ class VMHost(models.Model):
max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT
)
# List of VMs running on this host
vms = models.TextField(default='')
def get_vms(self):
return self.vms.split(',')
class VMProduct(Product):
vmhost = models.ForeignKey(
@ -51,6 +57,7 @@ class VMProduct(Product):
name = models.CharField(max_length=32)
cores = models.IntegerField()
ram_in_gb = models.FloatField()
vmid = models.IntegerField(null=True)
def recurring_price(self, recurring_period=RecurringPeriod.PER_MONTH):
# TODO: move magic numbers in variables
@ -97,7 +104,8 @@ class VMDiskImageProduct(models.Model):
size_in_gb = models.FloatField(null=True, blank=True)
import_url = models.URLField(null=True, blank=True)
image_source = models.CharField(max_length=128, null=True)
image_source_type = models.CharField(max_length=128, null=True)
storage_class = models.CharField(
max_length=32,
choices=(
@ -150,7 +158,7 @@ class VMDiskProduct(models.Model):
class VMNetworkCard(models.Model):
vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE)
mac_address = models.IntegerField()
mac_address = models.BigIntegerField()
ip_address = models.GenericIPAddressField(blank=True,
null=True)