ahmedbilal/uncloud
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

vmsnapshot progress

Browse source
This commit is contained in:
Nico Schottelius 2020-03-22 20:55:11 +01:00
parent 9961ca0446
commit 23203ff418
11 changed files with 175 additions and 125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

28
uncloud/uncloud/management/commands/uncloud.py Normal file
View file

@ -0,0 +1,28 @@
import sys
from datetime import datetime
from django.core.management.base import BaseCommand
from django.contrib.auth import get_user_model
from opennebula.models import VM as VMModel
from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster
import logging
log = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'General uncloud commands'
def add_arguments(self, parser):
parser.add_argument('--bootstrap', action='store_true', help='Bootstrap a typical uncloud installation')
def handle(self, *args, **options):
if options['bootstrap']:
self.bootstrap()
def bootstrap(self):
default_cluster = VMCluster.objects.get_or_create(name="default")
# local_host =

13
uncloud/uncloud/models.py
View file

@ -1,5 +1,6 @@
from django.db import models
from django.contrib.postgres.fields import JSONField
from django.utils.translation import gettext_lazy as _
class UncloudModel(models.Model):
"""
@ -20,3 +21,15 @@ class UncloudModel(models.Model):
class Meta:
abstract = True
# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types
class UncloudStatus(models.TextChoices):
PENDING = 'PENDING', _('Pending')
AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment')
BEING_CREATED = 'BEING_CREATED', _('Being created')
SCHEDULED = 'SCHEDULED', _('Scheduled') # resource selected, waiting for dispatching
ACTIVE = 'ACTIVE', _('Active')
MODIFYING = 'MODIFYING', _('Modifying') # Resource is being changed
DELETED = 'DELETED', _('Deleted') # Resource has been deleted
DISABLED = 'DISABLED', _('Disabled') # Is usable, but cannot be used for new things
UNUSABLE = 'UNUSABLE', _('Unusable'), # Has some kind of error

1
uncloud/uncloud/settings.py
View file

@ -59,6 +59,7 @@ INSTALLED_APPS = [
'django.contrib.staticfiles',
'django_extensions',
'rest_framework',
'uncloud',
'uncloud_pay',
'uncloud_auth',
'uncloud_storage',

19
uncloud/uncloud/urls.py
View file

@ -30,32 +30,16 @@ router = routers.DefaultRouter()
router.register(r'vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct')
router.register(r'vm/diskimage', vmviews.VMDiskImageProductViewSet, basename='vmdiskimageproduct')
router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct')
# images the provider provides :-)
# router.register(r'vm/image/official', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct')
router.register(r'vm/vm', vmviews.VMProductViewSet, basename='vmproduct')
# TBD
#router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct')
# creates VM from os image
#router.register(r'vm/ipv6onlyvm', vmviews.VMProductViewSet, basename='vmproduct')
# ... AND adds IPv4 mapping
#router.register(r'vm/dualstackvm', vmviews.VMProductViewSet, basename='vmproduct')
# allow vm creation from own images
# Services
router.register(r'service/matrix', serviceviews.MatrixServiceProductViewSet, basename='matrixserviceproduct')
# Pay
router.register(r'payment-method', payviews.PaymentMethodViewSet, basename='payment-method')
router.register(r'bill', payviews.BillViewSet, basename='bill')
@ -63,14 +47,13 @@ router.register(r'order', payviews.OrderViewSet, basename='order')
router.register(r'payment', payviews.PaymentViewSet, basename='payment')
router.register(r'payment-method', payviews.PaymentMethodViewSet, basename='payment-methods')
# VMs
router.register(r'vm/vm', vmviews.VMProductViewSet, basename='vm')
# admin/staff urls
router.register(r'admin/bill', payviews.AdminBillViewSet, basename='admin/bill')
router.register(r'admin/payment', payviews.AdminPaymentViewSet, basename='admin/payment')
router.register(r'admin/order', payviews.AdminOrderViewSet, basename='admin/order')
router.register(r'admin/vmhost', vmviews.VMHostViewSet)
router.register(r'admin/vmcluster', vmviews.VMClusterViewSet)
router.register(r'admin/opennebula', oneviews.VMViewSet, basename='opennebula')
# User/Account

13
uncloud/uncloud_pay/models.py
View file

@ -19,7 +19,7 @@ from decimal import Decimal
import uncloud_pay.stripe
from uncloud_pay.helpers import beginning_of_month, end_of_month
from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS
from uncloud.models import UncloudModel
from uncloud.models import UncloudModel, UncloudStatus
# Used to generate bill due dates.
@ -35,13 +35,6 @@ class RecurringPeriod(models.TextChoices):
PER_HOUR = 'HOUR', _('Per Hour')
PER_SECOND = 'SECOND', _('Per Second')
# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types
class ProductStatus(models.TextChoices):
PENDING = 'PENDING', _('Pending')
AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment')
BEING_CREATED = 'BEING_CREATED', _('Being created')
ACTIVE = 'ACTIVE', _('Active')
DELETED = 'DELETED', _('Deleted')
def get_balance_for_user(user):
@ -445,8 +438,8 @@ class Product(UncloudModel):
description = ""
status = models.CharField(max_length=32,
choices=ProductStatus.choices,
default=ProductStatus.PENDING)
choices=UncloudStatus.choices,
default=UncloudStatus.PENDING)
order = models.ForeignKey(Order,
on_delete=models.CASCADE,

35
uncloud/uncloud_vm/management/commands/vm-create-snapshots.py
View file

@ -1,35 +0,0 @@
import json
import uncloud.secrets as secrets
from django.core.management.base import BaseCommand
from django.contrib.auth import get_user_model
from uncloud_vm.models import VMSnapshotProduct
from datetime import datetime
class Command(BaseCommand):
help = 'Select VM Host for VMs'
def add_arguments(self, parser):
parser.add_argument('--this-hostname', required=True)
# parser.add_argument('--start-vms-here', action='store_true')
# parser.add_argument('--check-health', action='store_true')
# parser.add_argument('--vmhostname')
# print(parser)
def handle(self, *args, **options):
for snapshot in VMSnapshotProduct.objects.filter(status='PENDING'):
if not snapshot.extra_data:
snapshot.extra_data = {}
# TODO: implement locking here
if 'creating_hostname' in snapshot.extra_data:
pass
snapshot.extra_data['creating_hostname'] = options['this_hostname']
snapshot.extra_data['creating_start'] = str(datetime.now())
snapshot.save()
print(snapshot)

112
uncloud/uncloud_vm/management/commands/vm.py
View file

@ -5,73 +5,108 @@ import uncloud.secrets as secrets
from django.core.management.base import BaseCommand
from django.contrib.auth import get_user_model
from uncloud_vm.models import VMProduct, VMHost
from uncloud_vm.models import VMSnapshotProduct, VMProduct, VMHost
from datetime import datetime
class Command(BaseCommand):
help = 'Select VM Host for VMs'
def add_arguments(self, parser):
parser.add_argument('--this-hostname', required=True)
parser.add_argument('--this-cluster', required=True)
parser.add_argument('--create-vm-snapshots', action='store_true')
parser.add_argument('--schedule-vms', action='store_true')
parser.add_argument('--start-vms-here', action='store_true')
parser.add_argument('--check-health', action='store_true')
parser.add_argument('--vmhostname')
print(parser)
parser.add_argument('--start-vms', action='store_true')
def handle(self, *args, **options):
print(args)
print(options)
for cmd in [ 'create_vm_snapshots', 'schedule_vms', 'start_vms' ]:
if options[cmd]:
f = getattr(self, cmd)
f(args, options)
if options['schedule_vms']:
self.schedule_vms(args, option)
if options['start_vms_here']:
if not options['vmhostname']:
raise Exception("Argument vmhostname is required to know which vmhost we are on")
self.start_vms(args, options)
if options['check_health']:
self.check_health(args, option)
def schedule_vms(self, *args, **options):
for pending_vm in VMProduct.objects.filter(status='PENDING'):
cores_needed = pending_vm.cores
ram_needed = pending_vm.ram_in_gb
# Database filtering
possible_vmhosts = VMHost.objects.filter(physical_cores__gte=cores_needed)
# Logical filtering
possible_vmhosts = [ vmhost for vmhost in possible_vmhosts
if vmhost.available_cores >=cores_needed
and vmhost.available_ram_in_gb >= ram_needed ]
if not possible_vmhosts:
log.error("No suitable Host found - cannot schedule VM {}".format(pending_vm))
continue
vmhost = possible_vmhosts[0]
pending_vm.vmhost = vmhost
pending_vm.status = 'SCHEDULED'
pending_vm.save()
print("Scheduled VM {} on VMHOST {}".format(pending_vm, pending_vm.vmhost))
print(self)
def start_vms(self, *args, **options):
vmhost = VMHost.objects.get(status='active',
hostname=options['vmhostname'])
vmhost = VMHost.objects.get(hostname=options['this_hostname'])
if not vmhost:
print("No active vmhost {} exists".format(options['vmhostname']))
raise Exception("No vmhost {} exists".format(options['vmhostname']))
# not active? done here
if not vmhost.status = 'ACTIVE':
return
vms_to_start = VMProduct.objects.filter(vmhost=vmhost,
status='creating')
status='SCHEDULED')
for vm in vms_to_start:
""" run qemu:
check if VM is not already active / qemu running
prepare / create the Qemu arguments
"""
print("Starting VM {}".format(VM))
def check_vms(self, *args, **options):
"""
Check if all VMs that are supposed to run are running
"""
def schedule_vms(self, *args, **options)):
pending_vms = VMProduct.objects.filter(vmhost__isnull=True)
vmhosts = VMHost.objects.filter(status='active')
def modify_vms(self, *args, **options):
"""
Check all VMs that are requested to be modified and restart them
"""
for vm in pending_vms:
print(vm)
def create_vm_snapshots(self, *args, **options):
this_cluster = VMCluster(option['this_cluster'])
found_vmhost = False
for vmhost in vmhosts:
if vmhost.available_cores >= vm.cores and vmhost.available_ram_in_gb >= vm.ram_in_gb:
vm.vmhost = vmhost
vm.status = "creating"
vm.save()
found_vmhost = True
print("Scheduled VM {} on VMHOST {}".format(vm, vmhost))
break
for snapshot in VMSnapshotProduct.objects.filter(status='PENDING',
cluster=this_cluster):
if not snapshot.extra_data:
snapshot.extra_data = {}
if not found_vmhost:
print("Error: cannot schedule VM {}, no suitable host found".format(vm))
# TODO: implement locking here
if 'creating_hostname' in snapshot.extra_data:
pass
snapshot.extra_data['creating_hostname'] = options['this_hostname']
snapshot.extra_data['creating_start'] = str(datetime.now())
snapshot.save()
# something on the line of:
# for disk im vm.disks:
# rbd snap create pool/image-name@snapshot name
# snapshot.extra_data['snapshots']
# register the snapshot names in extra_data (?)
print(snapshot)
def check_health(self, *args, **options):
pending_vms = VMProduct.objects.filter(vmhost__isnull=True)
pending_vms = VMProduct.objects.filter(status='PENDING')
vmhosts = VMHost.objects.filter(status='active')
# 1. Check that all active hosts reported back N seconds ago
@ -81,5 +116,4 @@ class Command(BaseCommand):
# If VM snapshots exist without a VM -> notify user (?)
print("Nothing is good, you should implement me")

19
uncloud/uncloud_vm/migrations/0007_vmhost_vmcluster.py Normal file
View file

@ -0,0 +1,19 @@
# Generated by Django 3.0.3 on 2020-03-22 18:09
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('uncloud_vm', '0006_auto_20200322_1758'),
]
operations = [
migrations.AddField(
model_name='vmhost',
name='vmcluster',
field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMCluster'),
),
]

29
uncloud/uncloud_vm/models.py
View file

@ -8,21 +8,14 @@ from django.contrib.auth import get_user_model
# from django.core.exceptions import ValidationError
from uncloud_pay.models import Product, RecurringPeriod
from uncloud.models import UncloudModel
from uncloud.models import UncloudModel, UncloudStatus
import uncloud_pay.models as pay_models
import uncloud_storage.models
STATUS_CHOICES = (
('pending', 'Pending'), # Initial state
('creating', 'Creating'), # Creating VM/image/etc.
('active', 'Active'), # Is usable / active
('disabled', 'Disabled'), # Is usable, but cannot be used for new things
('unusable', 'Unusable'), # Has some kind of error
('deleted', 'Deleted'), # Does not exist anymore, only DB entry as a log
)
STATUS_DEFAULT = 'pending'
class VMCluster(UncloudModel):
uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
name = models.CharField(max_length=128, unique=True)
class VMHost(UncloudModel):
@ -31,6 +24,10 @@ class VMHost(UncloudModel):
# 253 is the maximum DNS name length
hostname = models.CharField(max_length=253, unique=True)
vmcluster = models.ForeignKey(
VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True
)
# indirectly gives a maximum number of cores / VM - f.i. 32
physical_cores = models.IntegerField(default=0)
@ -41,7 +38,7 @@ class VMHost(UncloudModel):
usable_ram_in_gb = models.FloatField(default=0)
status = models.CharField(
max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT
max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING
)
@property
@ -54,7 +51,7 @@ class VMHost(UncloudModel):
@property
def available_ram_in_gb(self):
return self.usable_ram_in_gb - sum([vm.ram_in_gb for vm in self.vms ])
return self.usable_ram_in_gb - self.used_ram_in_gb
@property
def available_cores(self):
@ -66,6 +63,10 @@ class VMProduct(Product):
VMHost, on_delete=models.CASCADE, editable=False, blank=True, null=True
)
vmcluster = models.ForeignKey(
VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True
)
# VM-specific. The name is only intended for customers: it's a pain to
# remember IDs (speaking from experience as ungleich customer)!
name = models.CharField(max_length=32, blank=True, null=True)
@ -131,7 +132,7 @@ class VMDiskImageProduct(UncloudModel):
default = uncloud_storage.models.StorageClass.SSD)
status = models.CharField(
max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT
max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING
)
def __str__(self):

12
uncloud/uncloud_vm/serializers.py
View file

@ -2,7 +2,7 @@ from django.contrib.auth import get_user_model
from rest_framework import serializers
from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct
from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster
from uncloud_pay.models import RecurringPeriod
GB_SSD_PER_DAY=0.012
@ -12,7 +12,7 @@ GB_SSD_PER_DAY=0.012
GB_HDD_PER_DAY=0.0006
class VMHostSerializer(serializers.ModelSerializer):
class VMHostSerializer(serializers.HyperlinkedModelSerializer):
vms = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
class Meta:
@ -20,6 +20,11 @@ class VMHostSerializer(serializers.ModelSerializer):
fields = '__all__'
read_only_fields = [ 'vms' ]
class VMClusterSerializer(serializers.HyperlinkedModelSerializer):
class Meta:
model = VMCluster
fields = '__all__'
class VMDiskProductSerializer(serializers.ModelSerializer):
class Meta:
@ -92,9 +97,6 @@ class VMProductSerializer(serializers.ModelSerializer):
recurring_period = serializers.ChoiceField(
choices=VMProduct.allowed_recurring_periods())
# snapshots = serializers.PrimaryKeyRelatedField(many=True,
# read_only=True)
snapshots = VMSnapshotProductSerializer(many=True,
read_only=True)

17
uncloud/uncloud_vm/views.py
View file

@ -8,12 +8,13 @@ from rest_framework import viewsets, permissions
from rest_framework.response import Response
from rest_framework.exceptions import ValidationError
from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct
from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster
from uncloud_pay.models import Order
from .serializers import (VMHostSerializer, VMProductSerializer,
VMSnapshotProductSerializer, VMDiskImageProductSerializer,
VMDiskProductSerializer, DCLVMProductSerializer)
VMDiskProductSerializer, DCLVMProductSerializer,
VMClusterSerializer)
from uncloud_pay.helpers import ProductViewSet
@ -24,6 +25,11 @@ class VMHostViewSet(viewsets.ModelViewSet):
queryset = VMHost.objects.all()
permission_classes = [permissions.IsAdminUser]
class VMClusterViewSet(viewsets.ModelViewSet):
serializer_class = VMClusterSerializer
queryset = VMCluster.objects.all()
permission_classes = [permissions.IsAdminUser]
class VMDiskImageProductViewSet(viewsets.ModelViewSet):
permission_classes = [permissions.IsAuthenticated]
serializer_class = VMDiskImageProductSerializer
@ -135,7 +141,12 @@ class VMSnapshotProductViewSet(viewsets.ModelViewSet):
serializer_class = VMSnapshotProductSerializer
def get_queryset(self):
return VMSnapshotProduct.objects.filter(owner=self.request.user)
if self.request.user.is_superuser:
obj = VMSnapshotProduct.objects.all()
else:
obj = VMSnapshotProduct.objects.filter(owner=self.request.user)
return obj
def create(self, request):
serializer = VMSnapshotProductSerializer(data=request.data, context={'request': request})