From 23203ff418051669351692067883eddcbc6e268c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 22 Mar 2020 20:55:11 +0100 Subject: [PATCH] vmsnapshot progress --- .../uncloud/management/commands/uncloud.py | 28 +++++ uncloud/uncloud/models.py | 13 ++ uncloud/uncloud/settings.py | 1 + uncloud/uncloud/urls.py | 19 +-- uncloud/uncloud_pay/models.py | 13 +- .../commands/vm-create-snapshots.py | 35 ------ uncloud/uncloud_vm/management/commands/vm.py | 114 ++++++++++++------ .../migrations/0007_vmhost_vmcluster.py | 19 +++ uncloud/uncloud_vm/models.py | 29 ++--- uncloud/uncloud_vm/serializers.py | 12 +- uncloud/uncloud_vm/views.py | 17 ++- 11 files changed, 175 insertions(+), 125 deletions(-) create mode 100644 uncloud/uncloud/management/commands/uncloud.py delete mode 100644 uncloud/uncloud_vm/management/commands/vm-create-snapshots.py create mode 100644 uncloud/uncloud_vm/migrations/0007_vmhost_vmcluster.py diff --git a/uncloud/uncloud/management/commands/uncloud.py b/uncloud/uncloud/management/commands/uncloud.py new file mode 100644 index 0000000..bd47c6b --- /dev/null +++ b/uncloud/uncloud/management/commands/uncloud.py @@ -0,0 +1,28 @@ +import sys +from datetime import datetime + +from django.core.management.base import BaseCommand + +from django.contrib.auth import get_user_model + +from opennebula.models import VM as VMModel +from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster + +import logging +log = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'General uncloud commands' + + def add_arguments(self, parser): + parser.add_argument('--bootstrap', action='store_true', help='Bootstrap a typical uncloud installation') + + def handle(self, *args, **options): + + if options['bootstrap']: + self.bootstrap() + + def bootstrap(self): + default_cluster = VMCluster.objects.get_or_create(name="default") +# local_host = diff --git a/uncloud/uncloud/models.py b/uncloud/uncloud/models.py index 7ca5dfa..bd7a931 100644 --- a/uncloud/uncloud/models.py +++ b/uncloud/uncloud/models.py @@ -1,5 +1,6 @@ from django.db import models from django.contrib.postgres.fields import JSONField +from django.utils.translation import gettext_lazy as _ class UncloudModel(models.Model): """ @@ -20,3 +21,15 @@ class UncloudModel(models.Model): class Meta: abstract = True + +# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types +class UncloudStatus(models.TextChoices): + PENDING = 'PENDING', _('Pending') + AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment') + BEING_CREATED = 'BEING_CREATED', _('Being created') + SCHEDULED = 'SCHEDULED', _('Scheduled') # resource selected, waiting for dispatching + ACTIVE = 'ACTIVE', _('Active') + MODIFYING = 'MODIFYING', _('Modifying') # Resource is being changed + DELETED = 'DELETED', _('Deleted') # Resource has been deleted + DISABLED = 'DISABLED', _('Disabled') # Is usable, but cannot be used for new things + UNUSABLE = 'UNUSABLE', _('Unusable'), # Has some kind of error diff --git a/uncloud/uncloud/settings.py b/uncloud/uncloud/settings.py index 99cf7a1..5b4744d 100644 --- a/uncloud/uncloud/settings.py +++ b/uncloud/uncloud/settings.py @@ -59,6 +59,7 @@ INSTALLED_APPS = [ 'django.contrib.staticfiles', 'django_extensions', 'rest_framework', + 'uncloud', 'uncloud_pay', 'uncloud_auth', 'uncloud_storage', diff --git a/uncloud/uncloud/urls.py b/uncloud/uncloud/urls.py index 50d59c3..a848dff 100644 --- a/uncloud/uncloud/urls.py +++ b/uncloud/uncloud/urls.py @@ -30,32 +30,16 @@ router = routers.DefaultRouter() router.register(r'vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct') router.register(r'vm/diskimage', vmviews.VMDiskImageProductViewSet, basename='vmdiskimageproduct') router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct') - - -# images the provider provides :-) -# router.register(r'vm/image/official', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct') - - - - router.register(r'vm/vm', vmviews.VMProductViewSet, basename='vmproduct') - -# TBD -#router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct') - # creates VM from os image #router.register(r'vm/ipv6onlyvm', vmviews.VMProductViewSet, basename='vmproduct') # ... AND adds IPv4 mapping #router.register(r'vm/dualstackvm', vmviews.VMProductViewSet, basename='vmproduct') -# allow vm creation from own images - - # Services router.register(r'service/matrix', serviceviews.MatrixServiceProductViewSet, basename='matrixserviceproduct') - # Pay router.register(r'payment-method', payviews.PaymentMethodViewSet, basename='payment-method') router.register(r'bill', payviews.BillViewSet, basename='bill') @@ -63,14 +47,13 @@ router.register(r'order', payviews.OrderViewSet, basename='order') router.register(r'payment', payviews.PaymentViewSet, basename='payment') router.register(r'payment-method', payviews.PaymentMethodViewSet, basename='payment-methods') -# VMs -router.register(r'vm/vm', vmviews.VMProductViewSet, basename='vm') # admin/staff urls router.register(r'admin/bill', payviews.AdminBillViewSet, basename='admin/bill') router.register(r'admin/payment', payviews.AdminPaymentViewSet, basename='admin/payment') router.register(r'admin/order', payviews.AdminOrderViewSet, basename='admin/order') router.register(r'admin/vmhost', vmviews.VMHostViewSet) +router.register(r'admin/vmcluster', vmviews.VMClusterViewSet) router.register(r'admin/opennebula', oneviews.VMViewSet, basename='opennebula') # User/Account diff --git a/uncloud/uncloud_pay/models.py b/uncloud/uncloud_pay/models.py index 532e130..945187b 100644 --- a/uncloud/uncloud_pay/models.py +++ b/uncloud/uncloud_pay/models.py @@ -19,7 +19,7 @@ from decimal import Decimal import uncloud_pay.stripe from uncloud_pay.helpers import beginning_of_month, end_of_month from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS -from uncloud.models import UncloudModel +from uncloud.models import UncloudModel, UncloudStatus # Used to generate bill due dates. @@ -35,13 +35,6 @@ class RecurringPeriod(models.TextChoices): PER_HOUR = 'HOUR', _('Per Hour') PER_SECOND = 'SECOND', _('Per Second') -# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types -class ProductStatus(models.TextChoices): - PENDING = 'PENDING', _('Pending') - AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment') - BEING_CREATED = 'BEING_CREATED', _('Being created') - ACTIVE = 'ACTIVE', _('Active') - DELETED = 'DELETED', _('Deleted') def get_balance_for_user(user): @@ -445,8 +438,8 @@ class Product(UncloudModel): description = "" status = models.CharField(max_length=32, - choices=ProductStatus.choices, - default=ProductStatus.PENDING) + choices=UncloudStatus.choices, + default=UncloudStatus.PENDING) order = models.ForeignKey(Order, on_delete=models.CASCADE, diff --git a/uncloud/uncloud_vm/management/commands/vm-create-snapshots.py b/uncloud/uncloud_vm/management/commands/vm-create-snapshots.py deleted file mode 100644 index bd3bb65..0000000 --- a/uncloud/uncloud_vm/management/commands/vm-create-snapshots.py +++ /dev/null @@ -1,35 +0,0 @@ -import json - -import uncloud.secrets as secrets - -from django.core.management.base import BaseCommand -from django.contrib.auth import get_user_model - -from uncloud_vm.models import VMSnapshotProduct -from datetime import datetime - -class Command(BaseCommand): - help = 'Select VM Host for VMs' - - def add_arguments(self, parser): - parser.add_argument('--this-hostname', required=True) - # parser.add_argument('--start-vms-here', action='store_true') - # parser.add_argument('--check-health', action='store_true') - # parser.add_argument('--vmhostname') - # print(parser) - - - def handle(self, *args, **options): - for snapshot in VMSnapshotProduct.objects.filter(status='PENDING'): - if not snapshot.extra_data: - snapshot.extra_data = {} - - # TODO: implement locking here - if 'creating_hostname' in snapshot.extra_data: - pass - - snapshot.extra_data['creating_hostname'] = options['this_hostname'] - snapshot.extra_data['creating_start'] = str(datetime.now()) - snapshot.save() - - print(snapshot) diff --git a/uncloud/uncloud_vm/management/commands/vm.py b/uncloud/uncloud_vm/management/commands/vm.py index c0e2783..667c5ad 100644 --- a/uncloud/uncloud_vm/management/commands/vm.py +++ b/uncloud/uncloud_vm/management/commands/vm.py @@ -5,73 +5,108 @@ import uncloud.secrets as secrets from django.core.management.base import BaseCommand from django.contrib.auth import get_user_model -from uncloud_vm.models import VMProduct, VMHost +from uncloud_vm.models import VMSnapshotProduct, VMProduct, VMHost +from datetime import datetime class Command(BaseCommand): help = 'Select VM Host for VMs' def add_arguments(self, parser): + parser.add_argument('--this-hostname', required=True) + parser.add_argument('--this-cluster', required=True) + + parser.add_argument('--create-vm-snapshots', action='store_true') parser.add_argument('--schedule-vms', action='store_true') - parser.add_argument('--start-vms-here', action='store_true') - parser.add_argument('--check-health', action='store_true') - parser.add_argument('--vmhostname') - print(parser) + parser.add_argument('--start-vms', action='store_true') def handle(self, *args, **options): - print(args) - print(options) + for cmd in [ 'create_vm_snapshots', 'schedule_vms', 'start_vms' ]: + if options[cmd]: + f = getattr(self, cmd) + f(args, options) - if options['schedule_vms']: - self.schedule_vms(args, option) - if options['start_vms_here']: - if not options['vmhostname']: - raise Exception("Argument vmhostname is required to know which vmhost we are on") - self.start_vms(args, options) - if options['check_health']: - self.check_health(args, option) + def schedule_vms(self, *args, **options): + for pending_vm in VMProduct.objects.filter(status='PENDING'): + cores_needed = pending_vm.cores + ram_needed = pending_vm.ram_in_gb + + # Database filtering + possible_vmhosts = VMHost.objects.filter(physical_cores__gte=cores_needed) + + # Logical filtering + possible_vmhosts = [ vmhost for vmhost in possible_vmhosts + if vmhost.available_cores >=cores_needed + and vmhost.available_ram_in_gb >= ram_needed ] + + if not possible_vmhosts: + log.error("No suitable Host found - cannot schedule VM {}".format(pending_vm)) + continue + + vmhost = possible_vmhosts[0] + pending_vm.vmhost = vmhost + pending_vm.status = 'SCHEDULED' + pending_vm.save() + + print("Scheduled VM {} on VMHOST {}".format(pending_vm, pending_vm.vmhost)) + + print(self) def start_vms(self, *args, **options): - vmhost = VMHost.objects.get(status='active', - hostname=options['vmhostname']) + vmhost = VMHost.objects.get(hostname=options['this_hostname']) if not vmhost: - print("No active vmhost {} exists".format(options['vmhostname'])) + raise Exception("No vmhost {} exists".format(options['vmhostname'])) + + # not active? done here + if not vmhost.status = 'ACTIVE': return vms_to_start = VMProduct.objects.filter(vmhost=vmhost, - status='creating') + status='SCHEDULED') for vm in vms_to_start: - """ run qemu: check if VM is not already active / qemu running prepare / create the Qemu arguments - - """ + print("Starting VM {}".format(VM)) - def schedule_vms(self, *args, **options)): - pending_vms = VMProduct.objects.filter(vmhost__isnull=True) - vmhosts = VMHost.objects.filter(status='active') + def check_vms(self, *args, **options): + """ + Check if all VMs that are supposed to run are running + """ - for vm in pending_vms: - print(vm) + def modify_vms(self, *args, **options): + """ + Check all VMs that are requested to be modified and restart them + """ - found_vmhost = False - for vmhost in vmhosts: - if vmhost.available_cores >= vm.cores and vmhost.available_ram_in_gb >= vm.ram_in_gb: - vm.vmhost = vmhost - vm.status = "creating" - vm.save() - found_vmhost = True - print("Scheduled VM {} on VMHOST {}".format(vm, vmhost)) - break + def create_vm_snapshots(self, *args, **options): + this_cluster = VMCluster(option['this_cluster']) - if not found_vmhost: - print("Error: cannot schedule VM {}, no suitable host found".format(vm)) + for snapshot in VMSnapshotProduct.objects.filter(status='PENDING', + cluster=this_cluster): + if not snapshot.extra_data: + snapshot.extra_data = {} + + # TODO: implement locking here + if 'creating_hostname' in snapshot.extra_data: + pass + + snapshot.extra_data['creating_hostname'] = options['this_hostname'] + snapshot.extra_data['creating_start'] = str(datetime.now()) + snapshot.save() + + # something on the line of: + # for disk im vm.disks: + # rbd snap create pool/image-name@snapshot name + # snapshot.extra_data['snapshots'] + # register the snapshot names in extra_data (?) + + print(snapshot) def check_health(self, *args, **options): - pending_vms = VMProduct.objects.filter(vmhost__isnull=True) + pending_vms = VMProduct.objects.filter(status='PENDING') vmhosts = VMHost.objects.filter(status='active') # 1. Check that all active hosts reported back N seconds ago @@ -81,5 +116,4 @@ class Command(BaseCommand): # If VM snapshots exist without a VM -> notify user (?) - print("Nothing is good, you should implement me") diff --git a/uncloud/uncloud_vm/migrations/0007_vmhost_vmcluster.py b/uncloud/uncloud_vm/migrations/0007_vmhost_vmcluster.py new file mode 100644 index 0000000..6766dd7 --- /dev/null +++ b/uncloud/uncloud_vm/migrations/0007_vmhost_vmcluster.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-03-22 18:09 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('uncloud_vm', '0006_auto_20200322_1758'), + ] + + operations = [ + migrations.AddField( + model_name='vmhost', + name='vmcluster', + field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMCluster'), + ), + ] diff --git a/uncloud/uncloud_vm/models.py b/uncloud/uncloud_vm/models.py index a4b7f2a..3b2c46b 100644 --- a/uncloud/uncloud_vm/models.py +++ b/uncloud/uncloud_vm/models.py @@ -8,21 +8,14 @@ from django.contrib.auth import get_user_model # from django.core.exceptions import ValidationError from uncloud_pay.models import Product, RecurringPeriod -from uncloud.models import UncloudModel +from uncloud.models import UncloudModel, UncloudStatus import uncloud_pay.models as pay_models import uncloud_storage.models -STATUS_CHOICES = ( - ('pending', 'Pending'), # Initial state - ('creating', 'Creating'), # Creating VM/image/etc. - ('active', 'Active'), # Is usable / active - ('disabled', 'Disabled'), # Is usable, but cannot be used for new things - ('unusable', 'Unusable'), # Has some kind of error - ('deleted', 'Deleted'), # Does not exist anymore, only DB entry as a log -) - -STATUS_DEFAULT = 'pending' +class VMCluster(UncloudModel): + uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False) + name = models.CharField(max_length=128, unique=True) class VMHost(UncloudModel): @@ -31,6 +24,10 @@ class VMHost(UncloudModel): # 253 is the maximum DNS name length hostname = models.CharField(max_length=253, unique=True) + vmcluster = models.ForeignKey( + VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + # indirectly gives a maximum number of cores / VM - f.i. 32 physical_cores = models.IntegerField(default=0) @@ -41,7 +38,7 @@ class VMHost(UncloudModel): usable_ram_in_gb = models.FloatField(default=0) status = models.CharField( - max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT + max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING ) @property @@ -54,7 +51,7 @@ class VMHost(UncloudModel): @property def available_ram_in_gb(self): - return self.usable_ram_in_gb - sum([vm.ram_in_gb for vm in self.vms ]) + return self.usable_ram_in_gb - self.used_ram_in_gb @property def available_cores(self): @@ -66,6 +63,10 @@ class VMProduct(Product): VMHost, on_delete=models.CASCADE, editable=False, blank=True, null=True ) + vmcluster = models.ForeignKey( + VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True + ) + # VM-specific. The name is only intended for customers: it's a pain to # remember IDs (speaking from experience as ungleich customer)! name = models.CharField(max_length=32, blank=True, null=True) @@ -131,7 +132,7 @@ class VMDiskImageProduct(UncloudModel): default = uncloud_storage.models.StorageClass.SSD) status = models.CharField( - max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT + max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING ) def __str__(self): diff --git a/uncloud/uncloud_vm/serializers.py b/uncloud/uncloud_vm/serializers.py index 6d26cbe..c0cca48 100644 --- a/uncloud/uncloud_vm/serializers.py +++ b/uncloud/uncloud_vm/serializers.py @@ -2,7 +2,7 @@ from django.contrib.auth import get_user_model from rest_framework import serializers -from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct +from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster from uncloud_pay.models import RecurringPeriod GB_SSD_PER_DAY=0.012 @@ -12,7 +12,7 @@ GB_SSD_PER_DAY=0.012 GB_HDD_PER_DAY=0.0006 -class VMHostSerializer(serializers.ModelSerializer): +class VMHostSerializer(serializers.HyperlinkedModelSerializer): vms = serializers.PrimaryKeyRelatedField(many=True, read_only=True) class Meta: @@ -20,6 +20,11 @@ class VMHostSerializer(serializers.ModelSerializer): fields = '__all__' read_only_fields = [ 'vms' ] +class VMClusterSerializer(serializers.HyperlinkedModelSerializer): + class Meta: + model = VMCluster + fields = '__all__' + class VMDiskProductSerializer(serializers.ModelSerializer): class Meta: @@ -92,9 +97,6 @@ class VMProductSerializer(serializers.ModelSerializer): recurring_period = serializers.ChoiceField( choices=VMProduct.allowed_recurring_periods()) - # snapshots = serializers.PrimaryKeyRelatedField(many=True, - # read_only=True) - snapshots = VMSnapshotProductSerializer(many=True, read_only=True) diff --git a/uncloud/uncloud_vm/views.py b/uncloud/uncloud_vm/views.py index 6d4e5a9..0672904 100644 --- a/uncloud/uncloud_vm/views.py +++ b/uncloud/uncloud_vm/views.py @@ -8,12 +8,13 @@ from rest_framework import viewsets, permissions from rest_framework.response import Response from rest_framework.exceptions import ValidationError -from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct +from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster from uncloud_pay.models import Order from .serializers import (VMHostSerializer, VMProductSerializer, VMSnapshotProductSerializer, VMDiskImageProductSerializer, - VMDiskProductSerializer, DCLVMProductSerializer) + VMDiskProductSerializer, DCLVMProductSerializer, + VMClusterSerializer) from uncloud_pay.helpers import ProductViewSet @@ -24,6 +25,11 @@ class VMHostViewSet(viewsets.ModelViewSet): queryset = VMHost.objects.all() permission_classes = [permissions.IsAdminUser] +class VMClusterViewSet(viewsets.ModelViewSet): + serializer_class = VMClusterSerializer + queryset = VMCluster.objects.all() + permission_classes = [permissions.IsAdminUser] + class VMDiskImageProductViewSet(viewsets.ModelViewSet): permission_classes = [permissions.IsAuthenticated] serializer_class = VMDiskImageProductSerializer @@ -135,7 +141,12 @@ class VMSnapshotProductViewSet(viewsets.ModelViewSet): serializer_class = VMSnapshotProductSerializer def get_queryset(self): - return VMSnapshotProduct.objects.filter(owner=self.request.user) + if self.request.user.is_superuser: + obj = VMSnapshotProduct.objects.all() + else: + obj = VMSnapshotProduct.objects.filter(owner=self.request.user) + + return obj def create(self, request): serializer = VMSnapshotProductSerializer(data=request.data, context={'request': request})