vmsnapshot progress

uncloud/uncloud/management/commands/uncloud.py

@@ -0,0 +1,28 @@
+import sys
+from datetime import datetime
+
+from django.core.management.base import BaseCommand
+
+from django.contrib.auth import get_user_model
+
+from opennebula.models import VM as VMModel
+from uncloud_vm.models import VMHost, VMProduct, VMNetworkCard, VMDiskImageProduct, VMDiskProduct, VMCluster
+
+import logging
+log = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = 'General uncloud commands'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--bootstrap', action='store_true', help='Bootstrap a typical uncloud installation')
+
+    def handle(self, *args, **options):
+
+        if options['bootstrap']:
+            self.bootstrap()
+
+    def bootstrap(self):
+        default_cluster = VMCluster.objects.get_or_create(name="default")
+#        local_host =

uncloud/uncloud/models.py

@@ -1,5 +1,6 @@
 from django.db import models
 from django.contrib.postgres.fields import JSONField
+from django.utils.translation import gettext_lazy as _
 
 class UncloudModel(models.Model):
     """
@@ -20,3 +21,15 @@ class UncloudModel(models.Model):
 
     class Meta:
         abstract = True
+
+# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types
+class UncloudStatus(models.TextChoices):
+    PENDING          = 'PENDING', _('Pending')
+    AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment')
+    BEING_CREATED    = 'BEING_CREATED', _('Being created')
+    SCHEDULED        = 'SCHEDULED', _('Scheduled') # resource selected, waiting for dispatching
+    ACTIVE           = 'ACTIVE', _('Active')
+    MODIFYING        = 'MODIFYING', _('Modifying') # Resource is being changed
+    DELETED          = 'DELETED', _('Deleted')     # Resource has been deleted
+    DISABLED         = 'DISABLED', _('Disabled')   # Is usable, but cannot be used for new things
+    UNUSABLE         = 'UNUSABLE', _('Unusable'),  # Has some kind of error

uncloud/uncloud/settings.py

@@ -59,6 +59,7 @@ INSTALLED_APPS = [
     'django.contrib.staticfiles',
     'django_extensions',
     'rest_framework',
+    'uncloud',
     'uncloud_pay',
     'uncloud_auth',
     'uncloud_storage',

uncloud/uncloud/urls.py

@@ -30,32 +30,16 @@ router = routers.DefaultRouter()
 router.register(r'vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct')
 router.register(r'vm/diskimage', vmviews.VMDiskImageProductViewSet, basename='vmdiskimageproduct')
 router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct')
-
-
-# images the provider provides :-)
-# router.register(r'vm/image/official', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct')
-
-
-
-
 router.register(r'vm/vm', vmviews.VMProductViewSet, basename='vmproduct')
 
-
-# TBD
-#router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct')
-
 # creates VM from os image
 #router.register(r'vm/ipv6onlyvm', vmviews.VMProductViewSet, basename='vmproduct')
 # ... AND adds IPv4 mapping
 #router.register(r'vm/dualstackvm', vmviews.VMProductViewSet, basename='vmproduct')
 
-# allow vm creation from own images
-
-
 # Services
 router.register(r'service/matrix', serviceviews.MatrixServiceProductViewSet, basename='matrixserviceproduct')
 
-
 # Pay
 router.register(r'payment-method', payviews.PaymentMethodViewSet, basename='payment-method')
 router.register(r'bill', payviews.BillViewSet, basename='bill')
@@ -63,14 +47,13 @@ router.register(r'order', payviews.OrderViewSet, basename='order')
 router.register(r'payment', payviews.PaymentViewSet, basename='payment')
 router.register(r'payment-method', payviews.PaymentMethodViewSet, basename='payment-methods')
 
-# VMs
-router.register(r'vm/vm', vmviews.VMProductViewSet, basename='vm')
 
 # admin/staff urls
 router.register(r'admin/bill', payviews.AdminBillViewSet, basename='admin/bill')
 router.register(r'admin/payment', payviews.AdminPaymentViewSet, basename='admin/payment')
 router.register(r'admin/order', payviews.AdminOrderViewSet, basename='admin/order')
 router.register(r'admin/vmhost', vmviews.VMHostViewSet)
+router.register(r'admin/vmcluster', vmviews.VMClusterViewSet)
 router.register(r'admin/opennebula', oneviews.VMViewSet, basename='opennebula')
 
 # User/Account

uncloud/uncloud_pay/models.py

@@ -19,7 +19,7 @@ from decimal import Decimal
 import uncloud_pay.stripe
 from uncloud_pay.helpers import beginning_of_month, end_of_month
 from uncloud import AMOUNT_DECIMALS, AMOUNT_MAX_DIGITS
-from uncloud.models import UncloudModel
+from uncloud.models import UncloudModel, UncloudStatus
 
 
 # Used to generate bill due dates.
@@ -35,13 +35,6 @@ class RecurringPeriod(models.TextChoices):
     PER_HOUR   = 'HOUR', _('Per Hour')
     PER_SECOND = 'SECOND', _('Per Second')
 
-# See https://docs.djangoproject.com/en/dev/ref/models/fields/#field-choices-enum-types
-class ProductStatus(models.TextChoices):
-    PENDING          = 'PENDING', _('Pending')
-    AWAITING_PAYMENT = 'AWAITING_PAYMENT', _('Awaiting payment')
-    BEING_CREATED    = 'BEING_CREATED', _('Being created')
-    ACTIVE           = 'ACTIVE', _('Active')
-    DELETED          = 'DELETED', _('Deleted')
 
 
 def get_balance_for_user(user):
@@ -445,8 +438,8 @@ class Product(UncloudModel):
     description = ""
 
     status = models.CharField(max_length=32,
-            choices=ProductStatus.choices,
+            choices=UncloudStatus.choices,
-            default=ProductStatus.PENDING)
+            default=UncloudStatus.PENDING)
 
     order = models.ForeignKey(Order,
                               on_delete=models.CASCADE,

uncloud/uncloud_vm/management/commands/vm-create-snapshots.py

@@ -1,35 +0,0 @@
-import json
-
-import uncloud.secrets as secrets
-
-from django.core.management.base import BaseCommand
-from django.contrib.auth import get_user_model
-
-from uncloud_vm.models import VMSnapshotProduct
-from datetime import datetime
-
-class Command(BaseCommand):
-    help = 'Select VM Host for VMs'
-
-    def add_arguments(self, parser):
-        parser.add_argument('--this-hostname', required=True)
-    #     parser.add_argument('--start-vms-here', action='store_true')
-    #     parser.add_argument('--check-health', action='store_true')
-    #     parser.add_argument('--vmhostname')
-    #     print(parser)
-
-
-    def handle(self, *args, **options):
-        for snapshot in VMSnapshotProduct.objects.filter(status='PENDING'):
-            if not snapshot.extra_data:
-                snapshot.extra_data = {}
-
-            # TODO: implement locking here
-            if 'creating_hostname' in snapshot.extra_data:
-                pass
-
-            snapshot.extra_data['creating_hostname'] = options['this_hostname']
-            snapshot.extra_data['creating_start'] = str(datetime.now())
-            snapshot.save()
-
-            print(snapshot)

uncloud/uncloud_vm/management/commands/vm.py

@@ -5,73 +5,108 @@ import uncloud.secrets as secrets
 from django.core.management.base import BaseCommand
 from django.contrib.auth import get_user_model
 
-from uncloud_vm.models import VMProduct, VMHost
+from uncloud_vm.models import VMSnapshotProduct, VMProduct, VMHost
+from datetime import datetime
 
 class Command(BaseCommand):
     help = 'Select VM Host for VMs'
 
     def add_arguments(self, parser):
+        parser.add_argument('--this-hostname', required=True)
+        parser.add_argument('--this-cluster', required=True)
+
+        parser.add_argument('--create-vm-snapshots', action='store_true')
         parser.add_argument('--schedule-vms', action='store_true')
-        parser.add_argument('--start-vms-here', action='store_true')
+        parser.add_argument('--start-vms', action='store_true')
-        parser.add_argument('--check-health', action='store_true')
-        parser.add_argument('--vmhostname')
-        print(parser)
 
 
     def handle(self, *args, **options):
-        print(args)
+        for cmd in [ 'create_vm_snapshots', 'schedule_vms', 'start_vms' ]:
-        print(options)
+            if options[cmd]:
+                f = getattr(self, cmd)
+                f(args, options)
 
-        if options['schedule_vms']:
+    def schedule_vms(self, *args, **options):
-            self.schedule_vms(args, option)
+        for pending_vm in VMProduct.objects.filter(status='PENDING'):
-        if options['start_vms_here']:
+            cores_needed = pending_vm.cores
-            if not options['vmhostname']:
+            ram_needed = pending_vm.ram_in_gb
-                raise Exception("Argument vmhostname is required to know which vmhost we are on")
+
-            self.start_vms(args, options)
+            # Database filtering
-        if options['check_health']:
+            possible_vmhosts = VMHost.objects.filter(physical_cores__gte=cores_needed)
-            self.check_health(args, option)
+
+            # Logical filtering
+            possible_vmhosts = [ vmhost for vmhost in possible_vmhosts
+                                 if vmhost.available_cores >=cores_needed
+                                 and vmhost.available_ram_in_gb >= ram_needed ]
+
+            if not possible_vmhosts:
+                log.error("No suitable Host found - cannot schedule VM {}".format(pending_vm))
+                continue
+
+            vmhost = possible_vmhosts[0]
+            pending_vm.vmhost = vmhost
+            pending_vm.status = 'SCHEDULED'
+            pending_vm.save()
+
+            print("Scheduled VM {} on VMHOST {}".format(pending_vm, pending_vm.vmhost))
+
+        print(self)
 
     def start_vms(self, *args, **options):
-        vmhost = VMHost.objects.get(status='active',
+        vmhost = VMHost.objects.get(hostname=options['this_hostname'])
-                                    hostname=options['vmhostname'])
 
         if not vmhost:
-            print("No active vmhost {} exists".format(options['vmhostname']))
+            raise Exception("No vmhost {} exists".format(options['vmhostname']))
+
+        # not active? done here
+        if not vmhost.status = 'ACTIVE':
             return
 
         vms_to_start = VMProduct.objects.filter(vmhost=vmhost,
-                                                status='creating')
+                                                status='SCHEDULED')
         for vm in vms_to_start:
-
             """ run qemu:
             check if VM is not already active / qemu running
             prepare / create the Qemu arguments
-
-
             """
+            print("Starting VM {}".format(VM))
 
-    def schedule_vms(self, *args, **options)):
+    def check_vms(self, *args, **options):
-        pending_vms = VMProduct.objects.filter(vmhost__isnull=True)
+        """
-        vmhosts = VMHost.objects.filter(status='active')
+        Check if all VMs that are supposed to run are running
+        """
 
-        for vm in pending_vms:
+    def modify_vms(self, *args, **options):
-            print(vm)
+        """
+        Check all VMs that are requested to be modified and restart them
+        """
 
-            found_vmhost = False
+    def create_vm_snapshots(self, *args, **options):
-            for vmhost in vmhosts:
+        this_cluster = VMCluster(option['this_cluster'])
-                if vmhost.available_cores >= vm.cores and vmhost.available_ram_in_gb >= vm.ram_in_gb:
-                    vm.vmhost = vmhost
-                    vm.status = "creating"
-                    vm.save()
-                    found_vmhost = True
-                    print("Scheduled VM {} on VMHOST {}".format(vm, vmhost))
-                    break
 
-            if not found_vmhost:
+        for snapshot in VMSnapshotProduct.objects.filter(status='PENDING',
-                print("Error: cannot schedule VM {}, no suitable host found".format(vm))
+                                                         cluster=this_cluster):
+            if not snapshot.extra_data:
+                snapshot.extra_data = {}
+
+            # TODO: implement locking here
+            if 'creating_hostname' in snapshot.extra_data:
+                pass
+
+            snapshot.extra_data['creating_hostname'] = options['this_hostname']
+            snapshot.extra_data['creating_start'] = str(datetime.now())
+            snapshot.save()
+
+            # something on the line of:
+            # for disk im vm.disks:
+            # rbd snap create pool/image-name@snapshot name
+            # snapshot.extra_data['snapshots']
+            # register the snapshot names in extra_data (?)
+
+            print(snapshot)
 
     def check_health(self, *args, **options):
-        pending_vms = VMProduct.objects.filter(vmhost__isnull=True)
+        pending_vms = VMProduct.objects.filter(status='PENDING')
         vmhosts = VMHost.objects.filter(status='active')
 
         # 1. Check that all active hosts reported back N seconds ago
@@ -81,5 +116,4 @@ class Command(BaseCommand):
 
         # If VM snapshots exist without a VM -> notify user (?)
 
-
         print("Nothing is good, you should implement me")

uncloud/uncloud_vm/migrations/0007_vmhost_vmcluster.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.3 on 2020-03-22 18:09
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0006_auto_20200322_1758'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmhost',
+            name='vmcluster',
+            field=models.ForeignKey(blank=True, editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMCluster'),
+        ),
+    ]

uncloud/uncloud_vm/models.py

@@ -8,21 +8,14 @@ from django.contrib.auth import get_user_model
 # from django.core.exceptions import ValidationError
 
 from uncloud_pay.models import Product, RecurringPeriod
-from uncloud.models import UncloudModel
+from uncloud.models import UncloudModel, UncloudStatus
 
 import uncloud_pay.models as pay_models
 import uncloud_storage.models
 
-STATUS_CHOICES = (
+class VMCluster(UncloudModel):
-    ('pending', 'Pending'),   # Initial state
+    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    ('creating', 'Creating'),  # Creating VM/image/etc.
+    name = models.CharField(max_length=128, unique=True)
-    ('active', 'Active'),      # Is usable / active
-    ('disabled', 'Disabled'),  # Is usable, but cannot be used for new things
-    ('unusable', 'Unusable'),  # Has some kind of error
-    ('deleted', 'Deleted'),   # Does not exist anymore, only DB entry as a log
-)
-
-STATUS_DEFAULT = 'pending'
 
 
 class VMHost(UncloudModel):
@@ -31,6 +24,10 @@ class VMHost(UncloudModel):
     # 253 is the maximum DNS name length
     hostname = models.CharField(max_length=253, unique=True)
 
+    vmcluster = models.ForeignKey(
+        VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True
+    )
+
     # indirectly gives a maximum number of cores / VM - f.i. 32
     physical_cores = models.IntegerField(default=0)
 
@@ -41,7 +38,7 @@ class VMHost(UncloudModel):
     usable_ram_in_gb = models.FloatField(default=0)
 
     status = models.CharField(
-        max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT
+        max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING
     )
 
     @property
@@ -54,7 +51,7 @@ class VMHost(UncloudModel):
 
     @property
     def available_ram_in_gb(self):
-        return self.usable_ram_in_gb - sum([vm.ram_in_gb for vm in self.vms ])
+        return self.usable_ram_in_gb - self.used_ram_in_gb
 
     @property
     def available_cores(self):
@@ -66,6 +63,10 @@ class VMProduct(Product):
         VMHost, on_delete=models.CASCADE, editable=False, blank=True, null=True
     )
 
+    vmcluster = models.ForeignKey(
+        VMCluster, on_delete=models.CASCADE, editable=False, blank=True, null=True
+    )
+
     # VM-specific. The name is only intended for customers: it's a pain to
     # remember IDs (speaking from experience as ungleich customer)!
     name = models.CharField(max_length=32, blank=True, null=True)
@@ -131,7 +132,7 @@ class VMDiskImageProduct(UncloudModel):
                                      default = uncloud_storage.models.StorageClass.SSD)
 
     status = models.CharField(
-        max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT
+        max_length=32, choices=UncloudStatus.choices, default=UncloudStatus.PENDING
     )
 
     def __str__(self):

uncloud/uncloud_vm/serializers.py

@@ -2,7 +2,7 @@ from django.contrib.auth import get_user_model
 
 from rest_framework import serializers
 
-from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct
+from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster
 from uncloud_pay.models import RecurringPeriod
 
 GB_SSD_PER_DAY=0.012
@@ -12,7 +12,7 @@ GB_SSD_PER_DAY=0.012
 GB_HDD_PER_DAY=0.0006
 
 
-class VMHostSerializer(serializers.ModelSerializer):
+class VMHostSerializer(serializers.HyperlinkedModelSerializer):
     vms = serializers.PrimaryKeyRelatedField(many=True, read_only=True)
 
     class Meta:
@@ -20,6 +20,11 @@ class VMHostSerializer(serializers.ModelSerializer):
         fields = '__all__'
         read_only_fields = [ 'vms' ]
 
+class VMClusterSerializer(serializers.HyperlinkedModelSerializer):
+    class Meta:
+        model = VMCluster
+        fields = '__all__'
+
 
 class VMDiskProductSerializer(serializers.ModelSerializer):
     class Meta:
@@ -92,9 +97,6 @@ class VMProductSerializer(serializers.ModelSerializer):
     recurring_period = serializers.ChoiceField(
         choices=VMProduct.allowed_recurring_periods())
 
-    # snapshots = serializers.PrimaryKeyRelatedField(many=True,
-    #                                                read_only=True)
-
     snapshots = VMSnapshotProductSerializer(many=True,
                                             read_only=True)
 

uncloud/uncloud_vm/views.py

@@ -8,12 +8,13 @@ from rest_framework import viewsets, permissions
 from rest_framework.response import Response
 from rest_framework.exceptions import ValidationError
 
-from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct
+from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct, VMCluster
 from uncloud_pay.models import Order
 
 from .serializers import (VMHostSerializer, VMProductSerializer,
                           VMSnapshotProductSerializer, VMDiskImageProductSerializer,
-                          VMDiskProductSerializer, DCLVMProductSerializer)
+                          VMDiskProductSerializer, DCLVMProductSerializer,
+                          VMClusterSerializer)
 from uncloud_pay.helpers import ProductViewSet
 
 
@@ -24,6 +25,11 @@ class VMHostViewSet(viewsets.ModelViewSet):
     queryset = VMHost.objects.all()
     permission_classes = [permissions.IsAdminUser]
 
+class VMClusterViewSet(viewsets.ModelViewSet):
+    serializer_class = VMClusterSerializer
+    queryset = VMCluster.objects.all()
+    permission_classes = [permissions.IsAdminUser]
+
 class VMDiskImageProductViewSet(viewsets.ModelViewSet):
     permission_classes = [permissions.IsAuthenticated]
     serializer_class = VMDiskImageProductSerializer
@@ -135,7 +141,12 @@ class VMSnapshotProductViewSet(viewsets.ModelViewSet):
     serializer_class = VMSnapshotProductSerializer
 
     def get_queryset(self):
-        return VMSnapshotProduct.objects.filter(owner=self.request.user)
+        if self.request.user.is_superuser:
+            obj = VMSnapshotProduct.objects.all()
+        else:
+            obj = VMSnapshotProduct.objects.filter(owner=self.request.user)
+
+        return obj
 
     def create(self, request):
         serializer = VMSnapshotProductSerializer(data=request.data, context={'request': request})