Merge branch 'vm-ordering-disk' into 'master'

Wire disk images to VM creation/ordering, make Order records transparents

See merge request uncloud/uncloud!4

.gitlab-ci.yml

@@ -13,7 +13,7 @@ run-tests:
     POSTGRES_HOST_AUTH_METHOD: trust
   coverage: /^TOTAL.+?(\d+\%)$/
   before_script:
-    - dnf install -y python3-devel python3-pip libpq-devel openldap-devel gcc
+    - dnf install -y python3-devel python3-pip python3-coverage libpq-devel openldap-devel gcc chromium
   script:
     - cd uncloud_django_based/uncloud
     - pip install -r requirements.txt

uncloud_django_based/uncloud/uncloud/settings.py

@@ -67,7 +67,7 @@ INSTALLED_APPS = [
     'uncloud_net',
     'uncloud_storage',
     'uncloud_vm',
-    'ungleich_service',
+    'uncloud_service',
     'opennebula'
 ]
 

uncloud_django_based/uncloud/uncloud/urls.py

@@ -27,7 +27,7 @@ from uncloud_auth import views as authviews
 from uncloud_net  import views as netviews
 from uncloud_pay  import views as payviews
 from uncloud_vm   import views as vmviews
-from ungleich_service import views as serviceviews
+from uncloud_service import views as serviceviews
 
 router = routers.DefaultRouter()
 

uncloud_django_based/uncloud/uncloud_pay/migrations/0005_auto_20200413_0924.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.0.5 on 2020-04-13 09:24
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_pay', '0004_auto_20200409_1225'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='order',
+            name='recurring_period',
+            field=models.CharField(choices=[('ONCE', 'Onetime'), ('YEAR', 'Per Year'), ('MONTH', 'Per Month'), ('WEEK', 'Per Week'), ('DAY', 'Per Day'), ('HOUR', 'Per Hour'), ('MINUTE', 'Per Minute'), ('SECOND', 'Per Second')], default='MONTH', max_length=32),
+        ),
+    ]

uncloud_django_based/uncloud/uncloud_pay/models.py

@@ -476,11 +476,13 @@ class Order(models.Model):
     def recurring_price(self):
         return reduce(lambda acc, record: acc + record.recurring_price, self.records, 0)
 
+    # Used by uncloud_pay tests.
     def add_record(self, one_time_price, recurring_price, description):
-        OrderRecord.objects.create(order=self,
+      OrderRecord.objects.create(order=self,
-                one_time_price=one_time_price,
+              one_time_price=one_time_price,
-                recurring_price=recurring_price,
+              recurring_price=recurring_price,
-                description=description)
+              description=description)
+
 
 
 class OrderRecord(models.Model):
@@ -543,6 +545,20 @@ class Product(UncloudModel):
     # Default period for all products
     default_recurring_period = RecurringPeriod.PER_MONTH
 
+    # Used to save records.
+    def save(self, *args, **kwargs):
+        # _state.adding is switched to false after super(...) call.
+        being_created = self._state.adding
+
+        super(Product, self).save(*args, **kwargs)
+
+        # Make sure we only create records on creation.
+        if being_created:
+            record = OrderRecord(
+                one_time_price=self.one_time_price,
+                recurring_price=self.recurring_price(self.recurring_period),
+                description=self.description)
+            self.order.orderrecord_set.add(record, bulk=False)
 
     @property
     def recurring_price(self, recurring_period=RecurringPeriod.PER_MONTH):

uncloud_django_based/uncloud/uncloud_service/migrations/0001_initial.py

@@ -1,6 +1,7 @@
-# Generated by Django 3.0.3 on 2020-03-09 07:57
+# Generated by Django 3.0.5 on 2020-04-13 09:38
 
 from django.conf import settings
+import django.contrib.postgres.fields.jsonb
 from django.db import migrations, models
 import django.db.models.deletion
 import uuid
@@ -11,8 +12,8 @@ class Migration(migrations.Migration):
     initial = True
 
     dependencies = [
-        ('uncloud_vm', '0003_remove_vmhost_vms'),
+        ('uncloud_pay', '0005_auto_20200413_0924'),
-        ('uncloud_pay', '0002_auto_20200305_1524'),
+        ('uncloud_vm', '0010_auto_20200413_0924'),
         migrations.swappable_dependency(settings.AUTH_USER_MODEL),
     ]
 
@@ -20,8 +21,9 @@ class Migration(migrations.Migration):
         migrations.CreateModel(
             name='MatrixServiceProduct',
             fields=[
+                ('extra_data', django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True)),
                 ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
-                ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted')], default='PENDING', max_length=32)),
+                ('status', models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32)),
                 ('domain', models.CharField(default='domain.tld', max_length=255)),
                 ('order', models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order')),
                 ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),

uncloud_django_based/uncloud/uncloud_service/models.py

@@ -2,7 +2,7 @@ import uuid
 
 from django.db import models
 from uncloud_pay.models import Product, RecurringPeriod
-from uncloud_vm.models import VMProduct
+from uncloud_vm.models import VMProduct, VMDiskImageProduct
 
 class MatrixServiceProduct(Product):
     monthly_managment_fee = 20
@@ -15,11 +15,14 @@ class MatrixServiceProduct(Product):
             )
     domain = models.CharField(max_length=255, default='domain.tld')
 
-    def recurring_price(self, recurring_period=RecurringPeriod.PER_MONTH):
+    # Default recurring price is PER_MONT, see Product class.
-        if recurring_period == RecurringPeriod.PER_MONTH:
+    def recurring_price(self):
-            return self.monthly_managment_fee
+        return self.monthly_managment_fee
-        else:
+
-            raise Exception('Invalid recurring period for VM Product pricing.')
+    @staticmethod
+    def base_image():
+        # TODO: find a way to safely reference debian 10 image.
+        return VMDiskImageProduct.objects.get(uuid="93e564c5-adb3-4741-941f-718f76075f02")
 
     @staticmethod
     def allowed_recurring_periods():

uncloud_django_based/uncloud/uncloud_service/views.py

@@ -7,7 +7,28 @@ from .serializers import MatrixServiceProductSerializer
 
 from uncloud_pay.helpers import ProductViewSet
 from uncloud_pay.models import Order
-from uncloud_vm.models import VMProduct
+from uncloud_vm.models import VMProduct, VMDiskProduct
+
+def create_managed_vm(cores, ram, disk_size, image, order):
+    # Create VM
+    disk = VMDiskProduct(
+            owner=order.owner,
+            order=order,
+            size_in_gb=disk_size,
+            image=image)
+    vm = VMProduct(
+            name="Managed Service Host",
+            owner=order.owner,
+            cores=cores,
+            ram_in_gb=ram,
+            primary_disk=disk)
+    disk.vm = vm
+
+    vm.save()
+    disk.save()
+
+    return vm
+
 
 class MatrixServiceProductViewSet(ProductViewSet):
     permission_classes = [permissions.IsAuthenticated]
@@ -23,39 +44,24 @@ class MatrixServiceProductViewSet(ProductViewSet):
         serializer.is_valid(raise_exception=True)
         order_recurring_period = serializer.validated_data.pop("recurring_period")
 
-        # Create base order.
+        # Create base order.)
         order = Order.objects.create(
                 recurring_period=order_recurring_period,
-                owner=request.user
+                owner=request.user)
-                )
-        order.save()
 
         # Create unerderlying VM.
-        # TODO: move this logic to a method for use with other
+        data = serializer.validated_data.pop('vm')
-        # products.
+        vm = create_managed_vm(
-        vm_data = serializer.validated_data.pop('vm')
+                order=order,
-        vm_data['owner'] = request.user
+                cores=data['cores'],
-        vm_data['order'] = order
+                ram=data['ram_in_gb'],
-        vm = VMProduct.objects.create(**vm_data)
+                disk_size=data['primary_disk']['size_in_gb'],
-
+                image=MatrixServiceProduct.base_image())
-        # XXX: Move this to some kind of on_create hook in parent
-        # Product class?
-        order.add_record(
-                vm.one_time_price,
-                vm.recurring_price(order.recurring_period),
-                vm.description)
 
         # Create service.
         service = serializer.save(
                 order=order,
-                owner=self.request.user,
+                owner=request.user,
                 vm=vm)
 
-        # XXX: Move this to some kind of on_create hook in parent
-        # Product class?
-        order.add_record(
-                service.one_time_price,
-                service.recurring_price(order.recurring_period),
-                service.description)
-
         return Response(serializer.data)

uncloud_django_based/uncloud/uncloud_vm/migrations/0004_vmproduct_primary_disk.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.0.3 on 2020-03-09 12:43
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0003_remove_vmhost_vms'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmproduct',
+            name='primary_disk',
+            field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMDiskProduct'),
+        ),
+    ]

uncloud_django_based/uncloud/uncloud_vm/migrations/0005_auto_20200309_1258.py

@@ -0,0 +1,25 @@
+# Generated by Django 3.0.3 on 2020-03-09 12:58
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_pay', '0001_initial'),
+        ('uncloud_vm', '0004_vmproduct_primary_disk'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmdiskproduct',
+            name='order',
+            field=models.ForeignKey(editable=False, null=True, on_delete=django.db.models.deletion.CASCADE, to='uncloud_pay.Order'),
+        ),
+        migrations.AddField(
+            model_name='vmdiskproduct',
+            name='status',
+            field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted')], default='PENDING', max_length=32),
+        ),
+    ]

uncloud_django_based/uncloud/uncloud_vm/migrations/0005_auto_20200321_1058.py

@@ -8,7 +8,7 @@ import django.db.models.deletion
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('uncloud_vm', '0004_remove_vmproduct_vmid'),
+            ('uncloud_vm', '0005_auto_20200309_1258'),
     ]
 
     operations = [

uncloud_django_based/uncloud/uncloud_vm/migrations/0009_merge_20200413_0857.py

@@ -0,0 +1,14 @@
+# Generated by Django 3.0.5 on 2020-04-13 08:57
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0004_remove_vmproduct_vmid'),
+        ('uncloud_vm', '0008_auto_20200403_1727'),
+    ]
+
+    operations = [
+    ]

uncloud_django_based/uncloud/uncloud_vm/migrations/0010_auto_20200413_0924.py

@@ -1,18 +1,24 @@
-# Generated by Django 3.0.5 on 2020-04-03 17:27
+# Generated by Django 3.0.5 on 2020-04-13 09:24
 
 from django.db import migrations, models
+import django.db.models.deletion
 
 
 class Migration(migrations.Migration):
 
     dependencies = [
-        ('ungleich_service', '0003_auto_20200322_1758'),
+        ('uncloud_vm', '0009_merge_20200413_0857'),
     ]
 
     operations = [
         migrations.AlterField(
-            model_name='matrixserviceproduct',
+            model_name='vmdiskproduct',
             name='status',
             field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('SCHEDULED', 'Scheduled'), ('ACTIVE', 'Active'), ('MODIFYING', 'Modifying'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32),
         ),
+        migrations.AlterField(
+            model_name='vmdiskproduct',
+            name='vm',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMProduct'),
+        ),
     ]

uncloud_django_based/uncloud/uncloud_vm/models.py

@@ -69,17 +69,11 @@ class VMProduct(Product):
     cores = models.IntegerField()
     ram_in_gb = models.FloatField()
 
+    primary_disk = models.ForeignKey('VMDiskProduct', on_delete=models.CASCADE, null=True)
 
-    def recurring_price(self, recurring_period=RecurringPeriod.PER_MONTH):
+    # Default recurring price is PER_MONTH, see uncloud_pay.models.Product.
-        # TODO: move magic numbers in variables
+    def recurring_price(self):
-        if recurring_period == RecurringPeriod.PER_MONTH:
+        return self.cores * 3 + self.ram_in_gb * 4
-            return self.cores * 3 + self.ram_in_gb * 4
-        elif recurring_period == RecurringPeriod.PER_HOUR:
-            return self.cores * 4.0/(30 * 24) + self.ram_in_gb * 4.5/(30* 24)
-        elif recurring_period == RecurringPeriod.PER_YEAR:
-            return (self.cores * 2.5 + self.ram_in_gb * 3.5) * 12
-        else:
-            raise Exception('Invalid recurring period for VM Product pricing.')
 
     def __str__(self):
         return "VM {} ({}): {} cores {} gb ram".format(self.uuid,
@@ -141,7 +135,7 @@ class VMDiskImageProduct(UncloudModel):
 
 
 
-class VMDiskProduct(UncloudModel):
+class VMDiskProduct(Product):
     """
     The VMDiskProduct is attached to a VM.
 
@@ -150,18 +144,27 @@ class VMDiskProduct(UncloudModel):
     It can be enlarged, but not shrinked compared to the VMDiskImageProduct.
     """
 
-    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE)
-    owner = models.ForeignKey(get_user_model(),
-                              on_delete=models.CASCADE,
-                              editable=False)
-
-    vm = models.ForeignKey(VMProduct,
-                           related_name='disks',
-                           on_delete=models.CASCADE)
     image = models.ForeignKey(VMDiskImageProduct, on_delete=models.CASCADE)
 
     size_in_gb = models.FloatField(blank=True)
 
+    @property
+    def description(self):
+        return "Disk for VM '{}': {}GB".format(self.vm.name, self.size_in_gb)
+
+    # TODO: move magic numbers in variables
+    def recurring_price(self, recurring_period=RecurringPeriod.PER_MONTH):
+        # TODO: move magic numbers in variables
+        if recurring_period == RecurringPeriod.PER_MONTH:
+            return (self.size_in_gb / 10) * 3.5
+        if recurring_period == RecurringPeriod.PER_YEAR:
+            return recurring_price(self, recurring_period.PER_MONTH) * 12
+        if recurring_period == RecurringPeriod.PER_HOUR:
+            return recurring_price(self, recurring_period.PER_MONTH) / 25
+        else:
+            raise Exception('Invalid recurring period for VM Disk Product pricing.')
+
     # Sample code for clean method
 
     # Ensures that a VMDiskProduct can only be created from a VMDiskImageProduct

uncloud_django_based/uncloud/uncloud_vm/serializers.py

@@ -31,13 +31,21 @@ class VMDiskProductSerializer(serializers.ModelSerializer):
         model = VMDiskProduct
         fields = '__all__'
 
+class CreateVMDiskProductSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = VMDiskProduct
+        fields = ['size_in_gb', 'image']
+
+class CreateManagedVMDiskProductSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = VMDiskProduct
+        fields = ['size_in_gb']
+
 class VMDiskImageProductSerializer(serializers.ModelSerializer):
     class Meta:
         model = VMDiskImageProduct
         fields = '__all__'
 
-
-
 class DCLVMProductSerializer(serializers.HyperlinkedModelSerializer):
     """
     Create an interface similar to standard DCL
@@ -57,9 +65,10 @@ class ManagedVMProductSerializer(serializers.ModelSerializer):
     """
     Managed VM serializer used in ungleich_service app.
     """
+    primary_disk = CreateManagedVMDiskProductSerializer()
     class Meta:
         model = VMProduct
-        fields = [ 'cores', 'ram_in_gb']
+        fields = [ 'cores', 'ram_in_gb', 'primary_disk']
 
 class VMSnapshotProductSerializer(serializers.ModelSerializer):
     class Meta:
@@ -84,18 +93,19 @@ class VMSnapshotProductSerializer(serializers.ModelSerializer):
     pricing['per_gb_hdd'] = 0.0006
     pricing['recurring_period'] = 'per_day'
 
-class VMProductSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = VMProduct
-        fields = ['uuid', 'order', 'owner', 'status', 'name',
-                  'cores', 'ram_in_gb', 'recurring_period',
-                  'snapshots', 'disks',
-                  'extra_data' ]
-        read_only_fields = ['uuid', 'order', 'owner', 'status' ]
 
+class VMProductSerializer(serializers.HyperlinkedModelSerializer):
     # Custom field used at creation (= ordering) only.
     recurring_period = serializers.ChoiceField(
-        choices=VMProduct.allowed_recurring_periods())
+            choices=VMProduct.allowed_recurring_periods())
+    primary_disk = CreateVMDiskProductSerializer()
+
+    class Meta:
+        model = VMProduct
+        fields = ['uuid', 'order', 'owner', 'status', 'name', \
+                'cores', 'ram_in_gb', 'recurring_period', 'primary_disk',
+                'snapshots', 'disks', 'extra_data' ]
+        read_only_fields = ['uuid', 'order', 'owner', 'status']
 
     snapshots = VMSnapshotProductSerializer(many=True,
                                             read_only=True)

uncloud_django_based/uncloud/uncloud_vm/views.py

@@ -89,7 +89,6 @@ class VMDiskProductViewSet(viewsets.ModelViewSet):
         if size_in_gb < serializer.validated_data['image'].size_in_gb:
             raise ValidationError(detail={ 'error_mesage': 'Size is smaller than original image' })
 
-
         serializer.save(owner=request.user, size_in_gb=size_in_gb)
         return Response(serializer.data)
 
@@ -122,15 +121,15 @@ class VMProductViewSet(ProductViewSet):
                 owner=request.user,
                 starting_date=timezone.now()
                 )
-        order.save()
+
+        # Create disk image.
+        disk = VMDiskProduct(owner=request.user, order=order,
+                **serializer.validated_data.pop("primary_disk"))
 
         # Create VM.
-        vm = serializer.save(owner=request.user, order=order)
+        vm = serializer.save(owner=request.user, order=order, primary_disk=disk)
-
+        disk.vm = vm
-        # Add Product record to order (VM is mutable, allows to keep history in order).
+        disk.save()
-        # XXX: Move this to some kind of on_create hook in parent Product class?
-        order.add_record(vm.one_time_price,
-                vm.recurring_price(order.recurring_period), vm.description)
 
         return Response(serializer.data)
 
@@ -215,9 +214,4 @@ class DCLCreateVMProductViewSet(ProductViewSet):
         # Create VM.
         vm = serializer.save(owner=request.user, order=order)
 
-        # Add Product record to order (VM is mutable, allows to keep history in order).
-        # XXX: Move this to some kind of on_create hook in parent Product class?
-        order.add_record(vm.one_time_price,
-                vm.recurring_price(order.recurring_period), vm.description)
-
         return Response(serializer.data)

uncloud_django_based/uncloud/ungleich_service/migrations/0002_matrixserviceproduct_extra_data.py

@@ -1,19 +0,0 @@
-# Generated by Django 3.0.3 on 2020-03-21 10:58
-
-import django.contrib.postgres.fields.jsonb
-from django.db import migrations
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('ungleich_service', '0001_initial'),
-    ]
-
-    operations = [
-        migrations.AddField(
-            model_name='matrixserviceproduct',
-            name='extra_data',
-            field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, editable=False, null=True),
-        ),
-    ]

uncloud_django_based/uncloud/ungleich_service/migrations/0003_auto_20200322_1758.py

@@ -1,18 +0,0 @@
-# Generated by Django 3.0.3 on 2020-03-22 17:58
-
-from django.db import migrations, models
-
-
-class Migration(migrations.Migration):
-
-    dependencies = [
-        ('ungleich_service', '0002_matrixserviceproduct_extra_data'),
-    ]
-
-    operations = [
-        migrations.AlterField(
-            model_name='matrixserviceproduct',
-            name='status',
-            field=models.CharField(choices=[('PENDING', 'Pending'), ('AWAITING_PAYMENT', 'Awaiting payment'), ('BEING_CREATED', 'Being created'), ('ACTIVE', 'Active'), ('DELETED', 'Deleted'), ('DISABLED', 'Disabled'), ('UNUSABLE', 'Unusable')], default='PENDING', max_length=32),
-        ),
-    ]