Merge remote-tracking branch 'origin/master' into fnux-hacks

uncloud/uncloud/settings.py

@@ -11,8 +11,6 @@ https://docs.djangoproject.com/en/3.0/ref/settings/
 """
 
 import os
-
-import stripe
 import ldap
 
 # Uncommitted file with secrets
@@ -174,10 +172,3 @@ USE_TZ = True
 # https://docs.djangoproject.com/en/3.0/howto/static-files/
 
 STATIC_URL = '/static/'
-
-stripe.api_key = uncloud.secrets.STRIPE_KEY
-
-############
-# Stripe
-
-STRIPE_API_KEY = uncloud.secrets.STRIPE_API_KEY

uncloud/uncloud/urls.py

@@ -27,11 +27,34 @@ router = routers.DefaultRouter()
 
 # VM
 router.register(r'vm/snapshot', vmviews.VMSnapshotProductViewSet, basename='vmsnapshotproduct')
+router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct')
+router.register(r'vm/image/mine', vmviews.VMDiskImageProductMineViewSet, basename='vmdiskimagemineproduct')
+router.register(r'vm/image/public', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct')
+
+# images the provider provides :-)
+# router.register(r'vm/image/official', vmviews.VMDiskImageProductPublicViewSet, basename='vmdiskimagepublicproduct')
+
+
+
+
 router.register(r'vm/vm', vmviews.VMProductViewSet, basename='vmproduct')
 
+
+# TBD
+#router.register(r'vm/disk', vmviews.VMDiskProductViewSet, basename='vmdiskproduct')
+
+# creates VM from os image
+#router.register(r'vm/ipv6onlyvm', vmviews.VMProductViewSet, basename='vmproduct')
+# ... AND adds IPv4 mapping
+#router.register(r'vm/dualstackvm', vmviews.VMProductViewSet, basename='vmproduct')
+
+# allow vm creation from own images
+
+
 # Services
 router.register(r'service/matrix', serviceviews.MatrixServiceProductViewSet, basename='matrixserviceproduct')
 
+
 # Pay
 router.register(r'user', payviews.UserViewSet, basename='user')
 router.register(r'payment-method', payviews.PaymentMethodViewSet, basename='payment-method')

uncloud/uncloud_pay/stripe.py

@@ -2,13 +2,12 @@ import stripe
 import stripe.error
 import logging
 
-from django.conf import settings
+import uncloud.secrets
 
 # Static stripe configuration used below.
 CURRENCY = 'chf'
 
-# Register stripe (secret) API key from config.
+stripe.api_key = uncloud.secrets.STRIPE_KEY
-stripe.api_key = settings.STRIPE_API_KEY
 
 # Helper (decorator) used to catch errors raised by stripe logic.
 def handle_stripe_error(f):

uncloud/uncloud_vm/migrations/0006_auto_20200229_1545.py

@@ -0,0 +1,53 @@
+# Generated by Django 3.0.3 on 2020-02-29 15:45
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+import uuid
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('uncloud_vm', '0005_auto_20200227_1230'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='VMDiskImageProduct',
+            fields=[
+                ('uuid', models.UUIDField(default=uuid.uuid4, editable=False, primary_key=True, serialize=False)),
+                ('name', models.CharField(max_length=256)),
+                ('is_os_image', models.BooleanField(default=False)),
+                ('is_public', models.BooleanField(default=False)),
+                ('size_in_gb', models.FloatField()),
+                ('storage_class', models.CharField(choices=[('hdd', 'HDD'), ('ssd', 'SSD')], default='ssd', max_length=32)),
+                ('owner', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+        migrations.RemoveField(
+            model_name='vmdiskproduct',
+            name='storage_class',
+        ),
+        migrations.AddField(
+            model_name='vmdiskproduct',
+            name='owner',
+            field=models.ForeignKey(default=1, editable=False, on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL),
+            preserve_default=False,
+        ),
+        migrations.AddField(
+            model_name='vmnetworkcard',
+            name='ip_address',
+            field=models.GenericIPAddressField(blank=True, null=True),
+        ),
+        migrations.DeleteModel(
+            name='OperatingSystemDisk',
+        ),
+        migrations.AddField(
+            model_name='vmdiskproduct',
+            name='image',
+            field=models.ForeignKey(default=0, on_delete=django.db.models.deletion.CASCADE, to='uncloud_vm.VMDiskImageProduct'),
+            preserve_default=False,
+        ),
+    ]

uncloud/uncloud_vm/migrations/0007_auto_20200229_1559.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.3 on 2020-02-29 15:59
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0006_auto_20200229_1545'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmdiskimageproduct',
+            name='import_url',
+            field=models.URLField(blank=True, null=True),
+        ),
+        migrations.AlterField(
+            model_name='vmdiskimageproduct',
+            name='size_in_gb',
+            field=models.FloatField(blank=True, null=True),
+        ),
+    ]

uncloud/uncloud_vm/migrations/0008_auto_20200229_1611.py

@@ -0,0 +1,23 @@
+# Generated by Django 3.0.3 on 2020-02-29 16:11
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('uncloud_vm', '0007_auto_20200229_1559'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vmdiskimageproduct',
+            name='status',
+            field=models.CharField(choices=[('pending', 'Pending'), ('creating', 'Creating'), ('active', 'Active'), ('disabled', 'Disabled'), ('unusable', 'Unusable'), ('deleted', 'Deleted')], default='pending', max_length=32),
+        ),
+        migrations.AlterField(
+            model_name='vmhost',
+            name='status',
+            field=models.CharField(choices=[('pending', 'Pending'), ('creating', 'Creating'), ('active', 'Active'), ('disabled', 'Disabled'), ('unusable', 'Unusable'), ('deleted', 'Deleted')], default='pending', max_length=32),
+        ),
+    ]

uncloud/uncloud_vm/models.py

@@ -1,10 +1,25 @@
+import uuid
+
 from django.db import models
 from django.contrib.auth import get_user_model
-import uuid
+
+# Uncomment if you override model's clean method
+# from django.core.exceptions import ValidationError
 
 from uncloud_pay.models import Product, RecurringPeriod
 import uncloud_pay.models as pay_models
 
+STATUS_CHOICES = (
+    ('pending', 'Pending'),   # Initial state
+    ('creating', 'Creating'),  # Creating VM/image/etc.
+    ('active', 'Active'),      # Is usable / active
+    ('disabled', 'Disabled'),  # Is usable, but cannot be used for new things
+    ('unusable', 'Unusable'),  # Has some kind of error
+    ('deleted', 'Deleted'),   # Does not exist anymore, only DB entry as a log
+)
+
+STATUS_DEFAULT = 'pending'
+
 
 class VMHost(models.Model):
     uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
@@ -21,24 +36,15 @@ class VMHost(models.Model):
     # ram that can be used of the server
     usable_ram_in_gb = models.FloatField(default=0)
 
-
+    status = models.CharField(
-    status = models.CharField(max_length=32,
+        max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT
-                              choices = (
-                                  ('pending', 'Pending'),
-                                  ('active', 'Active'),
-                                  ('unusable', 'Unusable'),
-                                  ('deleted', 'Deleted'),
-                              ),
-                              default='pending'
     )
 
 
 class VMProduct(Product):
-    vmhost   = models.ForeignKey(VMHost,
+    vmhost = models.ForeignKey(
-                                 on_delete=models.CASCADE,
+        VMHost, on_delete=models.CASCADE, editable=False, blank=True, null=True
-                                 editable=False,
+    )
-                                 blank=True,
-                                 null=True)
 
     # VM-specific. The name is only intended for customers: it's a pain te
     # remember IDs (speaking from experience as ungleich customer)!
@@ -69,26 +75,80 @@ class VMProduct(Product):
 class VMWithOSProduct(VMProduct):
     pass
 
-class VMDiskProduct(models.Model):
-    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
-    vm   = models.ForeignKey(VMProduct, on_delete=models.CASCADE)
-    size_in_gb = models.FloatField()
 
-    storage_class = models.CharField(max_length=32,
+class VMDiskImageProduct(models.Model):
-                              choices = (
+    """
-                                  ('hdd', 'HDD'),
+    Images are used for cloning/linking.
-                                  ('ssd', 'SSD'),
+
-                              ),
+    They are the base for images.
-                              default='ssd'
+
+    """
+
+    uuid = models.UUIDField(
+        primary_key=True, default=uuid.uuid4, editable=False
+    )
+    owner = models.ForeignKey(
+        get_user_model(), on_delete=models.CASCADE, editable=False
     )
 
-class OperatingSystemDisk(VMDiskProduct):
+    name = models.CharField(max_length=256)
-    """ Defines an Operating System Disk that can be cloned for a VM """
+    is_os_image = models.BooleanField(default=False)
-    os_name = models.CharField(max_length=128)
+    is_public = models.BooleanField(default=False)
+
+    size_in_gb = models.FloatField(null=True, blank=True)
+    import_url = models.URLField(null=True, blank=True)
+
+    storage_class = models.CharField(
+        max_length=32,
+        choices=(
+            ('hdd', 'HDD'),
+            ('ssd', 'SSD'),
+        ),
+        default='ssd'
+    )
+
+    status = models.CharField(
+        max_length=32, choices=STATUS_CHOICES, default=STATUS_DEFAULT
+    )
+
+
+class VMDiskProduct(models.Model):
+    """
+    The VMDiskProduct is attached to a VM.
+
+    It is based on a VMDiskImageProduct that will be used as a basis.
+
+    It can be enlarged, but not shrinked compared to the VMDiskImageProduct.
+    """
+
+    uuid = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)
+    owner = models.ForeignKey(get_user_model(),
+                              on_delete=models.CASCADE,
+                              editable=False)
+
+    vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE)
+    image = models.ForeignKey(VMDiskImageProduct, on_delete=models.CASCADE)
+
+    size_in_gb = models.FloatField(blank=True)
+
+    # Sample code for clean method
+
+    # Ensures that a VMDiskProduct can only be created from a VMDiskImageProduct
+    # that is in status 'active'
+
+    # def clean(self):
+    #     if self.image.status != 'active':
+    #         raise ValidationError({
+    #             'image': 'VM Disk must be created from an active disk image.'
+    #         })
+
+    def save(self, *args, **kwargs):
+        self.full_clean()
+        super().save(*args, **kwargs)
 
 
 class VMNetworkCard(models.Model):
-    vm   = models.ForeignKey(VMProduct, on_delete=models.CASCADE)
+    vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE)
 
     mac_address = models.IntegerField()
 
@@ -97,44 +157,7 @@ class VMNetworkCard(models.Model):
 
 
 class VMSnapshotProduct(Product):
-    price_per_gb_ssd = 0.35
-    price_per_gb_hdd = 1.5/100
-
-    # This we need to get from the VM
     gb_ssd = models.FloatField(editable=False)
     gb_hdd = models.FloatField(editable=False)
 
     vm = models.ForeignKey(VMProduct, on_delete=models.CASCADE)
-    #vm_uuid = models.UUIDField()
-
-    # Need to setup recurring_price and one_time_price and recurring period
-
-    sample_ssd = 10
-    sample_hdd = 100
-
-    def recurring_price(self):
-        return 0
-
-    def one_time_price(self):
-        return 0
-
-    @classmethod
-    def sample_price(cls):
-        return cls.sample_ssd * cls.price_per_gb_ssd + cls.sample_hdd * cls.price_per_gb_hdd
-
-    description = "Create snapshot of a VM"
-    recurring_period = "monthly"
-
-    @classmethod
-    def pricing_model(cls):
-        return """
-Pricing is on monthly basis and storage prices are equivalent to the storage
-price in the VM.
-
-Price per GB SSD is: {}
-Price per GB HDD is: {}
-
-
-Sample price for a VM with {} GB SSD and {} GB HDD VM is: {}.
-""".format(cls.price_per_gb_ssd, cls.price_per_gb_hdd,
-           cls.sample_ssd, cls.sample_hdd, cls.sample_price())

uncloud/uncloud_vm/serializers.py

@@ -1,15 +1,33 @@
 from django.contrib.auth import get_user_model
 
 from rest_framework import serializers
-from .models import VMHost, VMProduct, VMSnapshotProduct
+
+from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct
 from uncloud_pay.models import RecurringPeriod
 
-class VMHostSerializer(serializers.HyperlinkedModelSerializer):
+GB_SSD_PER_DAY=0.012
+GB_HDD_PER_DAY=0.0006
+
+GB_SSD_PER_DAY=0.012
+GB_HDD_PER_DAY=0.0006
+
+
+class VMHostSerializer(serializers.ModelSerializer):
     class Meta:
         model = VMHost
         fields = '__all__'
 
 
+class VMDiskProductSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = VMDiskProduct
+        fields = '__all__'
+
+class VMDiskImageProductSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = VMDiskImageProduct
+        fields = '__all__'
+
 class VMProductSerializer(serializers.HyperlinkedModelSerializer):
     # Custom field used at creation (= ordering) only.
     recurring_period = serializers.ChoiceField(
@@ -29,7 +47,6 @@ class ManagedVMProductSerializer(serializers.ModelSerializer):
         model = VMProduct
         fields = [ 'cores', 'ram_in_gb']
 
-
 class VMSnapshotProductSerializer(serializers.ModelSerializer):
     class Meta:
         model = VMSnapshotProduct
@@ -38,5 +55,17 @@ class VMSnapshotProductSerializer(serializers.ModelSerializer):
 
     # verify that vm.owner == user.request
     def validate_vm(self, value):
-        print(value)
+        if not value.owner == self.context['request'].user:
-        return True
+            raise serializers.ValidationError("VM {} not found for owner {}.".format(value,
+                                                                                     self.context['request'].user))
+        disks = VMDiskProduct.objects.filter(vm=value)
+
+        if len(disks) == 0:
+            raise serializers.ValidationError("VM {} does not have any disks, cannot snapshot".format(value.uuid))
+
+        return value
+
+    pricing = {}
+    pricing['per_gb_ssd'] = 0.012
+    pricing['per_gb_hdd'] = 0.0006
+    pricing['recurring_period'] = 'per_day'

uncloud/uncloud_vm/tests.py

@@ -0,0 +1,112 @@
+import datetime
+
+import parsedatetime
+
+from django.test import TestCase
+from django.contrib.auth import get_user_model
+from django.utils import timezone
+from django.core.exceptions import ValidationError
+
+from uncloud_vm.models import VMDiskImageProduct, VMDiskProduct, VMProduct, VMHost
+from uncloud_pay.models import Order
+
+User = get_user_model()
+cal = parsedatetime.Calendar()
+
+
+# If you want to check the test database using some GUI/cli tool
+# then use the following connecting parameters
+
+# host: localhost
+# database: test_uncloud
+# user: root
+# password:
+# port: 5432
+
+class VMTestCase(TestCase):
+    @classmethod
+    def setUpClass(cls):
+        # Setup vm host
+        cls.vm_host, created = VMHost.objects.get_or_create(
+            hostname='serverx.placey.ungleich.ch', physical_cores=32, usable_cores=320,
+            usable_ram_in_gb=512.0, status='active'
+        )
+        super().setUpClass()
+
+    def setUp(self) -> None:
+        # Setup two users as it is common to test with different user
+        self.user = User.objects.create_user(
+            username='testuser', email='test@test.com', first_name='Test', last_name='User'
+        )
+        self.user2 = User.objects.create_user(
+            username='Meow', email='meow123@test.com', first_name='Meow', last_name='Cat'
+        )
+        super().setUp()
+
+    def create_sample_vm(self, owner):
+        one_month_later, parse_status = cal.parse("1 month later")
+        return VMProduct.objects.create(
+            vmhost=self.vm_host, cores=2, ram_in_gb=4, owner=owner,
+            order=Order.objects.create(
+                owner=owner,
+                creation_date=datetime.datetime.now(tz=timezone.utc),
+                starting_date=datetime.datetime.now(tz=timezone.utc),
+                ending_date=datetime.datetime(*one_month_later[:6], tzinfo=timezone.utc),
+                recurring_price=4.0, one_time_price=5.0, recurring_period='per_month'
+            )
+        )
+
+    def test_disk_product(self):
+        """Ensures that a VMDiskProduct can only be created from a VMDiskImageProduct
+           that is in status 'active'"""
+
+        vm = self.create_sample_vm(owner=self.user)
+
+        pending_disk_image = VMDiskImageProduct.objects.create(
+            owner=self.user, name='pending_disk_image', is_os_image=True, is_public=True, size_in_gb=10,
+            status='pending'
+        )
+        try:
+            vm_disk_product = VMDiskProduct.objects.create(
+                owner=self.user, vm=vm, image=pending_disk_image, size_in_gb=10
+            )
+        except ValidationError:
+            vm_disk_product = None
+
+        self.assertIsNone(
+            vm_disk_product,
+            msg='VMDiskProduct created with disk image whose status is not active.'
+        )
+
+    def test_vm_disk_product_creation(self):
+        """Ensure that a user can only create a VMDiskProduct for an existing VM"""
+
+        disk_image = VMDiskImageProduct.objects.create(
+            owner=self.user, name='disk_image', is_os_image=True, is_public=True, size_in_gb=10,
+            status='active'
+        )
+
+        with self.assertRaises(ValidationError, msg='User created a VMDiskProduct for non-existing VM'):
+            # Create VMProduct object but don't save it in database
+            vm = VMProduct()
+
+            vm_disk_product = VMDiskProduct.objects.create(
+                owner=self.user, vm=vm, image=disk_image, size_in_gb=10
+            )
+
+    def test_vm_disk_product_creation_for_someone_else(self):
+        """Ensure that a user can only create a VMDiskProduct for his/her own VM"""
+
+        # Create a VM which is ownership of self.user2
+        someone_else_vm = self.create_sample_vm(owner=self.user2)
+
+        # 'self.user' would try to create a VMDiskProduct for 'user2's VM
+        with self.assertRaises(ValidationError, msg='User created a VMDiskProduct for someone else VM.'):
+            vm_disk_product = VMDiskProduct.objects.create(
+                owner=self.user, vm=someone_else_vm,
+                size_in_gb=10,
+                image=VMDiskImageProduct.objects.create(
+                    owner=self.user, name='disk_image', is_os_image=True, is_public=True, size_in_gb=10,
+                    status='active'
+                )
+            )

uncloud/uncloud_vm/views.py

@@ -6,10 +6,12 @@ from django.shortcuts import get_object_or_404
 
 from rest_framework import viewsets, permissions
 from rest_framework.response import Response
+from rest_framework.exceptions import ValidationError
 
-from .models import VMHost, VMProduct, VMSnapshotProduct
+from .models import VMHost, VMProduct, VMSnapshotProduct, VMDiskProduct, VMDiskImageProduct
-from uncloud_pay.models import Order, RecurringPeriod
+from uncloud_pay.models import Order
-from .serializers import VMHostSerializer, VMProductSerializer, VMSnapshotProductSerializer
+
+from .serializers import VMHostSerializer, VMProductSerializer, VMSnapshotProductSerializer, VMDiskImageProductSerializer, VMDiskProductSerializer
 from uncloud_pay.helpers import ProductViewSet
 
 
@@ -20,6 +22,61 @@ class VMHostViewSet(viewsets.ModelViewSet):
     queryset = VMHost.objects.all()
     permission_classes = [permissions.IsAdminUser]
 
+class VMDiskImageProductMineViewSet(viewsets.ModelViewSet):
+    permission_classes = [permissions.IsAuthenticated]
+    serializer_class = VMDiskImageProductSerializer
+
+    def get_queryset(self):
+        return VMDiskImageProduct.objects.filter(owner=self.request.user)
+
+    def create(self, request):
+        serializer = VMDiskImageProductSerializer(data=request.data, context={'request': request})
+        serializer.is_valid(raise_exception=True)
+
+        # did not specify size NOR import url?
+        if not serializer.validated_data['size_in_gb']:
+            if not serializer.validated_data['import_url']:
+                raise ValidationError(detail={ 'error_mesage': 'Specify either import_url or size_in_gb' })
+
+        serializer.save(owner=request.user)
+        return Response(serializer.data)
+
+
+class VMDiskImageProductPublicViewSet(viewsets.ReadOnlyModelViewSet):
+    permission_classes = [permissions.IsAuthenticated]
+    serializer_class = VMDiskImageProductSerializer
+
+    def get_queryset(self):
+        return VMDiskImageProduct.objects.filter(is_public=True)
+
+class VMDiskProductViewSet(viewsets.ModelViewSet):
+    """
+    Let a user modify their own VMDisks
+    """
+    permission_classes = [permissions.IsAuthenticated]
+    serializer_class = VMDiskProductSerializer
+
+    def get_queryset(self):
+        return VMDiskProduct.objects.filter(owner=self.request.user)
+
+    def create(self, request):
+        serializer = VMDiskProductSerializer(data=request.data, context={'request': request})
+        serializer.is_valid(raise_exception=True)
+
+        # get disk size from image, if not specified
+        if not 'size_in_gb' in serializer.validated_data:
+            size_in_gb = serializer.validated_data['image'].size_in_gb
+        else:
+            size_in_gb = serializer.validated_data['size_in_gb']
+
+        if size_in_gb < serializer.validated_data['image'].size_in_gb:
+            raise ValidationError(detail={ 'error_mesage': 'Size is smaller than original image' })
+
+
+        serializer.save(owner=request.user, size_in_gb=size_in_gb)
+        return Response(serializer.data)
+
+
 
 class VMProductViewSet(ProductViewSet):
     permission_classes = [permissions.IsAuthenticated]
@@ -64,22 +121,30 @@ class VMSnapshotProductViewSet(viewsets.ModelViewSet):
 
     def create(self, request):
         serializer = VMSnapshotProductSerializer(data=request.data, context={'request': request})
+
+        # This verifies that the VM belongs to the request user
         serializer.is_valid(raise_exception=True)
 
+        disks = VMDiskProduct.objects.filter(vm=serializer.validated_data['vm'])
+        ssds_size = sum([d.size_in_gb for d in disks if d.storage_class == 'ssd'])
+        hdds_size = sum([d.size_in_gb for d in disks if d.storage_class == 'hdd'])
+
+        recurring_price = serializer.pricing['per_gb_ssd'] * ssds_size + serializer.pricing['per_gb_hdd'] * hdds_size
+        recurring_period = serializer.pricing['recurring_period']
+
         # Create order
         now = datetime.datetime.now()
         order = Order(owner=request.user,
                       creation_date=now,
                       starting_date=now,
-                      recurring_price=20,
+                      recurring_price=recurring_price,
                       one_time_price=0,
-                      recurring_period="per_month")
+                      recurring_period=recurring_period)
         order.save()
 
-        # FIXME: calculate the gb_* values
         serializer.save(owner=request.user,
                         order=order,
-                        gb_ssd=12,
+                        gb_ssd=ssds_size,
-                        gb_hdd=20)
+                        gb_hdd=hdds_size)
 
         return Response(serializer.data)