Merge pull request #1 from ungleich/master

Update master

datacenterlight/tasks.py

@@ -0,0 +1,169 @@
+from dynamicweb.celery import app
+from celery.utils.log import get_task_logger
+from django.conf import settings
+from opennebula_api.models import OpenNebulaManager
+from opennebula_api.serializers import VirtualMachineSerializer
+from hosting.models import HostingOrder, HostingBill
+from utils.forms import UserBillingAddressForm
+from datetime import datetime
+from membership.models import StripeCustomer
+from django.core.mail import EmailMessage
+from utils.models import BillingAddress
+from celery.exceptions import MaxRetriesExceededError
+
+logger = get_task_logger(__name__)
+
+
+def retry_task(task, exception=None):
+    """Retries the specified task using a "backing off countdown",
+    meaning that the interval between retries grows exponentially
+    with every retry.
+
+    Arguments:
+        task:
+            The task to retry.
+
+        exception:
+            Optionally, the exception that caused the retry.
+    """
+
+    def backoff(attempts):
+        return 2 ** attempts
+
+    kwargs = {
+        'countdown': backoff(task.request.retries),
+    }
+
+    if exception:
+        kwargs['exc'] = exception
+
+    raise task.retry(**kwargs)
+
+
+@app.task(bind=True, max_retries=settings.CELERY_MAX_RETRIES)
+def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_id, billing_address_data,
+                   billing_address_id,
+                   charge):
+    vm_id = None
+    try:
+        final_price = specs.get('price')
+        billing_address = BillingAddress.objects.filter(id=billing_address_id).first()
+        customer = StripeCustomer.objects.filter(id=stripe_customer_id).first()
+        # Create OpenNebulaManager
+        manager = OpenNebulaManager(email=settings.OPENNEBULA_USERNAME,
+                                    password=settings.OPENNEBULA_PASSWORD)
+
+        # Create a vm using oneadmin, also specify the name
+        vm_id = manager.create_vm(
+            template_id=vm_template_id,
+            specs=specs,
+            ssh_key=settings.ONEADMIN_USER_SSH_PUBLIC_KEY,
+            vm_name="{email}-{template_name}-{date}".format(
+                email=user.get('email'),
+                template_name=template.get('name'),
+                date=int(datetime.now().strftime("%s")))
+        )
+
+        if vm_id is None:
+            raise Exception("Could not create VM")
+
+        # Create a Hosting Order
+        order = HostingOrder.create(
+            price=final_price,
+            vm_id=vm_id,
+            customer=customer,
+            billing_address=billing_address
+        )
+
+        # Create a Hosting Bill
+        HostingBill.create(
+            customer=customer, billing_address=billing_address)
+
+        # Create Billing Address for User if he does not have one
+        if not customer.user.billing_addresses.count():
+            billing_address_data.update({
+                'user': customer.user.id
+            })
+            billing_address_user_form = UserBillingAddressForm(
+                billing_address_data)
+            billing_address_user_form.is_valid()
+            billing_address_user_form.save()
+
+        # Associate an order with a stripe payment
+        charge_object = DictDotLookup(charge)
+        order.set_stripe_charge(charge_object)
+
+        # If the Stripe payment succeeds, set order status approved
+        order.set_approved()
+
+        vm = VirtualMachineSerializer(manager.get_vm(vm_id)).data
+
+        context = {
+            'name': user.get('name'),
+            'email': user.get('email'),
+            'cores': specs.get('cpu'),
+            'memory': specs.get('memory'),
+            'storage': specs.get('disk_size'),
+            'price': specs.get('price'),
+            'template': template.get('name'),
+            'vm.name': vm['name'],
+            'vm.id': vm['vm_id'],
+            'order.id': order.id
+        }
+        email_data = {
+            'subject': settings.DCL_TEXT + " Order from %s" % context['email'],
+            'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
+            'to': ['info@ungleich.ch'],
+            'body': "\n".join(["%s=%s" % (k, v) for (k, v) in context.items()]),
+            'reply_to': [context['email']],
+        }
+        email = EmailMessage(**email_data)
+        email.send()
+    except Exception as e:
+        logger.error(str(e))
+        try:
+            retry_task(self)
+        except MaxRetriesExceededError:
+            msg_text = 'Finished {} retries for create_vm_task'.format(self.request.retries)
+            logger.error(msg_text)
+            # Try sending email and stop
+            email_data = {
+                'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text),
+                'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
+                'to': ['info@ungleich.ch'],
+                'body': ',\n'.join(str(i) for i in self.request.args)
+            }
+            email = EmailMessage(**email_data)
+            email.send()
+            return
+
+    return vm_id
+
+
+class DictDotLookup(object):
+    """
+    Creates objects that behave much like a dictionaries, but allow nested
+    key access using object '.' (dot) lookups.
+    """
+
+    def __init__(self, d):
+        for k in d:
+            if isinstance(d[k], dict):
+                self.__dict__[k] = DictDotLookup(d[k])
+            elif isinstance(d[k], (list, tuple)):
+                l = []
+                for v in d[k]:
+                    if isinstance(v, dict):
+                        l.append(DictDotLookup(v))
+                    else:
+                        l.append(v)
+                self.__dict__[k] = l
+            else:
+                self.__dict__[k] = d[k]
+
+    def __getitem__(self, name):
+        if name in self.__dict__:
+            return self.__dict__[name]
+
+    def __iter__(self):
+        return iter(self.__dict__.keys())

datacenterlight/views.py

@@ -4,7 +4,6 @@ from .forms import BetaAccessForm
 from .models import BetaAccess, BetaAccessVMType, BetaAccessVM, VMTemplate
 from django.contrib import messages
 from django.core.urlresolvers import reverse
-from django.core.mail import EmailMessage
 from utils.mailer import BaseEmail
 from django.shortcuts import render
 from django.shortcuts import redirect
@@ -13,14 +12,14 @@ from django.core.exceptions import ValidationError
 from django.views.decorators.cache import cache_control
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
-from utils.forms import BillingAddressForm, UserBillingAddressForm
+from utils.forms import BillingAddressForm
 from utils.models import BillingAddress
-from hosting.models import HostingOrder, HostingBill
+from hosting.models import HostingOrder
 from utils.stripe_utils import StripeUtils
-from datetime import datetime
 from membership.models import CustomUser, StripeCustomer
 from opennebula_api.models import OpenNebulaManager
-from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer
+from opennebula_api.serializers import VirtualMachineTemplateSerializer, VMTemplateSerializer
+from datacenterlight.tasks import create_vm_task
 
 
 class LandingProgramView(TemplateView):
@@ -33,7 +32,6 @@ class SuccessView(TemplateView):
     def get(self, request, *args, **kwargs):
         if 'specs' not in request.session or 'user' not in request.session:
             return HttpResponseRedirect(reverse('datacenterlight:index'))
-
         elif 'token' not in request.session:
             return HttpResponseRedirect(reverse('datacenterlight:payment'))
         elif 'order_confirmation' not in request.session:
@@ -79,8 +77,7 @@ class PricingView(TemplateView):
         manager = OpenNebulaManager()
         template = manager.get_template(template_id)
 
-        request.session['template'] = VirtualMachineTemplateSerializer(
+        request.session['template'] = VirtualMachineTemplateSerializer(template).data
-            template).data
 
         if not request.user.is_authenticated():
             request.session['next'] = reverse('hosting:payment')
@@ -132,8 +129,7 @@ class BetaAccessView(FormView):
         email = BaseEmail(**email_data)
         email.send()
 
-        messages.add_message(
+        messages.add_message(self.request, messages.SUCCESS, self.success_message)
-            self.request, messages.SUCCESS, self.success_message)
         return render(self.request, 'datacenterlight/beta_success.html', {})
 
 
@@ -185,8 +181,7 @@ class BetaProgramView(CreateView):
         email = BaseEmail(**email_data)
         email.send()
 
-        messages.add_message(
+        messages.add_message(self.request, messages.SUCCESS, self.success_message)
-            self.request, messages.SUCCESS, self.success_message)
         return HttpResponseRedirect(self.get_success_url())
 
 
@@ -230,8 +225,7 @@ class IndexView(CreateView):
         storage_field = forms.IntegerField(validators=[self.validate_storage])
         price = request.POST.get('total')
         template_id = int(request.POST.get('config'))
-        template = VMTemplate.objects.filter(
+        template = VMTemplate.objects.filter(opennebula_vm_template_id=template_id).first()
-            opennebula_vm_template_id=template_id).first()
         template_data = VMTemplateSerializer(template).data
 
         name = request.POST.get('name')
@@ -243,40 +237,35 @@ class IndexView(CreateView):
             cores = cores_field.clean(cores)
         except ValidationError as err:
             msg = '{} : {}.'.format(cores, str(err))
-            messages.add_message(
+            messages.add_message(self.request, messages.ERROR, msg, extra_tags='cores')
-                self.request, messages.ERROR, msg, extra_tags='cores')
             return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
 
         try:
             memory = memory_field.clean(memory)
         except ValidationError as err:
             msg = '{} : {}.'.format(memory, str(err))
-            messages.add_message(
+            messages.add_message(self.request, messages.ERROR, msg, extra_tags='memory')
-                self.request, messages.ERROR, msg, extra_tags='memory')
             return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
 
         try:
             storage = storage_field.clean(storage)
         except ValidationError as err:
             msg = '{} : {}.'.format(storage, str(err))
-            messages.add_message(
+            messages.add_message(self.request, messages.ERROR, msg, extra_tags='storage')
-                self.request, messages.ERROR, msg, extra_tags='storage')
             return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
 
         try:
             name = name_field.clean(name)
         except ValidationError as err:
             msg = '{} {}.'.format(name, _('is not a proper name'))
-            messages.add_message(
+            messages.add_message(self.request, messages.ERROR, msg, extra_tags='name')
-                self.request, messages.ERROR, msg, extra_tags='name')
             return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
 
         try:
             email = email_field.clean(email)
         except ValidationError as err:
             msg = '{} {}.'.format(email, _('is not a proper email'))
-            messages.add_message(
+            messages.add_message(self.request, messages.ERROR, msg, extra_tags='email')
-                self.request, messages.ERROR, msg, extra_tags='email')
             return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
 
         specs = {
@@ -341,8 +330,7 @@ class IndexView(CreateView):
         email = BaseEmail(**email_data)
         email.send()
 
-        messages.add_message(
+        messages.add_message(self.request, messages.SUCCESS, self.success_message)
-            self.request, messages.SUCCESS, self.success_message)
         return super(IndexView, self).form_valid(form)
 
 
@@ -411,7 +399,6 @@ class PaymentOrderView(FormView):
 
             # Create Billing Address
             billing_address = form.save()
-
             request.session['billing_address_data'] = billing_address_data
             request.session['billing_address'] = billing_address.id
             request.session['token'] = token
@@ -436,13 +423,11 @@ class OrderConfirmationView(DetailView):
         stripe_customer_id = request.session.get('customer')
         customer = StripeCustomer.objects.filter(id=stripe_customer_id).first()
         stripe_utils = StripeUtils()
-        card_details = stripe_utils.get_card_details(
+        card_details = stripe_utils.get_card_details(customer.stripe_id, request.session.get('token'))
-            customer.stripe_id, request.session.get('token'))
         if not card_details.get('response_object') and not card_details.get('paid'):
             msg = card_details.get('error')
             messages.add_message(self.request, messages.ERROR, msg, extra_tags='failed_payment')
             return HttpResponseRedirect(reverse('datacenterlight:payment') + '#payment_error')
-
         context = {
             'site_url': reverse('datacenterlight:index'),
             'cc_last4': card_details.get('response_object').get('last4'),
@@ -458,8 +443,6 @@ class OrderConfirmationView(DetailView):
         customer = StripeCustomer.objects.filter(id=stripe_customer_id).first()
         billing_address_data = request.session.get('billing_address_data')
         billing_address_id = request.session.get('billing_address')
-        billing_address = BillingAddress.objects.filter(
-            id=billing_address_id).first()
         vm_template_id = template.get('id', 1)
         final_price = specs.get('price')
 
@@ -475,72 +458,8 @@ class OrderConfirmationView(DetailView):
             return HttpResponseRedirect(reverse('datacenterlight:payment') + '#payment_error')
 
         charge = charge_response.get('response_object')
-
+        create_vm_task.delay(vm_template_id, user, specs, template, stripe_customer_id, billing_address_data,
-        # Create OpenNebulaManager
+                             billing_address_id,
-        manager = OpenNebulaManager(email=settings.OPENNEBULA_USERNAME,
+                             charge)
-                                    password=settings.OPENNEBULA_PASSWORD)
-
-        # Create a vm using oneadmin, also specify the name
-        vm_id = manager.create_vm(
-            template_id=vm_template_id,
-            specs=specs,
-            ssh_key=settings.ONEADMIN_USER_SSH_PUBLIC_KEY,
-            vm_name="{email}-{template_name}-{date}".format(
-                email=user.get('email'),
-                template_name=template.get('name'),
-                date=int(datetime.now().strftime("%s")))
-        )
-
-        # Create a Hosting Order
-        order = HostingOrder.create(
-            price=final_price,
-            vm_id=vm_id,
-            customer=customer,
-            billing_address=billing_address
-        )
-
-        # Create a Hosting Bill
-        HostingBill.create(
-            customer=customer, billing_address=billing_address)
-
-        # Create Billing Address for User if he does not have one
-        if not customer.user.billing_addresses.count():
-            billing_address_data.update({
-                'user': customer.user.id
-            })
-            billing_address_user_form = UserBillingAddressForm(
-                billing_address_data)
-            billing_address_user_form.is_valid()
-            billing_address_user_form.save()
-
-        # Associate an order with a stripe payment
-        order.set_stripe_charge(charge)
-
-        # If the Stripe payment was successed, set order status approved
-        order.set_approved()
-
-        vm = VirtualMachineSerializer(manager.get_vm(vm_id)).data
-
-        context = {
-            'name': user.get('name'),
-            'email': user.get('email'),
-            'cores': specs.get('cpu'),
-            'memory': specs.get('memory'),
-            'storage': specs.get('disk_size'),
-            'price': specs.get('price'),
-            'template': template.get('name'),
-            'vm.name': vm['name'],
-            'vm.id': vm['vm_id'],
-            'order.id': order.id
-        }
-        email_data = {
-            'subject': settings.DCL_TEXT + " Order from %s" % context['email'],
-            'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
-            'to': ['info@ungleich.ch'],
-            'body': "\n".join(["%s=%s" % (k, v) for (k, v) in context.items()]),
-            'reply_to': [context['email']],
-        }
-        email = EmailMessage(**email_data)
-        email.send()
         request.session['order_confirmation'] = True
         return HttpResponseRedirect(reverse('datacenterlight:order_success'))

deploy.sh

@@ -13,6 +13,7 @@ while true; do
   case "$1" in
     -h | --help ) HELP=true; shift ;;
     -v | --verbose ) VERBOSE=true; shift ;;
+    -D | --dbmakemigrations ) DB_MAKE_MIGRATIONS=true; shift ;;
     -d | --dbmigrate ) DB_MIGRATE=true; shift ;;
     -n | --nogit ) NO_GIT=true; shift ;;
     -b | --branch ) BRANCH="$2"; shift 2 ;;
@@ -31,13 +32,15 @@ if [ "$HELP" == "true" ]; then
     echo "options are : "
     echo "        -h, --help: Print this help message"
     echo "        -v, --verbose: Show verbose output to stdout. Without this a deploy.log is written to ~/app folder"
-    echo "        -d, --dbmigrate: Do DB migrate"
+    echo "        -D, --dbmakemigrations: Do DB makemigrations"
-    echo "        -n, --nogit: Don't execute git commands. With this --branch has no effect."
+    echo "        -d, --dbmigrate: Do DB migrate. To do both makemigrations and migrate, supply both switches -D and -d"
+    echo "        -n, --nogit: Don't execute git commands. This is used to deploy the current code in the project repo. With this --branch has no effect."
     echo "        -b, --branch: The branch to pull from origin repo."
     exit
 fi
 
 echo "BRANCH="$BRANCH
+echo "DB_MAKE_MIGRATIONS="$DB_MAKE_MIGRATIONS
 echo "DB_MIGRATE="$DB_MIGRATE
 echo "NO_GIT="$NO_GIT
 echo "VERBOSE="$VERBOSE
@@ -45,7 +48,7 @@ echo "VERBOSE="$VERBOSE
 # The project directory exists, we pull the specified branch
 cd $APP_HOME_DIR
 if [ -z "$NO_GIT" ]; then
-    echo 'We are executing default git commands. Please -no_git to not use this.'
+    echo 'We are executing default git commands. Please add --nogit to not do this.'
     # Save any modified changes before git pulling
     git stash
     # Fetch all branches/tags
@@ -59,16 +62,23 @@ fi
 source ~/pyvenv/bin/activate
 pip install -r requirements.txt > deploy.log 2>&1
 echo "###" >> deploy.log
-if [ -z "$DB_MIGRATE" ]; then
+if [ -z "$DB_MAKE_MIGRATIONS" ]; then
-    echo 'We are not doing DB migration'
+    echo 'We are not doing DB makemigrations'
 else
+    echo 'Doing DB makemigrations'
     ./manage.py makemigrations >> deploy.log 2>&1
     echo "###" >> deploy.log
+fi
+if [ -z "$DB_MIGRATE" ]; then
+    echo 'We are not doing DB migrate'
+else
+    echo 'Doing DB migrate'
     ./manage.py migrate >> deploy.log 2>&1
     echo "###" >> deploy.log
 fi
 printf 'yes' | ./manage.py collectstatic >> deploy.log 2>&1
 echo "###" >> deploy.log
 django-admin compilemessages
+sudo systemctl restart celery.service
 sudo systemctl restart uwsgi
 

dynamicweb/__init__.py

@@ -0,0 +1,5 @@
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
+
+__all__ = ['celery_app']

dynamicweb/celery.py

@@ -0,0 +1,21 @@
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dynamicweb.settings')
+
+app = Celery('dynamicweb')
+
+# Using a string here means the worker don't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+#   should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks()
+
+
+@app.task(bind=True)
+def debug_task(self):
+    print('Request: {0!r}'.format(self.request))

dynamicweb/settings/base.py

@@ -10,6 +10,9 @@ from django.utils.translation import ugettext_lazy as _
 
 # dotenv
 import dotenv
+import logging
+
+logger = logging.getLogger(__name__)
 
 
 def gettext(s):
@@ -25,6 +28,21 @@ def bool_env(val):
     return True if os.environ.get(val, False) == 'True' else False
 
 
+def int_env(val, default_value=0):
+    """Replaces string based environment values with Python integers
+    Return default_value if val is not set or cannot be parsed, otherwise
+    returns the python integer equal to the passed val
+    """
+    return_value = default_value
+    try:
+        return_value = int(os.environ.get(val))
+    except Exception as e:
+        logger.error("Encountered exception trying to get env value for {}\nException details: {}".format(
+            val, str(e)))
+
+    return return_value
+
+
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 PROJECT_DIR = os.path.abspath(
@@ -120,7 +138,8 @@ INSTALLED_APPS = (
     'datacenterlight.templatetags',
     'alplora',
     'rest_framework',
-    'opennebula_api'
+    'opennebula_api',
+    'django_celery_results',
 )
 
 MIDDLEWARE_CLASSES = (
@@ -524,6 +543,15 @@ GOOGLE_ANALYTICS_PROPERTY_IDS = {
     'dynamicweb-staging.ungleich.ch': 'staging'
 }
 
+# CELERY Settings
+CELERY_BROKER_URL = env('CELERY_BROKER_URL')
+CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND')
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_TIMEZONE = 'Europe/Zurich'
+CELERY_MAX_RETRIES = int_env('CELERY_MAX_RETRIES', 5)
+
 ENABLE_DEBUG_LOGGING = bool_env('ENABLE_DEBUG_LOGGING')
 
 if ENABLE_DEBUG_LOGGING:

hosting/models.py

@@ -1,13 +1,9 @@
 import os
 import logging
 
-
 from django.db import models
 from django.utils.functional import cached_property
-
-
 from Crypto.PublicKey import RSA
-
 from membership.models import StripeCustomer, CustomUser
 from utils.models import BillingAddress
 from utils.mixins import AssignPermissionsMixin
@@ -42,7 +38,6 @@ class HostingPlan(models.Model):
 
 
 class HostingOrder(AssignPermissionsMixin, models.Model):
-
     ORDER_APPROVED_STATUS = 'Approved'
     ORDER_DECLINED_STATUS = 'Declined'
 
@@ -101,7 +96,7 @@ class HostingOrder(AssignPermissionsMixin, models.Model):
 class UserHostingKey(models.Model):
     user = models.ForeignKey(CustomUser)
     public_key = models.TextField()
-    private_key = models.FileField(upload_to='private_keys',  blank=True)
+    private_key = models.FileField(upload_to='private_keys', blank=True)
     created_at = models.DateTimeField(auto_now_add=True)
     name = models.CharField(max_length=100)