From d8592fc6d8d3e1c3efba54dfe478a1f0d582b917 Mon Sep 17 00:00:00 2001
From: PCoder <purple.coder@yahoo.co.uk>
Date: Sun, 6 Aug 2017 12:30:56 +0530
Subject: [PATCH 01/21] Added celery files
---
dynamicweb/__init__.py | 5 +++++
dynamicweb/celery.py | 22 ++++++++++++++++++++++
dynamicweb/settings/base.py | 9 +++++++++
requirements.txt | 2 ++
4 files changed, 38 insertions(+)
create mode 100644 dynamicweb/celery.py
diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py
index e69de29b..b64e43e8 100644
--- a/dynamicweb/__init__.py
+++ b/dynamicweb/__init__.py
@@ -0,0 +1,5 @@
+from __future__ import absolute_import
+
+# This will make sure the app is always imported when
+# Django starts so that shared_task will use this app.
+from .celery import app as celery_app
diff --git a/dynamicweb/celery.py b/dynamicweb/celery.py
new file mode 100644
index 00000000..749ffdef
--- /dev/null
+++ b/dynamicweb/celery.py
@@ -0,0 +1,22 @@
+from __future__ import absolute_import, unicode_literals
+import os
+from celery import Celery
+
+# set the default Django settings module for the 'celery' program.
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dynamicweb.settings')
+
+app = Celery('dynamicweb')
+
+# Using a string here means the worker don't have to serialize
+# the configuration object to child processes.
+# - namespace='CELERY' means all celery-related configuration keys
+# should have a `CELERY_` prefix.
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+# Load task modules from all registered Django app configs.
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+
+
+@app.task(bind=True)
+def debug_task(self):
+ print('Request: {0!r}'.format(self.request))
diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py
index 40187f84..f4c4b68d 100644
--- a/dynamicweb/settings/base.py
+++ b/dynamicweb/settings/base.py
@@ -521,6 +521,15 @@ GOOGLE_ANALYTICS_PROPERTY_IDS = {
'dynamicweb-staging.ungleich.ch': 'staging'
}
+# CELERY Settings
+#BROKER_URL = 'redis://localhost:6379'
+BROKER_URL = 'redis+socket:///var/run/redis/redis.sock'
+CELERY_RESULT_BACKEND = 'redis://localhost:6379'
+CELERY_ACCEPT_CONTENT = ['application/json']
+CELERY_TASK_SERIALIZER = 'json'
+CELERY_RESULT_SERIALIZER = 'json'
+CELERY_TIMEZONE = 'Europe/Zurich'
+
ENABLE_DEBUG_LOGGING = bool_env('ENABLE_DEBUG_LOGGING')
if ENABLE_DEBUG_LOGGING:
diff --git a/requirements.txt b/requirements.txt
index f392f4d9..2520d617 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -86,3 +86,5 @@ git+https://github.com/ungleich/python-oca.git#egg=python-oca
djangorestframework
flake8==3.3.0
python-memcached==1.58
+celery==4.0.2
+redis==2.10.5
From 3ad5928aa0930d7a6a41b0791f179cf847cf7a31 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 14:33:55 +0200
Subject: [PATCH 02/21] A working version of celery create_vm_task added
---
datacenterlight/views.py | 121 +++++++--------------------------------
1 file changed, 22 insertions(+), 99 deletions(-)
diff --git a/datacenterlight/views.py b/datacenterlight/views.py
index f7a07c7e..b2798844 100644
--- a/datacenterlight/views.py
+++ b/datacenterlight/views.py
@@ -4,7 +4,6 @@ from .forms import BetaAccessForm
from .models import BetaAccess, BetaAccessVMType, BetaAccessVM, VMTemplate
from django.contrib import messages
from django.core.urlresolvers import reverse
-from django.core.mail import EmailMessage
from utils.mailer import BaseEmail
from django.shortcuts import render
from django.shortcuts import redirect
@@ -13,14 +12,15 @@ from django.core.exceptions import ValidationError
from django.views.decorators.cache import cache_control
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
-from utils.forms import BillingAddressForm, UserBillingAddressForm
+from utils.forms import BillingAddressForm
from utils.models import BillingAddress
-from hosting.models import HostingOrder, HostingBill
+from hosting.models import HostingOrder
from utils.stripe_utils import StripeUtils
-from datetime import datetime
from membership.models import CustomUser, StripeCustomer
+
from opennebula_api.models import OpenNebulaManager
from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer
+from datacenterlight.tasks import create_vm_task
class LandingProgramView(TemplateView):
@@ -33,7 +33,6 @@ class SuccessView(TemplateView):
def get(self, request, *args, **kwargs):
if 'specs' not in request.session or 'user' not in request.session:
return HttpResponseRedirect(reverse('datacenterlight:index'))
-
elif 'token' not in request.session:
return HttpResponseRedirect(reverse('datacenterlight:payment'))
elif 'order_confirmation' not in request.session:
@@ -79,8 +78,7 @@ class PricingView(TemplateView):
manager = OpenNebulaManager()
template = manager.get_template(template_id)
- request.session['template'] = VirtualMachineTemplateSerializer(
- template).data
+ request.session['template'] = VirtualMachineTemplateSerializer(template).data
if not request.user.is_authenticated():
request.session['next'] = reverse('hosting:payment')
@@ -132,8 +130,7 @@ class BetaAccessView(FormView):
email = BaseEmail(**email_data)
email.send()
- messages.add_message(
- self.request, messages.SUCCESS, self.success_message)
+ messages.add_message(self.request, messages.SUCCESS, self.success_message)
return render(self.request, 'datacenterlight/beta_success.html', {})
@@ -185,8 +182,7 @@ class BetaProgramView(CreateView):
email = BaseEmail(**email_data)
email.send()
- messages.add_message(
- self.request, messages.SUCCESS, self.success_message)
+ messages.add_message(self.request, messages.SUCCESS, self.success_message)
return HttpResponseRedirect(self.get_success_url())
@@ -199,15 +195,15 @@ class IndexView(CreateView):
def validate_cores(self, value):
if (value > 48) or (value < 1):
- raise ValidationError(_('Invalid number of cores'))
+ raise ValidationError(_('Not a proper cores number'))
def validate_memory(self, value):
if (value > 200) or (value < 2):
- raise ValidationError(_('Invalid RAM size'))
+ raise ValidationError(_('Not a proper ram number'))
def validate_storage(self, value):
if (value > 2000) or (value < 10):
- raise ValidationError(_('Invalid storage size'))
+ raise ValidationError(_('Not a proper storage number'))
@cache_control(no_cache=True, must_revalidate=True, no_store=True)
def get(self, request, *args, **kwargs):
@@ -230,8 +226,7 @@ class IndexView(CreateView):
storage_field = forms.IntegerField(validators=[self.validate_storage])
price = request.POST.get('total')
template_id = int(request.POST.get('config'))
- template = VMTemplate.objects.filter(
- opennebula_vm_template_id=template_id).first()
+ template = VMTemplate.objects.filter(opennebula_vm_template_id=template_id).first()
template_data = VMTemplateSerializer(template).data
name = request.POST.get('name')
@@ -243,40 +238,35 @@ class IndexView(CreateView):
cores = cores_field.clean(cores)
except ValidationError as err:
msg = '{} : {}.'.format(cores, str(err))
- messages.add_message(
- self.request, messages.ERROR, msg, extra_tags='cores')
+ messages.add_message(self.request, messages.ERROR, msg, extra_tags='cores')
return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
try:
memory = memory_field.clean(memory)
except ValidationError as err:
msg = '{} : {}.'.format(memory, str(err))
- messages.add_message(
- self.request, messages.ERROR, msg, extra_tags='memory')
+ messages.add_message(self.request, messages.ERROR, msg, extra_tags='memory')
return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
try:
storage = storage_field.clean(storage)
except ValidationError as err:
msg = '{} : {}.'.format(storage, str(err))
- messages.add_message(
- self.request, messages.ERROR, msg, extra_tags='storage')
+ messages.add_message(self.request, messages.ERROR, msg, extra_tags='storage')
return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
try:
name = name_field.clean(name)
except ValidationError as err:
msg = '{} {}.'.format(name, _('is not a proper name'))
- messages.add_message(
- self.request, messages.ERROR, msg, extra_tags='name')
+ messages.add_message(self.request, messages.ERROR, msg, extra_tags='name')
return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
try:
email = email_field.clean(email)
except ValidationError as err:
msg = '{} {}.'.format(email, _('is not a proper email'))
- messages.add_message(
- self.request, messages.ERROR, msg, extra_tags='email')
+ messages.add_message(self.request, messages.ERROR, msg, extra_tags='email')
return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form")
specs = {
@@ -341,8 +331,7 @@ class IndexView(CreateView):
email = BaseEmail(**email_data)
email.send()
- messages.add_message(
- self.request, messages.SUCCESS, self.success_message)
+ messages.add_message(self.request, messages.SUCCESS, self.success_message)
return super(IndexView, self).form_valid(form)
@@ -411,7 +400,6 @@ class PaymentOrderView(FormView):
# Create Billing Address
billing_address = form.save()
-
request.session['billing_address_data'] = billing_address_data
request.session['billing_address'] = billing_address.id
request.session['token'] = token
@@ -436,8 +424,7 @@ class OrderConfirmationView(DetailView):
stripe_customer_id = request.session.get('customer')
customer = StripeCustomer.objects.filter(id=stripe_customer_id).first()
stripe_utils = StripeUtils()
- card_details = stripe_utils.get_card_details(
- customer.stripe_id, request.session.get('token'))
+ card_details = stripe_utils.get_card_details(customer.stripe_id, request.session.get('token'))
context = {
'site_url': reverse('datacenterlight:index'),
'cc_last4': card_details.get('response_object').get('last4'),
@@ -453,8 +440,7 @@ class OrderConfirmationView(DetailView):
customer = StripeCustomer.objects.filter(id=stripe_customer_id).first()
billing_address_data = request.session.get('billing_address_data')
billing_address_id = request.session.get('billing_address')
- billing_address = BillingAddress.objects.filter(
- id=billing_address_id).first()
+ billing_address = BillingAddress.objects.filter(id=billing_address_id).first()
vm_template_id = template.get('id', 1)
final_price = specs.get('price')
@@ -473,71 +459,8 @@ class OrderConfirmationView(DetailView):
return render(request, self.payment_template_name, context)
charge = charge_response.get('response_object')
-
- # Create OpenNebulaManager
- manager = OpenNebulaManager(email=settings.OPENNEBULA_USERNAME,
- password=settings.OPENNEBULA_PASSWORD)
-
- # Create a vm using oneadmin, also specify the name
- vm_id = manager.create_vm(
- template_id=vm_template_id,
- specs=specs,
- vm_name="{email}-{template_name}-{date}".format(
- email=user.get('email'),
- template_name=template.get('name'),
- date=int(datetime.now().strftime("%s")))
- )
-
- # Create a Hosting Order
- order = HostingOrder.create(
- price=final_price,
- vm_id=vm_id,
- customer=customer,
- billing_address=billing_address
- )
-
- # Create a Hosting Bill
- HostingBill.create(
- customer=customer, billing_address=billing_address)
-
- # Create Billing Address for User if he does not have one
- if not customer.user.billing_addresses.count():
- billing_address_data.update({
- 'user': customer.user.id
- })
- billing_address_user_form = UserBillingAddressForm(
- billing_address_data)
- billing_address_user_form.is_valid()
- billing_address_user_form.save()
-
- # Associate an order with a stripe payment
- order.set_stripe_charge(charge)
-
- # If the Stripe payment was successed, set order status approved
- order.set_approved()
-
- vm = VirtualMachineSerializer(manager.get_vm(vm_id)).data
-
- context = {
- 'name': user.get('name'),
- 'email': user.get('email'),
- 'cores': specs.get('cpu'),
- 'memory': specs.get('memory'),
- 'storage': specs.get('disk_size'),
- 'price': specs.get('price'),
- 'template': template.get('name'),
- 'vm.name': vm['name'],
- 'vm.id': vm['vm_id'],
- 'order.id': order.id
- }
- email_data = {
- 'subject': settings.DCL_TEXT + " Order from %s" % context['email'],
- 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
- 'to': ['info@ungleich.ch'],
- 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in context.items()]),
- 'reply_to': [context['email']],
- }
- email = EmailMessage(**email_data)
- email.send()
+ create_vm_task.delay(vm_template_id, user, specs, template, stripe_customer_id, billing_address_data,
+ billing_address_id,
+ charge)
request.session['order_confirmation'] = True
return HttpResponseRedirect(reverse('datacenterlight:order_success'))
From bd2edf599ae4b8e02cc43167821d9bf3adac75e3 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 15:08:05 +0200
Subject: [PATCH 03/21] Merged views from master
---
datacenterlight/views.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
diff --git a/datacenterlight/views.py b/datacenterlight/views.py
index b2798844..e37b2914 100644
--- a/datacenterlight/views.py
+++ b/datacenterlight/views.py
@@ -17,7 +17,6 @@ from utils.models import BillingAddress
from hosting.models import HostingOrder
from utils.stripe_utils import StripeUtils
from membership.models import CustomUser, StripeCustomer
-
from opennebula_api.models import OpenNebulaManager
from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer
from datacenterlight.tasks import create_vm_task
@@ -195,15 +194,15 @@ class IndexView(CreateView):
def validate_cores(self, value):
if (value > 48) or (value < 1):
- raise ValidationError(_('Not a proper cores number'))
+ raise ValidationError(_('Invalid number of cores'))
def validate_memory(self, value):
if (value > 200) or (value < 2):
- raise ValidationError(_('Not a proper ram number'))
+ raise ValidationError(_('Invalid RAM size'))
def validate_storage(self, value):
if (value > 2000) or (value < 10):
- raise ValidationError(_('Not a proper storage number'))
+ raise ValidationError(_('Invalid storage size'))
@cache_control(no_cache=True, must_revalidate=True, no_store=True)
def get(self, request, *args, **kwargs):
From d6d0b9c8c0eed4394d3610ccade37cbc9395fefc Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 15:08:46 +0200
Subject: [PATCH 04/21] Added default init code for celery
---
dynamicweb/__init__.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py
index b64e43e8..de98902b 100644
--- a/dynamicweb/__init__.py
+++ b/dynamicweb/__init__.py
@@ -1,5 +1,7 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, unicode_literals
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
+
+__all__ = ['celery_app']
\ No newline at end of file
From 477c8e81a40aeef502764082c5c22b02c77e512a Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 15:10:26 +0200
Subject: [PATCH 05/21] Added default celery.py code
---
dynamicweb/celery.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/dynamicweb/celery.py b/dynamicweb/celery.py
index 749ffdef..28f6af3b 100644
--- a/dynamicweb/celery.py
+++ b/dynamicweb/celery.py
@@ -14,7 +14,7 @@ app = Celery('dynamicweb')
app.config_from_object('django.conf:settings', namespace='CELERY')
# Load task modules from all registered Django app configs.
-app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)
+app.autodiscover_tasks()
@app.task(bind=True)
From 68505e73a8c834f3728e633c38dbf89e7b70de96 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 16:32:27 +0200
Subject: [PATCH 06/21] Added django-celery-results and setup redis as backend
for celery tasks
---
dynamicweb/settings/base.py | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py
index f4c4b68d..4847851f 100644
--- a/dynamicweb/settings/base.py
+++ b/dynamicweb/settings/base.py
@@ -120,7 +120,8 @@ INSTALLED_APPS = (
'datacenterlight.templatetags',
'alplora',
'rest_framework',
- 'opennebula_api'
+ 'opennebula_api',
+ 'django_celery_results',
)
MIDDLEWARE_CLASSES = (
@@ -522,9 +523,10 @@ GOOGLE_ANALYTICS_PROPERTY_IDS = {
}
# CELERY Settings
-#BROKER_URL = 'redis://localhost:6379'
-BROKER_URL = 'redis+socket:///var/run/redis/redis.sock'
-CELERY_RESULT_BACKEND = 'redis://localhost:6379'
+# BROKER_URL = 'redis://localhost:6379'
+CELERY_BROKER_URL = 'redis+socket:///tmp/redis.sock'
+# CELERY_RESULT_BACKEND = 'redis://localhost:6379'
+CELERY_RESULT_BACKEND = 'redis+socket:///tmp/redis.sock'
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
From 55177646b456be63805604eb5d2188d0c9bd5d4f Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 16:33:37 +0200
Subject: [PATCH 07/21] Reformatted code
---
hosting/models.py | 7 +------
1 file changed, 1 insertion(+), 6 deletions(-)
diff --git a/hosting/models.py b/hosting/models.py
index 88386913..8cdc6114 100644
--- a/hosting/models.py
+++ b/hosting/models.py
@@ -1,13 +1,9 @@
import os
import logging
-
from django.db import models
from django.utils.functional import cached_property
-
-
from Crypto.PublicKey import RSA
-
from membership.models import StripeCustomer, CustomUser
from utils.models import BillingAddress
from utils.mixins import AssignPermissionsMixin
@@ -42,7 +38,6 @@ class HostingPlan(models.Model):
class HostingOrder(AssignPermissionsMixin, models.Model):
-
ORDER_APPROVED_STATUS = 'Approved'
ORDER_DECLINED_STATUS = 'Declined'
@@ -101,7 +96,7 @@ class HostingOrder(AssignPermissionsMixin, models.Model):
class UserHostingKey(models.Model):
user = models.ForeignKey(CustomUser)
public_key = models.TextField()
- private_key = models.FileField(upload_to='private_keys', blank=True)
+ private_key = models.FileField(upload_to='private_keys', blank=True)
created_at = models.DateTimeField(auto_now_add=True)
name = models.CharField(max_length=100)
From 6d28783f9ffd526aa076289379ffdca927c693b1 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 16:34:23 +0200
Subject: [PATCH 08/21] Added django-celery-results to requirements
---
requirements.txt | 1 +
1 file changed, 1 insertion(+)
diff --git a/requirements.txt b/requirements.txt
index 2520d617..abce2271 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -88,3 +88,4 @@ flake8==3.3.0
python-memcached==1.58
celery==4.0.2
redis==2.10.5
+django-celery-results==1.0.1
From 5a5654be018239ae192864316ceb5a4e4c54be99 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 16:51:07 +0200
Subject: [PATCH 09/21] Moved CELERY_BROKER_URL and CELERY_RESULT_BACKEND to
.env file
---
dynamicweb/settings/base.py | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py
index 4847851f..fbe0c9ca 100644
--- a/dynamicweb/settings/base.py
+++ b/dynamicweb/settings/base.py
@@ -523,10 +523,8 @@ GOOGLE_ANALYTICS_PROPERTY_IDS = {
}
# CELERY Settings
-# BROKER_URL = 'redis://localhost:6379'
-CELERY_BROKER_URL = 'redis+socket:///tmp/redis.sock'
-# CELERY_RESULT_BACKEND = 'redis://localhost:6379'
-CELERY_RESULT_BACKEND = 'redis+socket:///tmp/redis.sock'
+CELERY_BROKER_URL = env('CELERY_BROKER_URL')
+CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND')
CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
From 26b4feb1d16296036ccb3fe75f11f96501f3e964 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 18:39:45 +0200
Subject: [PATCH 10/21] Introduced int_env and CELERY_MAX_RETRIES env variable
---
dynamicweb/settings/base.py | 18 ++++++++++++++++++
1 file changed, 18 insertions(+)
diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py
index fbe0c9ca..58609234 100644
--- a/dynamicweb/settings/base.py
+++ b/dynamicweb/settings/base.py
@@ -10,6 +10,9 @@ from django.utils.translation import ugettext_lazy as _
# dotenv
import dotenv
+import logging
+
+logger = logging.getLogger(__name__)
def gettext(s):
@@ -25,6 +28,20 @@ def bool_env(val):
return True if os.environ.get(val, False) == 'True' else False
+def int_env(val, default_value=0):
+ """Replaces string based environment values with Python integers
+ Return default_value if val is not set or cannot be parsed, otherwise
+ returns the python integer equal to the passed val
+ """
+ return_value = default_value
+ try:
+ return_value = int(os.environ.get(val))
+ except Exception as e:
+ logger.error(str(e))
+
+ return return_value
+
+
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PROJECT_DIR = os.path.abspath(
@@ -529,6 +546,7 @@ CELERY_ACCEPT_CONTENT = ['application/json']
CELERY_TASK_SERIALIZER = 'json'
CELERY_RESULT_SERIALIZER = 'json'
CELERY_TIMEZONE = 'Europe/Zurich'
+CELERY_MAX_RETRIES = int_env('CELERY_MAX_RETRIES', 5)
ENABLE_DEBUG_LOGGING = bool_env('ENABLE_DEBUG_LOGGING')
From f950c1defb23a0cba62bbbbb732a5428f2f416a5 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 18:42:27 +0200
Subject: [PATCH 11/21] Added datacenterlight/tasks.py
---
datacenterlight/tasks.py | 162 +++++++++++++++++++++++++++++++++++++++
1 file changed, 162 insertions(+)
create mode 100644 datacenterlight/tasks.py
diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py
new file mode 100644
index 00000000..69ba08ad
--- /dev/null
+++ b/datacenterlight/tasks.py
@@ -0,0 +1,162 @@
+from __future__ import absolute_import, unicode_literals
+from dynamicweb.celery import app
+from celery.utils.log import get_task_logger
+from django.conf import settings
+from opennebula_api.models import OpenNebulaManager
+from opennebula_api.serializers import VirtualMachineSerializer
+from hosting.models import HostingOrder, HostingBill
+from utils.forms import UserBillingAddressForm
+from datetime import datetime
+from membership.models import StripeCustomer
+from django.core.mail import EmailMessage
+from utils.models import BillingAddress
+
+logger = get_task_logger(__name__)
+
+
+def retry_task(task, exception=None):
+ """Retries the specified task using a "backing off countdown",
+ meaning that the interval between retries grows exponentially
+ with every retry.
+
+ Arguments:
+ task:
+ The task to retry.
+
+ exception:
+ Optionally, the exception that caused the retry.
+ """
+
+ def backoff(attempts):
+ return 2 ** attempts
+
+ kwargs = {
+ 'countdown': backoff(task.request.retries),
+ }
+
+ if exception:
+ kwargs['exc'] = exception
+
+ if task.request.retries > settings.CELERY_MAX_RETRIES:
+ msg_text = 'Finished {} retries for create_vm_task'.format(task.request.retries)
+ logger.log(msg_text)
+ # Try sending email and stop
+ email_data = {
+ 'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text),
+ 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
+ 'to': ['info@ungleich.ch'],
+ 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in kwargs.items()]),
+ }
+ email = EmailMessage(**email_data)
+ email.send()
+ return
+ else:
+ raise task.retry(**kwargs)
+
+
+@app.task(bind=True)
+def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_id, billing_address_data,
+ billing_address_id,
+ charge):
+ try:
+ final_price = specs.get('price')
+ billing_address = BillingAddress.objects.filter(id=billing_address_id).first()
+ customer = StripeCustomer.objects.filter(id=stripe_customer_id).first()
+ # Create OpenNebulaManager
+ manager = OpenNebulaManager(email=settings.OPENNEBULA_USERNAME,
+ password=settings.OPENNEBULA_PASSWORD)
+
+ # Create a vm using oneadmin, also specify the name
+ vm_id = manager.create_vm(
+ template_id=vm_template_id,
+ specs=specs,
+ vm_name="{email}-{template_name}-{date}".format(
+ email=user.get('email'),
+ template_name=template.get('name'),
+ date=int(datetime.now().strftime("%s")))
+ )
+
+ # Create a Hosting Order
+ order = HostingOrder.create(
+ price=final_price,
+ vm_id=vm_id,
+ customer=customer,
+ billing_address=billing_address
+ )
+
+ # Create a Hosting Bill
+ HostingBill.create(
+ customer=customer, billing_address=billing_address)
+
+ # Create Billing Address for User if he does not have one
+ if not customer.user.billing_addresses.count():
+ billing_address_data.update({
+ 'user': customer.user.id
+ })
+ billing_address_user_form = UserBillingAddressForm(
+ billing_address_data)
+ billing_address_user_form.is_valid()
+ billing_address_user_form.save()
+
+ # Associate an order with a stripe payment
+ charge_object = DictDotLookup(charge)
+ order.set_stripe_charge(charge_object)
+
+ # If the Stripe payment succeeds, set order status approved
+ order.set_approved()
+
+ vm = VirtualMachineSerializer(manager.get_vm(vm_id)).data
+
+ context = {
+ 'name': user.get('name'),
+ 'email': user.get('email'),
+ 'cores': specs.get('cpu'),
+ 'memory': specs.get('memory'),
+ 'storage': specs.get('disk_size'),
+ 'price': specs.get('price'),
+ 'template': template.get('name'),
+ 'vm.name': vm['name'],
+ 'vm.id': vm['vm_id'],
+ 'order.id': order.id
+ }
+ email_data = {
+ 'subject': settings.DCL_TEXT + " Order from %s" % context['email'],
+ 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
+ 'to': ['info@ungleich.ch'],
+ 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in context.items()]),
+ 'reply_to': [context['email']],
+ }
+ email = EmailMessage(**email_data)
+ email.send()
+ except Exception as e:
+ logger.error(str(e))
+ retry_task(self)
+
+
+class DictDotLookup(object):
+ """
+ Creates objects that behave much like a dictionaries, but allow nested
+ key access using object '.' (dot) lookups.
+ """
+
+ def __init__(self, d):
+ for k in d:
+ if isinstance(d[k], dict):
+ self.__dict__[k] = DictDotLookup(d[k])
+ elif isinstance(d[k], (list, tuple)):
+ l = []
+ for v in d[k]:
+ if isinstance(v, dict):
+ l.append(DictDotLookup(v))
+ else:
+ l.append(v)
+ self.__dict__[k] = l
+ else:
+ self.__dict__[k] = d[k]
+
+ def __getitem__(self, name):
+ if name in self.__dict__:
+ return self.__dict__[name]
+
+ def __iter__(self):
+ return iter(self.__dict__.keys())
From e7864f5d85788a44abab26ded38e5ec1808fc7a8 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 18:52:23 +0200
Subject: [PATCH 12/21] Added condition to check if create_vm succeeded
---
datacenterlight/tasks.py | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py
index 69ba08ad..9f0249a2 100644
--- a/datacenterlight/tasks.py
+++ b/datacenterlight/tasks.py
@@ -58,6 +58,7 @@ def retry_task(task, exception=None):
def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_id, billing_address_data,
billing_address_id,
charge):
+ vm_id = None
try:
final_price = specs.get('price')
billing_address = BillingAddress.objects.filter(id=billing_address_id).first()
@@ -76,6 +77,9 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_
date=int(datetime.now().strftime("%s")))
)
+ if vm_id is None:
+ raise Exception("Could not create VM")
+
# Create a Hosting Order
order = HostingOrder.create(
price=final_price,
@@ -132,6 +136,8 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_
logger.error(str(e))
retry_task(self)
+ return vm_id
+
class DictDotLookup(object):
"""
From 58bb75481491589d9024c399e448f3ae8336683d Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 21:54:12 +0200
Subject: [PATCH 13/21] Introduced MaxRetries and MaxRetriesExceededError
handling code
---
datacenterlight/tasks.py | 35 ++++++++++++++++++-----------------
1 file changed, 18 insertions(+), 17 deletions(-)
diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py
index 9f0249a2..342c87f4 100644
--- a/datacenterlight/tasks.py
+++ b/datacenterlight/tasks.py
@@ -10,6 +10,7 @@ from datetime import datetime
from membership.models import StripeCustomer
from django.core.mail import EmailMessage
from utils.models import BillingAddress
+from celery.exceptions import MaxRetriesExceededError
logger = get_task_logger(__name__)
@@ -37,24 +38,10 @@ def retry_task(task, exception=None):
if exception:
kwargs['exc'] = exception
- if task.request.retries > settings.CELERY_MAX_RETRIES:
- msg_text = 'Finished {} retries for create_vm_task'.format(task.request.retries)
- logger.log(msg_text)
- # Try sending email and stop
- email_data = {
- 'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text),
- 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
- 'to': ['info@ungleich.ch'],
- 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in kwargs.items()]),
- }
- email = EmailMessage(**email_data)
- email.send()
- return
- else:
- raise task.retry(**kwargs)
+ raise task.retry(**kwargs)
-@app.task(bind=True)
+@app.task(bind=True, max_retries=settings.CELERY_MAX_RETRIES)
def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_id, billing_address_data,
billing_address_id,
charge):
@@ -134,7 +121,21 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_
email.send()
except Exception as e:
logger.error(str(e))
- retry_task(self)
+ try:
+ retry_task(self)
+ except MaxRetriesExceededError:
+ msg_text = 'Finished {} retries for create_vm_task'.format(self.request.retries)
+ logger.error(msg_text)
+ # Try sending email and stop
+ email_data = {
+ 'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text),
+ 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
+ 'to': ['info@ungleich.ch'],
+ 'body':',\n'.join(str(i) for i in self.request.args)
+ }
+ email = EmailMessage(**email_data)
+ email.send()
+ return
return vm_id
From 29b28d551c2f3caf20929f6eb5f57187ed81c9f9 Mon Sep 17 00:00:00 2001
From: "M.Ravi" <mondi.ravi@gmail.com>
Date: Sun, 6 Aug 2017 22:19:27 +0200
Subject: [PATCH 14/21] Fixed flake8 lint warnings
---
datacenterlight/tasks.py | 2 +-
datacenterlight/views.py | 4 +---
dynamicweb/__init__.py | 2 +-
3 files changed, 3 insertions(+), 5 deletions(-)
diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py
index 342c87f4..dc71fcee 100644
--- a/datacenterlight/tasks.py
+++ b/datacenterlight/tasks.py
@@ -131,7 +131,7 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_
'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text),
'from_email': settings.DCL_SUPPORT_FROM_ADDRESS,
'to': ['info@ungleich.ch'],
- 'body':',\n'.join(str(i) for i in self.request.args)
+ 'body': ',\n'.join(str(i) for i in self.request.args)
}
email = EmailMessage(**email_data)
email.send()
diff --git a/datacenterlight/views.py b/datacenterlight/views.py
index e37b2914..2bae5c77 100644
--- a/datacenterlight/views.py
+++ b/datacenterlight/views.py
@@ -13,12 +13,11 @@ from django.views.decorators.cache import cache_control
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
from utils.forms import BillingAddressForm
-from utils.models import BillingAddress
from hosting.models import HostingOrder
from utils.stripe_utils import StripeUtils
from membership.models import CustomUser, StripeCustomer
from opennebula_api.models import OpenNebulaManager
-from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer
+from opennebula_api.serializers import VirtualMachineTemplateSerializer, VMTemplateSerializer
from datacenterlight.tasks import create_vm_task
@@ -439,7 +438,6 @@ class OrderConfirmationView(DetailView):
customer = StripeCustomer.objects.filter(id=stripe_customer_id).first()
billing_address_data = request.session.get('billing_address_data')
billing_address_id = request.session.get('billing_address')
- billing_address = BillingAddress.objects.filter(id=billing_address_id).first()
vm_template_id = template.get('id', 1)
final_price = specs.get('price')
diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py
index de98902b..3b91b070 100644
--- a/dynamicweb/__init__.py
+++ b/dynamicweb/__init__.py
@@ -4,4 +4,4 @@ from __future__ import absolute_import, unicode_literals
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
-__all__ = ['celery_app']
\ No newline at end of file
+__all__ = ['celery_app']
From 227d3a20a526f3e30915e99a2b0fc3d4ef0fbbe9 Mon Sep 17 00:00:00 2001
From: PCoder <purple.coder@yahoo.co.uk>
Date: Tue, 8 Aug 2017 01:57:29 +0530
Subject: [PATCH 15/21] Removed __future__
---
dynamicweb/__init__.py | 2 --
dynamicweb/celery.py | 1 -
2 files changed, 3 deletions(-)
diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py
index 3b91b070..1a6c551d 100644
--- a/dynamicweb/__init__.py
+++ b/dynamicweb/__init__.py
@@ -1,5 +1,3 @@
-from __future__ import absolute_import, unicode_literals
-
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery import app as celery_app
diff --git a/dynamicweb/celery.py b/dynamicweb/celery.py
index 28f6af3b..609ef5c4 100644
--- a/dynamicweb/celery.py
+++ b/dynamicweb/celery.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import, unicode_literals
import os
from celery import Celery
From 8029f058243c02d13f45c277d6c4f552b034c701 Mon Sep 17 00:00:00 2001
From: PCoder <purple.coder@yahoo.co.uk>
Date: Tue, 8 Aug 2017 02:12:13 +0530
Subject: [PATCH 16/21] Added env variable to logger message
---
dynamicweb/settings/base.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py
index 58609234..a0d31592 100644
--- a/dynamicweb/settings/base.py
+++ b/dynamicweb/settings/base.py
@@ -37,7 +37,8 @@ def int_env(val, default_value=0):
try:
return_value = int(os.environ.get(val))
except Exception as e:
- logger.error(str(e))
+ logger.error("Encountered exception trying to get env value for {}\nException details: {}".format(
+ val, str(e)))
return return_value
From 339a6283cd38694c144dc9cf670e9fc71b35a95d Mon Sep 17 00:00:00 2001
From: PCoder <purple.coder@yahoo.co.uk>
Date: Sat, 19 Aug 2017 03:32:30 +0530
Subject: [PATCH 17/21] Removed unused imports
---
datacenterlight/views.py | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
diff --git a/datacenterlight/views.py b/datacenterlight/views.py
index dec5add6..db7f2e53 100644
--- a/datacenterlight/views.py
+++ b/datacenterlight/views.py
@@ -4,7 +4,6 @@ from .forms import BetaAccessForm
from .models import BetaAccess, BetaAccessVMType, BetaAccessVM, VMTemplate
from django.contrib import messages
from django.core.urlresolvers import reverse
-from django.core.mail import EmailMessage
from utils.mailer import BaseEmail
from django.shortcuts import render
from django.shortcuts import redirect
@@ -13,14 +12,13 @@ from django.core.exceptions import ValidationError
from django.views.decorators.cache import cache_control
from django.conf import settings
from django.utils.translation import ugettext_lazy as _
-from utils.forms import BillingAddressForm, UserBillingAddressForm
+from utils.forms import BillingAddressForm
from utils.models import BillingAddress
-from hosting.models import HostingOrder, HostingBill
+from hosting.models import HostingOrder
from utils.stripe_utils import StripeUtils
-from datetime import datetime
from membership.models import CustomUser, StripeCustomer
from opennebula_api.models import OpenNebulaManager
-from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer
+from opennebula_api.serializers import VirtualMachineTemplateSerializer, VMTemplateSerializer
from datacenterlight.tasks import create_vm_task
@@ -445,8 +443,6 @@ class OrderConfirmationView(DetailView):
customer = StripeCustomer.objects.filter(id=stripe_customer_id).first()
billing_address_data = request.session.get('billing_address_data')
billing_address_id = request.session.get('billing_address')
- billing_address = BillingAddress.objects.filter(
- id=billing_address_id).first()
vm_template_id = template.get('id', 1)
final_price = specs.get('price')
From 2b717954312483b2064db7a9c03eb5ee12c79059 Mon Sep 17 00:00:00 2001
From: PCoder <purple.coder@yahoo.co.uk>
Date: Sat, 19 Aug 2017 18:02:55 +0530
Subject: [PATCH 18/21] Added missing oneadmin's ssh key statement
---
datacenterlight/tasks.py | 1 +
1 file changed, 1 insertion(+)
diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py
index dc71fcee..ee3b50c0 100644
--- a/datacenterlight/tasks.py
+++ b/datacenterlight/tasks.py
@@ -58,6 +58,7 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_
vm_id = manager.create_vm(
template_id=vm_template_id,
specs=specs,
+ ssh_key=settings.ONEADMIN_USER_SSH_PUBLIC_KEY,
vm_name="{email}-{template_name}-{date}".format(
email=user.get('email'),
template_name=template.get('name'),
From 422b0ca93e18d551ea69cf067f39136dc56575a7 Mon Sep 17 00:00:00 2001
From: PCoder <purple.coder@yahoo.co.uk>
Date: Sat, 19 Aug 2017 18:28:17 +0530
Subject: [PATCH 19/21] Updated deploy.sh - Added --dbmakemigrations to do
makemigrations - Added call to restart celery
---
deploy.sh | 20 +++++++++++++++-----
1 file changed, 15 insertions(+), 5 deletions(-)
diff --git a/deploy.sh b/deploy.sh
index f2a1d59e..0ce17c39 100755
--- a/deploy.sh
+++ b/deploy.sh
@@ -13,6 +13,7 @@ while true; do
case "$1" in
-h | --help ) HELP=true; shift ;;
-v | --verbose ) VERBOSE=true; shift ;;
+ -D | --dbmakemigrations ) DB_MAKE_MIGRATIONS=true; shift ;;
-d | --dbmigrate ) DB_MIGRATE=true; shift ;;
-n | --nogit ) NO_GIT=true; shift ;;
-b | --branch ) BRANCH="$2"; shift 2 ;;
@@ -31,13 +32,15 @@ if [ "$HELP" == "true" ]; then
echo "options are : "
echo " -h, --help: Print this help message"
echo " -v, --verbose: Show verbose output to stdout. Without this a deploy.log is written to ~/app folder"
- echo " -d, --dbmigrate: Do DB migrate"
- echo " -n, --nogit: Don't execute git commands. With this --branch has no effect."
+ echo " -D, --dbmakemigrations: Do DB makemigrations"
+ echo " -d, --dbmigrate: Do DB migrate. To do both makemigrations and migrate, supply both switches -D and -d"
+ echo " -n, --nogit: Don't execute git commands. This is used to deploy the current code in the project repo. With this --branch has no effect."
echo " -b, --branch: The branch to pull from origin repo."
exit
fi
echo "BRANCH="$BRANCH
+echo "DB_MAKE_MIGRATIONS="$DB_MAKE_MIGRATIONS
echo "DB_MIGRATE="$DB_MIGRATE
echo "NO_GIT="$NO_GIT
echo "VERBOSE="$VERBOSE
@@ -45,7 +48,7 @@ echo "VERBOSE="$VERBOSE
# The project directory exists, we pull the specified branch
cd $APP_HOME_DIR
if [ -z "$NO_GIT" ]; then
- echo 'We are executing default git commands. Please -no_git to not use this.'
+ echo 'We are executing default git commands. Please add --nogit to not do this.'
# Save any modified changes before git pulling
git stash
# Fetch all branches/tags
@@ -59,16 +62,23 @@ fi
source ~/pyvenv/bin/activate
pip install -r requirements.txt > deploy.log 2>&1
echo "###" >> deploy.log
+if [ -z "$DB_MAKE_MIGRATIONS" ]; then
+ echo 'We are not doing DB makemigrations'
+else
+ echo 'Doing DB makemigrations'
+ ./manage.py makemigrations >> deploy.log 2>&1
+ echo "###" >> deploy.log
+fi
if [ -z "$DB_MIGRATE" ]; then
echo 'We are not doing DB migration'
else
- ./manage.py makemigrations >> deploy.log 2>&1
- echo "###" >> deploy.log
+ echo 'Doing DB migrate'
./manage.py migrate >> deploy.log 2>&1
echo "###" >> deploy.log
fi
printf 'yes' | ./manage.py collectstatic >> deploy.log 2>&1
echo "###" >> deploy.log
django-admin compilemessages
+sudo systemctl restart celery.service
sudo systemctl restart uwsgi
From 777102c387fb01a98d3447e28cef80b49689527f Mon Sep 17 00:00:00 2001
From: PCoder <purple.coder@yahoo.co.uk>
Date: Sat, 19 Aug 2017 18:32:58 +0530
Subject: [PATCH 20/21] Updated deploy.sh: corrected terminology
---
deploy.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/deploy.sh b/deploy.sh
index 0ce17c39..04a7b04c 100755
--- a/deploy.sh
+++ b/deploy.sh
@@ -70,7 +70,7 @@ else
echo "###" >> deploy.log
fi
if [ -z "$DB_MIGRATE" ]; then
- echo 'We are not doing DB migration'
+ echo 'We are not doing DB migrate'
else
echo 'Doing DB migrate'
./manage.py migrate >> deploy.log 2>&1
From ddbaf902afbfaa0bd006a79febfcc8dbe3eb8414 Mon Sep 17 00:00:00 2001
From: PCoder <purple.coder@yahoo.co.uk>
Date: Sun, 20 Aug 2017 20:23:59 +0530
Subject: [PATCH 21/21] Removed __future__ datacenterlight/tasks
---
datacenterlight/tasks.py | 1 -
1 file changed, 1 deletion(-)
diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py
index ee3b50c0..b897c54a 100644
--- a/datacenterlight/tasks.py
+++ b/datacenterlight/tasks.py
@@ -1,4 +1,3 @@
-from __future__ import absolute_import, unicode_literals
from dynamicweb.celery import app
from celery.utils.log import get_task_logger
from django.conf import settings