From d8592fc6d8d3e1c3efba54dfe478a1f0d582b917 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sun, 6 Aug 2017 12:30:56 +0530 Subject: [PATCH 01/21] Added celery files --- dynamicweb/__init__.py | 5 +++++ dynamicweb/celery.py | 22 ++++++++++++++++++++++ dynamicweb/settings/base.py | 9 +++++++++ requirements.txt | 2 ++ 4 files changed, 38 insertions(+) create mode 100644 dynamicweb/celery.py diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py index e69de29b..b64e43e8 100644 --- a/dynamicweb/__init__.py +++ b/dynamicweb/__init__.py @@ -0,0 +1,5 @@ +from __future__ import absolute_import + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app diff --git a/dynamicweb/celery.py b/dynamicweb/celery.py new file mode 100644 index 00000000..749ffdef --- /dev/null +++ b/dynamicweb/celery.py @@ -0,0 +1,22 @@ +from __future__ import absolute_import, unicode_literals +import os +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dynamicweb.settings') + +app = Celery('dynamicweb') + +# Using a string here means the worker don't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) + + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py index 40187f84..f4c4b68d 100644 --- a/dynamicweb/settings/base.py +++ b/dynamicweb/settings/base.py @@ -521,6 +521,15 @@ GOOGLE_ANALYTICS_PROPERTY_IDS = { 'dynamicweb-staging.ungleich.ch': 'staging' } +# CELERY Settings +#BROKER_URL = 'redis://localhost:6379' +BROKER_URL = 'redis+socket:///var/run/redis/redis.sock' +CELERY_RESULT_BACKEND = 'redis://localhost:6379' +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = 'Europe/Zurich' + ENABLE_DEBUG_LOGGING = bool_env('ENABLE_DEBUG_LOGGING') if ENABLE_DEBUG_LOGGING: diff --git a/requirements.txt b/requirements.txt index f392f4d9..2520d617 100644 --- a/requirements.txt +++ b/requirements.txt @@ -86,3 +86,5 @@ git+https://github.com/ungleich/python-oca.git#egg=python-oca djangorestframework flake8==3.3.0 python-memcached==1.58 +celery==4.0.2 +redis==2.10.5 From 3ad5928aa0930d7a6a41b0791f179cf847cf7a31 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 14:33:55 +0200 Subject: [PATCH 02/21] A working version of celery create_vm_task added --- datacenterlight/views.py | 121 +++++++-------------------------------- 1 file changed, 22 insertions(+), 99 deletions(-) diff --git a/datacenterlight/views.py b/datacenterlight/views.py index f7a07c7e..b2798844 100644 --- a/datacenterlight/views.py +++ b/datacenterlight/views.py @@ -4,7 +4,6 @@ from .forms import BetaAccessForm from .models import BetaAccess, BetaAccessVMType, BetaAccessVM, VMTemplate from django.contrib import messages from django.core.urlresolvers import reverse -from django.core.mail import EmailMessage from utils.mailer import BaseEmail from django.shortcuts import render from django.shortcuts import redirect @@ -13,14 +12,15 @@ from django.core.exceptions import ValidationError from django.views.decorators.cache import cache_control from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from utils.forms import BillingAddressForm, UserBillingAddressForm +from utils.forms import BillingAddressForm from utils.models import BillingAddress -from hosting.models import HostingOrder, HostingBill +from hosting.models import HostingOrder from utils.stripe_utils import StripeUtils -from datetime import datetime from membership.models import CustomUser, StripeCustomer + from opennebula_api.models import OpenNebulaManager from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer +from datacenterlight.tasks import create_vm_task class LandingProgramView(TemplateView): @@ -33,7 +33,6 @@ class SuccessView(TemplateView): def get(self, request, *args, **kwargs): if 'specs' not in request.session or 'user' not in request.session: return HttpResponseRedirect(reverse('datacenterlight:index')) - elif 'token' not in request.session: return HttpResponseRedirect(reverse('datacenterlight:payment')) elif 'order_confirmation' not in request.session: @@ -79,8 +78,7 @@ class PricingView(TemplateView): manager = OpenNebulaManager() template = manager.get_template(template_id) - request.session['template'] = VirtualMachineTemplateSerializer( - template).data + request.session['template'] = VirtualMachineTemplateSerializer(template).data if not request.user.is_authenticated(): request.session['next'] = reverse('hosting:payment') @@ -132,8 +130,7 @@ class BetaAccessView(FormView): email = BaseEmail(**email_data) email.send() - messages.add_message( - self.request, messages.SUCCESS, self.success_message) + messages.add_message(self.request, messages.SUCCESS, self.success_message) return render(self.request, 'datacenterlight/beta_success.html', {}) @@ -185,8 +182,7 @@ class BetaProgramView(CreateView): email = BaseEmail(**email_data) email.send() - messages.add_message( - self.request, messages.SUCCESS, self.success_message) + messages.add_message(self.request, messages.SUCCESS, self.success_message) return HttpResponseRedirect(self.get_success_url()) @@ -199,15 +195,15 @@ class IndexView(CreateView): def validate_cores(self, value): if (value > 48) or (value < 1): - raise ValidationError(_('Invalid number of cores')) + raise ValidationError(_('Not a proper cores number')) def validate_memory(self, value): if (value > 200) or (value < 2): - raise ValidationError(_('Invalid RAM size')) + raise ValidationError(_('Not a proper ram number')) def validate_storage(self, value): if (value > 2000) or (value < 10): - raise ValidationError(_('Invalid storage size')) + raise ValidationError(_('Not a proper storage number')) @cache_control(no_cache=True, must_revalidate=True, no_store=True) def get(self, request, *args, **kwargs): @@ -230,8 +226,7 @@ class IndexView(CreateView): storage_field = forms.IntegerField(validators=[self.validate_storage]) price = request.POST.get('total') template_id = int(request.POST.get('config')) - template = VMTemplate.objects.filter( - opennebula_vm_template_id=template_id).first() + template = VMTemplate.objects.filter(opennebula_vm_template_id=template_id).first() template_data = VMTemplateSerializer(template).data name = request.POST.get('name') @@ -243,40 +238,35 @@ class IndexView(CreateView): cores = cores_field.clean(cores) except ValidationError as err: msg = '{} : {}.'.format(cores, str(err)) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='cores') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='cores') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") try: memory = memory_field.clean(memory) except ValidationError as err: msg = '{} : {}.'.format(memory, str(err)) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='memory') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='memory') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") try: storage = storage_field.clean(storage) except ValidationError as err: msg = '{} : {}.'.format(storage, str(err)) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='storage') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='storage') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") try: name = name_field.clean(name) except ValidationError as err: msg = '{} {}.'.format(name, _('is not a proper name')) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='name') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='name') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") try: email = email_field.clean(email) except ValidationError as err: msg = '{} {}.'.format(email, _('is not a proper email')) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='email') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='email') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") specs = { @@ -341,8 +331,7 @@ class IndexView(CreateView): email = BaseEmail(**email_data) email.send() - messages.add_message( - self.request, messages.SUCCESS, self.success_message) + messages.add_message(self.request, messages.SUCCESS, self.success_message) return super(IndexView, self).form_valid(form) @@ -411,7 +400,6 @@ class PaymentOrderView(FormView): # Create Billing Address billing_address = form.save() - request.session['billing_address_data'] = billing_address_data request.session['billing_address'] = billing_address.id request.session['token'] = token @@ -436,8 +424,7 @@ class OrderConfirmationView(DetailView): stripe_customer_id = request.session.get('customer') customer = StripeCustomer.objects.filter(id=stripe_customer_id).first() stripe_utils = StripeUtils() - card_details = stripe_utils.get_card_details( - customer.stripe_id, request.session.get('token')) + card_details = stripe_utils.get_card_details(customer.stripe_id, request.session.get('token')) context = { 'site_url': reverse('datacenterlight:index'), 'cc_last4': card_details.get('response_object').get('last4'), @@ -453,8 +440,7 @@ class OrderConfirmationView(DetailView): customer = StripeCustomer.objects.filter(id=stripe_customer_id).first() billing_address_data = request.session.get('billing_address_data') billing_address_id = request.session.get('billing_address') - billing_address = BillingAddress.objects.filter( - id=billing_address_id).first() + billing_address = BillingAddress.objects.filter(id=billing_address_id).first() vm_template_id = template.get('id', 1) final_price = specs.get('price') @@ -473,71 +459,8 @@ class OrderConfirmationView(DetailView): return render(request, self.payment_template_name, context) charge = charge_response.get('response_object') - - # Create OpenNebulaManager - manager = OpenNebulaManager(email=settings.OPENNEBULA_USERNAME, - password=settings.OPENNEBULA_PASSWORD) - - # Create a vm using oneadmin, also specify the name - vm_id = manager.create_vm( - template_id=vm_template_id, - specs=specs, - vm_name="{email}-{template_name}-{date}".format( - email=user.get('email'), - template_name=template.get('name'), - date=int(datetime.now().strftime("%s"))) - ) - - # Create a Hosting Order - order = HostingOrder.create( - price=final_price, - vm_id=vm_id, - customer=customer, - billing_address=billing_address - ) - - # Create a Hosting Bill - HostingBill.create( - customer=customer, billing_address=billing_address) - - # Create Billing Address for User if he does not have one - if not customer.user.billing_addresses.count(): - billing_address_data.update({ - 'user': customer.user.id - }) - billing_address_user_form = UserBillingAddressForm( - billing_address_data) - billing_address_user_form.is_valid() - billing_address_user_form.save() - - # Associate an order with a stripe payment - order.set_stripe_charge(charge) - - # If the Stripe payment was successed, set order status approved - order.set_approved() - - vm = VirtualMachineSerializer(manager.get_vm(vm_id)).data - - context = { - 'name': user.get('name'), - 'email': user.get('email'), - 'cores': specs.get('cpu'), - 'memory': specs.get('memory'), - 'storage': specs.get('disk_size'), - 'price': specs.get('price'), - 'template': template.get('name'), - 'vm.name': vm['name'], - 'vm.id': vm['vm_id'], - 'order.id': order.id - } - email_data = { - 'subject': settings.DCL_TEXT + " Order from %s" % context['email'], - 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, - 'to': ['info@ungleich.ch'], - 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in context.items()]), - 'reply_to': [context['email']], - } - email = EmailMessage(**email_data) - email.send() + create_vm_task.delay(vm_template_id, user, specs, template, stripe_customer_id, billing_address_data, + billing_address_id, + charge) request.session['order_confirmation'] = True return HttpResponseRedirect(reverse('datacenterlight:order_success')) From bd2edf599ae4b8e02cc43167821d9bf3adac75e3 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 15:08:05 +0200 Subject: [PATCH 03/21] Merged views from master --- datacenterlight/views.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/datacenterlight/views.py b/datacenterlight/views.py index b2798844..e37b2914 100644 --- a/datacenterlight/views.py +++ b/datacenterlight/views.py @@ -17,7 +17,6 @@ from utils.models import BillingAddress from hosting.models import HostingOrder from utils.stripe_utils import StripeUtils from membership.models import CustomUser, StripeCustomer - from opennebula_api.models import OpenNebulaManager from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer from datacenterlight.tasks import create_vm_task @@ -195,15 +194,15 @@ class IndexView(CreateView): def validate_cores(self, value): if (value > 48) or (value < 1): - raise ValidationError(_('Not a proper cores number')) + raise ValidationError(_('Invalid number of cores')) def validate_memory(self, value): if (value > 200) or (value < 2): - raise ValidationError(_('Not a proper ram number')) + raise ValidationError(_('Invalid RAM size')) def validate_storage(self, value): if (value > 2000) or (value < 10): - raise ValidationError(_('Not a proper storage number')) + raise ValidationError(_('Invalid storage size')) @cache_control(no_cache=True, must_revalidate=True, no_store=True) def get(self, request, *args, **kwargs): From d6d0b9c8c0eed4394d3610ccade37cbc9395fefc Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 15:08:46 +0200 Subject: [PATCH 04/21] Added default init code for celery --- dynamicweb/__init__.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py index b64e43e8..de98902b 100644 --- a/dynamicweb/__init__.py +++ b/dynamicweb/__init__.py @@ -1,5 +1,7 @@ -from __future__ import absolute_import +from __future__ import absolute_import, unicode_literals # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app + +__all__ = ['celery_app'] \ No newline at end of file From 477c8e81a40aeef502764082c5c22b02c77e512a Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 15:10:26 +0200 Subject: [PATCH 05/21] Added default celery.py code --- dynamicweb/celery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dynamicweb/celery.py b/dynamicweb/celery.py index 749ffdef..28f6af3b 100644 --- a/dynamicweb/celery.py +++ b/dynamicweb/celery.py @@ -14,7 +14,7 @@ app = Celery('dynamicweb') app.config_from_object('django.conf:settings', namespace='CELERY') # Load task modules from all registered Django app configs. -app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) +app.autodiscover_tasks() @app.task(bind=True) From 68505e73a8c834f3728e633c38dbf89e7b70de96 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 16:32:27 +0200 Subject: [PATCH 06/21] Added django-celery-results and setup redis as backend for celery tasks --- dynamicweb/settings/base.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py index f4c4b68d..4847851f 100644 --- a/dynamicweb/settings/base.py +++ b/dynamicweb/settings/base.py @@ -120,7 +120,8 @@ INSTALLED_APPS = ( 'datacenterlight.templatetags', 'alplora', 'rest_framework', - 'opennebula_api' + 'opennebula_api', + 'django_celery_results', ) MIDDLEWARE_CLASSES = ( @@ -522,9 +523,10 @@ GOOGLE_ANALYTICS_PROPERTY_IDS = { } # CELERY Settings -#BROKER_URL = 'redis://localhost:6379' -BROKER_URL = 'redis+socket:///var/run/redis/redis.sock' -CELERY_RESULT_BACKEND = 'redis://localhost:6379' +# BROKER_URL = 'redis://localhost:6379' +CELERY_BROKER_URL = 'redis+socket:///tmp/redis.sock' +# CELERY_RESULT_BACKEND = 'redis://localhost:6379' +CELERY_RESULT_BACKEND = 'redis+socket:///tmp/redis.sock' CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' From 55177646b456be63805604eb5d2188d0c9bd5d4f Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 16:33:37 +0200 Subject: [PATCH 07/21] Reformatted code --- hosting/models.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/hosting/models.py b/hosting/models.py index 88386913..8cdc6114 100644 --- a/hosting/models.py +++ b/hosting/models.py @@ -1,13 +1,9 @@ import os import logging - from django.db import models from django.utils.functional import cached_property - - from Crypto.PublicKey import RSA - from membership.models import StripeCustomer, CustomUser from utils.models import BillingAddress from utils.mixins import AssignPermissionsMixin @@ -42,7 +38,6 @@ class HostingPlan(models.Model): class HostingOrder(AssignPermissionsMixin, models.Model): - ORDER_APPROVED_STATUS = 'Approved' ORDER_DECLINED_STATUS = 'Declined' @@ -101,7 +96,7 @@ class HostingOrder(AssignPermissionsMixin, models.Model): class UserHostingKey(models.Model): user = models.ForeignKey(CustomUser) public_key = models.TextField() - private_key = models.FileField(upload_to='private_keys', blank=True) + private_key = models.FileField(upload_to='private_keys', blank=True) created_at = models.DateTimeField(auto_now_add=True) name = models.CharField(max_length=100) From 6d28783f9ffd526aa076289379ffdca927c693b1 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 16:34:23 +0200 Subject: [PATCH 08/21] Added django-celery-results to requirements --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index 2520d617..abce2271 100644 --- a/requirements.txt +++ b/requirements.txt @@ -88,3 +88,4 @@ flake8==3.3.0 python-memcached==1.58 celery==4.0.2 redis==2.10.5 +django-celery-results==1.0.1 From 5a5654be018239ae192864316ceb5a4e4c54be99 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 16:51:07 +0200 Subject: [PATCH 09/21] Moved CELERY_BROKER_URL and CELERY_RESULT_BACKEND to .env file --- dynamicweb/settings/base.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py index 4847851f..fbe0c9ca 100644 --- a/dynamicweb/settings/base.py +++ b/dynamicweb/settings/base.py @@ -523,10 +523,8 @@ GOOGLE_ANALYTICS_PROPERTY_IDS = { } # CELERY Settings -# BROKER_URL = 'redis://localhost:6379' -CELERY_BROKER_URL = 'redis+socket:///tmp/redis.sock' -# CELERY_RESULT_BACKEND = 'redis://localhost:6379' -CELERY_RESULT_BACKEND = 'redis+socket:///tmp/redis.sock' +CELERY_BROKER_URL = env('CELERY_BROKER_URL') +CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND') CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' From 26b4feb1d16296036ccb3fe75f11f96501f3e964 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 18:39:45 +0200 Subject: [PATCH 10/21] Introduced int_env and CELERY_MAX_RETRIES env variable --- dynamicweb/settings/base.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py index fbe0c9ca..58609234 100644 --- a/dynamicweb/settings/base.py +++ b/dynamicweb/settings/base.py @@ -10,6 +10,9 @@ from django.utils.translation import ugettext_lazy as _ # dotenv import dotenv +import logging + +logger = logging.getLogger(__name__) def gettext(s): @@ -25,6 +28,20 @@ def bool_env(val): return True if os.environ.get(val, False) == 'True' else False +def int_env(val, default_value=0): + """Replaces string based environment values with Python integers + Return default_value if val is not set or cannot be parsed, otherwise + returns the python integer equal to the passed val + """ + return_value = default_value + try: + return_value = int(os.environ.get(val)) + except Exception as e: + logger.error(str(e)) + + return return_value + + BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PROJECT_DIR = os.path.abspath( @@ -529,6 +546,7 @@ CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' CELERY_TIMEZONE = 'Europe/Zurich' +CELERY_MAX_RETRIES = int_env('CELERY_MAX_RETRIES', 5) ENABLE_DEBUG_LOGGING = bool_env('ENABLE_DEBUG_LOGGING') From f950c1defb23a0cba62bbbbb732a5428f2f416a5 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 18:42:27 +0200 Subject: [PATCH 11/21] Added datacenterlight/tasks.py --- datacenterlight/tasks.py | 162 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 162 insertions(+) create mode 100644 datacenterlight/tasks.py diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py new file mode 100644 index 00000000..69ba08ad --- /dev/null +++ b/datacenterlight/tasks.py @@ -0,0 +1,162 @@ +from __future__ import absolute_import, unicode_literals +from dynamicweb.celery import app +from celery.utils.log import get_task_logger +from django.conf import settings +from opennebula_api.models import OpenNebulaManager +from opennebula_api.serializers import VirtualMachineSerializer +from hosting.models import HostingOrder, HostingBill +from utils.forms import UserBillingAddressForm +from datetime import datetime +from membership.models import StripeCustomer +from django.core.mail import EmailMessage +from utils.models import BillingAddress + +logger = get_task_logger(__name__) + + +def retry_task(task, exception=None): + """Retries the specified task using a "backing off countdown", + meaning that the interval between retries grows exponentially + with every retry. + + Arguments: + task: + The task to retry. + + exception: + Optionally, the exception that caused the retry. + """ + + def backoff(attempts): + return 2 ** attempts + + kwargs = { + 'countdown': backoff(task.request.retries), + } + + if exception: + kwargs['exc'] = exception + + if task.request.retries > settings.CELERY_MAX_RETRIES: + msg_text = 'Finished {} retries for create_vm_task'.format(task.request.retries) + logger.log(msg_text) + # Try sending email and stop + email_data = { + 'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text), + 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, + 'to': ['info@ungleich.ch'], + 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in kwargs.items()]), + } + email = EmailMessage(**email_data) + email.send() + return + else: + raise task.retry(**kwargs) + + +@app.task(bind=True) +def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_id, billing_address_data, + billing_address_id, + charge): + try: + final_price = specs.get('price') + billing_address = BillingAddress.objects.filter(id=billing_address_id).first() + customer = StripeCustomer.objects.filter(id=stripe_customer_id).first() + # Create OpenNebulaManager + manager = OpenNebulaManager(email=settings.OPENNEBULA_USERNAME, + password=settings.OPENNEBULA_PASSWORD) + + # Create a vm using oneadmin, also specify the name + vm_id = manager.create_vm( + template_id=vm_template_id, + specs=specs, + vm_name="{email}-{template_name}-{date}".format( + email=user.get('email'), + template_name=template.get('name'), + date=int(datetime.now().strftime("%s"))) + ) + + # Create a Hosting Order + order = HostingOrder.create( + price=final_price, + vm_id=vm_id, + customer=customer, + billing_address=billing_address + ) + + # Create a Hosting Bill + HostingBill.create( + customer=customer, billing_address=billing_address) + + # Create Billing Address for User if he does not have one + if not customer.user.billing_addresses.count(): + billing_address_data.update({ + 'user': customer.user.id + }) + billing_address_user_form = UserBillingAddressForm( + billing_address_data) + billing_address_user_form.is_valid() + billing_address_user_form.save() + + # Associate an order with a stripe payment + charge_object = DictDotLookup(charge) + order.set_stripe_charge(charge_object) + + # If the Stripe payment succeeds, set order status approved + order.set_approved() + + vm = VirtualMachineSerializer(manager.get_vm(vm_id)).data + + context = { + 'name': user.get('name'), + 'email': user.get('email'), + 'cores': specs.get('cpu'), + 'memory': specs.get('memory'), + 'storage': specs.get('disk_size'), + 'price': specs.get('price'), + 'template': template.get('name'), + 'vm.name': vm['name'], + 'vm.id': vm['vm_id'], + 'order.id': order.id + } + email_data = { + 'subject': settings.DCL_TEXT + " Order from %s" % context['email'], + 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, + 'to': ['info@ungleich.ch'], + 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in context.items()]), + 'reply_to': [context['email']], + } + email = EmailMessage(**email_data) + email.send() + except Exception as e: + logger.error(str(e)) + retry_task(self) + + +class DictDotLookup(object): + """ + Creates objects that behave much like a dictionaries, but allow nested + key access using object '.' (dot) lookups. + """ + + def __init__(self, d): + for k in d: + if isinstance(d[k], dict): + self.__dict__[k] = DictDotLookup(d[k]) + elif isinstance(d[k], (list, tuple)): + l = [] + for v in d[k]: + if isinstance(v, dict): + l.append(DictDotLookup(v)) + else: + l.append(v) + self.__dict__[k] = l + else: + self.__dict__[k] = d[k] + + def __getitem__(self, name): + if name in self.__dict__: + return self.__dict__[name] + + def __iter__(self): + return iter(self.__dict__.keys()) From e7864f5d85788a44abab26ded38e5ec1808fc7a8 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 18:52:23 +0200 Subject: [PATCH 12/21] Added condition to check if create_vm succeeded --- datacenterlight/tasks.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index 69ba08ad..9f0249a2 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -58,6 +58,7 @@ def retry_task(task, exception=None): def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_id, billing_address_data, billing_address_id, charge): + vm_id = None try: final_price = specs.get('price') billing_address = BillingAddress.objects.filter(id=billing_address_id).first() @@ -76,6 +77,9 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_ date=int(datetime.now().strftime("%s"))) ) + if vm_id is None: + raise Exception("Could not create VM") + # Create a Hosting Order order = HostingOrder.create( price=final_price, @@ -132,6 +136,8 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_ logger.error(str(e)) retry_task(self) + return vm_id + class DictDotLookup(object): """ From 58bb75481491589d9024c399e448f3ae8336683d Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 21:54:12 +0200 Subject: [PATCH 13/21] Introduced MaxRetries and MaxRetriesExceededError handling code --- datacenterlight/tasks.py | 35 ++++++++++++++++++----------------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index 9f0249a2..342c87f4 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -10,6 +10,7 @@ from datetime import datetime from membership.models import StripeCustomer from django.core.mail import EmailMessage from utils.models import BillingAddress +from celery.exceptions import MaxRetriesExceededError logger = get_task_logger(__name__) @@ -37,24 +38,10 @@ def retry_task(task, exception=None): if exception: kwargs['exc'] = exception - if task.request.retries > settings.CELERY_MAX_RETRIES: - msg_text = 'Finished {} retries for create_vm_task'.format(task.request.retries) - logger.log(msg_text) - # Try sending email and stop - email_data = { - 'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text), - 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, - 'to': ['info@ungleich.ch'], - 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in kwargs.items()]), - } - email = EmailMessage(**email_data) - email.send() - return - else: - raise task.retry(**kwargs) + raise task.retry(**kwargs) -@app.task(bind=True) +@app.task(bind=True, max_retries=settings.CELERY_MAX_RETRIES) def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_id, billing_address_data, billing_address_id, charge): @@ -134,7 +121,21 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_ email.send() except Exception as e: logger.error(str(e)) - retry_task(self) + try: + retry_task(self) + except MaxRetriesExceededError: + msg_text = 'Finished {} retries for create_vm_task'.format(self.request.retries) + logger.error(msg_text) + # Try sending email and stop + email_data = { + 'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text), + 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, + 'to': ['info@ungleich.ch'], + 'body':',\n'.join(str(i) for i in self.request.args) + } + email = EmailMessage(**email_data) + email.send() + return return vm_id From 29b28d551c2f3caf20929f6eb5f57187ed81c9f9 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Sun, 6 Aug 2017 22:19:27 +0200 Subject: [PATCH 14/21] Fixed flake8 lint warnings --- datacenterlight/tasks.py | 2 +- datacenterlight/views.py | 4 +--- dynamicweb/__init__.py | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index 342c87f4..dc71fcee 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -131,7 +131,7 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_ 'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text), 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, 'to': ['info@ungleich.ch'], - 'body':',\n'.join(str(i) for i in self.request.args) + 'body': ',\n'.join(str(i) for i in self.request.args) } email = EmailMessage(**email_data) email.send() diff --git a/datacenterlight/views.py b/datacenterlight/views.py index e37b2914..2bae5c77 100644 --- a/datacenterlight/views.py +++ b/datacenterlight/views.py @@ -13,12 +13,11 @@ from django.views.decorators.cache import cache_control from django.conf import settings from django.utils.translation import ugettext_lazy as _ from utils.forms import BillingAddressForm -from utils.models import BillingAddress from hosting.models import HostingOrder from utils.stripe_utils import StripeUtils from membership.models import CustomUser, StripeCustomer from opennebula_api.models import OpenNebulaManager -from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer +from opennebula_api.serializers import VirtualMachineTemplateSerializer, VMTemplateSerializer from datacenterlight.tasks import create_vm_task @@ -439,7 +438,6 @@ class OrderConfirmationView(DetailView): customer = StripeCustomer.objects.filter(id=stripe_customer_id).first() billing_address_data = request.session.get('billing_address_data') billing_address_id = request.session.get('billing_address') - billing_address = BillingAddress.objects.filter(id=billing_address_id).first() vm_template_id = template.get('id', 1) final_price = specs.get('price') diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py index de98902b..3b91b070 100644 --- a/dynamicweb/__init__.py +++ b/dynamicweb/__init__.py @@ -4,4 +4,4 @@ from __future__ import absolute_import, unicode_literals # Django starts so that shared_task will use this app. from .celery import app as celery_app -__all__ = ['celery_app'] \ No newline at end of file +__all__ = ['celery_app'] From 227d3a20a526f3e30915e99a2b0fc3d4ef0fbbe9 Mon Sep 17 00:00:00 2001 From: PCoder Date: Tue, 8 Aug 2017 01:57:29 +0530 Subject: [PATCH 15/21] Removed __future__ --- dynamicweb/__init__.py | 2 -- dynamicweb/celery.py | 1 - 2 files changed, 3 deletions(-) diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py index 3b91b070..1a6c551d 100644 --- a/dynamicweb/__init__.py +++ b/dynamicweb/__init__.py @@ -1,5 +1,3 @@ -from __future__ import absolute_import, unicode_literals - # This will make sure the app is always imported when # Django starts so that shared_task will use this app. from .celery import app as celery_app diff --git a/dynamicweb/celery.py b/dynamicweb/celery.py index 28f6af3b..609ef5c4 100644 --- a/dynamicweb/celery.py +++ b/dynamicweb/celery.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import, unicode_literals import os from celery import Celery From 8029f058243c02d13f45c277d6c4f552b034c701 Mon Sep 17 00:00:00 2001 From: PCoder Date: Tue, 8 Aug 2017 02:12:13 +0530 Subject: [PATCH 16/21] Added env variable to logger message --- dynamicweb/settings/base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py index 58609234..a0d31592 100644 --- a/dynamicweb/settings/base.py +++ b/dynamicweb/settings/base.py @@ -37,7 +37,8 @@ def int_env(val, default_value=0): try: return_value = int(os.environ.get(val)) except Exception as e: - logger.error(str(e)) + logger.error("Encountered exception trying to get env value for {}\nException details: {}".format( + val, str(e))) return return_value From 339a6283cd38694c144dc9cf670e9fc71b35a95d Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 19 Aug 2017 03:32:30 +0530 Subject: [PATCH 17/21] Removed unused imports --- datacenterlight/views.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/datacenterlight/views.py b/datacenterlight/views.py index dec5add6..db7f2e53 100644 --- a/datacenterlight/views.py +++ b/datacenterlight/views.py @@ -4,7 +4,6 @@ from .forms import BetaAccessForm from .models import BetaAccess, BetaAccessVMType, BetaAccessVM, VMTemplate from django.contrib import messages from django.core.urlresolvers import reverse -from django.core.mail import EmailMessage from utils.mailer import BaseEmail from django.shortcuts import render from django.shortcuts import redirect @@ -13,14 +12,13 @@ from django.core.exceptions import ValidationError from django.views.decorators.cache import cache_control from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from utils.forms import BillingAddressForm, UserBillingAddressForm +from utils.forms import BillingAddressForm from utils.models import BillingAddress -from hosting.models import HostingOrder, HostingBill +from hosting.models import HostingOrder from utils.stripe_utils import StripeUtils -from datetime import datetime from membership.models import CustomUser, StripeCustomer from opennebula_api.models import OpenNebulaManager -from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer +from opennebula_api.serializers import VirtualMachineTemplateSerializer, VMTemplateSerializer from datacenterlight.tasks import create_vm_task @@ -445,8 +443,6 @@ class OrderConfirmationView(DetailView): customer = StripeCustomer.objects.filter(id=stripe_customer_id).first() billing_address_data = request.session.get('billing_address_data') billing_address_id = request.session.get('billing_address') - billing_address = BillingAddress.objects.filter( - id=billing_address_id).first() vm_template_id = template.get('id', 1) final_price = specs.get('price') From 2b717954312483b2064db7a9c03eb5ee12c79059 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 19 Aug 2017 18:02:55 +0530 Subject: [PATCH 18/21] Added missing oneadmin's ssh key statement --- datacenterlight/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index dc71fcee..ee3b50c0 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -58,6 +58,7 @@ def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_ vm_id = manager.create_vm( template_id=vm_template_id, specs=specs, + ssh_key=settings.ONEADMIN_USER_SSH_PUBLIC_KEY, vm_name="{email}-{template_name}-{date}".format( email=user.get('email'), template_name=template.get('name'), From 422b0ca93e18d551ea69cf067f39136dc56575a7 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 19 Aug 2017 18:28:17 +0530 Subject: [PATCH 19/21] Updated deploy.sh - Added --dbmakemigrations to do makemigrations - Added call to restart celery --- deploy.sh | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/deploy.sh b/deploy.sh index f2a1d59e..0ce17c39 100755 --- a/deploy.sh +++ b/deploy.sh @@ -13,6 +13,7 @@ while true; do case "$1" in -h | --help ) HELP=true; shift ;; -v | --verbose ) VERBOSE=true; shift ;; + -D | --dbmakemigrations ) DB_MAKE_MIGRATIONS=true; shift ;; -d | --dbmigrate ) DB_MIGRATE=true; shift ;; -n | --nogit ) NO_GIT=true; shift ;; -b | --branch ) BRANCH="$2"; shift 2 ;; @@ -31,13 +32,15 @@ if [ "$HELP" == "true" ]; then echo "options are : " echo " -h, --help: Print this help message" echo " -v, --verbose: Show verbose output to stdout. Without this a deploy.log is written to ~/app folder" - echo " -d, --dbmigrate: Do DB migrate" - echo " -n, --nogit: Don't execute git commands. With this --branch has no effect." + echo " -D, --dbmakemigrations: Do DB makemigrations" + echo " -d, --dbmigrate: Do DB migrate. To do both makemigrations and migrate, supply both switches -D and -d" + echo " -n, --nogit: Don't execute git commands. This is used to deploy the current code in the project repo. With this --branch has no effect." echo " -b, --branch: The branch to pull from origin repo." exit fi echo "BRANCH="$BRANCH +echo "DB_MAKE_MIGRATIONS="$DB_MAKE_MIGRATIONS echo "DB_MIGRATE="$DB_MIGRATE echo "NO_GIT="$NO_GIT echo "VERBOSE="$VERBOSE @@ -45,7 +48,7 @@ echo "VERBOSE="$VERBOSE # The project directory exists, we pull the specified branch cd $APP_HOME_DIR if [ -z "$NO_GIT" ]; then - echo 'We are executing default git commands. Please -no_git to not use this.' + echo 'We are executing default git commands. Please add --nogit to not do this.' # Save any modified changes before git pulling git stash # Fetch all branches/tags @@ -59,16 +62,23 @@ fi source ~/pyvenv/bin/activate pip install -r requirements.txt > deploy.log 2>&1 echo "###" >> deploy.log +if [ -z "$DB_MAKE_MIGRATIONS" ]; then + echo 'We are not doing DB makemigrations' +else + echo 'Doing DB makemigrations' + ./manage.py makemigrations >> deploy.log 2>&1 + echo "###" >> deploy.log +fi if [ -z "$DB_MIGRATE" ]; then echo 'We are not doing DB migration' else - ./manage.py makemigrations >> deploy.log 2>&1 - echo "###" >> deploy.log + echo 'Doing DB migrate' ./manage.py migrate >> deploy.log 2>&1 echo "###" >> deploy.log fi printf 'yes' | ./manage.py collectstatic >> deploy.log 2>&1 echo "###" >> deploy.log django-admin compilemessages +sudo systemctl restart celery.service sudo systemctl restart uwsgi From 777102c387fb01a98d3447e28cef80b49689527f Mon Sep 17 00:00:00 2001 From: PCoder Date: Sat, 19 Aug 2017 18:32:58 +0530 Subject: [PATCH 20/21] Updated deploy.sh: corrected terminology --- deploy.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy.sh b/deploy.sh index 0ce17c39..04a7b04c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -70,7 +70,7 @@ else echo "###" >> deploy.log fi if [ -z "$DB_MIGRATE" ]; then - echo 'We are not doing DB migration' + echo 'We are not doing DB migrate' else echo 'Doing DB migrate' ./manage.py migrate >> deploy.log 2>&1 From ddbaf902afbfaa0bd006a79febfcc8dbe3eb8414 Mon Sep 17 00:00:00 2001 From: PCoder Date: Sun, 20 Aug 2017 20:23:59 +0530 Subject: [PATCH 21/21] Removed __future__ datacenterlight/tasks --- datacenterlight/tasks.py | 1 - 1 file changed, 1 deletion(-) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index ee3b50c0..b897c54a 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -1,4 +1,3 @@ -from __future__ import absolute_import, unicode_literals from dynamicweb.celery import app from celery.utils.log import get_task_logger from django.conf import settings