diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py new file mode 100644 index 00000000..b897c54a --- /dev/null +++ b/datacenterlight/tasks.py @@ -0,0 +1,169 @@ +from dynamicweb.celery import app +from celery.utils.log import get_task_logger +from django.conf import settings +from opennebula_api.models import OpenNebulaManager +from opennebula_api.serializers import VirtualMachineSerializer +from hosting.models import HostingOrder, HostingBill +from utils.forms import UserBillingAddressForm +from datetime import datetime +from membership.models import StripeCustomer +from django.core.mail import EmailMessage +from utils.models import BillingAddress +from celery.exceptions import MaxRetriesExceededError + +logger = get_task_logger(__name__) + + +def retry_task(task, exception=None): + """Retries the specified task using a "backing off countdown", + meaning that the interval between retries grows exponentially + with every retry. + + Arguments: + task: + The task to retry. + + exception: + Optionally, the exception that caused the retry. + """ + + def backoff(attempts): + return 2 ** attempts + + kwargs = { + 'countdown': backoff(task.request.retries), + } + + if exception: + kwargs['exc'] = exception + + raise task.retry(**kwargs) + + +@app.task(bind=True, max_retries=settings.CELERY_MAX_RETRIES) +def create_vm_task(self, vm_template_id, user, specs, template, stripe_customer_id, billing_address_data, + billing_address_id, + charge): + vm_id = None + try: + final_price = specs.get('price') + billing_address = BillingAddress.objects.filter(id=billing_address_id).first() + customer = StripeCustomer.objects.filter(id=stripe_customer_id).first() + # Create OpenNebulaManager + manager = OpenNebulaManager(email=settings.OPENNEBULA_USERNAME, + password=settings.OPENNEBULA_PASSWORD) + + # Create a vm using oneadmin, also specify the name + vm_id = manager.create_vm( + template_id=vm_template_id, + specs=specs, + ssh_key=settings.ONEADMIN_USER_SSH_PUBLIC_KEY, + vm_name="{email}-{template_name}-{date}".format( + email=user.get('email'), + template_name=template.get('name'), + date=int(datetime.now().strftime("%s"))) + ) + + if vm_id is None: + raise Exception("Could not create VM") + + # Create a Hosting Order + order = HostingOrder.create( + price=final_price, + vm_id=vm_id, + customer=customer, + billing_address=billing_address + ) + + # Create a Hosting Bill + HostingBill.create( + customer=customer, billing_address=billing_address) + + # Create Billing Address for User if he does not have one + if not customer.user.billing_addresses.count(): + billing_address_data.update({ + 'user': customer.user.id + }) + billing_address_user_form = UserBillingAddressForm( + billing_address_data) + billing_address_user_form.is_valid() + billing_address_user_form.save() + + # Associate an order with a stripe payment + charge_object = DictDotLookup(charge) + order.set_stripe_charge(charge_object) + + # If the Stripe payment succeeds, set order status approved + order.set_approved() + + vm = VirtualMachineSerializer(manager.get_vm(vm_id)).data + + context = { + 'name': user.get('name'), + 'email': user.get('email'), + 'cores': specs.get('cpu'), + 'memory': specs.get('memory'), + 'storage': specs.get('disk_size'), + 'price': specs.get('price'), + 'template': template.get('name'), + 'vm.name': vm['name'], + 'vm.id': vm['vm_id'], + 'order.id': order.id + } + email_data = { + 'subject': settings.DCL_TEXT + " Order from %s" % context['email'], + 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, + 'to': ['info@ungleich.ch'], + 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in context.items()]), + 'reply_to': [context['email']], + } + email = EmailMessage(**email_data) + email.send() + except Exception as e: + logger.error(str(e)) + try: + retry_task(self) + except MaxRetriesExceededError: + msg_text = 'Finished {} retries for create_vm_task'.format(self.request.retries) + logger.error(msg_text) + # Try sending email and stop + email_data = { + 'subject': '{} CELERY TASK ERROR: {}'.format(settings.DCL_TEXT, msg_text), + 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, + 'to': ['info@ungleich.ch'], + 'body': ',\n'.join(str(i) for i in self.request.args) + } + email = EmailMessage(**email_data) + email.send() + return + + return vm_id + + +class DictDotLookup(object): + """ + Creates objects that behave much like a dictionaries, but allow nested + key access using object '.' (dot) lookups. + """ + + def __init__(self, d): + for k in d: + if isinstance(d[k], dict): + self.__dict__[k] = DictDotLookup(d[k]) + elif isinstance(d[k], (list, tuple)): + l = [] + for v in d[k]: + if isinstance(v, dict): + l.append(DictDotLookup(v)) + else: + l.append(v) + self.__dict__[k] = l + else: + self.__dict__[k] = d[k] + + def __getitem__(self, name): + if name in self.__dict__: + return self.__dict__[name] + + def __iter__(self): + return iter(self.__dict__.keys()) diff --git a/datacenterlight/views.py b/datacenterlight/views.py index 0dd5e5ad..db7f2e53 100644 --- a/datacenterlight/views.py +++ b/datacenterlight/views.py @@ -4,7 +4,6 @@ from .forms import BetaAccessForm from .models import BetaAccess, BetaAccessVMType, BetaAccessVM, VMTemplate from django.contrib import messages from django.core.urlresolvers import reverse -from django.core.mail import EmailMessage from utils.mailer import BaseEmail from django.shortcuts import render from django.shortcuts import redirect @@ -13,14 +12,14 @@ from django.core.exceptions import ValidationError from django.views.decorators.cache import cache_control from django.conf import settings from django.utils.translation import ugettext_lazy as _ -from utils.forms import BillingAddressForm, UserBillingAddressForm +from utils.forms import BillingAddressForm from utils.models import BillingAddress -from hosting.models import HostingOrder, HostingBill +from hosting.models import HostingOrder from utils.stripe_utils import StripeUtils -from datetime import datetime from membership.models import CustomUser, StripeCustomer from opennebula_api.models import OpenNebulaManager -from opennebula_api.serializers import VirtualMachineTemplateSerializer, VirtualMachineSerializer, VMTemplateSerializer +from opennebula_api.serializers import VirtualMachineTemplateSerializer, VMTemplateSerializer +from datacenterlight.tasks import create_vm_task class LandingProgramView(TemplateView): @@ -33,7 +32,6 @@ class SuccessView(TemplateView): def get(self, request, *args, **kwargs): if 'specs' not in request.session or 'user' not in request.session: return HttpResponseRedirect(reverse('datacenterlight:index')) - elif 'token' not in request.session: return HttpResponseRedirect(reverse('datacenterlight:payment')) elif 'order_confirmation' not in request.session: @@ -79,8 +77,7 @@ class PricingView(TemplateView): manager = OpenNebulaManager() template = manager.get_template(template_id) - request.session['template'] = VirtualMachineTemplateSerializer( - template).data + request.session['template'] = VirtualMachineTemplateSerializer(template).data if not request.user.is_authenticated(): request.session['next'] = reverse('hosting:payment') @@ -132,8 +129,7 @@ class BetaAccessView(FormView): email = BaseEmail(**email_data) email.send() - messages.add_message( - self.request, messages.SUCCESS, self.success_message) + messages.add_message(self.request, messages.SUCCESS, self.success_message) return render(self.request, 'datacenterlight/beta_success.html', {}) @@ -185,8 +181,7 @@ class BetaProgramView(CreateView): email = BaseEmail(**email_data) email.send() - messages.add_message( - self.request, messages.SUCCESS, self.success_message) + messages.add_message(self.request, messages.SUCCESS, self.success_message) return HttpResponseRedirect(self.get_success_url()) @@ -230,8 +225,7 @@ class IndexView(CreateView): storage_field = forms.IntegerField(validators=[self.validate_storage]) price = request.POST.get('total') template_id = int(request.POST.get('config')) - template = VMTemplate.objects.filter( - opennebula_vm_template_id=template_id).first() + template = VMTemplate.objects.filter(opennebula_vm_template_id=template_id).first() template_data = VMTemplateSerializer(template).data name = request.POST.get('name') @@ -243,40 +237,35 @@ class IndexView(CreateView): cores = cores_field.clean(cores) except ValidationError as err: msg = '{} : {}.'.format(cores, str(err)) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='cores') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='cores') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") try: memory = memory_field.clean(memory) except ValidationError as err: msg = '{} : {}.'.format(memory, str(err)) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='memory') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='memory') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") try: storage = storage_field.clean(storage) except ValidationError as err: msg = '{} : {}.'.format(storage, str(err)) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='storage') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='storage') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") try: name = name_field.clean(name) except ValidationError as err: msg = '{} {}.'.format(name, _('is not a proper name')) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='name') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='name') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") try: email = email_field.clean(email) except ValidationError as err: msg = '{} {}.'.format(email, _('is not a proper email')) - messages.add_message( - self.request, messages.ERROR, msg, extra_tags='email') + messages.add_message(self.request, messages.ERROR, msg, extra_tags='email') return HttpResponseRedirect(reverse('datacenterlight:index') + "#order_form") specs = { @@ -341,8 +330,7 @@ class IndexView(CreateView): email = BaseEmail(**email_data) email.send() - messages.add_message( - self.request, messages.SUCCESS, self.success_message) + messages.add_message(self.request, messages.SUCCESS, self.success_message) return super(IndexView, self).form_valid(form) @@ -411,7 +399,6 @@ class PaymentOrderView(FormView): # Create Billing Address billing_address = form.save() - request.session['billing_address_data'] = billing_address_data request.session['billing_address'] = billing_address.id request.session['token'] = token @@ -436,13 +423,11 @@ class OrderConfirmationView(DetailView): stripe_customer_id = request.session.get('customer') customer = StripeCustomer.objects.filter(id=stripe_customer_id).first() stripe_utils = StripeUtils() - card_details = stripe_utils.get_card_details( - customer.stripe_id, request.session.get('token')) + card_details = stripe_utils.get_card_details(customer.stripe_id, request.session.get('token')) if not card_details.get('response_object') and not card_details.get('paid'): msg = card_details.get('error') messages.add_message(self.request, messages.ERROR, msg, extra_tags='failed_payment') return HttpResponseRedirect(reverse('datacenterlight:payment') + '#payment_error') - context = { 'site_url': reverse('datacenterlight:index'), 'cc_last4': card_details.get('response_object').get('last4'), @@ -458,8 +443,6 @@ class OrderConfirmationView(DetailView): customer = StripeCustomer.objects.filter(id=stripe_customer_id).first() billing_address_data = request.session.get('billing_address_data') billing_address_id = request.session.get('billing_address') - billing_address = BillingAddress.objects.filter( - id=billing_address_id).first() vm_template_id = template.get('id', 1) final_price = specs.get('price') @@ -475,72 +458,8 @@ class OrderConfirmationView(DetailView): return HttpResponseRedirect(reverse('datacenterlight:payment') + '#payment_error') charge = charge_response.get('response_object') - - # Create OpenNebulaManager - manager = OpenNebulaManager(email=settings.OPENNEBULA_USERNAME, - password=settings.OPENNEBULA_PASSWORD) - - # Create a vm using oneadmin, also specify the name - vm_id = manager.create_vm( - template_id=vm_template_id, - specs=specs, - ssh_key=settings.ONEADMIN_USER_SSH_PUBLIC_KEY, - vm_name="{email}-{template_name}-{date}".format( - email=user.get('email'), - template_name=template.get('name'), - date=int(datetime.now().strftime("%s"))) - ) - - # Create a Hosting Order - order = HostingOrder.create( - price=final_price, - vm_id=vm_id, - customer=customer, - billing_address=billing_address - ) - - # Create a Hosting Bill - HostingBill.create( - customer=customer, billing_address=billing_address) - - # Create Billing Address for User if he does not have one - if not customer.user.billing_addresses.count(): - billing_address_data.update({ - 'user': customer.user.id - }) - billing_address_user_form = UserBillingAddressForm( - billing_address_data) - billing_address_user_form.is_valid() - billing_address_user_form.save() - - # Associate an order with a stripe payment - order.set_stripe_charge(charge) - - # If the Stripe payment was successed, set order status approved - order.set_approved() - - vm = VirtualMachineSerializer(manager.get_vm(vm_id)).data - - context = { - 'name': user.get('name'), - 'email': user.get('email'), - 'cores': specs.get('cpu'), - 'memory': specs.get('memory'), - 'storage': specs.get('disk_size'), - 'price': specs.get('price'), - 'template': template.get('name'), - 'vm.name': vm['name'], - 'vm.id': vm['vm_id'], - 'order.id': order.id - } - email_data = { - 'subject': settings.DCL_TEXT + " Order from %s" % context['email'], - 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, - 'to': ['info@ungleich.ch'], - 'body': "\n".join(["%s=%s" % (k, v) for (k, v) in context.items()]), - 'reply_to': [context['email']], - } - email = EmailMessage(**email_data) - email.send() + create_vm_task.delay(vm_template_id, user, specs, template, stripe_customer_id, billing_address_data, + billing_address_id, + charge) request.session['order_confirmation'] = True return HttpResponseRedirect(reverse('datacenterlight:order_success')) diff --git a/deploy.sh b/deploy.sh index f2a1d59e..04a7b04c 100755 --- a/deploy.sh +++ b/deploy.sh @@ -13,6 +13,7 @@ while true; do case "$1" in -h | --help ) HELP=true; shift ;; -v | --verbose ) VERBOSE=true; shift ;; + -D | --dbmakemigrations ) DB_MAKE_MIGRATIONS=true; shift ;; -d | --dbmigrate ) DB_MIGRATE=true; shift ;; -n | --nogit ) NO_GIT=true; shift ;; -b | --branch ) BRANCH="$2"; shift 2 ;; @@ -31,13 +32,15 @@ if [ "$HELP" == "true" ]; then echo "options are : " echo " -h, --help: Print this help message" echo " -v, --verbose: Show verbose output to stdout. Without this a deploy.log is written to ~/app folder" - echo " -d, --dbmigrate: Do DB migrate" - echo " -n, --nogit: Don't execute git commands. With this --branch has no effect." + echo " -D, --dbmakemigrations: Do DB makemigrations" + echo " -d, --dbmigrate: Do DB migrate. To do both makemigrations and migrate, supply both switches -D and -d" + echo " -n, --nogit: Don't execute git commands. This is used to deploy the current code in the project repo. With this --branch has no effect." echo " -b, --branch: The branch to pull from origin repo." exit fi echo "BRANCH="$BRANCH +echo "DB_MAKE_MIGRATIONS="$DB_MAKE_MIGRATIONS echo "DB_MIGRATE="$DB_MIGRATE echo "NO_GIT="$NO_GIT echo "VERBOSE="$VERBOSE @@ -45,7 +48,7 @@ echo "VERBOSE="$VERBOSE # The project directory exists, we pull the specified branch cd $APP_HOME_DIR if [ -z "$NO_GIT" ]; then - echo 'We are executing default git commands. Please -no_git to not use this.' + echo 'We are executing default git commands. Please add --nogit to not do this.' # Save any modified changes before git pulling git stash # Fetch all branches/tags @@ -59,16 +62,23 @@ fi source ~/pyvenv/bin/activate pip install -r requirements.txt > deploy.log 2>&1 echo "###" >> deploy.log -if [ -z "$DB_MIGRATE" ]; then - echo 'We are not doing DB migration' +if [ -z "$DB_MAKE_MIGRATIONS" ]; then + echo 'We are not doing DB makemigrations' else + echo 'Doing DB makemigrations' ./manage.py makemigrations >> deploy.log 2>&1 echo "###" >> deploy.log +fi +if [ -z "$DB_MIGRATE" ]; then + echo 'We are not doing DB migrate' +else + echo 'Doing DB migrate' ./manage.py migrate >> deploy.log 2>&1 echo "###" >> deploy.log fi printf 'yes' | ./manage.py collectstatic >> deploy.log 2>&1 echo "###" >> deploy.log django-admin compilemessages +sudo systemctl restart celery.service sudo systemctl restart uwsgi diff --git a/dynamicweb/__init__.py b/dynamicweb/__init__.py index e69de29b..1a6c551d 100644 --- a/dynamicweb/__init__.py +++ b/dynamicweb/__init__.py @@ -0,0 +1,5 @@ +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ['celery_app'] diff --git a/dynamicweb/celery.py b/dynamicweb/celery.py new file mode 100644 index 00000000..609ef5c4 --- /dev/null +++ b/dynamicweb/celery.py @@ -0,0 +1,21 @@ +import os +from celery import Celery + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dynamicweb.settings') + +app = Celery('dynamicweb') + +# Using a string here means the worker don't have to serialize +# the configuration object to child processes. +# - namespace='CELERY' means all celery-related configuration keys +# should have a `CELERY_` prefix. +app.config_from_object('django.conf:settings', namespace='CELERY') + +# Load task modules from all registered Django app configs. +app.autodiscover_tasks() + + +@app.task(bind=True) +def debug_task(self): + print('Request: {0!r}'.format(self.request)) diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py index a837f0f2..a724b38e 100644 --- a/dynamicweb/settings/base.py +++ b/dynamicweb/settings/base.py @@ -10,6 +10,9 @@ from django.utils.translation import ugettext_lazy as _ # dotenv import dotenv +import logging + +logger = logging.getLogger(__name__) def gettext(s): @@ -25,6 +28,21 @@ def bool_env(val): return True if os.environ.get(val, False) == 'True' else False +def int_env(val, default_value=0): + """Replaces string based environment values with Python integers + Return default_value if val is not set or cannot be parsed, otherwise + returns the python integer equal to the passed val + """ + return_value = default_value + try: + return_value = int(os.environ.get(val)) + except Exception as e: + logger.error("Encountered exception trying to get env value for {}\nException details: {}".format( + val, str(e))) + + return return_value + + BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PROJECT_DIR = os.path.abspath( @@ -120,7 +138,8 @@ INSTALLED_APPS = ( 'datacenterlight.templatetags', 'alplora', 'rest_framework', - 'opennebula_api' + 'opennebula_api', + 'django_celery_results', ) MIDDLEWARE_CLASSES = ( @@ -524,6 +543,15 @@ GOOGLE_ANALYTICS_PROPERTY_IDS = { 'dynamicweb-staging.ungleich.ch': 'staging' } +# CELERY Settings +CELERY_BROKER_URL = env('CELERY_BROKER_URL') +CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND') +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' +CELERY_TIMEZONE = 'Europe/Zurich' +CELERY_MAX_RETRIES = int_env('CELERY_MAX_RETRIES', 5) + ENABLE_DEBUG_LOGGING = bool_env('ENABLE_DEBUG_LOGGING') if ENABLE_DEBUG_LOGGING: diff --git a/hosting/models.py b/hosting/models.py index 88386913..8cdc6114 100644 --- a/hosting/models.py +++ b/hosting/models.py @@ -1,13 +1,9 @@ import os import logging - from django.db import models from django.utils.functional import cached_property - - from Crypto.PublicKey import RSA - from membership.models import StripeCustomer, CustomUser from utils.models import BillingAddress from utils.mixins import AssignPermissionsMixin @@ -42,7 +38,6 @@ class HostingPlan(models.Model): class HostingOrder(AssignPermissionsMixin, models.Model): - ORDER_APPROVED_STATUS = 'Approved' ORDER_DECLINED_STATUS = 'Declined' @@ -101,7 +96,7 @@ class HostingOrder(AssignPermissionsMixin, models.Model): class UserHostingKey(models.Model): user = models.ForeignKey(CustomUser) public_key = models.TextField() - private_key = models.FileField(upload_to='private_keys', blank=True) + private_key = models.FileField(upload_to='private_keys', blank=True) created_at = models.DateTimeField(auto_now_add=True) name = models.CharField(max_length=100)