diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index 281d5f45..2779f79b 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -8,13 +8,16 @@ from django.core.mail import EmailMessage from django.core.urlresolvers import reverse from django.utils import translation from django.utils.translation import ugettext_lazy as _ +from time import sleep from dynamicweb.celery import app from hosting.models import HostingOrder from membership.models import CustomUser from opennebula_api.models import OpenNebulaManager from opennebula_api.serializers import VirtualMachineSerializer -from utils.hosting_utils import get_all_public_keys, get_or_create_vm_detail +from utils.hosting_utils import ( + get_all_public_keys, get_or_create_vm_detail, ping_ok +) from utils.mailer import BaseEmail from utils.stripe_utils import StripeUtils from .models import VMPricing @@ -203,12 +206,45 @@ def create_vm_task(self, vm_template_id, user, specs, template, order_id): host=vm_ipv6, num_keys=len(keys) ) ) - # Let's delay the task by 75 seconds to be sure - # that we run the cdist configure after the host - # is up - manager.manage_public_key( - keys, hosts=[vm_ipv6], countdown=75 - ) + # Let's wait until the IP responds to ping before we + # run the cdist configure on the host + did_manage_public_key = False + for i in range(0, 15): + if ping_ok(vm_ipv6): + logger.debug( + "{} is pingable. Doing a " + "manage_public_key".format(vm_ipv6) + ) + sleep(10) + manager.manage_public_key( + keys, hosts=[vm_ipv6] + ) + did_manage_public_key = True + break + else: + logger.debug( + "Can't ping {}. Wait 5 secs".format( + vm_ipv6 + ) + ) + sleep(5) + if not did_manage_public_key: + emsg = ("Waited for over 75 seconds for {} to be " + "pingable. But the VM was not reachable. " + "So, gave up manage_public_key. Please do " + "this manually".format(vm_ipv6)) + logger.error(emsg) + email_data = { + 'subject': '{} CELERY TASK INCOMPLETE: {} not ' + 'pingable for 75 seconds'.format( + settings.DCL_TEXT, vm_ipv6 + ), + 'from_email': current_task.request.hostname, + 'to': settings.DCL_ERROR_EMAILS_TO_LIST, + 'body': emsg + } + email = EmailMessage(**email_data) + email.send() except Exception as e: logger.error(str(e)) try: diff --git a/datacenterlight/utils.py b/datacenterlight/utils.py index 1c54148e..8da408a0 100644 --- a/datacenterlight/utils.py +++ b/datacenterlight/utils.py @@ -1,3 +1,4 @@ +import logging from django.contrib.sites.models import Site from datacenterlight.tasks import create_vm_task @@ -8,6 +9,8 @@ from utils.models import BillingAddress from .cms_models import CMSIntegration from .models import VMPricing, VMTemplate +logger = logging.getLogger(__name__) + def get_cms_integration(name): current_site = Site.objects.get_current() diff --git a/utils/hosting_utils.py b/utils/hosting_utils.py index c0494b90..1859a82c 100644 --- a/utils/hosting_utils.py +++ b/utils/hosting_utils.py @@ -1,5 +1,6 @@ import decimal import logging +import subprocess from oca.pool import WrongIdError from datacenterlight.models import VMPricing @@ -130,6 +131,23 @@ def get_vm_price_with_vat(cpu, memory, ssd_size, hdd_size=0, return float(price), float(vat), float(vat_percent), discount +def ping_ok(host_ipv6): + """ + A utility method to check if a host responds to ping requests. Note: the + function relies on `ping6` utility of debian to check. + + :param host_ipv6 str type parameter that represets the ipv6 of the host to + checked + :return True if the host responds to ping else returns False + """ + try: + subprocess.check_output("ping6 -c 1 " + host_ipv6, shell=True) + except Exception as ex: + logger.debug(host_ipv6 + " not reachable via ping. Error = " + str(ex)) + return False + return True + + class HostingUtils: @staticmethod def clear_items_from_list(from_list, items_list):