Merge pull request #655 from pcoder/task/5473/ping_before_cdist

Task/5473/ping before cdist

datacenterlight/tasks.py

@@ -8,13 +8,16 @@ from django.core.mail import EmailMessage
 from django.core.urlresolvers import reverse
 from django.utils import translation
 from django.utils.translation import ugettext_lazy as _
+from time import sleep
 
 from dynamicweb.celery import app
 from hosting.models import HostingOrder
 from membership.models import CustomUser
 from opennebula_api.models import OpenNebulaManager
 from opennebula_api.serializers import VirtualMachineSerializer
-from utils.hosting_utils import get_all_public_keys, get_or_create_vm_detail
+from utils.hosting_utils import (
+    get_all_public_keys, get_or_create_vm_detail, ping_ok
+)
 from utils.mailer import BaseEmail
 from utils.stripe_utils import StripeUtils
 from .models import VMPricing
@@ -203,12 +206,45 @@ def create_vm_task(self, vm_template_id, user, specs, template, order_id):
                                 host=vm_ipv6, num_keys=len(keys)
                             )
                         )
-                        # Let's delay the task by 75 seconds to be sure
+                        # Let's wait until the IP responds to ping before we
-                        # that we run the cdist configure after the host
+                        # run the cdist configure on the host
-                        # is up
+                        did_manage_public_key = False
-                        manager.manage_public_key(
+                        for i in range(0, 15):
-                            keys, hosts=[vm_ipv6], countdown=75
+                            if ping_ok(vm_ipv6):
-                        )
+                                logger.debug(
+                                    "{} is pingable. Doing a "
+                                    "manage_public_key".format(vm_ipv6)
+                                )
+                                sleep(10)
+                                manager.manage_public_key(
+                                    keys, hosts=[vm_ipv6]
+                                )
+                                did_manage_public_key = True
+                                break
+                            else:
+                                logger.debug(
+                                    "Can't ping {}. Wait 5 secs".format(
+                                        vm_ipv6
+                                    )
+                                )
+                                sleep(5)
+                        if not did_manage_public_key:
+                            emsg = ("Waited for over 75 seconds for {} to be "
+                                    "pingable. But the VM was not reachable. "
+                                    "So, gave up manage_public_key. Please do "
+                                    "this manually".format(vm_ipv6))
+                            logger.error(emsg)
+                            email_data = {
+                                'subject': '{} CELERY TASK INCOMPLETE: {} not '
+                                           'pingable for 75 seconds'.format(
+                                                settings.DCL_TEXT, vm_ipv6
+                                            ),
+                                'from_email': current_task.request.hostname,
+                                'to': settings.DCL_ERROR_EMAILS_TO_LIST,
+                                'body': emsg
+                            }
+                            email = EmailMessage(**email_data)
+                            email.send()
     except Exception as e:
         logger.error(str(e))
         try:

datacenterlight/utils.py

@@ -1,3 +1,4 @@
+import logging
 from django.contrib.sites.models import Site
 
 from datacenterlight.tasks import create_vm_task
@@ -8,6 +9,8 @@ from utils.models import BillingAddress
 from .cms_models import CMSIntegration
 from .models import VMPricing, VMTemplate
 
+logger = logging.getLogger(__name__)
+
 
 def get_cms_integration(name):
     current_site = Site.objects.get_current()

utils/hosting_utils.py

@@ -1,5 +1,6 @@
 import decimal
 import logging
+import subprocess
 from oca.pool import WrongIdError
 
 from datacenterlight.models import VMPricing
@@ -130,6 +131,23 @@ def get_vm_price_with_vat(cpu, memory, ssd_size, hdd_size=0,
     return float(price), float(vat), float(vat_percent), discount
 
 
+def ping_ok(host_ipv6):
+    """
+    A utility method to check if a host responds to ping requests. Note: the
+    function relies on `ping6` utility of debian to check.
+
+    :param host_ipv6 str type parameter that represets the ipv6 of the host to
+           checked
+    :return True if the host responds to ping else returns False
+    """
+    try:
+        subprocess.check_output("ping6 -c 1 " + host_ipv6, shell=True)
+    except Exception as ex:
+        logger.debug(host_ipv6 + " not reachable via ping. Error = " + str(ex))
+        return False
+    return True
+
+
 class HostingUtils:
     @staticmethod
     def clear_items_from_list(from_list, items_list):