From 91021d761273db7230ae995faa595fd8b3043b5a Mon Sep 17 00:00:00 2001 From: PCoder Date: Tue, 21 Aug 2018 23:24:40 +0200 Subject: [PATCH 1/9] Add ping_ok utility method --- datacenterlight/utils.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/datacenterlight/utils.py b/datacenterlight/utils.py index 1c54148e..7734703b 100644 --- a/datacenterlight/utils.py +++ b/datacenterlight/utils.py @@ -1,3 +1,5 @@ +import logging +import subprocess from django.contrib.sites.models import Site from datacenterlight.tasks import create_vm_task @@ -8,6 +10,8 @@ from utils.models import BillingAddress from .cms_models import CMSIntegration from .models import VMPricing, VMTemplate +logger = logging.getLogger(__name__) + def get_cms_integration(name): current_site = Site.objects.get_current() @@ -91,3 +95,21 @@ def create_vm(billing_address_data, stripe_customer_id, specs, 'token', 'customer']: if session_var in request.session: del request.session[session_var] + + +def ping_ok(host_ipv6): + """ + A utility method to check if a host responds to ping requests. Note: the + function relies on `ping6` utility of debian to check. + + :param host_ipv6 str type parameter that represets the ipv6 of the host to + checked + :return True if the host responds to ping else returns False + """ + try: + output = subprocess.check_output("ping6 -c 1 -w 2 " + host_ipv6, + shell=True) + except Exception as ex: + logger.debug(host_ipv6 + " not reachable via ping. Error = " + str(ex)) + return False + return True From a55587dbf3024dacd673de34aeb0320e01660dc7 Mon Sep 17 00:00:00 2001 From: PCoder Date: Tue, 21 Aug 2018 23:49:47 +0200 Subject: [PATCH 2/9] Wait for VM to be pingable before doing a manage_public_key --- datacenterlight/tasks.py | 31 ++++++++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index 281d5f45..76216c16 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -8,6 +8,7 @@ from django.core.mail import EmailMessage from django.core.urlresolvers import reverse from django.utils import translation from django.utils.translation import ugettext_lazy as _ +from time import sleep from dynamicweb.celery import app from hosting.models import HostingOrder @@ -18,6 +19,7 @@ from utils.hosting_utils import get_all_public_keys, get_or_create_vm_detail from utils.mailer import BaseEmail from utils.stripe_utils import StripeUtils from .models import VMPricing +from .utils import ping_ok logger = get_task_logger(__name__) @@ -206,9 +208,32 @@ def create_vm_task(self, vm_template_id, user, specs, template, order_id): # Let's delay the task by 75 seconds to be sure # that we run the cdist configure after the host # is up - manager.manage_public_key( - keys, hosts=[vm_ipv6], countdown=75 - ) + did_manage_public_key = False + for i in range(0, 15): + if ping_ok(vm_ipv6): + logger.debug( + "{} is pingable. Doing a " + "manage_public_key".format(vm_ipv6) + ) + manager.manage_public_key( + keys, hosts=[vm_ipv6] + ) + did_manage_public_key = True + break + else: + logger.debug( + "Can't ping {}. Wait 5 secs".format( + vm_ipv6 + ) + ) + sleep(5) + if not did_manage_public_key: + logger.error( + "Waited for over 75 seconds for {} to be " + "pingable. But the VM was not reachable. So," + "gave up manage_public_key. Do this " + "manually".format(vm_ipv6) + ) except Exception as e: logger.error(str(e)) try: From 3b07687d3eb6e61e9182450f8b16ae9f4b4baaf8 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 22 Aug 2018 00:12:14 +0200 Subject: [PATCH 3/9] Refactor ping_ok from datacenterlight.utils to utils.hosting_utils --- datacenterlight/utils.py | 19 ------------------- utils/hosting_utils.py | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/datacenterlight/utils.py b/datacenterlight/utils.py index 7734703b..8da408a0 100644 --- a/datacenterlight/utils.py +++ b/datacenterlight/utils.py @@ -1,5 +1,4 @@ import logging -import subprocess from django.contrib.sites.models import Site from datacenterlight.tasks import create_vm_task @@ -95,21 +94,3 @@ def create_vm(billing_address_data, stripe_customer_id, specs, 'token', 'customer']: if session_var in request.session: del request.session[session_var] - - -def ping_ok(host_ipv6): - """ - A utility method to check if a host responds to ping requests. Note: the - function relies on `ping6` utility of debian to check. - - :param host_ipv6 str type parameter that represets the ipv6 of the host to - checked - :return True if the host responds to ping else returns False - """ - try: - output = subprocess.check_output("ping6 -c 1 -w 2 " + host_ipv6, - shell=True) - except Exception as ex: - logger.debug(host_ipv6 + " not reachable via ping. Error = " + str(ex)) - return False - return True diff --git a/utils/hosting_utils.py b/utils/hosting_utils.py index c0494b90..438dbc62 100644 --- a/utils/hosting_utils.py +++ b/utils/hosting_utils.py @@ -1,5 +1,6 @@ import decimal import logging +import subprocess from oca.pool import WrongIdError from datacenterlight.models import VMPricing @@ -130,6 +131,24 @@ def get_vm_price_with_vat(cpu, memory, ssd_size, hdd_size=0, return float(price), float(vat), float(vat_percent), discount +def ping_ok(host_ipv6): + """ + A utility method to check if a host responds to ping requests. Note: the + function relies on `ping6` utility of debian to check. + + :param host_ipv6 str type parameter that represets the ipv6 of the host to + checked + :return True if the host responds to ping else returns False + """ + try: + output = subprocess.check_output("ping6 -c 1 -w 2 " + host_ipv6, + shell=True) + except Exception as ex: + logger.debug(host_ipv6 + " not reachable via ping. Error = " + str(ex)) + return False + return True + + class HostingUtils: @staticmethod def clear_items_from_list(from_list, items_list): From 893c8ed08b1c5513b05ae37af1c5f3e37b4ee507 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 22 Aug 2018 00:14:17 +0200 Subject: [PATCH 4/9] Fix imports --- datacenterlight/tasks.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index 76216c16..c233f522 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -15,11 +15,12 @@ from hosting.models import HostingOrder from membership.models import CustomUser from opennebula_api.models import OpenNebulaManager from opennebula_api.serializers import VirtualMachineSerializer -from utils.hosting_utils import get_all_public_keys, get_or_create_vm_detail +from utils.hosting_utils import ( + get_all_public_keys, get_or_create_vm_detail, ping_ok +) from utils.mailer import BaseEmail from utils.stripe_utils import StripeUtils from .models import VMPricing -from .utils import ping_ok logger = get_task_logger(__name__) From 2ff7eaea65b6592e20785d89b093a578243f30a9 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 22 Aug 2018 00:25:59 +0200 Subject: [PATCH 5/9] Sleep additional 10 seconds before running manage_public_key --- datacenterlight/tasks.py | 1 + 1 file changed, 1 insertion(+) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index c233f522..b7834343 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -216,6 +216,7 @@ def create_vm_task(self, vm_template_id, user, specs, template, order_id): "{} is pingable. Doing a " "manage_public_key".format(vm_ipv6) ) + sleep(10) manager.manage_public_key( keys, hosts=[vm_ipv6] ) From f56933021fdd189550f4d0fbaee53cdfb2938dbe Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 22 Aug 2018 00:37:40 +0200 Subject: [PATCH 6/9] Also send an email if manage_ssh_key does not complete in time --- datacenterlight/tasks.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index b7834343..5f2390bb 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -230,12 +230,22 @@ def create_vm_task(self, vm_template_id, user, specs, template, order_id): ) sleep(5) if not did_manage_public_key: - logger.error( - "Waited for over 75 seconds for {} to be " - "pingable. But the VM was not reachable. So," - "gave up manage_public_key. Do this " - "manually".format(vm_ipv6) - ) + emsg = ("Waited for over 75 seconds for {} to be " + "pingable. But the VM was not reachable. " + "So, gave up manage_public_key. Please do " + "this manually".format(vm_ipv6)) + logger.error(emsg) + email_data = { + 'subject': '{} CELERY TASK INCOMPLETE: {} not ' + 'pingable for 75 seconds'.format( + settings.DCL_TEXT, vm_ipv6 + ), + 'from_email': current_task.request.hostname, + 'to': settings.DCL_ERROR_EMAILS_TO_LIST, + 'body': emsg + } + email = EmailMessage(**email_data) + email.send() except Exception as e: logger.error(str(e)) try: From 13876c24c178bf4eaaf4aabeaaaccee21287645a Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 22 Aug 2018 00:42:50 +0200 Subject: [PATCH 7/9] Update doc --- datacenterlight/tasks.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/datacenterlight/tasks.py b/datacenterlight/tasks.py index 5f2390bb..2779f79b 100644 --- a/datacenterlight/tasks.py +++ b/datacenterlight/tasks.py @@ -206,9 +206,8 @@ def create_vm_task(self, vm_template_id, user, specs, template, order_id): host=vm_ipv6, num_keys=len(keys) ) ) - # Let's delay the task by 75 seconds to be sure - # that we run the cdist configure after the host - # is up + # Let's wait until the IP responds to ping before we + # run the cdist configure on the host did_manage_public_key = False for i in range(0, 15): if ping_ok(vm_ipv6): From e66e8202db773a0935f694f7f1776c4e02773d34 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 22 Aug 2018 19:56:56 +0200 Subject: [PATCH 8/9] Remove unused variable --- utils/hosting_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/hosting_utils.py b/utils/hosting_utils.py index 438dbc62..a69af8f5 100644 --- a/utils/hosting_utils.py +++ b/utils/hosting_utils.py @@ -141,8 +141,7 @@ def ping_ok(host_ipv6): :return True if the host responds to ping else returns False """ try: - output = subprocess.check_output("ping6 -c 1 -w 2 " + host_ipv6, - shell=True) + subprocess.check_output("ping6 -c 1 -w 2 " + host_ipv6, shell=True) except Exception as ex: logger.debug(host_ipv6 + " not reachable via ping. Error = " + str(ex)) return False From 432b54a86d6c1f7f47b14272a4ee217e3b7e0a09 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 22 Aug 2018 20:27:19 +0200 Subject: [PATCH 9/9] Remove timeout parameter in ping6 request --- utils/hosting_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/hosting_utils.py b/utils/hosting_utils.py index a69af8f5..1859a82c 100644 --- a/utils/hosting_utils.py +++ b/utils/hosting_utils.py @@ -141,7 +141,7 @@ def ping_ok(host_ipv6): :return True if the host responds to ping else returns False """ try: - subprocess.check_output("ping6 -c 1 -w 2 " + host_ipv6, shell=True) + subprocess.check_output("ping6 -c 1 " + host_ipv6, shell=True) except Exception as ex: logger.debug(host_ipv6 + " not reachable via ping. Error = " + str(ex)) return False