From 464b48000d4d513eded877223d3b6be9f9c5f2ff Mon Sep 17 00:00:00 2001 From: PCoder Date: Sun, 27 Aug 2017 12:52:30 +0530 Subject: [PATCH 01/56] Added cdist requirement --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8d9c68c5..33bd51fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -95,4 +95,5 @@ pycodestyle==2.3.1 pyflakes==1.5.0 billiard==3.5.0.3 amqp==2.2.1 -vine==1.1.4 \ No newline at end of file +vine==1.1.4 +file:///home/test/projects/Nico/cdist-pcoder/cdist/#egg=cdist-web From 568f38dd6794c052d3109a35003d84842b9c4eac Mon Sep 17 00:00:00 2001 From: PCoder Date: Mon, 28 Aug 2017 12:25:39 +0530 Subject: [PATCH 02/56] Added utils/tasks.py --- utils/tasks.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 utils/tasks.py diff --git a/utils/tasks.py b/utils/tasks.py new file mode 100644 index 00000000..692e9e42 --- /dev/null +++ b/utils/tasks.py @@ -0,0 +1,31 @@ +from dynamicweb.celery import app +from celery.utils.log import get_task_logger +from django.conf import settings +from cdist.integration import configure_hosts_simple +import cdist +import tempfile +import pathlib + +logger = get_task_logger(__name__) + + +@app.task(bind=True, max_retries=settings.CELERY_MAX_RETRIES) +def save_ssh_key(hosts, keys): + """ + Saves ssh key into the VMs of a user using cdist + + :param hosts: A list of hosts to be configured + :param keys: A list of keys to be added + """ + # Generate manifest to be used for configuring the hosts + with tempfile.NamedTemporaryFile() as tmp_manifest: + tmp_manifest.writelines(['__ssh_authorized_keys root \\', + ' --key "{keys}"'.format( + keys='\n'.join(keys))]) + + f = pathlib.Path(tmp_manifest.name) + configure_hosts_simple(hosts, + tmp_manifest.name, + verbose=cdist.argparse.VERBOSE_TRACE) + + From 2b541da94b9ab75a234b2def31da13bbbbe1b76c Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Mon, 28 Aug 2017 11:39:46 +0200 Subject: [PATCH 03/56] Added SaveSSHKeyTestCase --- utils/tests.py | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/utils/tests.py b/utils/tests.py index c4608e73..362a84bc 100644 --- a/utils/tests.py +++ b/utils/tests.py @@ -1,16 +1,19 @@ import uuid +from time import sleep from unittest.mock import patch import stripe +from celery.result import AsyncResult +from django.conf import settings from django.http.request import HttpRequest from django.test import Client -from django.test import TestCase +from django.test import TestCase, override_settings from model_mommy import mommy from datacenterlight.models import StripePlan from membership.models import StripeCustomer from utils.stripe_utils import StripeUtils -from django.conf import settings +from .tasks import save_ssh_key class BaseTestCase(TestCase): @@ -235,3 +238,29 @@ class StripePlanTestCase(TestStripeCustomerDescription): 'response_object').stripe_plan_id}]) self.assertIsNone(result.get('response_object'), None) self.assertIsNotNone(result.get('error')) + + +class SaveSSHKeyTestCase(TestCase): + """ + A test case to test the celery save_ssh_key task + """ + + @override_settings( + task_eager_propagates=True, + task_always_eager=True, + ) + def setUp(self): + self.public_key = ["This is a test", ] + self.hosts = ['localhost'] + + def test_save_ssh_key(self): + async_task = save_ssh_key.delay(self.hosts, self.public_key) + save_ssh_key_result = None + for i in range(0, 10): + sleep(5) + res = AsyncResult(async_task.task_id) + if type(res.result) is bool: + save_ssh_key_result = res.result + break + self.assertIsNotNone(save_ssh_key, "save_ssh_key_result is None") + self.assertTrue(save_ssh_key_result, "save_ssh_key_result is False") From 21f51692c48de1dc93784d32877c2e859581b89c Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Mon, 28 Aug 2017 11:40:59 +0200 Subject: [PATCH 04/56] Improved save_ssh_key celery task --- requirements.txt | 3 ++- utils/tasks.py | 34 ++++++++++++++++++---------------- 2 files changed, 20 insertions(+), 17 deletions(-) diff --git a/requirements.txt b/requirements.txt index 33bd51fa..639ac881 100644 --- a/requirements.txt +++ b/requirements.txt @@ -96,4 +96,5 @@ pyflakes==1.5.0 billiard==3.5.0.3 amqp==2.2.1 vine==1.1.4 -file:///home/test/projects/Nico/cdist-pcoder/cdist/#egg=cdist-web +#git+https://github.com/pcoder/cdist.git#egg=cdist-web +file:///home/mravi/gitprojects/cdist-pcoder/cdist/#egg=cdist-web diff --git a/utils/tasks.py b/utils/tasks.py index 692e9e42..7e015df7 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -1,31 +1,33 @@ -from dynamicweb.celery import app -from celery.utils.log import get_task_logger -from django.conf import settings -from cdist.integration import configure_hosts_simple import cdist import tempfile -import pathlib +from cdist.integration import configure_hosts_simple +from celery.utils.log import get_task_logger +from django.conf import settings + +from dynamicweb.celery import app logger = get_task_logger(__name__) @app.task(bind=True, max_retries=settings.CELERY_MAX_RETRIES) -def save_ssh_key(hosts, keys): +def save_ssh_key(self, hosts, keys): """ Saves ssh key into the VMs of a user using cdist :param hosts: A list of hosts to be configured :param keys: A list of keys to be added """ - # Generate manifest to be used for configuring the hosts + return_value = True with tempfile.NamedTemporaryFile() as tmp_manifest: - tmp_manifest.writelines(['__ssh_authorized_keys root \\', + # Generate manifest to be used for configuring the hosts + tmp_manifest.writelines([b'__ssh_authorized_keys root \\', ' --key "{keys}"'.format( - keys='\n'.join(keys))]) - - f = pathlib.Path(tmp_manifest.name) - configure_hosts_simple(hosts, - tmp_manifest.name, - verbose=cdist.argparse.VERBOSE_TRACE) - - + keys='\n'.join(keys)).encode('utf-8')]) + try: + configure_hosts_simple(hosts, + tmp_manifest.name, + verbose=cdist.argparse.VERBOSE_TRACE) + except Exception as cdist_exception: + logger.error(cdist_exception) + return_value = False + return return_value From 12255364b28d254adc5fbb01e8bf157e3e1089c8 Mon Sep 17 00:00:00 2001 From: PCoder Date: Tue, 29 Aug 2017 13:31:05 +0530 Subject: [PATCH 05/56] Attempted to get hosts for adding ssh keys (to be tested) --- hosting/views.py | 95 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 64 insertions(+), 31 deletions(-) diff --git a/hosting/views.py b/hosting/views.py index 0747f134..f6150190 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -7,7 +7,8 @@ from django.shortcuts import render from django.http import Http404 from django.core.urlresolvers import reverse_lazy, reverse from django.contrib.auth.mixins import LoginRequiredMixin -from django.views.generic import View, CreateView, FormView, ListView, DetailView, \ +from django.views.generic import View, CreateView, FormView, ListView, \ + DetailView, \ DeleteView, TemplateView, UpdateView from django.http import HttpResponseRedirect from django.contrib import messages @@ -24,17 +25,23 @@ from django.utils.safestring import mark_safe from membership.models import CustomUser, StripeCustomer from utils.stripe_utils import StripeUtils -from utils.forms import BillingAddressForm, PasswordResetRequestForm, UserBillingAddressForm -from utils.views import PasswordResetViewMixin, PasswordResetConfirmViewMixin, LoginViewMixin +from utils.forms import BillingAddressForm, PasswordResetRequestForm, \ + UserBillingAddressForm +from utils.views import PasswordResetViewMixin, PasswordResetConfirmViewMixin, \ + LoginViewMixin from utils.mailer import BaseEmail from .models import HostingOrder, HostingBill, HostingPlan, UserHostingKey -from .forms import HostingUserSignupForm, HostingUserLoginForm, UserHostingKeyForm, generate_ssh_key_name +from .forms import HostingUserSignupForm, HostingUserLoginForm, \ + UserHostingKeyForm, generate_ssh_key_name from .mixins import ProcessVMSelectionMixin from opennebula_api.models import OpenNebulaManager from opennebula_api.serializers import VirtualMachineSerializer, \ VirtualMachineTemplateSerializer from django.utils.translation import ugettext_lazy as _ +import logging + +logger = logging.getLogger(__name__) CONNECTION_ERROR = "Your VMs cannot be displayed at the moment due to a backend \ connection error. please try again in a few minutes." @@ -284,7 +291,8 @@ class PasswordResetConfirmView(PasswordResetConfirmViewMixin): form = self.form_class(request.POST) - if user is not None and default_token_generator.check_token(user, token): + if user is not None and default_token_generator.check_token(user, + token): if form.is_valid(): new_password = form.cleaned_data['new_password2'] user.set_password(new_password) @@ -385,7 +393,8 @@ class SSHKeyListView(LoginRequiredMixin, ListView): def render_to_response(self, context, **response_kwargs): if not self.queryset: return HttpResponseRedirect(reverse('hosting:choice_ssh_keys')) - return super(SSHKeyListView, self).render_to_response(context, **response_kwargs) + return super(SSHKeyListView, self).render_to_response(context, + **response_kwargs) class SSHKeyChoiceView(LoginRequiredMixin, View): @@ -448,18 +457,29 @@ class SSHKeyCreateView(LoginRequiredMixin, FormView): }) owner = self.request.user - manager = OpenNebulaManager() + # Get all VMs belonging to the user + allorders = HostingOrder.objects.filter(customer__user=owner) + if len(allorders) > 0: + logger.debug("The user {} has 1 or more VMs. We need to configure " + "the ssh keys.".format(self.request.user.email)) + hosts = [order.vm_id for order in allorders] - # Get user ssh key - public_key = str(form.cleaned_data.get('public_key', '')) - # Add ssh key to user - try: - manager.add_public_key( - user=owner, public_key=public_key, merge=True) - except ConnectionError: - pass - except WrongNameError: - pass + + else: + logger.debug("The user {} has no VMs. We don't need to configure " + "the ssh keys.".format(self.request.user.email)) + # manager = OpenNebulaManager() + # + # # Get user ssh key + # public_key = str(form.cleaned_data.get('public_key', '')) + # # Add ssh key to user + # try: + # manager.add_public_key( + # user=owner, public_key=public_key, merge=True) + # except ConnectionError: + # pass + # except WrongNameError: + # pass return HttpResponseRedirect(self.success_url) @@ -558,8 +578,10 @@ class PaymentVMView(LoginRequiredMixin, FormView): token=token) if not customer: msg = _("Invalid credit card") - messages.add_message(self.request, messages.ERROR, msg, extra_tags='make_charge_error') - return HttpResponseRedirect(reverse('hosting:payment') + '#payment_error') + messages.add_message(self.request, messages.ERROR, msg, + extra_tags='make_charge_error') + return HttpResponseRedirect( + reverse('hosting:payment') + '#payment_error') # Create Billing Address billing_address = form.save() @@ -572,8 +594,10 @@ class PaymentVMView(LoginRequiredMixin, FormView): # Check if the payment was approved if not charge_response.get('response_object'): msg = charge_response.get('error') - messages.add_message(self.request, messages.ERROR, msg, extra_tags='make_charge_error') - return HttpResponseRedirect(reverse('hosting:payment') + '#payment_error') + messages.add_message(self.request, messages.ERROR, msg, + extra_tags='make_charge_error') + return HttpResponseRedirect( + reverse('hosting:payment') + '#payment_error') charge = charge_response.get('response_object') @@ -581,7 +605,8 @@ class PaymentVMView(LoginRequiredMixin, FormView): manager = OpenNebulaManager(email=owner.email, password=owner.password) # Get user ssh key - if not UserHostingKey.objects.filter(user=self.request.user).exists(): + if not UserHostingKey.objects.filter( + user=self.request.user).exists(): context.update({ 'sshError': 'error', 'form': form @@ -633,7 +658,8 @@ class PaymentVMView(LoginRequiredMixin, FormView): context = { 'vm': vm, 'order': order, - 'base_url': "{0}://{1}".format(request.scheme, request.get_host()) + 'base_url': "{0}://{1}".format(request.scheme, + request.get_host()) } email_data = { @@ -647,13 +673,15 @@ class PaymentVMView(LoginRequiredMixin, FormView): email.send() return HttpResponseRedirect( - "{url}?{query_params}".format(url=reverse('hosting:orders', kwargs={'pk': order.id}), - query_params='page=payment')) + "{url}?{query_params}".format( + url=reverse('hosting:orders', kwargs={'pk': order.id}), + query_params='page=payment')) else: return self.form_invalid(form) -class OrdersHostingDetailView(PermissionRequiredMixin, LoginRequiredMixin, DetailView): +class OrdersHostingDetailView(PermissionRequiredMixin, LoginRequiredMixin, + DetailView): template_name = "hosting/order_detail.html" context_object_name = "order" login_url = reverse_lazy('hosting:login') @@ -761,7 +789,8 @@ class CreateVirtualMachinesView(LoginRequiredMixin, View): configuration_options = HostingPlan.get_serialized_configs() context = { - 'templates': VirtualMachineTemplateSerializer(templates, many=True).data, + 'templates': VirtualMachineTemplateSerializer(templates, + many=True).data, 'configuration_options': configuration_options, } except: @@ -832,7 +861,8 @@ class VirtualMachineView(LoginRequiredMixin, View): serializer = VirtualMachineSerializer(vm) context = { 'virtual_machine': serializer.data, - 'order': HostingOrder.objects.get(vm_id=serializer.data['vm_id']) + 'order': HostingOrder.objects.get( + vm_id=serializer.data['vm_id']) } except: pass @@ -863,7 +893,8 @@ class VirtualMachineView(LoginRequiredMixin, View): context = { 'vm': vm, - 'base_url': "{0}://{1}".format(self.request.scheme, self.request.get_host()) + 'base_url': "{0}://{1}".format(self.request.scheme, + self.request.get_host()) } email_data = { 'subject': 'Virtual machine plan canceled', @@ -883,7 +914,8 @@ class VirtualMachineView(LoginRequiredMixin, View): return HttpResponseRedirect(self.get_success_url()) -class HostingBillListView(PermissionRequiredMixin, LoginRequiredMixin, ListView): +class HostingBillListView(PermissionRequiredMixin, LoginRequiredMixin, + ListView): template_name = "hosting/bills.html" login_url = reverse_lazy('hosting:login') permission_required = ['view_hostingview'] @@ -893,7 +925,8 @@ class HostingBillListView(PermissionRequiredMixin, LoginRequiredMixin, ListView) ordering = '-id' -class HostingBillDetailView(PermissionRequiredMixin, LoginRequiredMixin, DetailView): +class HostingBillDetailView(PermissionRequiredMixin, LoginRequiredMixin, + DetailView): template_name = "hosting/bill_detail.html" login_url = reverse_lazy('hosting:login') permission_required = ['view_hostingview'] From 34580b6514155970e1bd2c9800d450981f70c975 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Tue, 29 Aug 2017 17:51:58 +0200 Subject: [PATCH 06/56] Refactored saving public key functionality to opennebula_api.models.save_public_key. Also, now creating hosting VMs loaded by default with ONEADMIN_USER_SSH_PUBLIC_KEY --- hosting/views.py | 47 ++++++++++++++--------------------------------- 1 file changed, 14 insertions(+), 33 deletions(-) diff --git a/hosting/views.py b/hosting/views.py index f6150190..3acc3f11 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -27,8 +27,8 @@ from membership.models import CustomUser, StripeCustomer from utils.stripe_utils import StripeUtils from utils.forms import BillingAddressForm, PasswordResetRequestForm, \ UserBillingAddressForm -from utils.views import PasswordResetViewMixin, PasswordResetConfirmViewMixin, \ - LoginViewMixin +from utils.views import PasswordResetViewMixin, \ + PasswordResetConfirmViewMixin, LoginViewMixin from utils.mailer import BaseEmail from .models import HostingOrder, HostingBill, HostingPlan, UserHostingKey from .forms import HostingUserSignupForm, HostingUserLoginForm, \ @@ -43,8 +43,9 @@ import logging logger = logging.getLogger(__name__) -CONNECTION_ERROR = "Your VMs cannot be displayed at the moment due to a backend \ - connection error. please try again in a few minutes." +CONNECTION_ERROR = "Your VMs cannot be displayed at the moment due to a \ + backend connection error. please try again in a few \ + minutes." class DjangoHostingView(ProcessVMSelectionMixin, View): @@ -457,30 +458,14 @@ class SSHKeyCreateView(LoginRequiredMixin, FormView): }) owner = self.request.user - # Get all VMs belonging to the user - allorders = HostingOrder.objects.filter(customer__user=owner) - if len(allorders) > 0: - logger.debug("The user {} has 1 or more VMs. We need to configure " - "the ssh keys.".format(self.request.user.email)) - hosts = [order.vm_id for order in allorders] - - - else: - logger.debug("The user {} has no VMs. We don't need to configure " - "the ssh keys.".format(self.request.user.email)) - # manager = OpenNebulaManager() - # - # # Get user ssh key - # public_key = str(form.cleaned_data.get('public_key', '')) - # # Add ssh key to user - # try: - # manager.add_public_key( - # user=owner, public_key=public_key, merge=True) - # except ConnectionError: - # pass - # except WrongNameError: - # pass - + manager = OpenNebulaManager( + email=owner.email, + password=owner.password + ) + public_key = form.cleaned_data['public_key'] + if type(public_key) is bytes: + public_key = public_key.decode() + manager.save_public_key([public_key]) return HttpResponseRedirect(self.success_url) def post(self, request, *args, **kwargs): @@ -612,16 +597,12 @@ class PaymentVMView(LoginRequiredMixin, FormView): 'form': form }) return render(request, self.template_name, context) - # For now just get first one - user_key = UserHostingKey.objects.filter( - user=self.request.user).first() # Create a vm using logged user vm_id = manager.create_vm( template_id=vm_template_id, - # XXX: Confi specs=specs, - ssh_key=user_key.public_key, + ssh_key=settings.ONEADMIN_USER_SSH_PUBLIC_KEY, ) # Create a Hosting Order From d25bca0860bcabb7ebdb629c25edf807d1c1a76f Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Tue, 29 Aug 2017 18:02:56 +0200 Subject: [PATCH 07/56] Added save_public_key and reformatted code --- opennebula_api/models.py | 90 +++++++++++++++++++++++++++++----------- 1 file changed, 66 insertions(+), 24 deletions(-) diff --git a/opennebula_api/models.py b/opennebula_api/models.py index 60f3159c..a99d8353 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -9,6 +9,8 @@ from django.conf import settings from utils.models import CustomUser from .exceptions import KeyExistsError, UserExistsError, UserCredentialError +from hosting.models import HostingOrder +from utils.tasks import save_ssh_key logger = logging.getLogger(__name__) @@ -122,16 +124,19 @@ class OpenNebulaManager(): except WrongNameError: user_id = self.oneadmin_client.call(oca.User.METHODS['allocate'], - user.email, user.password, 'core') - logger.debug('Created a user for CustomObject: {user} with user id = {u_id}', - user=user, - u_id=user_id - ) + user.email, user.password, + 'core') + logger.debug( + 'Created a user for CustomObject: {user} with user id = {u_id}', + user=user, + u_id=user_id + ) return user_id except ConnectionRefusedError: - logger.error('Could not connect to host: {host} via protocol {protocol}'.format( - host=settings.OPENNEBULA_DOMAIN, - protocol=settings.OPENNEBULA_PROTOCOL) + logger.error( + 'Could not connect to host: {host} via protocol {protocol}'.format( + host=settings.OPENNEBULA_DOMAIN, + protocol=settings.OPENNEBULA_PROTOCOL) ) raise ConnectionRefusedError @@ -141,8 +146,9 @@ class OpenNebulaManager(): opennebula_user = user_pool.get_by_name(email) return opennebula_user except WrongNameError as wrong_name_err: - opennebula_user = self.oneadmin_client.call(oca.User.METHODS['allocate'], email, - password, 'core') + opennebula_user = self.oneadmin_client.call( + oca.User.METHODS['allocate'], email, + password, 'core') logger.debug( "User {0} does not exist. Created the user. User id = {1}", email, @@ -150,9 +156,10 @@ class OpenNebulaManager(): ) return opennebula_user except ConnectionRefusedError: - logger.info('Could not connect to host: {host} via protocol {protocol}'.format( - host=settings.OPENNEBULA_DOMAIN, - protocol=settings.OPENNEBULA_PROTOCOL) + logger.info( + 'Could not connect to host: {host} via protocol {protocol}'.format( + host=settings.OPENNEBULA_DOMAIN, + protocol=settings.OPENNEBULA_PROTOCOL) ) raise ConnectionRefusedError @@ -161,9 +168,10 @@ class OpenNebulaManager(): user_pool = oca.UserPool(self.oneadmin_client) user_pool.info() except ConnectionRefusedError: - logger.info('Could not connect to host: {host} via protocol {protocol}'.format( - host=settings.OPENNEBULA_DOMAIN, - protocol=settings.OPENNEBULA_PROTOCOL) + logger.info( + 'Could not connect to host: {host} via protocol {protocol}'.format( + host=settings.OPENNEBULA_DOMAIN, + protocol=settings.OPENNEBULA_PROTOCOL) ) raise return user_pool @@ -183,9 +191,10 @@ class OpenNebulaManager(): raise ConnectionRefusedError except ConnectionRefusedError: - logger.info('Could not connect to host: {host} via protocol {protocol}'.format( - host=settings.OPENNEBULA_DOMAIN, - protocol=settings.OPENNEBULA_PROTOCOL) + logger.info( + 'Could not connect to host: {host} via protocol {protocol}'.format( + host=settings.OPENNEBULA_DOMAIN, + protocol=settings.OPENNEBULA_PROTOCOL) ) raise ConnectionRefusedError # For now we'll just handle all other errors as connection errors @@ -258,7 +267,8 @@ class OpenNebulaManager(): vm_specs += "" if ssh_key: - vm_specs += "{ssh}".format(ssh=ssh_key) + vm_specs += "{ssh}".format( + ssh=ssh_key) vm_specs += """YES @@ -312,9 +322,11 @@ class OpenNebulaManager(): template_pool.info() return template_pool except ConnectionRefusedError: - logger.info('Could not connect to host: {host} via protocol {protocol}'.format( - host=settings.OPENNEBULA_DOMAIN, - protocol=settings.OPENNEBULA_PROTOCOL) + logger.info( + '''Could not connect to host: {host} via protocol \ + {protocol}'''.format( + host=settings.OPENNEBULA_DOMAIN, + protocol=settings.OPENNEBULA_PROTOCOL) ) raise ConnectionRefusedError except: @@ -347,7 +359,8 @@ class OpenNebulaManager(): except: raise ConnectionRefusedError - def create_template(self, name, cores, memory, disk_size, core_price, memory_price, + def create_template(self, name, cores, memory, disk_size, core_price, + memory_price, disk_size_price, ssh=''): """Create and add a new template to opennebula. :param name: A string representation describing the template. @@ -490,3 +503,32 @@ class OpenNebulaManager(): except ConnectionError: raise + + def save_public_key(self, keys): + """ + A function that saves the supplied keys to the authorized_keys file + + :param keys: List of ssh keys that are to be added + :return: + """ + owner = CustomUser.objects.filter( + email=self.opennebula_user.name).first() + all_orders = HostingOrder.objects.filter(customer__user=owner) + if len(all_orders) > 0: + logger.debug("The user {} has 1 or more VMs. We need to configure " + "the ssh keys.".format(self.opennebula_user.name)) + hosts = [] + for order in all_orders: + try: + vm = self.get_vm(order.vm_id) + for nic in vm.template.nics: + if hasattr(nic, 'ip'): + hosts.append(nic.ip) + except WrongIdError: + logger.debug( + "VM with ID {} does not exist".format(order.vm_id)) + if len(keys) > 0: + save_ssh_key.delay(hosts, keys) + else: + logger.debug("The user {} has no VMs. We don't need to configure " + "the ssh keys.".format(self.opennebula_user.name)) From 96be1245dcb94a98a565aecb5b91a4205914634e Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Tue, 29 Aug 2017 18:15:23 +0200 Subject: [PATCH 08/56] utils/tasks: Refactored code --- utils/tasks.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/utils/tasks.py b/utils/tasks.py index 7e015df7..a96decce 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -18,11 +18,13 @@ def save_ssh_key(self, hosts, keys): :param keys: A list of keys to be added """ return_value = True - with tempfile.NamedTemporaryFile() as tmp_manifest: + with tempfile.NamedTemporaryFile(delete=True) as tmp_manifest: # Generate manifest to be used for configuring the hosts - tmp_manifest.writelines([b'__ssh_authorized_keys root \\', - ' --key "{keys}"'.format( - keys='\n'.join(keys)).encode('utf-8')]) + lines_list = [' --key "{key}"\\\n'.format(key=key).encode('utf-8') for + key in keys] + lines_list.insert(0, b'__ssh_authorized_keys root \\\n') + tmp_manifest.writelines(lines_list) + tmp_manifest.flush() try: configure_hosts_simple(hosts, tmp_manifest.name, From 92e47b6f90803a52a6ea5ab4dd2a335eca007cde Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Wed, 30 Aug 2017 09:42:56 +0200 Subject: [PATCH 09/56] Added manage_public_key when deleting a key (However, this does not seem to work in cdist) --- hosting/views.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/hosting/views.py b/hosting/views.py index 3acc3f11..05a2b0dd 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -363,17 +363,14 @@ class SSHKeyDeleteView(LoginRequiredMixin, DeleteView): def delete(self, request, *args, **kwargs): owner = self.request.user - manager = OpenNebulaManager() + manager = OpenNebulaManager( + email=owner.email, + password=owner.password + ) pk = self.kwargs.get('pk') # Get user ssh key public_key = UserHostingKey.objects.get(pk=pk).public_key - # Add ssh key to user - try: - manager.remove_public_key(user=owner, public_key=public_key) - except ConnectionError: - pass - except WrongNameError: - pass + manager.manage_public_key([{'value': public_key, 'state': False}]) return super(SSHKeyDeleteView, self).delete(request, *args, **kwargs) @@ -465,7 +462,7 @@ class SSHKeyCreateView(LoginRequiredMixin, FormView): public_key = form.cleaned_data['public_key'] if type(public_key) is bytes: public_key = public_key.decode() - manager.save_public_key([public_key]) + manager.manage_public_key([{'value': public_key, 'state': True}]) return HttpResponseRedirect(self.success_url) def post(self, request, *args, **kwargs): From 666cf187eac933e89a94cd1fd0ca2161918fe623 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Wed, 30 Aug 2017 09:43:54 +0200 Subject: [PATCH 10/56] Refactored and added comments --- opennebula_api/models.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/opennebula_api/models.py b/opennebula_api/models.py index a99d8353..0167e000 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -504,11 +504,16 @@ class OpenNebulaManager(): except ConnectionError: raise - def save_public_key(self, keys): + def manage_public_key(self, keys): """ - A function that saves the supplied keys to the authorized_keys file + A function that manages the supplied keys in the authorized_keys file - :param keys: List of ssh keys that are to be added + :param keys: List of ssh keys that are to be added/removed + A key should be a dict of the form + { + 'value': 'sha-.....', # public key as string + 'state': True # whether key is to be added or + } # removed :return: """ owner = CustomUser.objects.filter( From a4a2b2a803ec90d2f9f752dfaeb70e94b105e6c3 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Wed, 30 Aug 2017 09:44:33 +0200 Subject: [PATCH 11/56] Refactored and commented utils/tasks --- utils/tasks.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/utils/tasks.py b/utils/tasks.py index a96decce..ad0a0e53 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -15,13 +15,22 @@ def save_ssh_key(self, hosts, keys): Saves ssh key into the VMs of a user using cdist :param hosts: A list of hosts to be configured - :param keys: A list of keys to be added + :param keys: A list of keys to be added. A key should be dict of the + form { + 'value': 'sha-.....', # public key as string + 'state': True # whether key is to be added or + } # removed + """ return_value = True with tempfile.NamedTemporaryFile(delete=True) as tmp_manifest: # Generate manifest to be used for configuring the hosts - lines_list = [' --key "{key}"\\\n'.format(key=key).encode('utf-8') for - key in keys] + lines_list = [ + ' --key "{key}" --state {state} \\\n'.format( + key=key['value'], + state='present' if key['state'] else 'absent' + ).encode('utf-8') + for key in keys] lines_list.insert(0, b'__ssh_authorized_keys root \\\n') tmp_manifest.writelines(lines_list) tmp_manifest.flush() From 51b1ee201344c1f425906e929c2feff2ed0ad889 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Wed, 30 Aug 2017 12:02:53 +0200 Subject: [PATCH 12/56] Added call to manage_public_key task when generating dcl generated keys and reorganized imports --- hosting/views.py | 62 ++++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/hosting/views.py b/hosting/views.py index 05a2b0dd..3b59a09f 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -1,45 +1,42 @@ +import logging import uuid -from django.core.files.base import ContentFile - -from oca.pool import WrongNameError, WrongIdError -from django.shortcuts import render -from django.http import Http404 -from django.core.urlresolvers import reverse_lazy, reverse +from django.conf import settings +from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin +from django.contrib.auth.tokens import default_token_generator +from django.core.files.base import ContentFile +from django.core.urlresolvers import reverse_lazy, reverse +from django.http import Http404 +from django.http import HttpResponseRedirect +from django.shortcuts import redirect +from django.shortcuts import render +from django.utils.http import urlsafe_base64_decode +from django.utils.safestring import mark_safe +from django.utils.translation import ugettext_lazy as _ from django.views.generic import View, CreateView, FormView, ListView, \ DetailView, \ DeleteView, TemplateView, UpdateView -from django.http import HttpResponseRedirect -from django.contrib import messages -from django.conf import settings -from django.shortcuts import redirect -from django.utils.http import urlsafe_base64_decode -from django.contrib.auth.tokens import default_token_generator - from guardian.mixins import PermissionRequiredMixin -from stored_messages.settings import stored_messages_settings -from stored_messages.models import Message +from oca.pool import WrongIdError from stored_messages.api import mark_read -from django.utils.safestring import mark_safe +from stored_messages.models import Message +from stored_messages.settings import stored_messages_settings from membership.models import CustomUser, StripeCustomer -from utils.stripe_utils import StripeUtils -from utils.forms import BillingAddressForm, PasswordResetRequestForm, \ - UserBillingAddressForm -from utils.views import PasswordResetViewMixin, \ - PasswordResetConfirmViewMixin, LoginViewMixin -from utils.mailer import BaseEmail -from .models import HostingOrder, HostingBill, HostingPlan, UserHostingKey -from .forms import HostingUserSignupForm, HostingUserLoginForm, \ - UserHostingKeyForm, generate_ssh_key_name -from .mixins import ProcessVMSelectionMixin - from opennebula_api.models import OpenNebulaManager from opennebula_api.serializers import VirtualMachineSerializer, \ VirtualMachineTemplateSerializer -from django.utils.translation import ugettext_lazy as _ -import logging +from utils.forms import BillingAddressForm, PasswordResetRequestForm, \ + UserBillingAddressForm +from utils.mailer import BaseEmail +from utils.stripe_utils import StripeUtils +from utils.views import PasswordResetViewMixin, \ + PasswordResetConfirmViewMixin, LoginViewMixin +from .forms import HostingUserSignupForm, HostingUserLoginForm, \ + UserHostingKeyForm, generate_ssh_key_name +from .mixins import ProcessVMSelectionMixin +from .models import HostingOrder, HostingBill, HostingPlan, UserHostingKey logger = logging.getLogger(__name__) @@ -411,6 +408,13 @@ class SSHKeyChoiceView(LoginRequiredMixin, View): user=request.user, public_key=public_key, name=name) filename = name + '_' + str(uuid.uuid4())[:8] + '_private.pem' ssh_key.private_key.save(filename, content) + owner = self.request.user + manager = OpenNebulaManager( + email=owner.email, + password=owner.password + ) + public_key_str = public_key.decode() + manager.manage_public_key([{'value': public_key_str, 'state': True}]) return redirect(reverse_lazy('hosting:ssh_keys'), foo='bar') From 79ecdadeb2b93ec055fbd3c1fc42c9e862982561 Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 31 Aug 2017 12:53:00 +0530 Subject: [PATCH 13/56] Added ssh public key validation --- hosting/forms.py | 45 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index 288a8caf..c9ceb514 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -1,13 +1,17 @@ +import base64 import datetime +import logging +import struct from django import forms -from membership.models import CustomUser from django.contrib.auth import authenticate - from django.utils.translation import ugettext_lazy as _ +from membership.models import CustomUser from .models import UserHostingKey +logger = logging.getLogger(__name__) + def generate_ssh_key_name(): return 'dcl-generated-key-' + datetime.datetime.now().strftime('%m%d%y%H%M') @@ -38,7 +42,7 @@ class HostingUserLoginForm(forms.Form): CustomUser.objects.get(email=email) return email except CustomUser.DoesNotExist: - raise forms.ValidationError("User does not exist") + raise forms.ValidationError(_("User does not exist")) else: return email @@ -51,7 +55,8 @@ class HostingUserSignupForm(forms.ModelForm): model = CustomUser fields = ['name', 'email', 'password'] widgets = { - 'name': forms.TextInput(attrs={'placeholder': 'Enter your name or company name'}), + 'name': forms.TextInput( + attrs={'placeholder': 'Enter your name or company name'}), } def clean_confirm_password(self): @@ -65,19 +70,45 @@ class HostingUserSignupForm(forms.ModelForm): class UserHostingKeyForm(forms.ModelForm): private_key = forms.CharField(widget=forms.HiddenInput(), required=False) public_key = forms.CharField(widget=forms.Textarea( - attrs={'class': 'form_public_key', 'placeholder': _('Paste here your public key')}), + attrs={'class': 'form_public_key', + 'placeholder': _('Paste here your public key')}), required=False, ) user = forms.models.ModelChoiceField(queryset=CustomUser.objects.all(), - required=False, widget=forms.HiddenInput()) + required=False, + widget=forms.HiddenInput()) name = forms.CharField(required=False, widget=forms.TextInput( - attrs={'class': 'form_key_name', 'placeholder': _('Give a name to your key')})) + attrs={'class': 'form_key_name', + 'placeholder': _('Give a name to your key')})) def __init__(self, *args, **kwargs): self.request = kwargs.pop("request") super(UserHostingKeyForm, self).__init__(*args, **kwargs) self.fields['name'].label = _('Key name') + def clean_public_key(self): + """ + A simple validation of ssh public key + See https://www.ietf.org/rfc/rfc4716.txt + :return: + """ + KEY_ERROR_MESSAGE = _("Please input a proper SSH key") + openssh_pubkey = self.data.get('public_key') + data = None + try: + key_type, key_string, comment = openssh_pubkey.split() + data = base64.decodebytes(key_string.encode('utf-8')) + except Exception as e: + logger.error("Exception while decoding ssh key {}".format(e)) + raise forms.ValidationError(KEY_ERROR_MESSAGE) + int_len = 4 + str_len = struct.unpack('>I', data[:int_len])[0] # this should return 7 + if str_len != 7: + raise forms.ValidationError(KEY_ERROR_MESSAGE) + if data[int_len:int_len + str_len] != key_type.encode('utf-8'): + raise forms.ValidationError(KEY_ERROR_MESSAGE) + return openssh_pubkey + def clean_name(self): return self.data.get('name') From fae96ec5acc4615df09074366b341172e65d1f46 Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 31 Aug 2017 12:54:08 +0530 Subject: [PATCH 14/56] Added email and password as attributes of the class OpenNebulaManager --- opennebula_api/models.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/opennebula_api/models.py b/opennebula_api/models.py index 0167e000..89a56774 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -1,16 +1,15 @@ -import oca -import socket import logging +import socket -from oca.pool import WrongNameError, WrongIdError -from oca.exceptions import OpenNebulaException - +import oca from django.conf import settings +from oca.exceptions import OpenNebulaException +from oca.pool import WrongNameError, WrongIdError -from utils.models import CustomUser -from .exceptions import KeyExistsError, UserExistsError, UserCredentialError from hosting.models import HostingOrder +from utils.models import CustomUser from utils.tasks import save_ssh_key +from .exceptions import KeyExistsError, UserExistsError, UserCredentialError logger = logging.getLogger(__name__) @@ -19,7 +18,8 @@ class OpenNebulaManager(): """This class represents an opennebula manager.""" def __init__(self, email=None, password=None): - + self.email = email + self.password = password # Get oneadmin client self.oneadmin_client = self._get_opennebula_client( settings.OPENNEBULA_USERNAME, @@ -517,11 +517,11 @@ class OpenNebulaManager(): :return: """ owner = CustomUser.objects.filter( - email=self.opennebula_user.name).first() + email=self.email).first() all_orders = HostingOrder.objects.filter(customer__user=owner) if len(all_orders) > 0: logger.debug("The user {} has 1 or more VMs. We need to configure " - "the ssh keys.".format(self.opennebula_user.name)) + "the ssh keys.".format(self.email)) hosts = [] for order in all_orders: try: @@ -536,4 +536,4 @@ class OpenNebulaManager(): save_ssh_key.delay(hosts, keys) else: logger.debug("The user {} has no VMs. We don't need to configure " - "the ssh keys.".format(self.opennebula_user.name)) + "the ssh keys.".format(self.email)) From b35e74e3551857f845cfdb9dd3352883bb47bd2f Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 31 Aug 2017 12:55:19 +0530 Subject: [PATCH 15/56] Updated cdist integration code with index parameter --- utils/tasks.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/utils/tasks.py b/utils/tasks.py index 1c6dd449..6a246d54 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -1,6 +1,7 @@ -import cdist import tempfile -from cdist.integration import configure_hosts_simple + +import cdist +import cdist.integration as cdist_integration from celery.utils.log import get_task_logger from django.conf import settings from django.core.mail import EmailMessage @@ -50,9 +51,13 @@ def save_ssh_key(self, hosts, keys): tmp_manifest.writelines(lines_list) tmp_manifest.flush() try: - configure_hosts_simple(hosts, - tmp_manifest.name, - verbose=cdist.argparse.VERBOSE_TRACE) + cdist_instance_index = cdist_integration.instance_index + cdist_index = next(cdist_instance_index) + cdist_integration.configure_hosts_simple(hosts, + tmp_manifest.name, + index=cdist_index, + verbose=cdist.argparse.VERBOSE_TRACE) + cdist_instance_index.free(cdist_index) except Exception as cdist_exception: logger.error(cdist_exception) return_value = False From 60561be3b9f726f7cc7cb057c4ee8b57ce2c0c1f Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 31 Aug 2017 12:20:00 +0200 Subject: [PATCH 16/56] Added missing condition and reformatted code --- hosting/forms.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index c9ceb514..8541e1f1 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -1,8 +1,8 @@ -import base64 import datetime import logging -import struct +import base64 +import struct from django import forms from django.contrib.auth import authenticate from django.utils.translation import ugettext_lazy as _ @@ -14,7 +14,8 @@ logger = logging.getLogger(__name__) def generate_ssh_key_name(): - return 'dcl-generated-key-' + datetime.datetime.now().strftime('%m%d%y%H%M') + return 'dcl-generated-key-' + datetime.datetime.now().strftime( + '%m%d%y%H%M') class HostingUserLoginForm(forms.Form): @@ -92,6 +93,8 @@ class UserHostingKeyForm(forms.ModelForm): See https://www.ietf.org/rfc/rfc4716.txt :return: """ + if 'generate' in self.request.POST: + return self.data.get('public_key') KEY_ERROR_MESSAGE = _("Please input a proper SSH key") openssh_pubkey = self.data.get('public_key') data = None @@ -102,7 +105,7 @@ class UserHostingKeyForm(forms.ModelForm): logger.error("Exception while decoding ssh key {}".format(e)) raise forms.ValidationError(KEY_ERROR_MESSAGE) int_len = 4 - str_len = struct.unpack('>I', data[:int_len])[0] # this should return 7 + str_len = struct.unpack('>I', data[:int_len])[0] if str_len != 7: raise forms.ValidationError(KEY_ERROR_MESSAGE) if data[int_len:int_len + str_len] != key_type.encode('utf-8'): From 89ba4b6d0406b9e913245387a21701761b9a72fa Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 31 Aug 2017 15:05:45 +0200 Subject: [PATCH 17/56] Added CdistUtils class to manage indices --- opennebula_api/models.py | 3 ++- utils/cdist_utils.py | 26 ++++++++++++++++++++++++++ utils/tasks.py | 34 +++++++++++++++++++++++----------- 3 files changed, 51 insertions(+), 12 deletions(-) create mode 100644 utils/cdist_utils.py diff --git a/opennebula_api/models.py b/opennebula_api/models.py index 89a56774..7a8a333a 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -7,6 +7,7 @@ from oca.exceptions import OpenNebulaException from oca.pool import WrongNameError, WrongIdError from hosting.models import HostingOrder +from utils.cdist_utils import CdistUtilts from utils.models import CustomUser from utils.tasks import save_ssh_key from .exceptions import KeyExistsError, UserExistsError, UserCredentialError @@ -533,7 +534,7 @@ class OpenNebulaManager(): logger.debug( "VM with ID {} does not exist".format(order.vm_id)) if len(keys) > 0: - save_ssh_key.delay(hosts, keys) + save_ssh_key.delay(hosts, keys, CdistUtilts.get_cdist_index()) else: logger.debug("The user {} has no VMs. We don't need to configure " "the ssh keys.".format(self.email)) diff --git a/utils/cdist_utils.py b/utils/cdist_utils.py new file mode 100644 index 00000000..801a02d5 --- /dev/null +++ b/utils/cdist_utils.py @@ -0,0 +1,26 @@ +import cdist.integration as cdist_integration + + +class CdistUtilts(): + @staticmethod + def get_cdist_index(): + """ + Returns the next available instance index. + This is useful while making simultaneous configurations of + the same host. + :return: the next available index + """ + cdist_instance_index = cdist_integration.instance_index + cdist_index = next(cdist_instance_index) + return cdist_index + + @staticmethod + def free_cdist_index(cdist_index): + """ + Frees up the index that was used during configuring a host + using cdist. + :param cdist_index: The index to be freed + :return: + """ + cdist_instance_index = cdist_integration.instance_index + cdist_instance_index.free(cdist_index) diff --git a/utils/tasks.py b/utils/tasks.py index 6a246d54..d261ff31 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -1,12 +1,12 @@ -import tempfile - import cdist -import cdist.integration as cdist_integration +import tempfile +from cdist.integration import configure_hosts_simple from celery.utils.log import get_task_logger from django.conf import settings from django.core.mail import EmailMessage from dynamicweb.celery import app +from utils.cdist_utils import CdistUtilts logger = get_task_logger(__name__) @@ -26,7 +26,7 @@ def send_plain_email_task(self, email_data): @app.task(bind=True, max_retries=settings.CELERY_MAX_RETRIES) -def save_ssh_key(self, hosts, keys): +def save_ssh_key(self, hosts, keys, index): """ Saves ssh key into the VMs of a user using cdist @@ -36,8 +36,22 @@ def save_ssh_key(self, hosts, keys): 'value': 'sha-.....', # public key as string 'state': True # whether key is to be added or } # removed + :param index: An integer that uniquely identifies simultaneous cdist + configurations being run on a host """ + logger.debug("""Running save_ssh_key task for + Hosts: {hosts_str} + Keys: {keys_str} + index: {index}""".format(hosts_str=", ".join(hosts), + keys_str=", ".join([ + "{value}->{state}".format( + value=key.get('value'), + state=str( + key.get('state'))) + for key in keys]), + index=index) + ) return_value = True with tempfile.NamedTemporaryFile(delete=True) as tmp_manifest: # Generate manifest to be used for configuring the hosts @@ -51,14 +65,12 @@ def save_ssh_key(self, hosts, keys): tmp_manifest.writelines(lines_list) tmp_manifest.flush() try: - cdist_instance_index = cdist_integration.instance_index - cdist_index = next(cdist_instance_index) - cdist_integration.configure_hosts_simple(hosts, - tmp_manifest.name, - index=cdist_index, - verbose=cdist.argparse.VERBOSE_TRACE) - cdist_instance_index.free(cdist_index) + configure_hosts_simple(hosts, + tmp_manifest.name, + index=index, + verbose=cdist.argparse.VERBOSE_TRACE) except Exception as cdist_exception: logger.error(cdist_exception) return_value = False + CdistUtilts.free_cdist_index(index) return return_value From 5a46cea307dec77be5ca2c8f41675cd30782ae5c Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 31 Aug 2017 18:20:02 +0200 Subject: [PATCH 18/56] Added functionality to add keys on creating a new vm --- hosting/views.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/hosting/views.py b/hosting/views.py index 768bc19e..e34faaf8 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -29,6 +29,7 @@ from opennebula_api.serializers import VirtualMachineSerializer, \ VirtualMachineTemplateSerializer from utils.forms import BillingAddressForm, PasswordResetRequestForm, \ UserBillingAddressForm +from utils.hosting_utils import get_all_public_keys from utils.mailer import BaseEmail from utils.stripe_utils import StripeUtils from utils.views import PasswordResetViewMixin, PasswordResetConfirmViewMixin, \ @@ -666,6 +667,19 @@ class PaymentVMView(LoginRequiredMixin, FormView): email = BaseEmail(**email_data) email.send() + # try to see if we have the IP and that if the ssh keys can + # be configured + new_host = manager.get_primary_ipv4(vm_id) + if new_host is not None: + public_keys = get_all_public_keys(owner) + keys = [{'value': key, 'state': True} for key in public_keys] + logger.debug( + "Calling configure on {host} for {num_keys} keys".format( + host=new_host, num_keys=len(keys))) + # Let's delay the task by 75 seconds to be sure that we run + # the cdist configure after the host is up + manager.manage_public_key(keys, hosts=[new_host], countdown=75) + return HttpResponseRedirect( "{url}?{query_params}".format( url=reverse('hosting:orders', kwargs={'pk': order.id}), @@ -858,7 +872,8 @@ class VirtualMachineView(LoginRequiredMixin, View): 'order': HostingOrder.objects.get( vm_id=serializer.data['vm_id']) } - except: + except Exception as ex: + logger.debug("Exception generated {}".format(str(ex))) pass return render(request, self.template_name, context) From 9a56b66282d94909b1303e58485ff513fea5294c Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 31 Aug 2017 18:33:07 +0200 Subject: [PATCH 19/56] Added function get_all_hosts --- opennebula_api/models.py | 59 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 6 deletions(-) diff --git a/opennebula_api/models.py b/opennebula_api/models.py index 7a8a333a..1e199016 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -218,6 +218,33 @@ class OpenNebulaManager(): except: raise ConnectionRefusedError + def get_primary_ipv4(self, vm_id): + """ + Returns the primary IPv4 of the given vm. + To be changed later. + + :return: An IP address string, if it exists else returns None + """ + all_ipv4s = self.get_vm_ipv4_addresses(vm_id) + if len(all_ipv4s) > 0: + return all_ipv4s[0] + else: + return None + + def get_vm_ipv4_addresses(self, vm_id): + """ + Returns a list of IPv4 addresses of the given vm + + :param vm_id: The ID of the vm + :return: + """ + ipv4s = [] + vm = self.get_vm(vm_id) + for nic in vm.template.nics: + if hasattr(nic, 'ip'): + ipv4s.append(nic.ip) + return ipv4s + def create_vm(self, template_id, specs, ssh_key=None, vm_name=None): template = self.get_template(template_id) @@ -505,25 +532,46 @@ class OpenNebulaManager(): except ConnectionError: raise - def manage_public_key(self, keys): + def manage_public_key(self, keys, hosts=None, countdown=0): """ - A function that manages the supplied keys in the authorized_keys file + A function that manages the supplied keys in the + authorized_keys file of the given list of hosts. If hosts + parameter is not supplied, all hosts of this customer + will be configured with the supplied keys - :param keys: List of ssh keys that are to be added/removed + :param keys: A list of ssh keys that are to be added/removed A key should be a dict of the form { 'value': 'sha-.....', # public key as string 'state': True # whether key is to be added or } # removed + :param hosts: A list of hosts IP addresses + :param countdown: Parameter to be passed to celery apply_async + Allows to delay a task by `countdown` number of seconds :return: """ + if hosts is None: + hosts = self.get_all_hosts() + + if len(hosts) > 0 and len(keys) > 0: + save_ssh_key.apply_async( + (hosts, keys, CdistUtilts.get_cdist_index()), + countdown=countdown) + else: + logger.debug("Keys and hosts are empty, so not managing any keys") + + def get_all_hosts(self): + """ + A utility function to obtain all hosts of this owner + :return: A list of hosts IP addresses, empty if none exist + """ owner = CustomUser.objects.filter( email=self.email).first() all_orders = HostingOrder.objects.filter(customer__user=owner) + hosts = [] if len(all_orders) > 0: logger.debug("The user {} has 1 or more VMs. We need to configure " "the ssh keys.".format(self.email)) - hosts = [] for order in all_orders: try: vm = self.get_vm(order.vm_id) @@ -533,8 +581,7 @@ class OpenNebulaManager(): except WrongIdError: logger.debug( "VM with ID {} does not exist".format(order.vm_id)) - if len(keys) > 0: - save_ssh_key.delay(hosts, keys, CdistUtilts.get_cdist_index()) else: logger.debug("The user {} has no VMs. We don't need to configure " "the ssh keys.".format(self.email)) + return hosts From 75a778e65f031463b26f6bab0d08cf402879719d Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 31 Aug 2017 18:35:19 +0200 Subject: [PATCH 20/56] Added hosting_utils.py --- utils/hosting_utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 utils/hosting_utils.py diff --git a/utils/hosting_utils.py b/utils/hosting_utils.py new file mode 100644 index 00000000..d73fb6b0 --- /dev/null +++ b/utils/hosting_utils.py @@ -0,0 +1,11 @@ +from hosting.models import UserHostingKey + + +def get_all_public_keys(customer): + """ + Returns all the public keys of the user + :param customer: The customer whose public keys are needed + :return: A list of public keys + """ + return UserHostingKey.objects.filter( + user__site__customuser=customer).values_list("public_key", flat=True) From ab31ad37d3dd74a047c103c4a5556270e4005a40 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 31 Aug 2017 18:57:05 +0200 Subject: [PATCH 21/56] Not passing index to cdist configure --- opennebula_api/models.py | 5 +---- utils/cdist_utils.py | 26 -------------------------- utils/tasks.py | 24 ++++++++---------------- 3 files changed, 9 insertions(+), 46 deletions(-) delete mode 100644 utils/cdist_utils.py diff --git a/opennebula_api/models.py b/opennebula_api/models.py index 1e199016..c812c7cd 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -7,7 +7,6 @@ from oca.exceptions import OpenNebulaException from oca.pool import WrongNameError, WrongIdError from hosting.models import HostingOrder -from utils.cdist_utils import CdistUtilts from utils.models import CustomUser from utils.tasks import save_ssh_key from .exceptions import KeyExistsError, UserExistsError, UserCredentialError @@ -554,9 +553,7 @@ class OpenNebulaManager(): hosts = self.get_all_hosts() if len(hosts) > 0 and len(keys) > 0: - save_ssh_key.apply_async( - (hosts, keys, CdistUtilts.get_cdist_index()), - countdown=countdown) + save_ssh_key.apply_async((hosts, keys), countdown=countdown) else: logger.debug("Keys and hosts are empty, so not managing any keys") diff --git a/utils/cdist_utils.py b/utils/cdist_utils.py deleted file mode 100644 index 801a02d5..00000000 --- a/utils/cdist_utils.py +++ /dev/null @@ -1,26 +0,0 @@ -import cdist.integration as cdist_integration - - -class CdistUtilts(): - @staticmethod - def get_cdist_index(): - """ - Returns the next available instance index. - This is useful while making simultaneous configurations of - the same host. - :return: the next available index - """ - cdist_instance_index = cdist_integration.instance_index - cdist_index = next(cdist_instance_index) - return cdist_index - - @staticmethod - def free_cdist_index(cdist_index): - """ - Frees up the index that was used during configuring a host - using cdist. - :param cdist_index: The index to be freed - :return: - """ - cdist_instance_index = cdist_integration.instance_index - cdist_instance_index.free(cdist_index) diff --git a/utils/tasks.py b/utils/tasks.py index d261ff31..5c6aaeab 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -6,7 +6,6 @@ from django.conf import settings from django.core.mail import EmailMessage from dynamicweb.celery import app -from utils.cdist_utils import CdistUtilts logger = get_task_logger(__name__) @@ -26,7 +25,7 @@ def send_plain_email_task(self, email_data): @app.task(bind=True, max_retries=settings.CELERY_MAX_RETRIES) -def save_ssh_key(self, hosts, keys, index): +def save_ssh_key(self, hosts, keys): """ Saves ssh key into the VMs of a user using cdist @@ -36,21 +35,16 @@ def save_ssh_key(self, hosts, keys, index): 'value': 'sha-.....', # public key as string 'state': True # whether key is to be added or } # removed - :param index: An integer that uniquely identifies simultaneous cdist - configurations being run on a host - """ logger.debug("""Running save_ssh_key task for Hosts: {hosts_str} - Keys: {keys_str} - index: {index}""".format(hosts_str=", ".join(hosts), - keys_str=", ".join([ - "{value}->{state}".format( - value=key.get('value'), - state=str( - key.get('state'))) - for key in keys]), - index=index) + Keys: {keys_str}""".format(hosts_str=", ".join(hosts), + keys_str=", ".join([ + "{value}->{state}".format( + value=key.get('value'), + state=str( + key.get('state'))) + for key in keys])) ) return_value = True with tempfile.NamedTemporaryFile(delete=True) as tmp_manifest: @@ -67,10 +61,8 @@ def save_ssh_key(self, hosts, keys, index): try: configure_hosts_simple(hosts, tmp_manifest.name, - index=index, verbose=cdist.argparse.VERBOSE_TRACE) except Exception as cdist_exception: logger.error(cdist_exception) return_value = False - CdistUtilts.free_cdist_index(index) return return_value From 6933ad590090a343b4df85c76f3675044544f355 Mon Sep 17 00:00:00 2001 From: PCoder Date: Fri, 1 Sep 2017 00:51:58 +0530 Subject: [PATCH 22/56] Updated requirements for cdist --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 639ac881..c6ac7d48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -96,5 +96,4 @@ pyflakes==1.5.0 billiard==3.5.0.3 amqp==2.2.1 vine==1.1.4 -#git+https://github.com/pcoder/cdist.git#egg=cdist-web -file:///home/mravi/gitprojects/cdist-pcoder/cdist/#egg=cdist-web +git+https://github.com/darko-poljak/cdist.git@cdist-web#egg=cdist-web From 8eca78b0435714a1a8921f190bf2e8984b73b97f Mon Sep 17 00:00:00 2001 From: PCoder Date: Fri, 1 Sep 2017 00:58:30 +0530 Subject: [PATCH 23/56] Reorganized imports hosting/forms.py --- hosting/forms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index 8541e1f1..cf8dd9b7 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -1,8 +1,8 @@ +import base64 import datetime import logging - -import base64 import struct + from django import forms from django.contrib.auth import authenticate from django.utils.translation import ugettext_lazy as _ From a72287be5c7edf318993c44424dc76ca239b3602 Mon Sep 17 00:00:00 2001 From: PCoder Date: Fri, 1 Sep 2017 01:32:31 +0530 Subject: [PATCH 24/56] Changed a string format --- opennebula_api/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennebula_api/models.py b/opennebula_api/models.py index c812c7cd..c115f225 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -350,8 +350,8 @@ class OpenNebulaManager(): return template_pool except ConnectionRefusedError: logger.info( - '''Could not connect to host: {host} via protocol \ - {protocol}'''.format( + """Could not connect to host: {host} via protocol + {protocol}""".format( host=settings.OPENNEBULA_DOMAIN, protocol=settings.OPENNEBULA_PROTOCOL) ) From 1bc78ceb5da49586672bd1a5bf9e3a7555abf187 Mon Sep 17 00:00:00 2001 From: PCoder Date: Fri, 1 Sep 2017 02:33:31 +0530 Subject: [PATCH 25/56] Added save_ssh_key_error_handler --- opennebula_api/models.py | 5 +++-- utils/tasks.py | 20 +++++++++++++++++++- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/opennebula_api/models.py b/opennebula_api/models.py index c115f225..f88f3b15 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -8,7 +8,7 @@ from oca.pool import WrongNameError, WrongIdError from hosting.models import HostingOrder from utils.models import CustomUser -from utils.tasks import save_ssh_key +from utils.tasks import save_ssh_key, save_ssh_key_error_handler from .exceptions import KeyExistsError, UserExistsError, UserCredentialError logger = logging.getLogger(__name__) @@ -553,7 +553,8 @@ class OpenNebulaManager(): hosts = self.get_all_hosts() if len(hosts) > 0 and len(keys) > 0: - save_ssh_key.apply_async((hosts, keys), countdown=countdown) + save_ssh_key.apply_async((hosts, keys), countdown=countdown, + link_error=save_ssh_key_error_handler.s()) else: logger.debug("Keys and hosts are empty, so not managing any keys") diff --git a/utils/tasks.py b/utils/tasks.py index 5c6aaeab..1844bc16 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -1,6 +1,8 @@ -import cdist import tempfile + +import cdist from cdist.integration import configure_hosts_simple +from celery.result import AsyncResult from celery.utils.log import get_task_logger from django.conf import settings from django.core.mail import EmailMessage @@ -66,3 +68,19 @@ def save_ssh_key(self, hosts, keys): logger.error(cdist_exception) return_value = False return return_value + + +@app.task +def save_ssh_key_error_handler(uuid): + result = AsyncResult(uuid) + exc = result.get(propagate=False) + logger.error('Task {0} raised exception: {1!r}\n{2!r}'.format( + uuid, exc, result.traceback)) + email_data = { + 'subject': "[celery error] Save SSH key error {0}".format(uuid), + 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, + 'to': ['info@ungleich.ch'], + 'body': "Task Id: {0}\nResult: {1}\nTraceback: {2}".format( + uuid, exc, result.traceback), + } + send_plain_email_task(email_data) From 277fcdbac12a346b4f0e20d9c44b73e405a30677 Mon Sep 17 00:00:00 2001 From: PCoder Date: Tue, 5 Sep 2017 00:34:22 +0530 Subject: [PATCH 26/56] cdist: Updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c6ac7d48..73f0bc39 100644 --- a/requirements.txt +++ b/requirements.txt @@ -96,4 +96,4 @@ pyflakes==1.5.0 billiard==3.5.0.3 amqp==2.2.1 vine==1.1.4 -git+https://github.com/darko-poljak/cdist.git@cdist-web#egg=cdist-web +git+https://github.com/ungleich/cdist.git#egg=cdist From a32bb9b774783edfebba52a9ca6f5f034c9b67dc Mon Sep 17 00:00:00 2001 From: PCoder Date: Tue, 5 Sep 2017 02:26:00 +0530 Subject: [PATCH 27/56] We use the default timezone available to celery --- dynamicweb/settings/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py index 08ce457d..a2b60026 100644 --- a/dynamicweb/settings/base.py +++ b/dynamicweb/settings/base.py @@ -559,7 +559,7 @@ CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND') CELERY_ACCEPT_CONTENT = ['application/json'] CELERY_TASK_SERIALIZER = 'json' CELERY_RESULT_SERIALIZER = 'json' -CELERY_TIMEZONE = 'Europe/Zurich' +#CELERY_TIMEZONE = 'Europe/Zurich' CELERY_MAX_RETRIES = int_env('CELERY_MAX_RETRIES', 5) ENABLE_DEBUG_LOGGING = bool_env('ENABLE_DEBUG_LOGGING') From dc6fa5428e40572bcf2a879328d0cfd086f20f60 Mon Sep 17 00:00:00 2001 From: PCoder Date: Tue, 5 Sep 2017 03:29:12 +0530 Subject: [PATCH 28/56] Corrected filtering public ssh keys condition --- utils/hosting_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/hosting_utils.py b/utils/hosting_utils.py index d73fb6b0..7c1a83ad 100644 --- a/utils/hosting_utils.py +++ b/utils/hosting_utils.py @@ -7,5 +7,5 @@ def get_all_public_keys(customer): :param customer: The customer whose public keys are needed :return: A list of public keys """ - return UserHostingKey.objects.filter( - user__site__customuser=customer).values_list("public_key", flat=True) + return UserHostingKey.objects.filter(user_id=customer.id).values_list( + "public_key", flat=True) From cf6bd8a7c18140b9a9f8bd10b5b050c0a938113c Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 7 Sep 2017 00:31:09 +0200 Subject: [PATCH 29/56] Validating ssh public keys using sshpubkeys --- hosting/forms.py | 27 ++++++++++++------------- hosting/locale/de/LC_MESSAGES/django.po | 8 +++++++- requirements.txt | 1 + 3 files changed, 21 insertions(+), 15 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index cf8dd9b7..a4076339 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -6,6 +6,8 @@ import struct from django import forms from django.contrib.auth import authenticate from django.utils.translation import ugettext_lazy as _ +from sshpubkeys import SSHKey +from sshpubkeys.exceptions import InvalidKeyException from membership.models import CustomUser from .models import UserHostingKey @@ -89,28 +91,25 @@ class UserHostingKeyForm(forms.ModelForm): def clean_public_key(self): """ - A simple validation of ssh public key - See https://www.ietf.org/rfc/rfc4716.txt + A function that validates a public ssh key using sshpubkeys module :return: """ if 'generate' in self.request.POST: return self.data.get('public_key') KEY_ERROR_MESSAGE = _("Please input a proper SSH key") - openssh_pubkey = self.data.get('public_key') - data = None + openssh_pubkey_str = self.data.get('public_key') + ssh_key = SSHKey(openssh_pubkey_str) try: - key_type, key_string, comment = openssh_pubkey.split() - data = base64.decodebytes(key_string.encode('utf-8')) - except Exception as e: - logger.error("Exception while decoding ssh key {}".format(e)) + ssh_key.parse() + except InvalidKeyException as err: + logger.error( + "InvalidKeyException while parsing ssh key {0}".format(err)) raise forms.ValidationError(KEY_ERROR_MESSAGE) - int_len = 4 - str_len = struct.unpack('>I', data[:int_len])[0] - if str_len != 7: + except NotImplementedError as err: + logger.error( + "NotImplementedError while parsing ssh key {0}".format(err)) raise forms.ValidationError(KEY_ERROR_MESSAGE) - if data[int_len:int_len + str_len] != key_type.encode('utf-8'): - raise forms.ValidationError(KEY_ERROR_MESSAGE) - return openssh_pubkey + return openssh_pubkey_str def clean_name(self): return self.data.get('name') diff --git a/hosting/locale/de/LC_MESSAGES/django.po b/hosting/locale/de/LC_MESSAGES/django.po index f905d905..eb7db5fe 100644 --- a/hosting/locale/de/LC_MESSAGES/django.po +++ b/hosting/locale/de/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-09-02 03:08+0530\n" +"POT-Creation-Date: 2017-09-06 22:27+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -24,6 +24,9 @@ msgstr "Dein Benutzername und/oder Dein Passwort ist falsch." msgid "Your account is not activated yet." msgstr "Dein Account wurde noch nicht aktiviert." +msgid "User does not exist" +msgstr "" + msgid "Paste here your public key" msgstr "Füge deinen Public Key ein" @@ -33,6 +36,9 @@ msgstr "Gebe deinem SSH-Key einen Name" msgid "Key name" msgstr "Key-Name" +msgid "Please input a proper SSH key" +msgstr "" + msgid "My Virtual Machines" msgstr "Meine virtuellen Maschinen" diff --git a/requirements.txt b/requirements.txt index 73f0bc39..89123b46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -97,3 +97,4 @@ billiard==3.5.0.3 amqp==2.2.1 vine==1.1.4 git+https://github.com/ungleich/cdist.git#egg=cdist +sshpubkeys \ No newline at end of file From ec300a76eb6b9591edfe81e1bce875ef8a5668d8 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 7 Sep 2017 00:44:39 +0200 Subject: [PATCH 30/56] Handled some more errors that can be generated when creating SSHKey --- hosting/forms.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/hosting/forms.py b/hosting/forms.py index a4076339..64de0276 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -98,8 +98,8 @@ class UserHostingKeyForm(forms.ModelForm): return self.data.get('public_key') KEY_ERROR_MESSAGE = _("Please input a proper SSH key") openssh_pubkey_str = self.data.get('public_key') - ssh_key = SSHKey(openssh_pubkey_str) try: + ssh_key = SSHKey(openssh_pubkey_str) ssh_key.parse() except InvalidKeyException as err: logger.error( @@ -109,6 +109,14 @@ class UserHostingKeyForm(forms.ModelForm): logger.error( "NotImplementedError while parsing ssh key {0}".format(err)) raise forms.ValidationError(KEY_ERROR_MESSAGE) + except UnicodeDecodeError as u: + logger.error( + "UnicodeDecodeError while parsing ssh key {0}".format(u)) + raise forms.ValidationError(KEY_ERROR_MESSAGE) + except ValueError as v: + logger.error( + "ValueError while parsing ssh key {0}".format(v)) + raise forms.ValidationError(KEY_ERROR_MESSAGE) return openssh_pubkey_str def clean_name(self): From 5a4f0241c2c1fca7d63be8ff572038b658bea946 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Tue, 12 Sep 2017 20:10:34 +0200 Subject: [PATCH 31/56] Updated requirements.txt for fetching appropriate cdist and python-sshpubkeys. --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 89123b46..cba80468 100644 --- a/requirements.txt +++ b/requirements.txt @@ -96,5 +96,6 @@ pyflakes==1.5.0 billiard==3.5.0.3 amqp==2.2.1 vine==1.1.4 -git+https://github.com/ungleich/cdist.git#egg=cdist -sshpubkeys \ No newline at end of file +#git+https://github.com/ungleich/cdist.git#egg=cdist +file:///home/app/cdist#egg=cdist +https://github.com/pcoder/python-sshpubkeys.git#egg=sshpubkeys \ No newline at end of file From abb419fdd7077ee0613a8b236244437fc400eea6 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 00:04:18 +0530 Subject: [PATCH 32/56] Updated requirements.txt --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index cba80468..c04b143b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -98,4 +98,4 @@ amqp==2.2.1 vine==1.1.4 #git+https://github.com/ungleich/cdist.git#egg=cdist file:///home/app/cdist#egg=cdist -https://github.com/pcoder/python-sshpubkeys.git#egg=sshpubkeys \ No newline at end of file +git+https://github.com/pcoder/python-sshpubkeys.git#egg=sshpubkeys From 825a99497935bf882d64062c30d87c39f8d04d64 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 00:10:45 +0530 Subject: [PATCH 33/56] Updated hosting django.po --- hosting/locale/de/LC_MESSAGES/django.po | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/hosting/locale/de/LC_MESSAGES/django.po b/hosting/locale/de/LC_MESSAGES/django.po index b1c5b55c..bb88f269 100644 --- a/hosting/locale/de/LC_MESSAGES/django.po +++ b/hosting/locale/de/LC_MESSAGES/django.po @@ -312,9 +312,6 @@ msgstr "Konfiguration" msgid "Dashboard" msgstr "Mein Dashboard" -msgid "Logout" -msgstr "Abmelden" - msgid "Don't have an account yet ? " msgstr "Besitzt du kein Benutzerkonto?" @@ -668,24 +665,6 @@ msgstr "VM Kündigung" msgid "VM %(VM_ID)s terminated successfully" msgstr "VM %(VM_ID)s erfolgreich beendet" -#~ msgid "My Virtual Machines" -#~ msgstr "Meine virtuellen Maschinen" - -#~ msgid "My Orders" -#~ msgstr "Meine Bestellungen" - -#~ msgid "SSH Keys" -#~ msgstr "SSH Key" - -#~ msgid "Notifications " -#~ msgstr "Benachrichtigungen" - -#~ msgid "REMOVE CARD" -#~ msgstr "KARTE ENTFERNEN" - -#~ msgid "EDIT CARD" -#~ msgstr "BEARBEITEN" - #~ msgid "Add a new Card." #~ msgstr "Neue Kreditkarte hinzufügen." From bb9d22b0f14d4a43e0104d54693459206cf4e0ee Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 01:08:22 +0530 Subject: [PATCH 34/56] Updated requirements.txt --- requirements.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index c04b143b..6446a5c9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -98,4 +98,3 @@ amqp==2.2.1 vine==1.1.4 #git+https://github.com/ungleich/cdist.git#egg=cdist file:///home/app/cdist#egg=cdist -git+https://github.com/pcoder/python-sshpubkeys.git#egg=sshpubkeys From 4ce914b1780011ba56d43ee1ca6b9228bcd7304e Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 01:11:25 +0530 Subject: [PATCH 35/56] Removed some code that was not used --- hosting/forms.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index 64de0276..464e2059 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -1,13 +1,9 @@ -import base64 import datetime import logging -import struct from django import forms from django.contrib.auth import authenticate from django.utils.translation import ugettext_lazy as _ -from sshpubkeys import SSHKey -from sshpubkeys.exceptions import InvalidKeyException from membership.models import CustomUser from .models import UserHostingKey @@ -46,8 +42,6 @@ class HostingUserLoginForm(forms.Form): return email except CustomUser.DoesNotExist: raise forms.ValidationError(_("User does not exist")) - else: - return email class HostingUserSignupForm(forms.ModelForm): From 261615e7018867f0279659c3f7c84fc477f8524c Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 02:22:29 +0530 Subject: [PATCH 36/56] Using ssh-keygen to validate public key --- hosting/forms.py | 38 +++++++++++++++++++------------------- utils/tasks.py | 5 +++++ 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index 464e2059..458bee6f 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -1,5 +1,8 @@ import datetime import logging +import subprocess +import tempfile +import os from django import forms from django.contrib.auth import authenticate @@ -92,25 +95,22 @@ class UserHostingKeyForm(forms.ModelForm): return self.data.get('public_key') KEY_ERROR_MESSAGE = _("Please input a proper SSH key") openssh_pubkey_str = self.data.get('public_key') - try: - ssh_key = SSHKey(openssh_pubkey_str) - ssh_key.parse() - except InvalidKeyException as err: - logger.error( - "InvalidKeyException while parsing ssh key {0}".format(err)) - raise forms.ValidationError(KEY_ERROR_MESSAGE) - except NotImplementedError as err: - logger.error( - "NotImplementedError while parsing ssh key {0}".format(err)) - raise forms.ValidationError(KEY_ERROR_MESSAGE) - except UnicodeDecodeError as u: - logger.error( - "UnicodeDecodeError while parsing ssh key {0}".format(u)) - raise forms.ValidationError(KEY_ERROR_MESSAGE) - except ValueError as v: - logger.error( - "ValueError while parsing ssh key {0}".format(v)) - raise forms.ValidationError(KEY_ERROR_MESSAGE) + + with tempfile.NamedTemporaryFile(delete=True) as tmp_public_key_file: + tmp_public_key_file.writelines(openssh_pubkey_str) + tmp_public_key_file.flush() + try: + out = subprocess.check_output( + ['ssh-keygen', '-lf', tmp_public_key_file.name]) + except subprocess.CalledProcessError as cpe: + logger.debug( + "Not a correct ssh format {error} {out}".format( + error=str(cpe), out=out)) + raise forms.ValidationError(KEY_ERROR_MESSAGE) + try: + os.remove(tmp_public_key_file.name) + except OSError: + pass return openssh_pubkey_str def clean_name(self): diff --git a/utils/tasks.py b/utils/tasks.py index 1844bc16..21d2c9b3 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -1,4 +1,5 @@ import tempfile +import os import cdist from cdist.integration import configure_hosts_simple @@ -67,6 +68,10 @@ def save_ssh_key(self, hosts, keys): except Exception as cdist_exception: logger.error(cdist_exception) return_value = False + try: + os.remove(tmp_manifest.name) + except OSError: + pass return return_value From 3770489a493d43e980fa275b3bb16b807ed4e065 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 02:31:19 +0530 Subject: [PATCH 37/56] Writing bytes to NamedTemporaryFile --- hosting/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/forms.py b/hosting/forms.py index 458bee6f..bcc957b9 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -97,7 +97,7 @@ class UserHostingKeyForm(forms.ModelForm): openssh_pubkey_str = self.data.get('public_key') with tempfile.NamedTemporaryFile(delete=True) as tmp_public_key_file: - tmp_public_key_file.writelines(openssh_pubkey_str) + tmp_public_key_file.write(openssh_pubkey_str.encode('utf-8')) tmp_public_key_file.flush() try: out = subprocess.check_output( From 2983940d0f22f978a7d37a4d90132dd6e6a5dba9 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 02:33:21 +0530 Subject: [PATCH 38/56] Removed unwanted code --- hosting/forms.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index bcc957b9..36d05431 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -1,8 +1,8 @@ import datetime import logging +import os import subprocess import tempfile -import os from django import forms from django.contrib.auth import authenticate @@ -104,8 +104,7 @@ class UserHostingKeyForm(forms.ModelForm): ['ssh-keygen', '-lf', tmp_public_key_file.name]) except subprocess.CalledProcessError as cpe: logger.debug( - "Not a correct ssh format {error} {out}".format( - error=str(cpe), out=out)) + "Not a correct ssh format {error}".format(error=str(cpe))) raise forms.ValidationError(KEY_ERROR_MESSAGE) try: os.remove(tmp_public_key_file.name) From 511a1551d593982b9c105c729ecabb2a4774a82f Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 02:41:19 +0530 Subject: [PATCH 39/56] Added logger debug message when file not found when deleting --- hosting/forms.py | 6 ++++-- utils/tasks.py | 8 +++++--- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index 36d05431..8aff59d4 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -108,8 +108,10 @@ class UserHostingKeyForm(forms.ModelForm): raise forms.ValidationError(KEY_ERROR_MESSAGE) try: os.remove(tmp_public_key_file.name) - except OSError: - pass + except FileNotFoundError: + logger.debug( + "{} could not be deleted because it doesn't exist".format( + tmp_public_key_file.name)) return openssh_pubkey_str def clean_name(self): diff --git a/utils/tasks.py b/utils/tasks.py index 21d2c9b3..ecad5c71 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -1,5 +1,5 @@ -import tempfile import os +import tempfile import cdist from cdist.integration import configure_hosts_simple @@ -70,8 +70,10 @@ def save_ssh_key(self, hosts, keys): return_value = False try: os.remove(tmp_manifest.name) - except OSError: - pass + except FileNotFoundError: + logger.debug( + "{} could not be deleted because it doesn't exist".format( + tmp_manifest.name)) return return_value From 221af96c8b1c0d80ae39f02b239a2ad4f257b84e Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 02:46:54 +0530 Subject: [PATCH 40/56] Removed explicit deleting of file with NamedTemporaryFile --- hosting/forms.py | 6 ------ utils/tasks.py | 6 ------ 2 files changed, 12 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index 8aff59d4..23f7b3f6 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -106,12 +106,6 @@ class UserHostingKeyForm(forms.ModelForm): logger.debug( "Not a correct ssh format {error}".format(error=str(cpe))) raise forms.ValidationError(KEY_ERROR_MESSAGE) - try: - os.remove(tmp_public_key_file.name) - except FileNotFoundError: - logger.debug( - "{} could not be deleted because it doesn't exist".format( - tmp_public_key_file.name)) return openssh_pubkey_str def clean_name(self): diff --git a/utils/tasks.py b/utils/tasks.py index ecad5c71..d9b40ac3 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -68,12 +68,6 @@ def save_ssh_key(self, hosts, keys): except Exception as cdist_exception: logger.error(cdist_exception) return_value = False - try: - os.remove(tmp_manifest.name) - except FileNotFoundError: - logger.debug( - "{} could not be deleted because it doesn't exist".format( - tmp_manifest.name)) return return_value From f94ed011e5fdc6dbf0eedecc8adedf052e0f2f3d Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 05:30:05 +0530 Subject: [PATCH 41/56] Remove keys from known_hosts on delete of a vm --- hosting/views.py | 54 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/hosting/views.py b/hosting/views.py index 09e45371..012040f4 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -1,4 +1,5 @@ import logging +import subprocess import uuid from django.conf import settings @@ -18,10 +19,10 @@ from django.views.generic import View, CreateView, FormView, ListView, \ DetailView, \ DeleteView, TemplateView, UpdateView from guardian.mixins import PermissionRequiredMixin -from oca.pool import WrongNameError, WrongIdError -from stored_messages.settings import stored_messages_settings -from stored_messages.models import Message +from oca.pool import WrongIdError from stored_messages.api import mark_read +from stored_messages.models import Message +from stored_messages.settings import stored_messages_settings from membership.models import CustomUser, StripeCustomer from opennebula_api.models import OpenNebulaManager @@ -981,6 +982,53 @@ class VirtualMachineView(LoginRequiredMixin, View): 'VM_ID': opennebula_vm_id} ) + # Remove all keys belonging to the IP(s) + # ssh-keygen -R ip_address + if vm_data.ipv4 is not None: + if ', ' in vm_data.ipv4: + vm_ips = vm_data.ipv4.split(', ') + for ip_address in vm_ips: + try: + subprocess.check_output( + ['ssh-keygen', '-R', ip_address]) + except subprocess.CalledProcessError as cpe: + logger.debug( + """Could not remove key belonging to {ip}. + Error details: {details}""".format(ip=ip_address, + details=str( + cpe))) + else: + try: + subprocess.check_output( + ['ssh-keygen', '-R', vm_data.ipv4]) + except subprocess.CalledProcessError as cpe: + logger.debug( + """Could not remove key belonging to {ip}. + Error details: {details}""".format(ip=vm_data.ipv4, + details=str(cpe))) + if vm_data.ipv6 is not None: + if ', ' in vm_data.ipv6: + vm_ips = vm_data.ipv6.split(', ') + for ip_address in vm_ips: + try: + subprocess.check_output( + ['ssh-keygen', '-R', ip_address]) + except subprocess.CalledProcessError as cpe: + logger.debug( + """Could not remove key belonging to {ip}. + Error details: {details}""".format(ip=ip_address, + details=str( + cpe))) + else: + try: + subprocess.check_output( + ['ssh-keygen', '-R', vm_data.ipv6]) + except subprocess.CalledProcessError as cpe: + logger.debug( + """Could not remove key belonging to {ip}. + Error details: {details}""".format(ip=vm_data.ipv6, + details=str(cpe))) + return HttpResponseRedirect(self.get_success_url()) From 612e11736e350fa593485ced7dd26d7f2d1493af Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 05:37:26 +0530 Subject: [PATCH 42/56] Corrected obtaining values from vm_data dict --- hosting/views.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/hosting/views.py b/hosting/views.py index 012040f4..e6d68a9e 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -984,9 +984,9 @@ class VirtualMachineView(LoginRequiredMixin, View): # Remove all keys belonging to the IP(s) # ssh-keygen -R ip_address - if vm_data.ipv4 is not None: - if ', ' in vm_data.ipv4: - vm_ips = vm_data.ipv4.split(', ') + if vm_data['ipv4'] is not None: + if ', ' in vm_data['ipv4']: + vm_ips = vm_data['ipv4'].split(', ') for ip_address in vm_ips: try: subprocess.check_output( @@ -1000,15 +1000,15 @@ class VirtualMachineView(LoginRequiredMixin, View): else: try: subprocess.check_output( - ['ssh-keygen', '-R', vm_data.ipv4]) + ['ssh-keygen', '-R', vm_data['ipv4']]) except subprocess.CalledProcessError as cpe: logger.debug( """Could not remove key belonging to {ip}. - Error details: {details}""".format(ip=vm_data.ipv4, + Error details: {details}""".format(ip=vm_data['ipv4'], details=str(cpe))) - if vm_data.ipv6 is not None: - if ', ' in vm_data.ipv6: - vm_ips = vm_data.ipv6.split(', ') + if vm_data['ipv6'] is not None: + if ', ' in vm_data['ipv6']: + vm_ips = vm_data['ipv6'].split(', ') for ip_address in vm_ips: try: subprocess.check_output( @@ -1022,11 +1022,11 @@ class VirtualMachineView(LoginRequiredMixin, View): else: try: subprocess.check_output( - ['ssh-keygen', '-R', vm_data.ipv6]) + ['ssh-keygen', '-R', vm_data['ipv6']]) except subprocess.CalledProcessError as cpe: logger.debug( """Could not remove key belonging to {ip}. - Error details: {details}""".format(ip=vm_data.ipv6, + Error details: {details}""".format(ip=vm_data['ipv6'], details=str(cpe))) return HttpResponseRedirect(self.get_success_url()) From 662219f52414dec898bcf96cb537b399a0659ce6 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 06:07:12 +0530 Subject: [PATCH 43/56] Added trim function to clean public ssh key before validating and saving it --- hosting/forms.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index 23f7b3f6..f572b693 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -94,13 +94,13 @@ class UserHostingKeyForm(forms.ModelForm): if 'generate' in self.request.POST: return self.data.get('public_key') KEY_ERROR_MESSAGE = _("Please input a proper SSH key") - openssh_pubkey_str = self.data.get('public_key') + openssh_pubkey_str = self.data.get('public_key').trim() with tempfile.NamedTemporaryFile(delete=True) as tmp_public_key_file: tmp_public_key_file.write(openssh_pubkey_str.encode('utf-8')) tmp_public_key_file.flush() try: - out = subprocess.check_output( + subprocess.check_output( ['ssh-keygen', '-lf', tmp_public_key_file.name]) except subprocess.CalledProcessError as cpe: logger.debug( From ee5150dc6e01cab7b4cb221fd2cb870e100f5a47 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 06:27:46 +0530 Subject: [PATCH 44/56] strip instead of trim --- hosting/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hosting/forms.py b/hosting/forms.py index f572b693..dae911e4 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -94,7 +94,7 @@ class UserHostingKeyForm(forms.ModelForm): if 'generate' in self.request.POST: return self.data.get('public_key') KEY_ERROR_MESSAGE = _("Please input a proper SSH key") - openssh_pubkey_str = self.data.get('public_key').trim() + openssh_pubkey_str = self.data.get('public_key').strip() with tempfile.NamedTemporaryFile(delete=True) as tmp_public_key_file: tmp_public_key_file.write(openssh_pubkey_str.encode('utf-8')) From 94a59879852bce93500437c7a34d8310f16c4e03 Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 06:35:07 +0530 Subject: [PATCH 45/56] Removed an unwanted print --- hosting/views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/hosting/views.py b/hosting/views.py index e6d68a9e..b174acad 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -485,7 +485,6 @@ class SSHKeyCreateView(LoginRequiredMixin, FormView): return HttpResponseRedirect(self.success_url) def post(self, request, *args, **kwargs): - print(self.request.POST.dict()) form = self.get_form() required = 'add_ssh' in self.request.POST form.fields['name'].required = required From 01795dcad0b8c185efa7324382cd4baec569492c Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 06:57:15 +0530 Subject: [PATCH 46/56] Verifying if a public key exists already --- hosting/forms.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index dae911e4..1b68a7df 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -1,6 +1,5 @@ import datetime import logging -import os import subprocess import tempfile @@ -9,6 +8,7 @@ from django.contrib.auth import authenticate from django.utils.translation import ugettext_lazy as _ from membership.models import CustomUser +from utils.hosting_utils import get_all_public_keys from .models import UserHostingKey logger = logging.getLogger(__name__) @@ -88,7 +88,7 @@ class UserHostingKeyForm(forms.ModelForm): def clean_public_key(self): """ - A function that validates a public ssh key using sshpubkeys module + A function that validates a public ssh key using ssh-keygen :return: """ if 'generate' in self.request.POST: @@ -96,6 +96,14 @@ class UserHostingKeyForm(forms.ModelForm): KEY_ERROR_MESSAGE = _("Please input a proper SSH key") openssh_pubkey_str = self.data.get('public_key').strip() + if openssh_pubkey_str in get_all_public_keys(self.request.user): + key_name = UserHostingKey.objects.filter( + public_key=openssh_pubkey_str).first().name + KEY_EXISTS_MESSAGE = _( + "This key exists already with the name %(name)s") % { + 'name': key_name} + raise forms.ValidationError(KEY_EXISTS_MESSAGE) + with tempfile.NamedTemporaryFile(delete=True) as tmp_public_key_file: tmp_public_key_file.write(openssh_pubkey_str.encode('utf-8')) tmp_public_key_file.flush() From d92e850f3e7f18dba9b734e2ae27d5fa245f555e Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 07:03:49 +0530 Subject: [PATCH 47/56] Improved the condition of fetching key name --- hosting/forms.py | 1 + 1 file changed, 1 insertion(+) diff --git a/hosting/forms.py b/hosting/forms.py index 1b68a7df..2ebc6f48 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -98,6 +98,7 @@ class UserHostingKeyForm(forms.ModelForm): if openssh_pubkey_str in get_all_public_keys(self.request.user): key_name = UserHostingKey.objects.filter( + user_id=self.request.user.id, public_key=openssh_pubkey_str).first().name KEY_EXISTS_MESSAGE = _( "This key exists already with the name %(name)s") % { From d40403c15c7b4e7095ba4d36fecd8c10adce298d Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 07:31:22 +0530 Subject: [PATCH 48/56] Updated hosting django.po --- hosting/locale/de/LC_MESSAGES/django.po | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/hosting/locale/de/LC_MESSAGES/django.po b/hosting/locale/de/LC_MESSAGES/django.po index bb88f269..0e8aeec6 100644 --- a/hosting/locale/de/LC_MESSAGES/django.po +++ b/hosting/locale/de/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-09-09 06:04+0000\n" +"POT-Creation-Date: 2017-09-13 01:59+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -39,6 +39,10 @@ msgstr "Key-Name" msgid "Please input a proper SSH key" msgstr "Bitte verwende einen gültigen SSH-Key" +#, python-format +msgid "This key exists already with the name %(name)s" +msgstr "" + msgid "My Virtual Machines" msgstr "Meine virtuellen Maschinen" From f4dfee230652941e44868a59dc37f3d76edfffab Mon Sep 17 00:00:00 2001 From: PCoder Date: Wed, 13 Sep 2017 07:45:16 +0530 Subject: [PATCH 49/56] Send an email in case of a failed save_ssh_key task --- utils/tasks.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/utils/tasks.py b/utils/tasks.py index d9b40ac3..28c00c6e 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -1,4 +1,3 @@ -import os import tempfile import cdist @@ -68,6 +67,15 @@ def save_ssh_key(self, hosts, keys): except Exception as cdist_exception: logger.error(cdist_exception) return_value = False + email_data = { + 'subject': "celery save_ssh_key error - task id {0}".format( + self.request.id.__str__()), + 'from_email': settings.DCL_SUPPORT_FROM_ADDRESS, + 'to': ['info@ungleich.ch'], + 'body': "Task Id: {0}\nResult: {1}\nTraceback: {2}".format( + self.request.id.__str__(), False, str(cdist_exception)), + } + send_plain_email_task(email_data) return return_value From f0d0f2abccaab5823b07b05df17de0497298a752 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Wed, 13 Sep 2017 12:24:08 +0200 Subject: [PATCH 50/56] Updated logger message --- opennebula_api/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/opennebula_api/models.py b/opennebula_api/models.py index f88f3b15..424ff464 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -556,7 +556,8 @@ class OpenNebulaManager(): save_ssh_key.apply_async((hosts, keys), countdown=countdown, link_error=save_ssh_key_error_handler.s()) else: - logger.debug("Keys and hosts are empty, so not managing any keys") + logger.debug( + "Keys and/or hosts are empty, so not managing any keys") def get_all_hosts(self): """ From a08104ba2c1c4d4cbc78eeba3c8560950ab41adf Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Wed, 13 Sep 2017 14:31:13 +0200 Subject: [PATCH 51/56] Updated a comment --- hosting/forms.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index 2ebc6f48..f6d9b4d0 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -1,8 +1,8 @@ import datetime import logging import subprocess -import tempfile +import tempfile from django import forms from django.contrib.auth import authenticate from django.utils.translation import ugettext_lazy as _ @@ -88,7 +88,9 @@ class UserHostingKeyForm(forms.ModelForm): def clean_public_key(self): """ - A function that validates a public ssh key using ssh-keygen + Validates a public ssh key using `ssh-keygen -lf key.pub` + Also checks if a given key already exists in the database and + alerts the user of it. :return: """ if 'generate' in self.request.POST: From 86ed0e531992a92cc2ef575035ce5e33f2f1e519 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Wed, 13 Sep 2017 14:32:33 +0200 Subject: [PATCH 52/56] Removed code related removing keys from known_hosts --- hosting/views.py | 48 ------------------------------------------------ 1 file changed, 48 deletions(-) diff --git a/hosting/views.py b/hosting/views.py index b174acad..9f041333 100644 --- a/hosting/views.py +++ b/hosting/views.py @@ -1,5 +1,4 @@ import logging -import subprocess import uuid from django.conf import settings @@ -981,53 +980,6 @@ class VirtualMachineView(LoginRequiredMixin, View): 'VM_ID': opennebula_vm_id} ) - # Remove all keys belonging to the IP(s) - # ssh-keygen -R ip_address - if vm_data['ipv4'] is not None: - if ', ' in vm_data['ipv4']: - vm_ips = vm_data['ipv4'].split(', ') - for ip_address in vm_ips: - try: - subprocess.check_output( - ['ssh-keygen', '-R', ip_address]) - except subprocess.CalledProcessError as cpe: - logger.debug( - """Could not remove key belonging to {ip}. - Error details: {details}""".format(ip=ip_address, - details=str( - cpe))) - else: - try: - subprocess.check_output( - ['ssh-keygen', '-R', vm_data['ipv4']]) - except subprocess.CalledProcessError as cpe: - logger.debug( - """Could not remove key belonging to {ip}. - Error details: {details}""".format(ip=vm_data['ipv4'], - details=str(cpe))) - if vm_data['ipv6'] is not None: - if ', ' in vm_data['ipv6']: - vm_ips = vm_data['ipv6'].split(', ') - for ip_address in vm_ips: - try: - subprocess.check_output( - ['ssh-keygen', '-R', ip_address]) - except subprocess.CalledProcessError as cpe: - logger.debug( - """Could not remove key belonging to {ip}. - Error details: {details}""".format(ip=ip_address, - details=str( - cpe))) - else: - try: - subprocess.check_output( - ['ssh-keygen', '-R', vm_data['ipv6']]) - except subprocess.CalledProcessError as cpe: - logger.debug( - """Could not remove key belonging to {ip}. - Error details: {details}""".format(ip=vm_data['ipv6'], - details=str(cpe))) - return HttpResponseRedirect(self.get_success_url()) From face045090fdaebb00938a7cb9c9d33604e67833 Mon Sep 17 00:00:00 2001 From: PCoder Date: Thu, 14 Sep 2017 00:50:24 +0530 Subject: [PATCH 53/56] Updated hosting django.po --- hosting/forms.py | 2 +- hosting/locale/de/LC_MESSAGES/django.po | 11 +++++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/hosting/forms.py b/hosting/forms.py index f6d9b4d0..056d0004 100644 --- a/hosting/forms.py +++ b/hosting/forms.py @@ -103,7 +103,7 @@ class UserHostingKeyForm(forms.ModelForm): user_id=self.request.user.id, public_key=openssh_pubkey_str).first().name KEY_EXISTS_MESSAGE = _( - "This key exists already with the name %(name)s") % { + "This key exists already with the name \"%(name)s\"") % { 'name': key_name} raise forms.ValidationError(KEY_EXISTS_MESSAGE) diff --git a/hosting/locale/de/LC_MESSAGES/django.po b/hosting/locale/de/LC_MESSAGES/django.po index 0e8aeec6..18f33ccb 100644 --- a/hosting/locale/de/LC_MESSAGES/django.po +++ b/hosting/locale/de/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-09-13 01:59+0000\n" +"POT-Creation-Date: 2017-09-13 19:15+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -40,6 +40,9 @@ msgid "Please input a proper SSH key" msgstr "Bitte verwende einen gültigen SSH-Key" #, python-format +msgid "This key exists already with the name \"%(name)s\"" +msgstr "Der SSH-Key mit dem Name \"%(name)s\" existiert bereits" + msgid "This key exists already with the name %(name)s" msgstr "" @@ -55,9 +58,6 @@ msgstr "SSH Key" msgid "Notifications " msgstr "Benachrichtigungen" -msgid "Logout" -msgstr "Abmelden" - msgid "All Rights Reserved" msgstr "Alle Rechte vorbehalten" @@ -316,6 +316,9 @@ msgstr "Konfiguration" msgid "Dashboard" msgstr "Mein Dashboard" +msgid "Logout" +msgstr "Abmelden" + msgid "Don't have an account yet ? " msgstr "Besitzt du kein Benutzerkonto?" From bd45577c90f702b0de087e6ace3861841fe5a4c0 Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 14 Sep 2017 14:28:52 +0200 Subject: [PATCH 54/56] Removed unwanted translations in hosting django.po --- hosting/locale/de/LC_MESSAGES/django.po | 51 +++++++++---------------- 1 file changed, 19 insertions(+), 32 deletions(-) diff --git a/hosting/locale/de/LC_MESSAGES/django.po b/hosting/locale/de/LC_MESSAGES/django.po index 18f33ccb..d684bec7 100644 --- a/hosting/locale/de/LC_MESSAGES/django.po +++ b/hosting/locale/de/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: PACKAGE VERSION\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2017-09-13 19:15+0000\n" +"POT-Creation-Date: 2017-09-14 12:27+0000\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: FULL NAME \n" "Language-Team: LANGUAGE \n" @@ -43,21 +43,6 @@ msgstr "Bitte verwende einen gültigen SSH-Key" msgid "This key exists already with the name \"%(name)s\"" msgstr "Der SSH-Key mit dem Name \"%(name)s\" existiert bereits" -msgid "This key exists already with the name %(name)s" -msgstr "" - -msgid "My Virtual Machines" -msgstr "Meine virtuellen Maschinen" - -msgid "My Orders" -msgstr "Meine Bestellungen" - -msgid "SSH Keys" -msgstr "SSH Key" - -msgid "Notifications " -msgstr "Benachrichtigungen" - msgid "All Rights Reserved" msgstr "Alle Rechte vorbehalten" @@ -248,16 +233,6 @@ msgstr "" "dann ignoriere diese E-Mail.
\n" "Dankeschön!\n" -msgid "Please go to the following page and choose a new password:" -msgstr "" - -msgid "Thanks for using our site!" -msgstr "" - -#, python-format -msgid "The %(site_name)s team" -msgstr "" - #, python-format msgid "" "You're receiving this email because you requested a password reset for your " @@ -475,12 +450,6 @@ msgstr "" msgid "Type" msgstr "Kartentyp" -msgid "REMOVE CARD" -msgstr "KARTE ENTFERNEN" - -msgid "EDIT CARD" -msgstr "BEARBEITEN" - msgid "No Credit Cards Added" msgstr "Es wurde keine Kreditkarte hinzugefügt" @@ -672,6 +641,24 @@ msgstr "VM Kündigung" msgid "VM %(VM_ID)s terminated successfully" msgstr "VM %(VM_ID)s erfolgreich beendet" +#~ msgid "My Virtual Machines" +#~ msgstr "Meine virtuellen Maschinen" + +#~ msgid "My Orders" +#~ msgstr "Meine Bestellungen" + +#~ msgid "SSH Keys" +#~ msgstr "SSH Key" + +#~ msgid "Notifications " +#~ msgstr "Benachrichtigungen" + +#~ msgid "REMOVE CARD" +#~ msgstr "KARTE ENTFERNEN" + +#~ msgid "EDIT CARD" +#~ msgstr "BEARBEITEN" + #~ msgid "Add a new Card." #~ msgstr "Neue Kreditkarte hinzufügen." From a4fe2e2db95bd0e2932a069ba702265f1417ddca Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 14 Sep 2017 15:05:28 +0200 Subject: [PATCH 55/56] Updated SaveSSHKeyTestCase to include add/remove ssh keys --- dynamicweb/settings/base.py | 3 +++ utils/tests.py | 37 +++++++++++++++++++++++++++++++++---- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/dynamicweb/settings/base.py b/dynamicweb/settings/base.py index a2b60026..29533211 100644 --- a/dynamicweb/settings/base.py +++ b/dynamicweb/settings/base.py @@ -585,6 +585,9 @@ if ENABLE_DEBUG_LOGGING: }, } +TEST_MANAGE_SSH_KEY_PUBKEY = env('TEST_MANAGE_SSH_KEY_PUBKEY') +TEST_MANAGE_SSH_KEY_HOST = env('TEST_MANAGE_SSH_KEY_HOST') + DEBUG = bool_env('DEBUG') if DEBUG: diff --git a/utils/tests.py b/utils/tests.py index 362a84bc..d5c2d726 100644 --- a/utils/tests.py +++ b/utils/tests.py @@ -8,6 +8,7 @@ from django.conf import settings from django.http.request import HttpRequest from django.test import Client from django.test import TestCase, override_settings +from unittest import skipIf from model_mommy import mommy from datacenterlight.models import StripePlan @@ -250,11 +251,39 @@ class SaveSSHKeyTestCase(TestCase): task_always_eager=True, ) def setUp(self): - self.public_key = ["This is a test", ] - self.hosts = ['localhost'] + self.public_key = settings.TEST_MANAGE_SSH_KEY_PUBKEY + self.hosts = settings.TEST_MANAGE_SSH_KEY_HOST - def test_save_ssh_key(self): - async_task = save_ssh_key.delay(self.hosts, self.public_key) + @skipIf(settings.TEST_MANAGE_SSH_KEY_PUBKEY is None or + settings.TEST_MANAGE_SSH_KEY_PUBKEY == "" or + settings.TEST_MANAGE_SSH_KEY_HOST is None or + settings.TEST_MANAGE_SSH_KEY_HOST is "", + """Skipping test_save_ssh_key_add because either host + or public key were not specified or were empty""") + def test_save_ssh_key_add(self): + async_task = save_ssh_key.delay([self.hosts], + [{'value': self.public_key, + 'state': True}]) + save_ssh_key_result = None + for i in range(0, 10): + sleep(5) + res = AsyncResult(async_task.task_id) + if type(res.result) is bool: + save_ssh_key_result = res.result + break + self.assertIsNotNone(save_ssh_key, "save_ssh_key_result is None") + self.assertTrue(save_ssh_key_result, "save_ssh_key_result is False") + + @skipIf(settings.TEST_MANAGE_SSH_KEY_PUBKEY is None or + settings.TEST_MANAGE_SSH_KEY_PUBKEY == "" or + settings.TEST_MANAGE_SSH_KEY_HOST is None or + settings.TEST_MANAGE_SSH_KEY_HOST is "", + """Skipping test_save_ssh_key_add because either host + or public key were not specified or were empty""") + def test_save_ssh_key_remove(self): + async_task = save_ssh_key.delay([self.hosts], + [{'value': self.public_key, + 'state': False}]) save_ssh_key_result = None for i in range(0, 10): sleep(5) From 5c3cf7682f50ab027a1b36c061a46ba7c8dd0dcf Mon Sep 17 00:00:00 2001 From: "M.Ravi" Date: Thu, 14 Sep 2017 15:27:25 +0200 Subject: [PATCH 56/56] Fixed some flake8 warnings --- datacenterlight/tests.py | 4 ++-- hosting/urls.py | 1 - opennebula_api/models.py | 4 ++-- utils/tasks.py | 2 +- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/datacenterlight/tests.py b/datacenterlight/tests.py index 7c2f7353..c34c56ba 100644 --- a/datacenterlight/tests.py +++ b/datacenterlight/tests.py @@ -115,8 +115,8 @@ class CeleryTaskTestCase(TestCase): 'response_object').stripe_plan_id}]) stripe_subscription_obj = subscription_result.get('response_object') # Check if the subscription was approved and is active - if stripe_subscription_obj is None or \ - stripe_subscription_obj.status != 'active': + if stripe_subscription_obj is None \ + or stripe_subscription_obj.status != 'active': msg = subscription_result.get('error') raise Exception("Creating subscription failed: {}".format(msg)) diff --git a/hosting/urls.py b/hosting/urls.py index 10e09dd0..455aa97f 100644 --- a/hosting/urls.py +++ b/hosting/urls.py @@ -11,7 +11,6 @@ from .views import ( SSHKeyChoiceView, DashboardView, SettingsView) - urlpatterns = [ url(r'index/?$', IndexView.as_view(), name='index'), url(r'django/?$', DjangoHostingView.as_view(), name='djangohosting'), diff --git a/opennebula_api/models.py b/opennebula_api/models.py index 424ff464..d584bf26 100644 --- a/opennebula_api/models.py +++ b/opennebula_api/models.py @@ -350,8 +350,8 @@ class OpenNebulaManager(): return template_pool except ConnectionRefusedError: logger.info( - """Could not connect to host: {host} via protocol - {protocol}""".format( + """Could not connect to host: {host} via protocol + {protocol}""".format( host=settings.OPENNEBULA_DOMAIN, protocol=settings.OPENNEBULA_PROTOCOL) ) diff --git a/utils/tasks.py b/utils/tasks.py index 28c00c6e..d66c37ee 100644 --- a/utils/tasks.py +++ b/utils/tasks.py @@ -38,7 +38,7 @@ def save_ssh_key(self, hosts, keys): 'state': True # whether key is to be added or } # removed """ - logger.debug("""Running save_ssh_key task for + logger.debug("""Running save_ssh_key task for Hosts: {hosts_str} Keys: {keys_str}""".format(hosts_str=", ".join(hosts), keys_str=", ".join([