From 054886fd9c486c231b3ee9d919d09b9e8da705d2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 20 Dec 2020 12:20:54 +0100 Subject: [PATCH] begin phasing in config of vpn via cdist --- doc/uncloud-manual-2020-08-01.org | 49 +++++++++++++++++++++++++++- uncloud/settings.py | 30 +++++++++++++---- uncloud_net/models.py | 23 +++++++++++++ uncloud_net/tasks.py | 54 +++++++++++++------------------ uncloud_net/views.py | 2 ++ 5 files changed, 119 insertions(+), 39 deletions(-) diff --git a/doc/uncloud-manual-2020-08-01.org b/doc/uncloud-manual-2020-08-01.org index d48ce5e..83fa65e 100644 --- a/doc/uncloud-manual-2020-08-01.org +++ b/doc/uncloud-manual-2020-08-01.org @@ -170,7 +170,7 @@ VPNNetworks can be managed by all authenticated users. * Developer Handbook The following section describe decisions / architecture of uncloud. These chapters are intended to be read by developers. -** Documentation +** This Documentation This documentation is written in org-mode. To compile it to html/pdf, just open emacs and press *C-c C-e l p*. ** Models @@ -234,6 +234,53 @@ VPNNetworks can be managed by all authenticated users. *** Decision We use integers, because they are easy. +** Distributing/Dispatching/Orchestrating +*** Variant 1: using cdist + - The uncloud server can git commit things + - The uncloud server loads cdist and configures the server + - Advantages + - Fully integrated into normal flow + - Disadvantage + - web frontend has access to more data than it needs + - On compromise of the machine, more data leaks + - Some cdist usual delay +*** Variant 2: via celery + - The uncloud server dispatches via celery + - Every decentral node also runs celery/connects to the broker + - Summary brokers: + - If local only celery -> good to use redis - Broker + - If remote: probably better to use rabbitmq + - redis + - simpler + - rabbitmq + - more versatile + - made for remote connections + - quorom queues would be nice, but not clear if supported + - https://github.com/celery/py-amqp/issues/302 + - https://github.com/celery/celery/issues/6067 + - Cannot be installed on alpine Linux at the moment + - Advantage + - Very python / django integrated + - Rather instant + - Disadvantages + - Every decentral node needs to have the uncloud code available + - Decentral nodes *might* need to access the database + - Tasks can probably be written to work without that + (i.e. only strings/bytes) + +**** log/tests + (venv) [19:54] vpn-2a0ae5c1200:~/uncloud$ celery -A uncloud -b redis://bridge.place7.ungleich.ch worker -n worker1@%h --logfile ~/celery.log - +Q vpn-2a0ae5c1200.ungleich.ch + + +*** Variant 3: dedicated cdist instance via message broker + - A separate VM/machine + - Has Checkout of ~/.cdist + - Has cdist checkout + - Tiny API for management + - Not directly web accessible + - "cdist" queue + ** Milestones :uncloud: *** 1.1 (cleanup 1) **** TODO [#C] Unify ValidationError, FieldError - define proper Exception diff --git a/uncloud/settings.py b/uncloud/settings.py index cd60354..bc99743 100644 --- a/uncloud/settings.py +++ b/uncloud/settings.py @@ -11,6 +11,7 @@ https://docs.djangoproject.com/en/3.0/ref/settings/ """ import os +import re import ldap from django.core.management.utils import get_random_secret_key @@ -199,14 +200,29 @@ AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=example,dc=com", # where to create customers LDAP_CUSTOMER_DN="ou=customer,dc=example,dc=com" -CELERY_TASK_ROUTES = { - '*': { - 'queue': 'vpn1' - } -} +def route_task(name, args, kwargs, options, task=None, **kw): + print(f"{name} - {args} - {kwargs}") +# if name == 'myapp.tasks.compress_video': + return {'queue': 'vpn1' } +# 'exchange_type': 'topic', +# 'routing_key': 'video.compress'} -CELERY_BROKER_URL = 'redis://bridge.place7.ungleich.ch:6379/0' -CELERY_RESULT_BACKEND = 'redis://bridge.place7.ungleich.ch:6379/0' + +CELERY_TASK_ROUTES = (route_task,) + +# CELERY_TASK_ROUTES = { +# '*': { +# 'queue': 'vpn1' +# } +# } + + +CELERY_BROKER_URL = 'redis://:uncloud.example.com:6379/0' +CELERY_RESULT_BACKEND = 'redis://:uncloud.example.com:6379/0' + +CELERY_TASK_ROUTES = { + (re.compile(r'.*\.cdist\..*'), { 'queue': 'cdist' } + } # CELERY_TASK_CREATE_MISSING_QUEUES = False diff --git a/uncloud_net/models.py b/uncloud_net/models.py index 5c1da3d..92499dd 100644 --- a/uncloud_net/models.py +++ b/uncloud_net/models.py @@ -53,6 +53,29 @@ class WireGuardVPNPool(models.Model): def __str__(self): return f"{self.ip_network} (subnets: /{self.subnetwork_mask})" + @property + def wireguard_config(self): + wireguard_config = [ + "[Interface]\nListenPort = 51820\nPrivateKey = {self.wireguard_private_key}\n".format( + privatekey=self.wireguard_private_key) + ] + + peers = [] + + for vpn in self.wireguardvpn_set.all(): + public_key = vpn.wireguard_public_key + peer_network = "{}/{}".format(vpn.address, self.subnetwork_mask) + owner = vpn.owner + + peers.append("# Owner: {owner}\n[Peer]\nPublicKey = {public_key}\nAllowedIPs = {peer_network}\n\n".format( + owner=owner, + public_key=public_key, + peer_network=peer_network)) + + wireguard_config.extend(peers) + + return "\n".join(wireguard_config) + class WireGuardVPN(models.Model): """ diff --git a/uncloud_net/tasks.py b/uncloud_net/tasks.py index 60c1e7d..3b07b48 100644 --- a/uncloud_net/tasks.py +++ b/uncloud_net/tasks.py @@ -8,38 +8,30 @@ def whereami(): print(os.uname()) return os.uname() +def configure_wireguard_server(wireguardvpnpool): + """ + - Create wireguard config (DB query -> string) + - Submit config to cdist worker + - Change config locally on worker / commit / shared + + """ + + config = wireguardvpnpool.wireguard_config + server = wireguardvpnpool.vpn_server_hostname + + print(f"Configuring {vpnpool.vpn_server_hostname}: {osa}") + cdist_configure_wireguard_server(config, server): + + @shared_task -def configure_wireguard_server(vpnpool): - print(f"Configuring {vpnpool.vpn_server_hostname}") +def cdist_configure_wireguard_server(config, server): + """ + Create config and configure server. - wireguard_config_filename = '/etc/wireguard/{}.conf'.format(vpnpool.network) + To be executed on the cdist workers. + """ - @property - def wireguard_config(self): - wireguard_config = [ - """ -[Interface] -ListenPort = 51820 -PrivateKey = {privatekey} -""".format(privatekey=self.wireguard_private_key) ] + fname = f"/home/app/.cdist/type/__ungleich_wireguard/files/{server}" - peers = [] - - for reservation in self.vpnnetworkreservation_set.filter(status='used'): - public_key = reservation.vpnnetwork_set.first().wireguard_public_key - peer_network = "{}/{}".format(reservation.address, self.subnetwork_size) - owner = reservation.vpnnetwork_set.first().owner - - peers.append(""" -# Owner: {owner} -[Peer] -PublicKey = {public_key} -AllowedIPs = {peer_network} -""".format( - owner=owner, - public_key=public_key, - peer_network=peer_network)) - - wireguard_config.extend(peers) - - return "\n".join(wireguard_config) + with open(fname, "w") as fd: + fd.write(config) diff --git a/uncloud_net/views.py b/uncloud_net/views.py index f91ff7c..e06e9bd 100644 --- a/uncloud_net/views.py +++ b/uncloud_net/views.py @@ -12,6 +12,7 @@ from .serializers import * from .selectors import * from .services import * from .forms import * +from .tasks import * # class VPNPoolViewSet(viewsets.ModelViewSet): # serializer_class = VPNPoolSerializer @@ -39,6 +40,7 @@ class WireGuardVPNViewSet(viewsets.ModelViewSet): public_key=serializer.validated_data['wireguard_public_key'], network_mask=serializer.validated_data['network_mask'] ) + configure_wireguard_server.apply_async((vpn.vpnpool,)) return Response(WireGuardVPNSerializer(vpn).data)