begin phasing in config of vpn via cdist

doc/uncloud-manual-2020-08-01.org

@@ -170,7 +170,7 @@ VPNNetworks can be managed by all authenticated users.
 * Developer Handbook
   The following section describe decisions / architecture of
   uncloud. These chapters are intended to be read by developers.
-** Documentation
+** This Documentation
    This documentation is written in org-mode. To compile it to
    html/pdf, just open emacs and press *C-c C-e l p*.
 ** Models
@@ -234,6 +234,53 @@ VPNNetworks can be managed by all authenticated users.
 *** Decision
     We use integers, because they are easy.
 
+** Distributing/Dispatching/Orchestrating
+*** Variant 1: using cdist
+    - The uncloud server can git commit things
+    - The uncloud server loads cdist and configures the server
+    - Advantages
+      - Fully integrated into normal flow
+    - Disadvantage
+      - web frontend has access to more data than it needs
+      - On compromise of the machine, more data leaks
+      - Some cdist usual delay
+*** Variant 2: via celery
+    - The uncloud server dispatches via celery
+    - Every decentral node also runs celery/connects to the broker
+    - Summary brokers:
+      - If local only celery -> good to use redis    - Broker
+      - If remote: probably better to use rabbitmq
+    - redis
+      - simpler
+    - rabbitmq
+      - more versatile
+      - made for remote connections
+      - quorom queues would be nice, but not clear if supported
+        - https://github.com/celery/py-amqp/issues/302
+        - https://github.com/celery/celery/issues/6067
+      - Cannot be installed on alpine Linux at the moment
+    - Advantage
+      - Very python / django integrated
+      - Rather instant
+    - Disadvantages
+      - Every decentral node needs to have the uncloud code available
+      - Decentral nodes *might* need to access the database
+        - Tasks can probably be written to work without that
+          (i.e. only strings/bytes)
+
+**** log/tests
+     (venv) [19:54] vpn-2a0ae5c1200:~/uncloud$ celery -A uncloud -b redis://bridge.place7.ungleich.ch worker -n worker1@%h --logfile ~/celery.log -
+Q vpn-2a0ae5c1200.ungleich.ch
+
+
+*** Variant 3: dedicated cdist instance via message broker
+    - A separate VM/machine
+    - Has Checkout of ~/.cdist
+    - Has cdist checkout
+    - Tiny API for management
+    - Not directly web accessible
+    - "cdist" queue
+
 ** Milestones                                                       :uncloud:
 *** 1.1 (cleanup 1)
 **** TODO [#C] Unify ValidationError, FieldError - define proper Exception

uncloud/settings.py

@@ -11,6 +11,7 @@ https://docs.djangoproject.com/en/3.0/ref/settings/
 """
 
 import os
+import re
 import ldap
 
 from django.core.management.utils import get_random_secret_key
@@ -199,14 +200,29 @@ AUTH_LDAP_USER_SEARCH = LDAPSearch("dc=example,dc=com",
 # where to create customers
 LDAP_CUSTOMER_DN="ou=customer,dc=example,dc=com"
 
-CELERY_TASK_ROUTES = {
+def route_task(name, args, kwargs, options, task=None, **kw):
-    '*': {
+    print(f"{name} - {args} - {kwargs}")
-        'queue': 'vpn1'
+#    if name == 'myapp.tasks.compress_video':
-    }
+    return {'queue': 'vpn1' }
-}
+#            'exchange_type': 'topic',
+#            'routing_key': 'video.compress'}
 
-CELERY_BROKER_URL = 'redis://bridge.place7.ungleich.ch:6379/0'
+
-CELERY_RESULT_BACKEND = 'redis://bridge.place7.ungleich.ch:6379/0'
+CELERY_TASK_ROUTES = (route_task,)
+
+# CELERY_TASK_ROUTES = {
+#     '*': {
+#         'queue': 'vpn1'
+#     }
+# }
+
+
+CELERY_BROKER_URL = 'redis://:uncloud.example.com:6379/0'
+CELERY_RESULT_BACKEND = 'redis://:uncloud.example.com:6379/0'
+
+CELERY_TASK_ROUTES = {
+    (re.compile(r'.*\.cdist\..*'), { 'queue': 'cdist' }
+     }
 
 # CELERY_TASK_CREATE_MISSING_QUEUES = False
 

uncloud_net/models.py

@@ -53,6 +53,29 @@ class WireGuardVPNPool(models.Model):
     def __str__(self):
         return f"{self.ip_network} (subnets: /{self.subnetwork_mask})"
 
+    @property
+    def wireguard_config(self):
+        wireguard_config = [
+            "[Interface]\nListenPort = 51820\nPrivateKey = {self.wireguard_private_key}\n".format(
+                privatekey=self.wireguard_private_key)
+                            ]
+
+        peers = []
+
+        for vpn in self.wireguardvpn_set.all():
+            public_key = vpn.wireguard_public_key
+            peer_network = "{}/{}".format(vpn.address, self.subnetwork_mask)
+            owner = vpn.owner
+
+            peers.append("# Owner: {owner}\n[Peer]\nPublicKey = {public_key}\nAllowedIPs = {peer_network}\n\n".format(
+                owner=owner,
+                public_key=public_key,
+                peer_network=peer_network))
+
+        wireguard_config.extend(peers)
+
+        return "\n".join(wireguard_config)
+
 
 class WireGuardVPN(models.Model):
     """

uncloud_net/tasks.py

@@ -8,38 +8,30 @@ def whereami():
     print(os.uname())
     return os.uname()
 
+def configure_wireguard_server(wireguardvpnpool):
+    """
+    - Create wireguard config (DB query -> string)
+    - Submit config to cdist worker
+    - Change config locally on worker / commit / shared
+
+    """
+
+    config = wireguardvpnpool.wireguard_config
+    server = wireguardvpnpool.vpn_server_hostname
+
+    print(f"Configuring {vpnpool.vpn_server_hostname}: {osa}")
+    cdist_configure_wireguard_server(config, server):
+
+
 @shared_task
-def configure_wireguard_server(vpnpool):
+def cdist_configure_wireguard_server(config, server):
-    print(f"Configuring {vpnpool.vpn_server_hostname}")
+    """
+    Create config and configure server.
 
-    wireguard_config_filename =  '/etc/wireguard/{}.conf'.format(vpnpool.network)
+    To be executed on the cdist workers.
+    """
 
-    @property
+    fname = f"/home/app/.cdist/type/__ungleich_wireguard/files/{server}"
-    def wireguard_config(self):
-        wireguard_config = [
-            """
-[Interface]
-ListenPort = 51820
-PrivateKey = {privatekey}
-""".format(privatekey=self.wireguard_private_key) ]
 
-        peers = []
+    with open(fname, "w") as fd:
-
+        fd.write(config)
-        for reservation in self.vpnnetworkreservation_set.filter(status='used'):
-            public_key = reservation.vpnnetwork_set.first().wireguard_public_key
-            peer_network = "{}/{}".format(reservation.address, self.subnetwork_size)
-            owner = reservation.vpnnetwork_set.first().owner
-
-            peers.append("""
-# Owner: {owner}
-[Peer]
-PublicKey = {public_key}
-AllowedIPs = {peer_network}
-""".format(
-    owner=owner,
-    public_key=public_key,
-    peer_network=peer_network))
-
-        wireguard_config.extend(peers)
-
-        return "\n".join(wireguard_config)

uncloud_net/views.py

@@ -12,6 +12,7 @@ from .serializers import *
 from .selectors   import *
 from .services    import *
 from .forms       import *
+from .tasks       import *
 
 # class VPNPoolViewSet(viewsets.ModelViewSet):
 #     serializer_class = VPNPoolSerializer
@@ -39,6 +40,7 @@ class WireGuardVPNViewSet(viewsets.ModelViewSet):
             public_key=serializer.validated_data['wireguard_public_key'],
             network_mask=serializer.validated_data['network_mask']
         )
+        configure_wireguard_server.apply_async((vpn.vpnpool,))
         return Response(WireGuardVPNSerializer(vpn).data)