More Networking Implementation

Pipfile

@@ -19,6 +19,7 @@ pyotp = "*"
 sshtunnel = "*"
 helper = "*"
 sphinx = "*"
+pynetbox = "*"
 
 [requires]
 python_version = "3.5"

Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "45db72f1a666be82e7dc044ced7e7ad7a5b5a6efbb8b8103e6ad04c93a7d017a"
+            "sha256": "5e4aa65086afdf9ac2f1479e9e35684f767dfbbd13877c4e4a23dd471aef6c13"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -377,6 +377,13 @@
             ],
             "version": "==1.3.0"
         },
+        "pynetbox": {
+            "hashes": [
+                "sha256:e171380b36bedb7e0cd6a735fe8193d5809b373897b6905a2de43342761426c7"
+            ],
+            "index": "pypi",
+            "version": "==4.0.8"
+        },
         "pyotp": {
             "hashes": [
                 "sha256:c88f37fd47541a580b744b42136f387cdad481b560ef410c0d85c957eb2a2bc0",
@@ -522,10 +529,10 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
+                "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
-                "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
+                "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
             ],
-            "version": "==1.25.6"
+            "version": "==1.25.7"
         },
         "werkzeug": {
             "hashes": [
@@ -896,10 +903,10 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:3de946ffbed6e6746608990594d08faac602528ac7015ac28d33cee6a45b7398",
+                "sha256:a8a318824cc77d1fd4b2bec2ded92646630d7fe8619497b142c84a9e6f5a7293",
-                "sha256:9a107b99a5393caf59c7aa3c1249c16e6879447533d0887f4336dde834c7be86"
+                "sha256:f3c5fd51747d450d4dcf6f923c81f78f811aab8205fda64b0aba34a4e48b0745"
             ],
-            "version": "==1.25.6"
+            "version": "==1.25.7"
         },
         "vulture": {
             "hashes": [

TODO.md

@@ -2,4 +2,5 @@
 
 - Check for `etcd3.exceptions.ConnectionFailedError` when calling some etcd operation to
   avoid crashing whole application
-- Throw KeyError instead of returning None when some key is not found in etcd
+- Throw KeyError instead of returning None when some key is not found in etcd
+- Expose more details in ListUserFiles

api/helper.py

@@ -163,3 +163,21 @@ def get_etcd_counter(etcd_client, key):
     if kv:
         return int(kv.value)
     return None
+
+def mac2ipv6(mac, prefix):
+    # only accept MACs separated by a colon
+    parts = mac.split(":")
+
+    # modify parts to match IPv6 value
+    parts.insert(3, "ff")
+    parts.insert(4, "fe")
+    parts[0] = "%x" % (int(parts[0], 16) ^ 2)
+
+    # format output
+    ipv6Parts = [str(0)]*4
+    for i in range(0, len(parts), 2):
+        ipv6Parts.append("".join(parts[i:i+2]))
+    
+    lower_part = ipaddress.IPv6Address(":".join(ipv6Parts))
+    prefix = ipaddress.IPv6Address(prefix)
+    return str(prefix + int(lower_part))

api/main.py

@@ -1,6 +1,8 @@
 import json
 import subprocess
 import os
+import pynetbox
+import decouple
 
 import schemas
 
@@ -12,7 +14,8 @@ from flask_restful import Resource, Api
 from ucloud_common.vm import VMStatus
 from ucloud_common.request import RequestEntry, RequestType
 
-from helper import generate_mac, get_ip_addr, get_etcd_counter, increment_etcd_counter
+from helper import (generate_mac, get_ip_addr, get_etcd_counter,
+                    increment_etcd_counter, mac2ipv6)
 
 from config import (
     etcd_client,
@@ -46,7 +49,7 @@ class CreateVM(Resource):
                 'os-ssd': validator.specs['os-ssd'],
                 'hdd': validator.specs['hdd']
             }
-
+            macs = [generate_mac() for i in range(len(data["network"]))]
             vm_entry = {
                 "name": data["vm_name"],
                 "owner": data["name"],
@@ -57,7 +60,7 @@ class CreateVM(Resource):
                 "image_uuid": validator.image_uuid,
                 "log": [],
                 "vnc_socket": "",
-                "network": data["network"],
+                "network": list(zip(data["network"], macs)),
                 "metadata": {
                     "ssh-keys": []
                 },
@@ -80,7 +83,13 @@ class VmStatus(Resource):
         if validator.is_valid():
             vm = VM_POOL.get(os.path.join(VM_PREFIX, data["uuid"]))
             vm_value = vm.value.copy()
-            # vm_value["ip"] = list(map(str, get_ip_addr(vm.mac, "br0")))
+            vm_value["ip"] = []
+            for network_and_mac in vm.network:
+                network_name, mac = network_and_mac
+                network = etcd_client.get(os.path.join(NETWORK_PREFIX, data["name"], network_name),
+                                          value_in_json=True)
+                ipv6_addr = network.value.get("ipv6").split("::")[0] + "::"
+                vm_value["ip"].append(mac2ipv6(mac, ipv6_addr))
             vm.value = vm_value
             return vm.value
         else:
@@ -296,7 +305,8 @@ class GetSSHKeys(Resource):
             if not validator.key_name.value:
 
                 # {user_prefix}/{realm}/{name}/key/
-                etcd_key = os.path.join(USER_PREFIX, data["realm"], data["name"], "key")
+                etcd_key = os.path.join(decouple.config("USER_PREFIX"), data["realm"],
+                                        data["name"], "key")
                 etcd_entry = etcd_client.get_prefix(etcd_key, value_in_json=True)
                 
                 keys = {key.key.split("/")[-1]: key.value for key in etcd_entry}
@@ -304,8 +314,8 @@ class GetSSHKeys(Resource):
             else:
 
                 # {user_prefix}/{realm}/{name}/key/{key_name}
-                etcd_key = os.path.join(USER_PREFIX, data["realm"], data["name"],
+                etcd_key = os.path.join(decouple.config("USER_PREFIX"), data["realm"],
-                                        "key", data["key_name"])
+                                        data["name"], "key", data["key_name"])
                 etcd_entry = etcd_client.get(etcd_key, value_in_json=True)
                 
                 if etcd_entry:
@@ -367,8 +377,25 @@ class CreateNetwork(Resource):
             
             network_entry = {
                 "id": increment_etcd_counter(etcd_client, "/v1/counter/vxlan"),
-                "type": data["type"]
+                "type": data["type"],
             }
+            if validator.user.value:
+                nb = pynetbox.api(url=decouple.config("NETBOX_URL"),
+                                  token=decouple.config("NETBOX_TOKEN"))
+                nb_prefix = nb.ipam.prefixes.get(prefix=decouple.config("PREFIX"))
+
+                prefix = nb_prefix.available_prefixes.create(data=
+                    {
+                        "prefix_length": decouple.config("PREFIX_LENGTH", cast=int),
+                        "description": "{}'s network \"{}\"".format(data["name"],
+                                                                    data["network_name"]),
+                        "is_pool": True
+                    }
+                )
+                network_entry["ipv6"] = prefix["prefix"]
+            else:
+                network_entry["ipv6"] = "fd00::/64"
+            
             network_key = os.path.join(NETWORK_PREFIX, data["name"], data["network_name"])
             etcd_client.put(network_key, network_entry, value_in_json=True)
             return {"message": "Network successfully added."}

api/schemas.py

@@ -438,11 +438,12 @@ class CreateNetwork(OTPSchema):
     def __init__(self, data):
         self.network_name = Field("network_name", str, data.get("network_name", KeyError))
         self.type = Field("type", str, data.get("type", KeyError))
+        self.user = Field("user", bool, bool(data.get("user", False)))
 
         self.network_name.validation = self.network_name_validation
         self.type.validation = self.network_type_validation
 
-        fields = [self.network_name, self.type]
+        fields = [self.network_name, self.type, self.user]
         super().__init__(data, fields=fields)
     
     def network_name_validation(self):

docs/Makefile

@@ -7,10 +7,12 @@ SPHINXOPTS    ?=
 SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = source/
 BUILDDIR      = build/
-DESTINATION=root@[2a0a:e5c0:2:12:0:f0ff:fea9:c3d9]:/home/app/static/ucloud
+DESTINATION=root@staticweb.ungleich.ch:/home/services/www/ungleichstatic/staticcms.ungleich.ch/www/ucloud/
+
+.PHONY: all build clean
 
 publish: build
-	rsync -av $(BUILDDIR)/ $(DESTINATION)
+	rsync -av $(BUILDDIR) $(DESTINATION)
 
 build:
 	$(SPHINXBUILD) "$(SOURCEDIR)" "$(BUILDDIR)"

docs/source/conf.py

@@ -18,8 +18,8 @@
 # -- Project information -----------------------------------------------------
 
 project = 'ucloud'
-copyright = '2019, Ahmed Bilal Khalid'
+copyright = '2019, ungleich'
-author = 'Ahmed Bilal Khalid'
+author = 'ungleich'
 
 
 # -- General configuration ---------------------------------------------------
@@ -28,6 +28,7 @@ author = 'Ahmed Bilal Khalid'
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
 extensions = [
+    'sphinx.ext.autodoc'
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/source/index.rst

@@ -12,7 +12,9 @@ Welcome to ucloud's documentation!
 
    introduction/introduction
    introduction/installation
-   introduction/usage
+   usage/usage-for-admins
+   usage/usage-for-users
+
 
 Indices and tables
 ==================

docs/source/introduction/installation.rst

@@ -7,7 +7,7 @@ Installation
     The instructions assumes the following things
     
     * User is **root**.
-    * Base Directory is `/root/`.
+    * Base Directory is :file:`/root/`.
 
 Alpine
 ------

docs/source/introduction/introduction.rst

@@ -1,5 +1,5 @@
-Introduction
+What is ucloud?
-============
+===============
 
 **Open** + **Simple** + **Easy to hack** + **IPv6 First**
 
@@ -18,6 +18,7 @@ Tech Stack
 * QEMU (+ kvm acceleration) as hypervisor.
 * etcd for key/value storage (specifically all metadata e.g Virtual Machine Specifications, Networks Specifications, Images Specifications etc.).
 * Ceph for image storage.
+* uotp for user authentication.
 
 Components
 ----------

docs/source/usage/usage-for-admins.rst

@@ -1,5 +1,5 @@
-Usage
+Usage Guide For Administrators
-=====
+==============================
 
 Start API
 ----------
@@ -95,14 +95,14 @@ An image belongs to an image store. There are two types of store
 * Private Image Store (Not Implemented Yet)
 
 .. note::
-    **Quick Quiz** Have we create an image store yet?
+    **Quick Quiz** Have we created an image store yet?
 
-The answer is **No, we haven't**. Creating an example image store is very easy.
+The answer is **No, we haven't**. Creating a sample image store is very easy.
 Just execute the following command
 
 .. code-block:: sh
 
-    pipenv run python ~/ucloud/api/create_image_store.py
+    (cd ~/ucloud && pipenv run python api/create_image_store.py)
 
 An image store (with name = "images") would be created. Now, we are fully ready for creating our
 very own image. Executing the following command to create image using the file uploaded earlier
@@ -132,73 +132,3 @@ output something like the following
             }
         ]
     }
-
-Create VM
----------
-
-The following command would create a Virtual Machine (name: meow) with following specs
-
-* CPU: 1
-* RAM: 1GB
-* OS-SSD: 4GB
-* OS: Alpine Linux
-
-.. code-block:: sh
-
-    ucloud-cli vm create --vm-name meow --cpu 1 --ram '1gb' --os-ssd '4gb' --image images:alpine
-
-Check VM Status
----------------
-
-.. code-block:: sh
-
-    ucloud-cli vm status --vm-name meow
-
-
-.. code-block:: json
-
-    {
-        "hostname": "/v1/host/74c21c332f664972bf5078e8de080eea",
-        "image_uuid": "3f75bd20-45d6-4013-89c4-7fceaedc8dda",
-        "in_migration": null,
-        "log": [
-            "2019-11-12T09:11:09.800798 - Started successfully"
-        ],
-        "metadata": {
-            "ssh-keys": []
-        },
-        "name": "meow",
-        "network": [],
-        "owner": "admin",
-        "owner_realm": "ungleich-admin",
-        "specs": {
-            "cpu": 1,
-            "hdd": [],
-            "os-ssd": "4.0 GB",
-            "ram": "1.0 GB"
-        },
-        "status": "RUNNING",
-        "vnc_socket": "/tmp/tmpj1k6sdo_"
-    }
-
-Create Network
---------------
-
-.. code-block:: sh
-
-    ucloud-cli network create --network-name mynet --network-type vxlan
-
-
-.. code-block:: json
-
-    {
-        "message": "Network successfully added."
-    }
-
-Create VM using this network
-
-.. code-block:: sh
-
-    ucloud-cli vm create --vm-name meow2 --cpu 1 --ram '1gb' --os-ssd '4gb' --image images:alpine --network mynet
-
-

docs/source/usage/usage-for-users.rst

@@ -0,0 +1,89 @@
+Usage Guide For End Users
+=========================
+
+Create VM
+---------
+
+The following command would create a Virtual Machine (name: meow) with following specs
+
+* CPU: 1
+* RAM: 1GB
+* OS-SSD: 4GB
+* OS: Alpine Linux
+
+.. code-block:: sh
+
+    ucloud-cli vm create --vm-name meow --cpu 1 --ram '1gb' --os-ssd '4gb' --image images:alpine
+
+
+.. _how-to-check-vm-status:
+
+Check VM Status
+---------------
+
+.. code-block:: sh
+
+    ucloud-cli vm status --vm-name meow
+
+.. code-block:: json
+
+    {
+        "hostname": "/v1/host/74c21c332f664972bf5078e8de080eea",
+        "image_uuid": "3f75bd20-45d6-4013-89c4-7fceaedc8dda",
+        "in_migration": null,
+        "log": [
+            "2019-11-12T09:11:09.800798 - Started successfully"
+        ],
+        "metadata": {
+            "ssh-keys": []
+        },
+        "name": "meow",
+        "network": [],
+        "owner": "admin",
+        "owner_realm": "ungleich-admin",
+        "specs": {
+            "cpu": 1,
+            "hdd": [],
+            "os-ssd": "4.0 GB",
+            "ram": "1.0 GB"
+        },
+        "status": "RUNNING",
+        "vnc_socket": "/tmp/tmpj1k6sdo_"
+    }
+
+
+Connect to VM using VNC
+-----------------------
+
+We would need **socat** utility and a remote desktop client e.g Remmina, KRDC etc.
+We can get the vnc socket path by getting its status, see :ref:`how-to-check-vm-status`.
+
+
+.. code-block:: sh
+
+    socat TCP-LISTEN:1234,reuseaddr,fork UNIX-CLIENT:/tmp/tmpj1k6sdo_
+
+
+Then, launch your remote desktop client and connect to vnc://localhost:1234.
+
+Create Network
+--------------
+
+.. code-block:: sh
+
+    ucloud-cli network create --network-name mynet --network-type vxlan
+
+
+.. code-block:: json
+
+    {
+        "message": "Network successfully added."
+    }
+
+Create VM using this network
+
+.. code-block:: sh
+
+    ucloud-cli vm create --vm-name meow2 --cpu 1 --ram '1gb' --os-ssd '4gb' --image images:alpine --network mynet
+
+

host/virtualmachine.py

@@ -10,23 +10,26 @@ import subprocess as sp
 import tempfile
 import time
 import random
+import ipaddress
 
 from functools import wraps
 from os.path import join
 from typing import Union
-from decouple import config
+from string import Template
 
 import bitmath
 import sshtunnel
-
 import qmp
-from config import (WITHOUT_CEPH, VM_PREFIX, VM_DIR, IMAGE_DIR,
+
-                    NETWORK_PREFIX, etcd_client, logging,
+from decouple import config
-                    request_pool, running_vms, vm_pool)
 from ucloud_common.helpers import get_ipv4_address
 from ucloud_common.request import RequestEntry, RequestType
 from ucloud_common.vm import VMEntry, VMStatus
 
+from config import (WITHOUT_CEPH, VM_PREFIX, VM_DIR, IMAGE_DIR,
+                    NETWORK_PREFIX, etcd_client, logging,
+                    request_pool, running_vms, vm_pool)
+
 
 class VM:
     def __init__(self, key, handle, vnc_socket_file):
@@ -38,10 +41,12 @@ class VM:
         return "VM({})".format(self.key)
 
 
-def create_dev(script, _id, dev):
+def create_dev(script, _id, dev, ip=None):
-    assert isinstance(_id, str) and isinstance(dev, str), "_id and dev both must be string"
+    command = [script, _id, dev]
+    if ip: 
+        command.append(ip)
     try:
-        output = sp.check_output([script, _id, dev], stderr=sp.PIPE)
+        output = sp.check_output(command, stderr=sp.PIPE)
     except Exception as e:
         print(e.stderr)
         return None
@@ -49,13 +54,13 @@ def create_dev(script, _id, dev):
         return output.decode("utf-8").strip()
 
 
-def create_vxlan_br_tap(_id, _dev):
+def create_vxlan_br_tap(_id, _dev, ip=None):
     network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network')
     vxlan = create_dev(script=os.path.join(network_script_base, 'create-vxlan.sh'),
                         _id=_id, dev=_dev)
     if vxlan:
         bridge = create_dev(script=os.path.join(network_script_base, 'create-bridge.sh'),
-                            _id=_id, dev=vxlan)
+                            _id=_id, dev=vxlan, ip=ip)
         if bridge:
             tap = create_dev(script=os.path.join(network_script_base, 'create-tap.sh'),
                             _id=str(random.randint(1, 100000)), dev=bridge)
@@ -85,6 +90,28 @@ def generate_mac(uaa=False, multicast=False, oui=None, separator=':', byte_fmt='
     return separator.join(byte_fmt % b for b in mac)
 
 
+def update_radvd_conf(etcd_client):
+    network_script_base = os.path.join(os.path.dirname(os.path.dirname(__file__)), 'network')
+
+    networks =  {
+                    net.value['ipv6']:net.value['id'] 
+                    for net in etcd_client.get_prefix('/v1/network/', value_in_json=True)
+                    if net.value.get('ipv6')
+                }
+    radvd_template = open(os.path.join(network_script_base,
+                                       'radvd-template.conf'), 'r').read()
+    radvd_template = Template(radvd_template)
+
+    content = [radvd_template.safe_substitute(bridge='br{}'.format(networks[net]),
+                                              prefix=net) 
+               for net in networks if networks.get(net)]
+    
+    with open('/etc/radvd.conf', 'w') as radvd_conf:
+        radvd_conf.writelines(content)
+
+    sp.check_output(['systemctl', 'restart', 'radvd'])
+
+
 def get_start_command_args(
     vm_entry, vnc_sock_filename: str, migration=False, migration_port=4444,
 ):
@@ -94,7 +121,6 @@ def get_start_command_args(
     vm_uuid = vm_entry.uuid
     vm_networks = vm_entry.network
 
-
     if WITHOUT_CEPH:
         command = "-drive file={},format=raw,if=virtio,cache=none".format(
             os.path.join(VM_DIR, vm_uuid)
@@ -114,18 +140,22 @@ def get_start_command_args(
         command += " -incoming tcp:0:{}".format(migration_port)
     
     tap = None
-    for network_name in vm_networks:
+    for network_and_mac in vm_networks:
+        network_name, mac = network_and_mac
+        
         _key = os.path.join(NETWORK_PREFIX, vm_entry.owner, network_name)
         network = etcd_client.get(_key, value_in_json=True)
         network_type = network.value["type"]
         network_id = str(network.value["id"])
+        network_ipv6 = network.value["ipv6"]
 
         if network_type == "vxlan":
-            tap = create_vxlan_br_tap(network_id, config("VXLAN_PHY_DEV"))
+            tap = create_vxlan_br_tap(network_id, config("VXLAN_PHY_DEV"), network_ipv6)
+            update_radvd_conf(etcd_client)
         
         command += " -netdev tap,id=vmnet{net_id},ifname={tap},script=no,downscript=no"\
                    " -device virtio-net-pci,netdev=vmnet{net_id},mac={mac}"\
-                       .format(tap=tap, net_id=network_id, mac=generate_mac())
+                       .format(tap=tap, net_id=network_id, mac=mac)
 
     return command.split(" ")
 

metadata/main.py

@@ -9,7 +9,7 @@ api = Api(app)
 
 
 def get_vm_entry(mac_addr):
-    return next(filter(lambda vm: vm.mac == mac_addr, VM_POOL.vms), None)
+    return next(filter(lambda vm: mac_addr in list(zip(*vm.network))[1], VM_POOL.vms), None)
 
 
 # https://stackoverflow.com/questions/37140846/how-to-convert-ipv6-link-local-address-to-mac-address-in-python

network/create-bridge.sh

@@ -1,14 +1,15 @@
 #!/bin/sh
 
-if [ $# -ne 2 ]; then
+if [ $# -ne 3 ]; then
-    echo "$0 brid dev"
+    echo "$0 brid dev ip"
-    echo "f.g. $0 100 vxlan100"
+    echo "f.g. $0 100 vxlan100 fd00:/64"
     echo "Missing arguments" >&2
     exit 1
 fi
 
 brid=$1; shift
 dev=$1; shift
+ip=$1; shift
 bridge=br${brid}
 
 sysctl net.ipv6.conf.all.forwarding=1 > /dev/null
@@ -17,7 +18,7 @@ if ! ip link show $bridge > /dev/null 2> /dev/null; then
     ip link add name $bridge type bridge
     ip link set $bridge up
     ip link set $dev master $bridge
-    ip address add fd00:/64 dev $bridge
+    ip address add $ip dev $bridge
 fi
 
 echo $bridge

network/radvd-template.conf

@@ -0,0 +1,13 @@
+interface $bridge
+{
+  AdvSendAdvert on;
+  MinRtrAdvInterval 3;
+  MaxRtrAdvInterval 5;
+  AdvDefaultLifetime 10;
+
+  prefix $prefix { };
+
+  RDNSS 2a0a:e5c0:2:1::5 2a0a:e5c0:2:1::6  { AdvRDNSSLifetime 6000; };
+  DNSSL place6.ungleich.ch {  AdvDNSSLLifetime 6000; } ;
+};
+