From a2547bcd8361254403d50ed7b38a30050938dfaf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 1 Nov 2019 17:51:06 +0100 Subject: [PATCH 1/4] begin networking --- network/README | 60 ++++++++++++++++++++++++++++++++++ network/create-vxlan-on-dev.sh | 17 ++++++++++ 2 files changed, 77 insertions(+) create mode 100644 network/README create mode 100644 network/create-vxlan-on-dev.sh diff --git a/network/README b/network/README new file mode 100644 index 0000000..e32acd9 --- /dev/null +++ b/network/README @@ -0,0 +1,60 @@ +The network base - experimental + + +We want to have 1 "main" network for convience. + +We want to be able to create networks automatically, once a new +customer is created -> need hooks! + + +Mapping: + +- each network is a "virtual" network. We use vxlan by default, but + could be any technology! +- we need a counter for vxlan mappings / network IDs -> cannot use + +Model in etcd: + +/v1/networks/ + + +Tests +see +https://vincent.bernat.ch/en/blog/2017-vxlan-linux + + +# local 2001:db8:1::1 \ + + +netid=100 +dev=wlp2s0 +dev=wlp0s20f3 +ip -6 link add vxlan${netid} type vxlan \ + id ${netid} \ + dstport 4789 \ + group ff05::${netid} \ + dev ${dev} \ + ttl 5 + +[root@diamond ~]# ip addr add 2a0a:e5c0:5::1/48 dev vxlan100 +root@manager:~/.ssh# ip addr add 2a0a:e5c0:5::2/48 dev vxlan100 +root@manager:~/.ssh# ping -c3 2a0a:e5c0:5::1 +PING 2a0a:e5c0:5::1(2a0a:e5c0:5::1) 56 data bytes +64 bytes from 2a0a:e5c0:5::1: icmp_seq=1 ttl=64 time=15.6 ms +64 bytes from 2a0a:e5c0:5::1: icmp_seq=2 ttl=64 time=30.3 ms +64 bytes from 2a0a:e5c0:5::1: icmp_seq=3 ttl=64 time=84.4 ms + +--- 2a0a:e5c0:5::1 ping statistics --- +3 packets transmitted, 3 received, 0% packet loss, time 2003ms +rtt min/avg/max/mdev = 15.580/43.437/84.417/29.594 ms + +--> work even via wifi + + +-------------------------------------------------------------------------------- + +Creating a network: + +1) part of the initialisation / demo data (?) + +We should probably provide some demo sets that can easily be used. diff --git a/network/create-vxlan-on-dev.sh b/network/create-vxlan-on-dev.sh new file mode 100644 index 0000000..87687c9 --- /dev/null +++ b/network/create-vxlan-on-dev.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 vxlanid dev" + echo "f.i. $0 100 eth0" + exit 1 +fi + +netid=$1; shift +dev=$1; shift + +ip -6 link add vxlan${netid} type vxlan \ + id ${netid} \ + dstport 4789 \ + group ff05::${netid} \ + dev ${dev} \ + ttl 5 From b27f1b62f380a8fbc086111bfbfb406607097c11 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 1 Nov 2019 17:54:02 +0100 Subject: [PATCH 2/4] network: up the dev some kernels do that automatically, some don't --- network/create-vxlan-on-dev.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/network/create-vxlan-on-dev.sh b/network/create-vxlan-on-dev.sh index 87687c9..b366392 100644 --- a/network/create-vxlan-on-dev.sh +++ b/network/create-vxlan-on-dev.sh @@ -15,3 +15,5 @@ ip -6 link add vxlan${netid} type vxlan \ group ff05::${netid} \ dev ${dev} \ ttl 5 + +ip link set ${dev} up From 1a76150d4d0dedaf00b039125b6c1e8e3725c834 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 1 Nov 2019 21:51:28 +0100 Subject: [PATCH 3/4] ++ network readme update --- network/README | 112 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) diff --git a/network/README b/network/README index e32acd9..29863ab 100644 --- a/network/README +++ b/network/README @@ -58,3 +58,115 @@ Creating a network: 1) part of the initialisation / demo data (?) We should probably provide some demo sets that can easily be used. + +2) manual/hook based request + +- hosts might have different network interfaces (?) + -> this will make things very tricky -> don't support it +- endpoint needs only support + +-------------------------------------------------------------------------------- + +IPAM + +IP address management (IPAM) is related to networks, but needs to be +decoupled to allow pure L2 networks. + +From a customer point of view, we probably want to do something like: + +- ORDERING an IPv6 network can include creating a virtual network and + an IPAM service + +Maybe "orders" should always be the first class citizen and ucloud +internally "hooks" or binds things together. + +-------------------------------------------------------------------------------- + +testing / hacking: + +- starting etcd as storage + + +[18:07] diamond:~% etcdctl put /v1/network/200 "{ some_network }" +OK +[18:08] diamond:~% etcdctl watch -w=json --prefix /v1/network +{"Header":{"cluster_id":14841639068965178418,"member_id":10276657743932975437,"revision":6,"raft_term":2},"Events":[{"kv":{"key":"L3YxL25ldHdvcmsvMjAw","create_revision":5,"mod_revision":6,"version":2,"value":"eyBzb21lX25ldHdvcmsgfQ=="}}],"CompactRevision":0,"Canceled":false,"Created":false} + + +-------------------------------------------------------------------------------- + +Flow for using and creating networks: + +- a network is created -> entry in etcd is created + -> we need to keep a counter/lock so that 2 processes don't create + the same network [Ahmed] + -> nothing to be done on the hosts +- a VM using a network is created +- a VM using a network is scheduled to some host +- the local "spawn a VM" process needs to check whether there is a + vxlan interface existing -> if no, create it before creating the VM. + -> if no, also create the bridge + -> possibly adjusting the MTU (??) + -> both names should be in hexadecimal (i.e. brff01 or vxlanff01) + --> this way they are consistent with the multicast ipv6 address + --> attention, ip -6 link ... id XXX expects DECIMAL input + +-------------------------------------------------------------------------------- + + + + +Example + +-------------------------------------------------------------------------------- + +TODOs + +- create-vxlan-on-dev.sh -> the multicast group + needs to be ff05:: +int(vxlan_id) + +-------------------------------------------------------------------------------- + +Python hints: + +>>> vxlan_id = 3400 +>>> b = ipaddress.IPv6Network("ff05::/16") +>>> b[vxlan_id] +IPv6Address('ff05::d48') + +we need / should assign hex values for vxlan ids in etcd! +--> easier to read + +>>> b[0x3400] +IPv6Address('ff05::3400') + + +-------------------------------------------------------------------------------- + +Bridge names are limited to 15 characters + + +Maximum/highest number of vxlan: + +>>> 2**24 +16777216 +>>> (2**25)-1 +33554431 + +>>> b[33554431] +IPv6Address('ff05::1ff:ffff') + +Last interface: +br1ffffff +vxlan1ffffff + +root@manager:~/ucloud/network# ip -6 link add vxlan1ffffff type vxlan id 33554431 dstport 4789 group ff05::1ff:ffff dev wlp2s0 ttl 5 +Error: argument "33554431" is wrong: invalid id + +root@manager:~/ucloud/network# ip -6 link add vxlanffffff type vxlan id 16777215 dstport 4789 group ff05::ff:ffff dev wlp2s0 ttl 5 + + +# id needs to be decimal +root@manager:~# ip -6 link add vxlanff01 type vxlan id ff01 dstport 4789 group ff05::ff01 dev ttl 5 +Error: argument "ff01" is wrong: invalid id +root@manager:~# ip -6 link add vxlanff01 type vxlan id 65281 dstport 4789 group ff05::ff01 dev wlp2s0 ttl 5 From 583bbe34bc17df51f62362b720318fd2da058d44 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 1 Nov 2019 23:13:40 +0100 Subject: [PATCH 4/4] ++ network ideas --- network/README | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/network/README b/network/README index 29863ab..dca25d1 100644 --- a/network/README +++ b/network/README @@ -112,9 +112,32 @@ Flow for using and creating networks: --> attention, ip -6 link ... id XXX expects DECIMAL input -------------------------------------------------------------------------------- +If we also supply IPAM: + +- ipam needs to be created *after* the network is created +- ipam is likely to be coupled to netbox (?) + --> we need a "get next /64 prefix" function +- when an ipam service is created in etcd, we need to create a new + radvd instance on all routers (this will be a different service on + BSDs) +- we will need to create a new vxlan device on the routers +- we need to create a new / modify radvd.conf +- only after all of the routers reloaded radvd the ipam service is + available! + + +-------------------------------------------------------------------------------- +If the user requests an IPv4 VM: + +- we need to get the next free IPv4 address (again, netbox?) +- we need to create a mapping entry on the routers for NAT64 + --> this requires the VM to be in a network with IPAM + --> we always assume that the VM embeds itself using EUI64 +-------------------------------------------------------------------------------- +mac address handling! Example