From 9c06d0c7f434064e087b9a919d62e8bfc9b62f5e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 5 Aug 2019 18:26:04 +0200 Subject: [PATCH 1/3] results and such --- ..._entries.py => gen_nat64_table_entries.sh} | 18 + doc/appendix.tex | 321 +++++++++++++++++- 2 files changed, 336 insertions(+), 3 deletions(-) rename bin/{gen_nat64_table_entries.py => gen_nat64_table_entries.sh} (61%) mode change 100644 => 100755 diff --git a/bin/gen_nat64_table_entries.py b/bin/gen_nat64_table_entries.sh old mode 100644 new mode 100755 similarity index 61% rename from bin/gen_nat64_table_entries.py rename to bin/gen_nat64_table_entries.sh index ab2cd9a..a93b752 --- a/bin/gen_nat64_table_entries.py +++ b/bin/gen_nat64_table_entries.sh @@ -1,5 +1,23 @@ #!/bin/sh + + +# egress for 10.0.0.42 +echo 'table_cam_add_entry realmain_v4_networks_0 realmain.set_egress_port 167772202 => 16 0 0 0 0' + +# egress for 2001:db8:42::42 +echo 'table_cam_add_entry realmain_v6_networks_0 realmain.set_egress_port 42540766411362381960998550477184434242 => 64 0 0 0 0' + +# nat64: 2001:db8:42::a00:2a to be translated to 10.0.0.66 +echo 'table_cam_add_entry realmain_nat64_0 realmain.nat64_static 42540766411362381960998550477352206378 => 42540766411362381960998550477184434176 167772160 42540766411362381960998550477184434176 0' + +# nat64: 10.0.0.66 to be translated to 2001:db8:42:42 +echo 'table_cam_add_entry realmain_nat46_0 realmain.nat46_static 167772226 => 42540766411362381960998550477184434176 167772160 42540766411362381960998550477184434176 0' + +exit 0 + +# outdated + cat < 16 0 0 0 0 diff --git a/doc/appendix.tex b/doc/appendix.tex index bc1a124..629e689 100644 --- a/doc/appendix.tex +++ b/doc/appendix.tex @@ -211,6 +211,7 @@ Step 1: getting IP address values 42540766411362381960998550477184434182L >>> int(ipaddress.IPv6Address(u"2001:db8:42::42")) 42540766411362381960998550477184434242L + \end{verbatim} Step 2: setting table entries @@ -296,6 +297,9 @@ Step 1: getting IP address values via Python >>> int(ipaddress.IPv6Address(u"2001:db8:42::")) 42540766411362381960998550477184434176L +>>> int(ipaddress.IPv6Address(u"2001:db8:42::a00:2a")) +42540766411362381960998550477352206378 + >>> int(ipaddress.IPv4Address(u"10.0.0.0")) 167772160 @@ -337,9 +341,8 @@ success \end{verbatim} Add table entry for 2001:db8:42::a00:2a to be translated to 10.0.0.66: - \begin{verbatim} - +table_cam_add_entry realmain_nat64_0 realmain.nat64_static 42540766411362381960998550477352206378 => 42540766411362381960998550477184434176 167772160 42540766411362381960998550477184434176 0 \end{verbatim} Add table entry for 10.0.0.66 to be translated to 2001:db8:42:42: @@ -1029,7 +1032,7 @@ UDP v6->v4, again 100% cpu P1: 5.81 gbit/s P20: 9.40 gbit/s -P50: 19.6 Gbits/sec +P50: 19.6 gbits/sec On the line only ca. 3600 mbit/s seen @@ -1143,6 +1146,318 @@ nico@nsg-System:~/jool-4.0.1$ sudo apt install iptables-dev Compiles! +\begin{verbatim} +nico@nsg-System:~/jool-4.0.1$ sudo make install + +\end{verbatim} + +\subsection{\label{benchmark:jool:config}Jool Configuration} + +Loading module: +\begin{verbatim} +nico@nsg-System:~/jool-4.0.1$ sudo modprobe jool_siit + + +\end{verbatim} + +enabling forwarding: +\begin{verbatim} +sysctl -w net.ipv4.conf.all.forwarding=1 +sysctl -w net.ipv6.conf.all.forwarding=1 +\end{verbatim} + +Mapping configuration: + +\begin{verbatim} +nico@nsg-System:~/jool-4.0.1$ sudo jool_siit instance add example --iptables --pool6 2001:db8:23::/96 + +nico@nsg-System:~/jool-4.0.1$ sudo ip6tables -t mangle -A PREROUTING \ + -s 2001:db8:42::/64 -d 2001:db8:23::/96 -j JOOL_SIIT --instance example + +nico@nsg-System:~/jool-4.0.1$ sudo iptables -t mangle -A PREROUTING \ + -s 10.0.0.0/24 -j JOOL_SIIT --instance example + +\end{verbatim} + + + +Debugging: + +\begin{verbatim} +[16:39] nsg-System:~# lsmod| grep jool +jool_siit 147456 2 +x_tables 40960 5 jool_siit,ip6_tables,ip_tables,ip6table_mangle,iptable_mangle +[16:39] nsg-System:~# + +[16:41] nsg-System:~# jool_siit -i example stats display --explain +JSTAT64_DST: 276 +Translations cancelled: IPv6 packet's destination address did not match pool6 nor any EAMT entries, or the resulting address was blacklist4ed. + + +\end{verbatim} + +Try 2 w/ eamt: + +\begin{verbatim} +[16:53] nsg-System:~# modprobe jool_siit +[16:54] nsg-System:~# jool_siit instance add "example" --iptables +[16:54] nsg-System:~# jool_siit -i example eamt add 2001:db8:42::/120 10.0.1.0/24 +[16:55] nsg-System:~# jool_siit -i example eamt add 2001:db8:23::/120 10.0.0.0/24 +[16:57] nsg-System:~# ip6tables -t mangle -A PREROUTING -s 2001:db8:42::/120 -d 2001:db8:23::/120 -j JOOL_SIIT --instance example +[16:57] nsg-System:~# iptables -t mangle -A PREROUTING -s 10.0.0.0/24 -d 10.0.1.0/24 -j JOOL_SIIT --instance example +[16:57] nsg-System:~# + +\end{verbatim} + +Testing NAT64: + +\begin{verbatim} +nico@ESPRIMO-P956:~/master-thesis/iperf$ ping6 2001:db8:23::2a +PING 2001:db8:23::2a(2001:db8:23::2a) 56 data bytes +64 bytes from 2001:db8:23::2a: icmp_seq=1 ttl=63 time=0.199 ms +64 bytes from 2001:db8:23::2a: icmp_seq=2 ttl=63 time=0.282 ms +64 bytes from 2001:db8:23::2a: icmp_seq=3 ttl=63 time=0.186 ms +^C +--- 2001:db8:23::2a ping statistics --- +3 packets transmitted, 3 received, 0% packet loss, time 2040ms +rtt min/avg/max/mdev = 0.186/0.222/0.282/0.044 ms +nico@ESPRIMO-P956:~/master-thesis/iperf$ ping 10.0.1.66 +PING 10.0.1.66 (10.0.1.66) 56(84) bytes of data. +64 bytes from 10.0.1.66: icmp_seq=1 ttl=63 time=0.218 ms +64 bytes from 10.0.1.66: icmp_seq=2 ttl=63 time=0.281 ms +64 bytes from 10.0.1.66: icmp_seq=3 ttl=63 time=0.280 ms +^C +--- 10.0.1.66 ping statistics --- +3 packets transmitted, 3 received, 0% packet loss, time 2051ms +rtt min/avg/max/mdev = 0.218/0.259/0.281/0.034 ms +nico@ESPRIMO-P956:~/master-thesis/iperf$ + +\end{verbatim} + + +\subsection{\label{benchmark:jool:benchmark}Jool Benchmarks} + + + +v4->v6 tcp + +P1: 8.24 gbit/s no cpu load visible +P20: 8.26 gbit/s iperf 42 + 10% cpu no cpu load visible +P50: 8.29 gbit/s + +v6->v4 tcp + +P1: 8.22 +P20: 8.22 15/60% iperf +P50: 8.23 iperf: 73/16% + +Commands: + +\begin{verbatim} +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -B 2001:db8:42::42 -s | tee iperf-jool-v4tov6-server-tcp-P50 + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -t 70 -O 10 -P1 -c 10.0.1.66 | tee iperf-jool-v4tov6-client-tcp-P1 + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -t 70 -O 10 -P20 -c 10.0.1.66 | tee iperf-jool-v4tov6-client-tcp-P20 +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -t 70 -O 10 -P50 -c 10.0.1.66 | tee iperf-jool-v4tov6-client-tcp-P50 + +Other way: + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -B 10.0.0.42 -s | tee iperf-jool-v6tov4-server-tcp-P1 + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -t 70 -O 10 -P1 -c 2001:db8:23::2a | tee iperf-jool-v6tov4-client-tcp-P1 + +... + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -t 70 -O 10 -P1 -b0 -u -c 2001:db8:23::2a | tee iperf-jool-v6tov4-client-tcp-P1 + + +\end{verbatim} + + +v4->v6 udp + +P1: 4.46 iperf 30% cpu nat64: 100% cpu +P20: 18.8 iperf 100% nat64: 100% +P50: 22.8 iperf 100% nat64: 100% atop: ca. 5gbit/s + +\begin{verbatim} +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -B 2001:db8:42::42 -s | tee iperf-jool-v4tov6-server-udp-P1 + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -t 70 -O 10 -P1 -c 10.0.1.66 -u -b0 | tee iperf-jool-v4tov6-client-udp-P1 + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -t 70 -O 10 -P20 -c 10.0.1.66 -u -b0 | tee iperf-jool-v4tov6-client-udp-P20 + + +\end{verbatim} + +v6->v4 udp + +P1: 6.67 gbit/s iperf 50/50%, nat64: 100% cpu! +P20: 16.8 nat64: iperf: ? 100% cpu +P50: 20.5 Gbits/sec nat64: 100% (1 core) iperf: 100%/50% atop: 7000mbit/s in / out + +\begin{verbatim} + +\end{verbatim} + +Turning off offloading, redoing tcp: + +\begin{verbatim} +root@ESPRIMO-P956:~# ethtool -K enp2s0f0 gso off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +root@ESPRIMO-P956:~# ethtool -K enp2s0f0 rx off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +root@ESPRIMO-P956:~# ethtool -K enp2s0f0 tx off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +Actual changes: +tx-checksumming: off + tx-checksum-ip-generic: off + tx-checksum-sctp: off +tcp-segmentation-offload: off + tx-tcp-segmentation: off [requested on] + tx-tcp6-segmentation: off [requested on] +root@ESPRIMO-P956:~# ethtool -K enp2s0f1 tx off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +Actual changes: +tx-checksumming: off + tx-checksum-ip-generic: off + tx-checksum-sctp: off +tcp-segmentation-offload: off + tx-tcp-segmentation: off [requested on] + tx-tcp6-segmentation: off [requested on] +root@ESPRIMO-P956:~# ethtool -K enp2s0f1 rx off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +root@ESPRIMO-P956:~# ethtool -K enp2s0f1 gso off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +root@ESPRIMO-P956:~# + +[17:26] nsg-System:~# ethtool -K eth1 tx off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +Actual changes: +tx-checksumming: off + tx-checksum-ip-generic: off + tx-checksum-sctp: off +tcp-segmentation-offload: off + tx-tcp-segmentation: off [requested on] + tx-tcp6-segmentation: off [requested on] +[17:26] nsg-System:~# ethtool -K eth1 gso off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +[17:26] nsg-System:~# ethtool -K eth2 gso off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +[17:26] nsg-System:~# ethtool -K eth2 rx off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +[17:26] nsg-System:~# ethtool -K eth2 tx off +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +Actual changes: +tx-checksumming: off + tx-checksum-ip-generic: off + tx-checksum-sctp: off +tcp-segmentation-offload: off + tx-tcp-segmentation: off [requested on] + tx-tcp6-segmentation: off [requested on] +[17:26] nsg-System:~# + +\end{verbatim} + +Retesting using -P50: + +Still no cpu load with tcp, 100% cpu load iperf + +result: 7.96 gbit/s + +\begin{verbatim} +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -B 10.0.0.42 -s | tee iperf-jool-v6tov4-server-tcp-P50-no-offload +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -t 70 -O 10 -P50 -c 2001:db8:23::2a | tee iperf-jool-v6tov4-client-tcp-P50-no-offload + + +\end{verbatim} + +\subsection{\label{benchmark:netfpga:benchmark}NetPFGA Benchmarks} + +Only 1 test did have offloading on esprimo off, was redone + +v4->v6 tcp + +P1: 7.41 gbit/s iperf 50% / 20% +P1-offload-on-esprimo: 8.43 gbit/s +P20: iperf: 66/20% +P50: + +v4->v6 udp + +P1: +P20: +P50: + + +v6->v4 tcp + +P1: iperf 50% / 20% +P20: +P50: + +v6->v4 udp + +P1: +P20: +P50: + + +After first netpfga, tcp v4->v6 p1 turned offloading on again + +\begin{verbatim} +root@ESPRIMO-P956:~# ethtool -K enp2s0f1 tx-checksum-ipv6 on +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +Could not change any device features +root@ESPRIMO-P956:~# ethtool -K enp2s0f1 tx on +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +Actual changes: +tx-checksumming: on + tx-checksum-ip-generic: on + tx-checksum-sctp: on +tcp-segmentation-offload: on + tx-tcp-segmentation: on + tx-tcp6-segmentation: on +root@ESPRIMO-P956:~# ethtool -K enp2s0f1 rx on +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +root@ESPRIMO-P956:~# ethtool -K enp2s0f1 gso on +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +root@ESPRIMO-P956:~# ethtool -K enp2s0f0 gso on +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +root@ESPRIMO-P956:~# ethtool -K enp2s0f0 tx on +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +Actual changes: +tx-checksumming: on + tx-checksum-ip-generic: on + tx-checksum-sctp: on +tcp-segmentation-offload: on + tx-tcp-segmentation: on + tx-tcp6-segmentation: on +root@ESPRIMO-P956:~# ethtool -K enp2s0f0 rx on +Cannot get device udp-fragmentation-offload settings: Operation not supported +Cannot get device udp-fragmentation-offload settings: Operation not supported +root@ESPRIMO-P956:~# + +\end{verbatim} + %Entries for the list of abbrevations: % %To generate the list of abbrevations, execute: From 646ac097cd22a1dbdc2d45731af660c1b69e8353 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 5 Aug 2019 18:28:38 +0200 Subject: [PATCH 2/3] ++stuff --- doc/appendix.tex | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/appendix.tex b/doc/appendix.tex index 629e689..54cc1a7 100644 --- a/doc/appendix.tex +++ b/doc/appendix.tex @@ -1392,7 +1392,7 @@ v4->v6 tcp P1: 7.41 gbit/s iperf 50% / 20% P1-offload-on-esprimo: 8.43 gbit/s -P20: iperf: 66/20% +P20: 9.29 gbit/s iperf: 66/20% P50: v4->v6 udp From 2d2ca29637ac507d7dd1f05021ec3bd1f5484bf2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 5 Aug 2019 18:51:02 +0200 Subject: [PATCH 3/3] ++cc --- doc/appendix.tex | 47 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/doc/appendix.tex b/doc/appendix.tex index 54cc1a7..e852287 100644 --- a/doc/appendix.tex +++ b/doc/appendix.tex @@ -1237,17 +1237,15 @@ nico@ESPRIMO-P956:~/master-thesis/iperf$ \subsection{\label{benchmark:jool:benchmark}Jool Benchmarks} - - v4->v6 tcp -P1: 8.24 gbit/s no cpu load visible +P1: 8.24 gbit/s no cpu load visible P20: 8.26 gbit/s iperf 42 + 10% cpu no cpu load visible P50: 8.29 gbit/s v6->v4 tcp -P1: 8.22 +P1: 8.22 P20: 8.22 15/60% iperf P50: 8.23 iperf: 73/16% @@ -1277,7 +1275,7 @@ nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -t 70 -O 10 -P1 -b0 - v4->v6 udp -P1: 4.46 iperf 30% cpu nat64: 100% cpu +P1: 4.46 iperf 30% cpu nat64: 100% cpu P20: 18.8 iperf 100% nat64: 100% P50: 22.8 iperf 100% nat64: 100% atop: ca. 5gbit/s @@ -1381,6 +1379,18 @@ result: 7.96 gbit/s nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -B 10.0.0.42 -s | tee iperf-jool-v6tov4-server-tcp-P50-no-offload nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -t 70 -O 10 -P50 -c 2001:db8:23::2a | tee iperf-jool-v6tov4-client-tcp-P50-no-offload +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -t 70 -O 10 -P20 -u -b0 -c 10.0.0.66 | tee iperf-netpfga-v4tov6-client-udp-P20 + + + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -B 10.0.0.42 -s | tee iperf-netfpga-v6tov4-server-tcp-P1 + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -t 70 -O 10 -P1 -c 2001:db8:42::a00:2a | tee iperf-netfpga-v6tov4-client-tcp-P1 + + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -B 10.0.0.42 -s | tee iperf-netfpga-v6tov4-server-udp-P1 + +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -6 -p 2345 -t 70 -O 10 -P1 -b0 -u -c 2001:db8:42::a00:2a | tee iperf-netfpga-v6tov4-client-udp-P1 \end{verbatim} @@ -1393,28 +1403,35 @@ v4->v6 tcp P1: 7.41 gbit/s iperf 50% / 20% P1-offload-on-esprimo: 8.43 gbit/s P20: 9.29 gbit/s iperf: 66/20% -P50: +P50: 9.29 gbit/s 84/42% iperf v4->v6 udp -P1: -P20: -P50: +P1: 7.4gbit/s 100% iperf +P20: 17.7gbit/s iperf 100% atop: ca. 9800 Mbit/s +P50: 21.5 gbit/s iperf 100% ca. 9800 Mbit/s v6->v4 tcp -P1: iperf 50% / 20% -P20: -P50: +P1: 9.28 gbit/s atop ~9800 mbit/s iperf 44% +P20: 9.29 gbit/s atop ~9800 mbit/s iperf 70% +P50: 9.29 gbit/s atop ~9800 mbit/s iperf 90% v6->v4 udp -P1: -P20: -P50: +P1: 7.96 gbit/s atop ~8200mbit/s iperf 70% +P20: 13.4 gbit/s atop 9800 mbit/s iperf 100% +P50: 19.0 gbit/s atop 9800 mbit/s iperf 100% + +Commands: +\begin{verbatim} +nico@ESPRIMO-P956:~/master-thesis/iperf$ iperf3 -4 -p 2345 -t 70 -O 10 -P1 -u -b0 -c 10.0.0.66 | tee iperf-netpfga-v4tov6-client-udp-P1 + +\end{verbatim} + After first netpfga, tcp v4->v6 p1 turned offloading on again \begin{verbatim}