From fd319a934220130af57da3da3eda94071ef63889 Mon Sep 17 00:00:00 2001 From: llnu Date: Fri, 6 Dec 2019 18:06:37 +0100 Subject: [PATCH 001/543] ipxe script updated for alpine --- rebuild-ipxe.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rebuild-ipxe.sh b/rebuild-ipxe.sh index 561cd46..2d1ad5a 100755 --- a/rebuild-ipxe.sh +++ b/rebuild-ipxe.sh @@ -1,7 +1,8 @@ #!/bin/sh # Nico Schottelius, 2019-09-20, Seoul, Coffebean, 23:56 # Copying: GPLv3 - +echo "If you are running alpine, these packages are needed:" +echo "apk add alpine-sdk xz-dev" set -x set -e From a666916a72609551f1c6e39dd4546d5a928741f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 08:48:31 +0100 Subject: [PATCH 002/543] Fedora image: initialize /etc/hosts --- fedora-build-opennebula-image.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 77d6a0b..40f5d12 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -110,6 +110,13 @@ mount --bind /sys /mnt/sys # TODO: use non-$BIGCORP DNS service. echo 'nameserver 1.1.1.1' >> /mnt/etc/resolv.conf +# Initialize /etc/hosts. +cat > /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + # See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. # network-scripts.x86_64 : Legacy scripts for manipulating of network devices run_root dnf -y install network-scripts From 157379235c42f7bce8672a8122bd023ce99511c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 09:53:14 +0100 Subject: [PATCH 003/543] Fedora image: reset systemd machine-id and random-seed, clean dnf database --- fedora-build-opennebula-image.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 40f5d12..6222efc 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -134,6 +134,8 @@ run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service # Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... run_root dnf -y install kernel grub2 run_root grub2-install --target=i386-pc "${NBD_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg @@ -150,5 +152,14 @@ UUID=$boot_uuid /boot ext4 rw,relatime,data=ordered 0 2 UUID=$root_uuid / ext4 rw,relatime,data=ordered 0 1 EOF +# Reset systemd's environment. +run_root rm -f /etc/machine-id +run_root touch /etc/machine-id +rm -f /var/lib/systemd/random-seed + +# Remove temporary files and reclaim freed disk space. +# Note: build logs could be removed as well. +run_root dnf clean all + # Make sure everything is written to disk before exiting. sync From 7e01fa5f06ba53c7d19d25a3ab83ab18ab598328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 12:08:11 +0100 Subject: [PATCH 004/543] Fedora image: add virtio-blk driver to initramfs --- fedora-build-opennebula-image.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 6222efc..ac26d7d 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -137,6 +137,14 @@ run_root systemctl enable systemd-timesyncd.service # Note: linux-firmware is not required our environment and takes almost 200M # uncompressed but is a direct dependency of kernel-core... run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf < Date: Mon, 9 Dec 2019 14:57:42 +0100 Subject: [PATCH 005/543] Fedora image: mention build-time depency on e2fsprogs --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index ac26d7d..b120dc6 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -9,7 +9,7 @@ # definitely opinionated. # Depends on the following packages (as of Fedora 31): -# qemu-img util-linux coreutils dnf curl +# qemu-img util-linux coreutils dnf curl e2fsprogs # Run locally (without network) with: # qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 From aa01350594f48a88dc82236f60f87a8dca8eefb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 18:58:41 +0100 Subject: [PATCH 006/543] Fedora image: use `date +%+F` instead of `date --iso-8601` --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index b120dc6..4d22522 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -20,7 +20,7 @@ set -x # XXX: Handle command-line arguments? RELEASE=31 ARCH=x86_64 -IMAGE_PATH=fedora-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd1 From 366c9c65b2d2ca05329494254915ae3ee124d0a8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:39:03 +0100 Subject: [PATCH 007/543] begin devuan netboo script --- devuan-netboot.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh new file mode 100644 index 0000000..6e7f39c --- /dev/null +++ b/devuan-netboot.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +date=$(date +%F) +suite=ascii + +dir=${suit}-${date} + +debootstrap ${suite} From be9616a17f25e263656021bab4183de1f3a90f2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:43:41 +0100 Subject: [PATCH 008/543] ++ netboot devuan script base --- devuan-netboot.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh new file mode 100644 index 0000000..c47c92b --- /dev/null +++ b/devuan-netboot.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Nico Schottelius, 2019-12-09 + + +date=$(date +%F) +suite=ascii + +dir=${suit}-${date} + +debootstrap "${suite}" "${dir}" +chroot ${dir} apt install -y openssh-server rdnssd + +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet6 auto +EOF From 5e5e37a7d737f6743572656f5e24a7d3c83026eb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:57:57 +0100 Subject: [PATCH 009/543] [netboot] add ssh keys --- devuan-netboot.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index c47c92b..066472e 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -5,10 +5,12 @@ date=$(date +%F) suite=ascii -dir=${suit}-${date} +dir=${suite}-${date} +kernel=${dir}-kernel +keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 cat > ${dir}/etc/network/interfaces << EOF auto lo @@ -17,3 +19,13 @@ iface lo inet loopback auto eth0 iface eth0 inet6 auto EOF + +mv ${dir}/boot/vmlinuz-* ${kernel} +rm ${dir}/boot/initrd* +mkdir -p ${dir}/root/.ssh + +for key in balazs dominique jinguk nico; do + curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys +done + +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 2970833df53656ffb1fb0b726c3e7b8529125b8f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:01:46 +0100 Subject: [PATCH 010/543] pass in suite from outside --- devuan-netboot.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index 066472e..c627ec1 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -1,9 +1,14 @@ #!/bin/sh # Nico Schottelius, 2019-12-09 +if [ $# -ne 1 ]; then + echo $0 suite + echo suite is for instance ascii, beowulf, etc +fi + +suite=$1; shift date=$(date +%F) -suite=ascii dir=${suite}-${date} kernel=${dir}-kernel From 1870aeca15f8777bd23f18fb1dbb36d2748c159e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:02:21 +0100 Subject: [PATCH 011/543] +exit error --- devuan-netboot.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index c627ec1..3dbb17f 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -4,6 +4,7 @@ if [ $# -ne 1 ]; then echo $0 suite echo suite is for instance ascii, beowulf, etc + exit 1 fi suite=$1; shift From caca8ea43f7cabdaaeebc5b29c2d2c527b0ae844 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:02:50 +0100 Subject: [PATCH 012/543] +perms --- build-alpine-chroot.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build-alpine-chroot.sh diff --git a/build-alpine-chroot.sh b/build-alpine-chroot.sh old mode 100644 new mode 100755 From 6cfdf9d40ee30f74f0017e020b27ad9bfe42a77e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:03:30 +0100 Subject: [PATCH 013/543] +perms --- devuan-netboot.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh old mode 100644 new mode 100755 From 8ed587eb0ad3c15d1477bb2805f93f04f3d3044e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:04:46 +0100 Subject: [PATCH 014/543] rename - script is suitable for debian and devuan --- devuan-netboot.sh | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100755 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh deleted file mode 100755 index 3dbb17f..0000000 --- a/devuan-netboot.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Nico Schottelius, 2019-12-09 - -if [ $# -ne 1 ]; then - echo $0 suite - echo suite is for instance ascii, beowulf, etc - exit 1 -fi - -suite=$1; shift - -date=$(date +%F) - -dir=${suite}-${date} -kernel=${dir}-kernel -keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files - -debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 - -cat > ${dir}/etc/network/interfaces << EOF -auto lo -iface lo inet loopback - -auto eth0 -iface eth0 inet6 auto -EOF - -mv ${dir}/boot/vmlinuz-* ${kernel} -rm ${dir}/boot/initrd* -mkdir -p ${dir}/root/.ssh - -for key in balazs dominique jinguk nico; do - curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys -done - -( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 0c56febefe7dd40d0f801d5af9111c8746ad0df9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 21:40:42 +0100 Subject: [PATCH 015/543] [fedora] use resolv.conf from the host --- fedora-build-opennebula-image.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 4d22522..f078042 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -107,8 +107,7 @@ mount --bind /sys /mnt/sys # systemd-networkd. # Required to resolve package mirror in chroot. -# TODO: use non-$BIGCORP DNS service. -echo 'nameserver 1.1.1.1' >> /mnt/etc/resolv.conf +cp /etc/resolv.conf /mnt/etc/resolv.conf # Initialize /etc/hosts. cat > /mnt/etc/hosts << EOF From 7bcdd02c9ec619dd450e6f0e171442e94127e835 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:11:48 +0100 Subject: [PATCH 016/543] Add new file for getting VM List --- vm_list | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 vm_list diff --git a/vm_list b/vm_list new file mode 100644 index 0000000..7d77eee --- /dev/null +++ b/vm_list @@ -0,0 +1,19 @@ +#!/bin/bash -e +#option $1 is ldap password +#option $2 is ou + + +uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) + +for ((i=0; i<${#uid_list[@]}; i++)) do + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=customer,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_vmid=() + list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) + for ((j=0; j<${#list_vmid[@]}; j++)) do + temp=$(onevm show ${list_vmid[$j]} | grep PORT) + temp1="${temp#*\"}" + port="${temp1%%\"*}" + host=$(onevm show ${list_vmid[$j]} | grep HOST | grep ungleich | awk '{print $3}') + echo ${uid_list[$i]} ${list_vmid[$j]} $port $host >> ~/vm_vnc_list + done +done \ No newline at end of file From cdd7779e2ce2654a089e20ad09deab5e68d0f2ab Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:12:52 +0100 Subject: [PATCH 017/543] Update vm_list --- vm_list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_list b/vm_list index 7d77eee..95f64be 100644 --- a/vm_list +++ b/vm_list @@ -6,7 +6,7 @@ uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) for ((i=0; i<${#uid_list[@]}; i++)) do - list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=customer,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=$2,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) list_vmid=() list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) for ((j=0; j<${#list_vmid[@]}; j++)) do From c7f02bce5c9d0307a7e1d07006d9a87c490b5bb4 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:23:28 +0100 Subject: [PATCH 018/543] Add new file --- create-guacamole-session-ldap-DB | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 create-guacamole-session-ldap-DB diff --git a/create-guacamole-session-ldap-DB b/create-guacamole-session-ldap-DB new file mode 100644 index 0000000..25819e2 --- /dev/null +++ b/create-guacamole-session-ldap-DB @@ -0,0 +1,39 @@ +#!/bin/bash +#option $1 is vm_list file name +#option $2 id DB location +#option $3 is DB user +#option $4 is DB name + +host='localhost' + +user_arr=( $(cat $1 | awk '{print $1}' )) +vmid_arr=( $(cat $1 | awk '{print $2}' )) +port_arr=( $(cat $1 | awk '{print $3}' )) +place_arr=( $(cat $1 | awk '{print $4}' )) + +for ((i=0; i<${#user_arr[@]}; i++)) do + #create user + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_entity (name, type) VALUES ('${user_arr[i]}','USER');" + en_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT entity_id FROM guacamole_entity WHERE name = '${user_arr[i]}';") + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('$en_id', '\x74657374', now());" + + #create connection + cn=${user_arr[i]}${vmid_arr[i]} + echo $cn + if [ 0 -eq $(psql -h $2 -U $3 -d $4 -tAc "SELECT connection_id FROM guacamole_connection WHERE connection_name = '$cn';" | wc -l) ]; then + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');" + cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") + + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');" + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');" + + #connection permission + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');" + + else + cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';" + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';" + fi + +done \ No newline at end of file From 36cc9b3c50ace54519c62bae43f32c2deeda4f1d Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:29:47 +0100 Subject: [PATCH 019/543] Add new file --- create-guacamole-session-ldap-file | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 create-guacamole-session-ldap-file diff --git a/create-guacamole-session-ldap-file b/create-guacamole-session-ldap-file new file mode 100644 index 0000000..c11b4bc --- /dev/null +++ b/create-guacamole-session-ldap-file @@ -0,0 +1,38 @@ +#!/bin/bash +#option $1 is vm_list file name +#option $2 is DB name +#this script should be run on guacamole server + + +host='localhost' +user_arr=( $(cat $1 | awk '{print $1}' )) +vmid_arr=( $(cat $1 | awk '{print $2}' )) +port_arr=( $(cat $1 | awk '{print $3}' )) +place_arr=( $(cat $1 | awk '{print $4}' )) + +for ((i=0; i<${#user_arr[@]}; i++)) do + #create user + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_entity (name, type) VALUES ('${user_arr[i]}','USER');\"" + en_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT entity_id FROM guacamole_entity WHERE name = '${user_arr[i]}';\"") + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('$en_id', '\x74657374', now());\"" + + #create connection + cn=${user_arr[i]}${vmid_arr[i]} + + if [ 0 -eq $(su - postgres -c "psql postgres -d $2 -tAc \"SELECT connection_id FROM guacamole_connection WHERE connection_name = '$cn';\"" | wc -l) ]; then + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');\"" + cn_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;\"") + + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');\"" + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');\"" + + #connection permission + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');\"" + + else + cn_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;\"") + su - postgres -c "psql postgres -d $2 -tAc \"UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';\"" + su - postgres -c "psql postgres -d $2 -tAc \"UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';\"" + fi + +done \ No newline at end of file From 7d37b7d5a6960c35ac787aea753b5d98144c84fd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 11 Dec 2019 15:33:24 +0100 Subject: [PATCH 020/543] ++ script for debian netboot --- debian-devuan-netboot.sh | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 debian-devuan-netboot.sh diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh new file mode 100755 index 0000000..9585709 --- /dev/null +++ b/debian-devuan-netboot.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Nico Schottelius, 2019-12-09 + +if [ $# -ne 1 ]; then + echo $0 suite + echo suite is for instance ascii, beowulf, etc + exit 1 +fi + +suite=$1; shift + +date=$(date +%F) + +dir=${suite}-${date} +kernel=${dir}-kernel +keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files + +debootstrap "${suite}" "${dir}" +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 + +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet6 auto +EOF + +mv ${dir}/boot/vmlinuz-* ${kernel} +rm ${dir}/boot/initrd* +mkdir -p ${dir}/root/.ssh + +for key in balazs dominique jinguk nico; do + curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys +done + +# ensure there is /init in the initramfs -> otherwise there is a kernel panic +# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init +# in our case, they are just the same +ln -s /sbin/init ${dir}/init + +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 96f0f94cfd4b0bd13058be8febba73c2b9443095 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 11 Dec 2019 22:56:54 +0100 Subject: [PATCH 021/543] not working state --- debian-devuan-netboot.sh | 31 ++++++++++++++++++++++++++++--- magiccommand | 2 ++ 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100755 magiccommand diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 9585709..799ee11 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -1,12 +1,16 @@ #!/bin/sh # Nico Schottelius, 2019-12-09 +# the ugly code is llnu + +#this can only run in the ungleich-tools directory because of the cat magiccommand........ if [ $# -ne 1 ]; then - echo $0 suite + echo $0 suite # rootpw echo suite is for instance ascii, beowulf, etc +# echo rootpw: set root password exit 1 fi - +#pw=$2 suite=$1; shift date=$(date +%F) @@ -16,7 +20,8 @@ kernel=${dir}-kernel keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 # ; \ +# echo "root:${pw}" | chgpasswd cat > ${dir}/etc/network/interfaces << EOF auto lo @@ -26,6 +31,23 @@ auto eth0 iface eth0 inet6 auto EOF + +#set hostname +echo "unconfigured-host" > ${dir}/etc/hostname + + +#add script to display eth0's ip +# the pretty but ugly code and not working +#echo 'echo $(ip -o -6 addr show | grep -E -v "lo|one" | awk '{print $1" " $2": "$4}') >> /etc/issue' > ${dir}/etc/rc.local + +#the also not working +#echo '* * * * * root ip -o -6 addr show | grep -E -v "lo|one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr + +# +#echo '* * * * * root ' > ${dir}/etc/cron.d/ipv6addr +cp ./magiccommand ${dir}/etc/cron.d/ipv6addr + + mv ${dir}/boot/vmlinuz-* ${kernel} rm ${dir}/boot/initrd* mkdir -p ${dir}/root/.ssh @@ -40,3 +62,6 @@ done ln -s /sbin/init ${dir}/init ( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) + +# tried to clear history but doesnt work :/ +# history -c diff --git a/magiccommand b/magiccommand new file mode 100755 index 0000000..e724d8e --- /dev/null +++ b/magiccommand @@ -0,0 +1,2 @@ +* * * * * root ip -o -6 addr show | grep -E -v "lo |one" | awk '{print $1" " $2": "$4}' >> /dev/tty1 + From fbcecc1bf2fe46cce7f7051fac4c60d258486b98 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 16:47:35 +0100 Subject: [PATCH 022/543] make cp instead of rm --- debian-devuan-netboot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 9585709..8937872 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -26,8 +26,8 @@ auto eth0 iface eth0 inet6 auto EOF -mv ${dir}/boot/vmlinuz-* ${kernel} -rm ${dir}/boot/initrd* +cp ${dir}/boot/vmlinuz-* ${kernel} +#rm ${dir}/boot/initrd* mkdir -p ${dir}/root/.ssh for key in balazs dominique jinguk nico; do From b549e1059dc7dfe0c88cf213c1d85e11426de3dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 16:57:25 +0100 Subject: [PATCH 023/543] prepend out-dir for building image --- debian-devuan-netboot.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 8eb9817..fdc43f4 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -4,19 +4,21 @@ #this can only run in the ungleich-tools directory because of the cat magiccommand........ -if [ $# -ne 1 ]; then - echo $0 suite # rootpw +if [ $# -ne 2 ]; then + echo $0 suite out-directory + echo out-directory: into which directory to place resulting files echo suite is for instance ascii, beowulf, etc -# echo rootpw: set root password exit 1 fi -#pw=$2 + suite=$1; shift +outdir=$1; shift date=$(date +%F) -dir=${suite}-${date} +dir=${outdir}/${suite}-${date} kernel=${dir}-kernel +initramfs=${dir}-initramfs keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" @@ -33,7 +35,7 @@ EOF cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v "lo|one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v 'lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh @@ -46,4 +48,4 @@ done # in our case, they are just the same ln -s /sbin/init ${dir}/init -( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From 3824bb6d94a6a7841b31debb92a28621a28dffdf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:02:42 +0100 Subject: [PATCH 024/543] prefix kernel + initramfs --- debian-devuan-netboot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index fdc43f4..c5d254a 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -17,8 +17,8 @@ outdir=$1; shift date=$(date +%F) dir=${outdir}/${suite}-${date} -kernel=${dir}-kernel -initramfs=${dir}-initramfs +kernel=kernel-${dir} +initramfs=initramfs-${dir} keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" From 279f86a1f0f042e8d5feeb513916aa7807dc58c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:14:13 +0100 Subject: [PATCH 025/543] read boot interface at boot --- debian-devuan-netboot.sh | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index c5d254a..6fdeecb 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -23,15 +23,11 @@ keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/ debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 +# need non-free for firmware-bnx2 +echo 'deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free' > ${dir}/etc/apt/sources.list -cat > ${dir}/etc/network/interfaces << EOF -auto lo -iface lo inet loopback +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 -auto eth0 -iface eth0 inet6 auto -EOF cp ${dir}/boot/vmlinuz-* ${kernel} @@ -43,6 +39,33 @@ for key in balazs dominique jinguk nico; do curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys done +################################################################################ +# networking + +# always lo +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +EOF + +# find the boot interfaces at boot +cat > ${dir}/etc/rc.local < /etc/network/interfaces.d/bootinterface << eof +auto \$dev +iface \$dev inet6 auto +eof + +ifup "\${dev}" + +exit 0 + + +EOF + # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same From 3f2dece3f5bf566712a084ccee766427e6d7c1c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:24:25 +0100 Subject: [PATCH 026/543] ++update --- debian-devuan-netboot.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 6fdeecb..0d366fa 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -16,22 +16,24 @@ outdir=$1; shift date=$(date +%F) -dir=${outdir}/${suite}-${date} -kernel=kernel-${dir} -initramfs=initramfs-${dir} +basename=${suite}-${date} +dir=${outdir}/${basename} +kernel=${outdir}/kernel-${basename} +initramfs=${outdir}/initramfs-${basename} keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" # need non-free for firmware-bnx2 -echo 'deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free' > ${dir}/etc/apt/sources.list +echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${dir}/etc/apt/sources.list +chroot ${dir} apt update chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v 'lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v ' lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh @@ -62,8 +64,6 @@ eof ifup "\${dev}" exit 0 - - EOF # ensure there is /init in the initramfs -> otherwise there is a kernel panic @@ -71,4 +71,5 @@ EOF # in our case, they are just the same ln -s /sbin/init ${dir}/init +# Finally building the initramfs ( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From d1ded3265ca4aeb7ca38224c0113bf731d65ded2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:26:49 +0100 Subject: [PATCH 027/543] do not nest single quotes --- debian-devuan-netboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 0d366fa..a9e0228 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -33,7 +33,7 @@ chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-b cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v ' lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh From b97d667753a2a0f5a427bf53e933d281039b8692 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:42:12 +0100 Subject: [PATCH 028/543] include network config cfiles --- debian-devuan-netboot.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index a9e0228..d64de5e 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -49,6 +49,7 @@ cat > ${dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback +source-directory /etc/network/interfaces.d EOF # find the boot interfaces at boot @@ -66,6 +67,8 @@ ifup "\${dev}" exit 0 EOF +chmod a+rx ${dir}/etc/rc.local" + # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same From 09a05b6a56f55fe27a37d26aaab13c65aa82a9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 15 Dec 2019 14:42:56 +0100 Subject: [PATCH 029/543] Add e2fsprogs to fedora image (used by one-context to resize fs) --- fedora-build-opennebula-image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index f078042..3dddae1 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -125,6 +125,9 @@ curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + # Initalize base services. run_root systemd-machine-id-setup run_root systemctl enable systemd-networkd.service From 65f5bfb99699328e5bfc5d3d28399f27512aac7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 16 Dec 2019 14:51:14 +0100 Subject: [PATCH 030/543] Do not enable unused systemd-networkd on Fedora image --- fedora-build-opennebula-image.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 3dddae1..219ad6e 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -130,7 +130,6 @@ run_root dnf -y install e2fsprogs # Initalize base services. run_root systemd-machine-id-setup -run_root systemctl enable systemd-networkd.service run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service From d11620a9fb7a1550df37d221711413cd2960704b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 17 Dec 2019 10:18:40 +0100 Subject: [PATCH 031/543] Fedora image: be sure to use right kernel version when generating initramfs --- fedora-build-opennebula-image.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 219ad6e..837ba6f 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -143,7 +143,8 @@ run_root dnf -y install kernel grub2 cat > /mnt/etc/dracut.conf.d/virtio-blk.conf < Date: Tue, 17 Dec 2019 11:21:54 +0100 Subject: [PATCH 032/543] Remove now irrelevant comment from Fedora image definition --- fedora-build-opennebula-image.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 837ba6f..5197acc 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -83,8 +83,6 @@ mount "${NBD_DEVICE}p2" /mnt mkdir /mnt/boot mount "${NBD_DEVICE}p1" /mnt/boot -# XXX: dnf has a lot a weird (libX11?) dependencies, use microdnf instead? - dnf -y \ --releasever=$RELEASE \ --installroot=/mnt \ From 5e438aafe83a10444fc68f35b4074748df4905c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 17 Dec 2019 13:56:05 +0100 Subject: [PATCH 033/543] Fedora image: increase /boot from 100M to 500M --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 5197acc..45c1629 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -68,7 +68,7 @@ trap cleanup EXIT # Create partition table, format partitions. sfdisk --no-reread "$NBD_DEVICE" < Date: Mon, 16 Dec 2019 13:46:34 +0100 Subject: [PATCH 034/543] Initial CentOS image build script --- centos-build-opennebula-image.sh | 170 +++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100755 centos-build-opennebula-image.sh diff --git a/centos-build-opennebula-image.sh b/centos-build-opennebula-image.sh new file mode 100755 index 0000000..6a8fe31 --- /dev/null +++ b/centos-build-opennebula-image.sh @@ -0,0 +1,170 @@ +#!/bin/sh + +# This script generates CentOS images for OpenNebula. + +# Depends on the following packages (as of CentOS 8): +# qemu-img util-linux coreutils dnf curl e2fsprogs + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=8 +ARCH=x86_64 +IMAGE_PATH=centos-$RELEASE-$(date --iso-8601).img +IMAGE_SIZE=10G +LOOPBACK_DEVICE=/dev/loop0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + losetup -d "$LOOPBACK_DEVICE" +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/centos-release' ]; then + echo "WARNING: this script has been designed to run on a CentOS system." >&2 + echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base RAW image (no LOOPBACK support in RHEL/CentOS). +qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" +losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +{ +sfdisk --no-reread "$LOOPBACK_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. +# network-scripts.x86_64 : Legacy scripts for manipulating of network devices +run_root dnf -y install network-scripts + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install and configure NTP client. +run_root dnf install -y chrony +run_root systemctl enable chronyd.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <>/mnt/etc/fstab < Date: Thu, 26 Dec 2019 12:16:38 +0100 Subject: [PATCH 035/543] [vpn stats] added number of configured/active VPNs --- vpn-statistics.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vpn-statistics.sh b/vpn-statistics.sh index a1e7960..c721cf9 100755 --- a/vpn-statistics.sh +++ b/vpn-statistics.sh @@ -9,3 +9,10 @@ done # countries with counter ( for ip in $(wg | grep endpoint | sed -e 's/endpoint: //' -e 's/\(.*\):[0-9]*/\1/' -e 's/\[//' -e 's/\]//'); do curl -s ipinfo.io/$ip | grep -e country ; done ) | sort | uniq -c | sort -g + +# Get number of configured VPNs +configured_vpns=$(wg show | grep ^peer | wc -l) +active_vpns=$(wg show | grep endpoint | wc -l) + +echo "Configured VPNs: ${configured_vpns}" +echo "Active VPNs: ${active_vpns}" From 1d88aac14ee6a53288aa95fd6de4ad552b68a411 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Fri, 27 Dec 2019 11:06:45 +0100 Subject: [PATCH 036/543] [script]Add new file for ceph pg repair --- pg_repair | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 pg_repair diff --git a/pg_repair b/pg_repair new file mode 100644 index 0000000..428923f --- /dev/null +++ b/pg_repair @@ -0,0 +1,14 @@ +#!/bin/bash -e + +pglist_arr=( $(ceph health detail | grep pg | grep active | awk '{print $2}' )) + +echo ${pglist_arr[*]} + +for ((i=0; i<${#pglist_arr[@]}; i++)) do + if [ 1 -eq $(ceph pg repair ${pglist_arr[$i]} | grep repair | grep instructing | wc -l) ]; then + echo repair error + break + fi + echo ${pglist_arr[$i]} repair ok + sleep 10 +done \ No newline at end of file From 693b1bed0063ebd97586746d08cb4a9fb386b9dc Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Fri, 27 Dec 2019 11:08:24 +0100 Subject: [PATCH 037/543] [script]Update pg_repair for error message --- pg_repair | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_repair b/pg_repair index 428923f..caa9953 100644 --- a/pg_repair +++ b/pg_repair @@ -6,9 +6,9 @@ echo ${pglist_arr[*]} for ((i=0; i<${#pglist_arr[@]}; i++)) do if [ 1 -eq $(ceph pg repair ${pglist_arr[$i]} | grep repair | grep instructing | wc -l) ]; then - echo repair error + echo repair script error break fi - echo ${pglist_arr[$i]} repair ok + echo ${pglist_arr[$i]} repair script done sleep 10 done \ No newline at end of file From 54e4bc1d49f951a57576b86f9a9620d7b3df5a8a Mon Sep 17 00:00:00 2001 From: llnu Date: Thu, 2 Jan 2020 20:46:15 +0100 Subject: [PATCH 038/543] added alpine compatible version --- ceph-osd-create-start-alpine | 107 +++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 ceph-osd-create-start-alpine diff --git a/ceph-osd-create-start-alpine b/ceph-osd-create-start-alpine new file mode 100755 index 0000000..a19e1da --- /dev/null +++ b/ceph-osd-create-start-alpine @@ -0,0 +1,107 @@ +#!/bin/sh +# 17:19, 2018-02-09 +# Nico Schottelius + +# Based on ceph-disk -v prepare --bluestore /dev/sdc --osd-id ${ID} --osd-uuid $(uuidgen) --crush-device-class "ssd" + +# Create: +# - block -> link to partuuid +# - block_uuid -e> uuid if the block +# - ceph_fsid -> get from ceph-conf +# crush_device_class -> ssd, hdd +# fsid -> uuidgen! +# magic -> string "ceph osd volume v026" +# type -> bluestore + +fsid=$(ceph-conf --cluster=ceph --name=osd. --lookup fsid) +fs_uuid=$(uuidgen) +magic="ceph osd volume v026" + +set -x +set -e + +if [ $# -lt 2 ]; then + echo "$0 disk class [osdweight]" + echo "class = hdd or ssd" + exit 1 +fi + +export DEV=$1;shift +export CLASS=$1; shift + + +uuid_metadata=$(uuidgen) +uuid_block=$(uuidgen) + +osd_id=$(ceph osd create) + +dev_metadata="/dev/disk/by-partuuid/$uuid_metadata" +dev_block="/dev/disk/by-partuuid/$uuid_block" + +/usr/bin/sgdisk --new=0:0:+100M --change-name="0:ceph data" \ + --partition-guid="0:$uuid_metadata" \ + --typecode=0:4fbd7e29-9d25-41b8-afd0-062c0ceff05d \ + --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + +# Using gdisk --largest-new does not change the name or set guid; +# So use 2 steps instead +/usr/bin/sgdisk --largest-new=0 --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + + +lastpart=$(gdisk -l $DEV | tail -n1 | awk '{ print $1 }') +/usr/bin/sgdisk --change-name="${lastpart}:ceph block" \ + --partition-guid="${lastpart}:$uuid_block" \ + --typecode="${lastpart}:cafecafe-9b03-4f30-b4c6-b4b80ceff106" \ + --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + +#echo $1 +#echo $(blkid | grep $1"2") + +#cblock=$(blkid | grep $1"2" | cut -d'"' -f4) +#echo $cblock + +/sbin/mkfs -t xfs -f -i size=2048 -- "$dev_metadata" + +mountpath=/var/lib/ceph/osd/ceph-${osd_id} + +mkdir -p "$mountpath" +mount "$dev_metadata" "$mountpath" + +ln -s $dev_block "$mountpath/block" + +echo "$uuid_block" > "$mountpath/block_uuid" +echo "$fsid" > "$mountpath/ceph_fsid" +echo "$magic" > "$mountpath/magic" +echo "$CLASS" > "$mountpath/crush_device_class" +echo $(echo $dev_block | cut -c23-) > "$mountpath/fsid" + + +# Important, otherwise --mkfs later will try to create filestore +echo bluestore > "$mountpath/type" + +ceph auth get-or-create "osd.${osd_id}" osd \ + 'allow *' mon 'allow profile osd' > $mountpath/keyring + +echo ${osd_id} > "$mountpath/whoami" +touch "$mountpath/openrc" + +ceph-osd --cluster ceph -i "${osd_id}" --mkfs +chown -R ceph:ceph "$mountpath" + +if [ $# -eq 1 ]; then + WEIGHT=$1; shift +else + devname=$(readlink -f $dev_block) + nodev=$(echo $devname | sed 's,/dev/,,') + WEIGHT=$(lsblk -l -b | awk "/^$nodev/ { print \$4/(1024^4) }") +fi + +ceph osd crush add osd.${osd_id} ${WEIGHT} host=$(hostname) + +echo "$metadata_dev /var/lib/ceph/osd/ceph-${osd_id} xfs noatime 0 0" >> /etc/fstab + +# Starting with monit, if available +ceph-osd -i ${osd_id} From fe27d294b1ca5884586a8713c169780030d3b7bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 7 Jan 2020 18:07:34 +0100 Subject: [PATCH 039/543] Add script to detect DNS64 prefix --- detect-dns64-prefix.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 detect-dns64-prefix.py diff --git a/detect-dns64-prefix.py b/detect-dns64-prefix.py new file mode 100644 index 0000000..e5bd179 --- /dev/null +++ b/detect-dns64-prefix.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# Nico Schottelius, 2020-01-07 +# Detect the DNS64 prefix +# Based on https://tools.ietf.org/html/draft-ietf-behave-nat64-discovery-heuristic-05 + +import dns.resolver +import ipaddress + + +if __name__ == '__main__': + dns64_prefix = None + answers = dns.resolver.query('ipv4only.arpa', 'AAAA') + + for rdata in answers: + address = str(rdata) + network = ipaddress.IPv6Network("{}/96".format(address), + strict=False) + # print("{}: {}".format(rdata, network)) + print("{}".format(network)) From e56e1d587800cf31f7aa9d3f8f0023fb710fb8bf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 7 Jan 2020 18:08:53 +0100 Subject: [PATCH 040/543] + explanation --- detect-dns64-prefix.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/detect-dns64-prefix.py b/detect-dns64-prefix.py index e5bd179..1179ca4 100644 --- a/detect-dns64-prefix.py +++ b/detect-dns64-prefix.py @@ -2,6 +2,12 @@ # Nico Schottelius, 2020-01-07 # Detect the DNS64 prefix # Based on https://tools.ietf.org/html/draft-ietf-behave-nat64-discovery-heuristic-05 +# +# How it works: +# - ipv4only.arpa only has A records. +# - a DNS64 server will add AAAA records +# - we take this response (if any) and derive the IPv6 prefix from it +# import dns.resolver import ipaddress From 86ad491bbc404d3ec02fda72d798c8b3dbadf065 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 00:27:03 +0100 Subject: [PATCH 041/543] [alpine] version bump --- alpine-rebuild-initramfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 643cc3f..b56454b 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -3,8 +3,8 @@ set -e set -x -MAJOR_VERSION=3.10 -MINOR_VERSION=3 +MAJOR_VERSION=3.11 +MINOR_VERSION=2 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz SSH_KEYS=$(cat ~/.ssh/id_rsa.pub) RESOLVCONF=/etc/resolv.conf From edb8a359e8caa81333ae37145f085608189c4e59 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 16 Jan 2020 15:47:50 +0100 Subject: [PATCH 042/543] Update create-guacamole-session-ldap-DB --- create-guacamole-session-ldap-DB | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/create-guacamole-session-ldap-DB b/create-guacamole-session-ldap-DB index 25819e2..ce1e5cd 100644 --- a/create-guacamole-session-ldap-DB +++ b/create-guacamole-session-ldap-DB @@ -4,7 +4,7 @@ #option $3 is DB user #option $4 is DB name -host='localhost' +#host='localhost' user_arr=( $(cat $1 | awk '{print $1}' )) vmid_arr=( $(cat $1 | awk '{print $2}' )) @@ -24,15 +24,17 @@ for ((i=0; i<${#user_arr[@]}; i++)) do psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');" cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") - psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');" + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','${place_arr[i]}');" psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');" #connection permission psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');" + #clipboard-encoding + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','clipboard-encoding','UTF-8');" else cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") - psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';" + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${place_arr[i]}' where connection_id='$cn_id' and parameter_name='hostname';" psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';" fi From ddc9ebaeaf509525c4f38b7079efa3c9ae1a0776 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 16 Jan 2020 15:59:26 +0100 Subject: [PATCH 043/543] Add new file --- vm_list_dual_uid | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 vm_list_dual_uid diff --git a/vm_list_dual_uid b/vm_list_dual_uid new file mode 100644 index 0000000..bf21c0b --- /dev/null +++ b/vm_list_dual_uid @@ -0,0 +1,20 @@ +#!/bin/bash -e +#option $1 is ldap password +#option $2 is ou + + +uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) + +for ((i=0; i<${#uid_list[@]}; i++)) do + uid_temp=$(echo ${uid_list[i]} | sed "s/b'//g" | sed "s/'//g") + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=$2,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_vmid=() + list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) + for ((j=0; j<${#list_vmid[@]}; j++)) do + temp=$(onevm show ${list_vmid[$j]} | grep PORT) + temp1="${temp#*\"}" + port="${temp1%%\"*}" + host=$(onevm show ${list_vmid[$j]} | grep HOST | grep ungleich | awk '{print $3}') + echo $uid_temp ${list_vmid[$j]} $port $host >> ~/vm_vnc_list + done +done From 0e91be0d69b3fd47425921bb3a2667f825efbb91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:22:56 +0100 Subject: [PATCH 044/543] Add ldap-get-emails script --- ldap-get-emails | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 ldap-get-emails diff --git a/ldap-get-emails b/ldap-get-emails new file mode 100755 index 0000000..733811a --- /dev/null +++ b/ldap-get-emails @@ -0,0 +1,31 @@ +#!/bin/sh +# +# List mail addresses found under base DN $1 (defaults to dc=ungleich,dc=ch) + +set -e + +# Hardcoded parameters. +LDAP_SERVER="ldaps://ldap1.ungleich.ch" +LDAP_BIND_DN="cn=manager,dc=ungleich,dc=ch" + +if [ "$1" != "" ]; then + LDAP_SEARCH_BASE="$1" +else + LDAP_SEARCH_BASE="dc=ungleich,dc=ch" +fi + +# Read secrets from environment. +if [ "$LDAP_BIND_PASSWD" = "" ]; then + echo "You have to define LDAP_BIND_PASSWD before launching this script." >&2 + exit 1 +fi + +# Extract mail addresses from LDAP directory. +ldap_search_result="$( + ldapsearch -x -H "$LDAP_SERVER" \ + -D "$LDAP_BIND_DN" \ + -w "$LDAP_BIND_PASSWD" \ + -b "$LDAP_SEARCH_BASE" mail + )" + +echo "$ldap_search_result" | grep 'mail:' | cut -d ' ' -f 2 - From f3a219755c382d81d7dc385bce237932401c5bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:25:39 +0100 Subject: [PATCH 045/543] Add one-get-instances scripts --- one-get-instances | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 one-get-instances diff --git a/one-get-instances b/one-get-instances new file mode 100755 index 0000000..653fed6 --- /dev/null +++ b/one-get-instances @@ -0,0 +1,18 @@ +#!/bin/sh +# +# This script extract VM IDs and filter them if a pattern is provided as first +# argument. + +set -e + +# Extract instances from ONE. +instances=$(onevm list --csv | tail -n +2) + +# Filter them is a pattern has been provided. +if [ "$1" != "" ]; then + filtered_instances="$(echo "$instances" | grep -E "$1")" + instances="$filtered_instances" +fi + +# Outputs instance IDs. +echo "$instances" | cut -d ',' -f 1 - From 1f2ea6305f43c3a817f027fb218a872710175a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:40:54 +0100 Subject: [PATCH 046/543] Add one-inspect-instance-network script --- one-inspect-instance-network | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 one-inspect-instance-network diff --git a/one-inspect-instance-network b/one-inspect-instance-network new file mode 100755 index 0000000..70e5795 --- /dev/null +++ b/one-inspect-instance-network @@ -0,0 +1,18 @@ +#!/bin/sh +# +# This script is expected to run on the ONE server (i.e. +# opennebula.ungleich.ch). + +set -e + +# Fetch instance list from STDIN. +instances=$(cat -) + +# For every instance, extract relevant information: +for id in $instances; do + nics_raw="$(onevm show --xml $id | xml_grep 'NIC')" + networks="$(echo $nics_raw | xml_grep --text_only 'NETWORK' | tr '\n' ',' | sed 's/,$//')" + ip="$(echo $nics_raw | xml_grep --text_only 'IP' | tr '\n' ',' | sed 's/,$//')" + ip6="$(echo $nics_raw | xml_grep --text_only 'IP6_GLOBAL' | tr '\n' ',' | sed 's/,$//')" + echo "$id,$networks,$ip,$ip6" +done From fea52e0295056d5e5f9606c4cba027eca5ad567c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 18 Jan 2020 19:56:06 +0100 Subject: [PATCH 047/543] Add ubuntu ONE image definition --- ubuntu-build-opennebula-image.sh | 153 +++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 ubuntu-build-opennebula-image.sh diff --git a/ubuntu-build-opennebula-image.sh b/ubuntu-build-opennebula-image.sh new file mode 100644 index 0000000..6535f66 --- /dev/null +++ b/ubuntu-build-opennebula-image.sh @@ -0,0 +1,153 @@ +#!/bin/sh + +# This script generates Ubuntu images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=eoan # 19.10 +ARCH=amd64 +IMAGE_PATH=ubuntu-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Ubuntu" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat >/mnt/etc/apt/sources.list < "/mnt$ONE_CONTEXT_DEB_PATH" +run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" +run_root rm "$ONE_CONTEXT_DEB_PATH" + +# Manually install legacy network scripts used by one-context. +run_root apt-get -y install ifupdown + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and bootloader. Do not autoconfigure grub. +run_root echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections +run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-base linux-image-generic grub-pc + +# Configure grub. +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root apt-get -y install openssh-server + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Sat, 25 Jan 2020 10:17:49 +0100 Subject: [PATCH 048/543] Add hacking script for import vm info into etcd --- etcd_import_opennebula_vm.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 etcd_import_opennebula_vm.py diff --git a/etcd_import_opennebula_vm.py b/etcd_import_opennebula_vm.py new file mode 100644 index 0000000..d2c94c9 --- /dev/null +++ b/etcd_import_opennebula_vm.py @@ -0,0 +1,28 @@ +import json +import pprint +#import etcd3 + +with open("nico-vm-one.json", "r") as fd: + vmcontent = fd.read() + +#vm = json.loads(vmcontent.decode('utf-8')) +vm = json.loads(vmcontent) +pprint.pprint(vm['TEMPLATE']['DISK']) + +# storing info + +for_etcd={} +for_etcd['data_version'] = "1" +for_etcd['vm_id'] = vm['ID'] +for_etcd['owner'] = vm['UNAME'] + +for_etcd['disks'] = [] +for disk in vm['TEMPLATE']['DISK']: + disk_etcd = {} + disk_etcd['image_name'] = disk['IMAGE'] + disk_etcd['image_id'] = disk['IMAGE_ID'] + disk_etcd['datastore_name'] = disk['DATASTORE'] + disk_etcd['datastore_id'] = disk['DATASTORE_ID'] + for_etcd['disks'].append(disk_etcd) + +pprint.pprint(for_etcd) From 1ffc6f57681db8cc1eae76038be324afcceca97d Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 15:34:09 +0500 Subject: [PATCH 049/543] Added scripts related to getting info of OpenNebula VM into etcd and related queries --- .gitignore | 6 + opennebula-vm-etcd/config-and-secrets.conf | 18 +++ opennebula-vm-etcd/config.py | 12 ++ opennebula-vm-etcd/etcd_wrapper.py | 75 ++++++++++++ opennebula-vm-etcd/put-vm-info-into-etcd.py | 126 ++++++++++++++++++++ opennebula-vm-etcd/vm-queries.py | 55 +++++++++ 6 files changed, 292 insertions(+) create mode 100644 .gitignore create mode 100644 opennebula-vm-etcd/config-and-secrets.conf create mode 100644 opennebula-vm-etcd/config.py create mode 100644 opennebula-vm-etcd/etcd_wrapper.py create mode 100644 opennebula-vm-etcd/put-vm-info-into-etcd.py create mode 100644 opennebula-vm-etcd/vm-queries.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c093faa --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +opennebula-snapshot/config-and-secrets.conf + +*.pyc + +.idea/ +.vscode/ \ No newline at end of file diff --git a/opennebula-vm-etcd/config-and-secrets.conf b/opennebula-vm-etcd/config-and-secrets.conf new file mode 100644 index 0000000..b7faa08 --- /dev/null +++ b/opennebula-vm-etcd/config-and-secrets.conf @@ -0,0 +1,18 @@ +# Do not put single/double quotation mark for string as they are +# aslo considered as normal character. + +[oca] +client_secrets = ahmedbilal96@gmail.com:d00359fa33a74fcb5ea40bb088e299fd2ab85126 + +[etcd] +# url = localhost +# port = 2379 +# ca_cert +# cert_cert +# cert_key + +url = etcd1.ungleich.ch +port = 2379 +ca_cert = /home/meow/.cdist/files/etcd/ca.pem +cert_cert = /home/meow/.cdist/files/etcd/developer.pem +cert_key = /home/meow/.cdist/files/etcd/developer-key.pem \ No newline at end of file diff --git a/opennebula-vm-etcd/config.py b/opennebula-vm-etcd/config.py new file mode 100644 index 0000000..b329f4f --- /dev/null +++ b/opennebula-vm-etcd/config.py @@ -0,0 +1,12 @@ +import configparser + +from etcd_wrapper import EtcdWrapper + +config = configparser.ConfigParser(allow_no_value=True) +config.read('config-and-secrets.conf') + +etcd_client = EtcdWrapper( + host=config['etcd']['url'], port=config['etcd']['port'], + ca_cert=config['etcd']['ca_cert'], cert_key=config['etcd']['cert_key'], + cert_cert=config['etcd']['cert_cert'] +) diff --git a/opennebula-vm-etcd/etcd_wrapper.py b/opennebula-vm-etcd/etcd_wrapper.py new file mode 100644 index 0000000..9624677 --- /dev/null +++ b/opennebula-vm-etcd/etcd_wrapper.py @@ -0,0 +1,75 @@ +import etcd3 +import json + +from functools import wraps + +from uncloud import UncloudException +from uncloud.common import logger + + +class EtcdEntry: + def __init__(self, meta_or_key, value, value_in_json=True): + if hasattr(meta_or_key, 'key'): + # if meta has attr 'key' then get it + self.key = meta_or_key.key.decode('utf-8') + else: + # otherwise meta is the 'key' + self.key = meta_or_key + self.value = value.decode('utf-8') + + if value_in_json: + self.value = json.loads(self.value) + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError: + raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') + except etcd3.exceptions.ConnectionTimeoutError as err: + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err + except Exception as err: + logger.exception('Some etcd error occured. See syslog for details.', err) + + return wrapper + + +class EtcdWrapper: + @readable_errors + def __init__(self, *args, **kwargs): + self.client = etcd3.client(*args, **kwargs) + + @readable_errors + def get(self, *args, value_in_json=True, **kwargs): + _value, _key = self.client.get(*args, **kwargs) + if _key is None or _value is None: + return None + return EtcdEntry(_key, _value, value_in_json=value_in_json) + + @readable_errors + def put(self, *args, value_in_json=True, **kwargs): + _key, _value = args + if value_in_json: + _value = json.dumps(_value) + + if not isinstance(_key, str): + _key = _key.decode('utf-8') + + return self.client.put(_key, _value, **kwargs) + + @readable_errors + def get_prefix(self, *args, value_in_json=True, **kwargs): + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) + + @readable_errors + def watch_prefix(self, key, value_in_json=True): + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = getattr('e', '_event') + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py new file mode 100644 index 0000000..70a1b09 --- /dev/null +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -0,0 +1,126 @@ +import pyone + +from enum import IntEnum +from config import config, etcd_client + +# How to get client secrets? +# 1. Login to OpenNebula +# 2. Go to Settings then Auth +# 3. Click on "Manage login tokens" button +# 4. Click on "Get a new token" button + +one_client = pyone.OneServer( + uri='https://opennebula.ungleich.ch:2634/RPC2', + session=config['oca']['client_secrets'] +) + + +def get_hostname_of_vm(vm_id): + host_pool = { + host.NAME: { + 'name': host.NAME, + 'id': host.ID, + 'cluster': { + 'name': host.CLUSTER, + 'id': host.CLUSTER_ID + }, + 'vms': host.VMS.ID + } + for host in one_client.hostpool.info().HOST + } + for hostname, host in host_pool.items(): + if vm_id in host['vms']: + return host + + return None + + +def put_under_list(obj): + if not isinstance(obj, list): + return [obj] + return obj + + +class Snapshot: + def __init__(self, disk_id, snapshot): + self.active = bool(snapshot.ACTIVE) + self.date = snapshot.DATE + self.id = snapshot.ID + self.name = snapshot.NAME + self.size = snapshot.SIZE + self.disk_id = disk_id + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + +class VM: + def __init__(self, vm): + self.name = vm.get_NAME() + self.id = vm.get_ID() + self.owner = { + 'name': vm.get_UNAME(), + 'id': vm.get_UID(), + } + + template = vm.get_TEMPLATE() + host = get_hostname_of_vm(self.id) + + self.vcpu = template.get('VCPU', None) + self.memory = template.get('MEMORY', None) + self.disks = [dict(disk) for disk in put_under_list(template.get('DISK', []))] + self.graphics = [dict(graphics) for graphics in put_under_list(template.get('GRAPHICS', []))] + self.nics = [dict(nic) for nic in put_under_list(template.get('NIC', []))] + self.status = pyone.VM_STATE(vm.get_STATE()).name.lower() + self.snapshots = [] + + for disk in one_client.vm.info(self.id).SNAPSHOTS: + disk_id = disk.DISK_ID + for snapshot in disk.SNAPSHOT: + self.snapshots.append(Snapshot(disk_id, snapshot).get_data()) + + if host: + self.host = { + 'name': host['name'], + 'id': host['id'] + } + else: + self.host = host + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + def __repr__(self): + return str(self.get_data()) + + +class VmFilterFlag(IntEnum): + UIDUserResources = 0 # UID User’s Resources + UserAndItsGroupsResources = -1 # Resources belonging to the user and any of his groups + AllResources = -2 # All resources + UserResources = -3 # Resources belonging to the user + UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group + + +def main(): + VM_STATES = list(pyone.VM_STATE) + START_ID = -1 # First id whatever it is + END_ID = -1 # Last id whatever it is + + for VM_STATE in VM_STATES: + vm_pool = one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, VM_STATE) + for i, vm in enumerate(vm_pool.VM): + vm = VM(vm) + etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + + +if __name__ == "__main__": + main() diff --git a/opennebula-vm-etcd/vm-queries.py b/opennebula-vm-etcd/vm-queries.py new file mode 100644 index 0000000..e92ef14 --- /dev/null +++ b/opennebula-vm-etcd/vm-queries.py @@ -0,0 +1,55 @@ +from pprint import pprint + +from config import config, etcd_client + + +def get_vm_by_ip(vms, ip, status='active'): + vms_by_status = { + vm_id: vm + for vm_id, vm in vms.items() + if vm['status'] == status + } + for vm_id, vm in vms_by_status.items(): + vm_ips = [] + for nic in vm.get('nics', []): + global_ipv6 = nic.get('IP6_GLOBAL', None) + local_ipv6 = nic.get('IP6_LINK', None) + ipv4 = nic.get('IP', None) + vm_ips += [global_ipv6, local_ipv6, ipv4] + + if ip in vm_ips: + return {vm_id: vm} + return None + + +def main(): + vm_prefix = '/opennebula/vm/' + + vms = { + int(vm.key.split('/')[-1]): vm.value + for vm in etcd_client.get_prefix(vm_prefix) + } + + VM_ID = 10761 # One of nico's VM + + # Get all data related to a VM + pprint(vms.get(VM_ID)) + + # Get host of a VM + print(vms.get(VM_ID).get('host').get('name')) + + # Get VNC Port of a VM + print(vms.get(VM_ID).get('graphics')[0].get('PORT')) + + # Get all disks attached with VM + pprint(vms.get(VM_ID).get('disks')) + + # Who is owner of a VM? + print(vms.get(VM_ID).get('owner').get('name')) + + # Get VM who has 2a0a:e5c0:0:5:0:78ff:fe11:d75f + search_ungleich_ch = get_vm_by_ip(vms, '2a0a:e5c0:0:5:0:78ff:fe11:d75f') + pprint(search_ungleich_ch) + +if __name__ == '__main__': + main() From 89244dc4ee89414db01621965e29a070512359f5 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 15:51:31 +0500 Subject: [PATCH 050/543] remove uncloud specific things --- opennebula-vm-etcd/etcd_wrapper.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/opennebula-vm-etcd/etcd_wrapper.py b/opennebula-vm-etcd/etcd_wrapper.py index 9624677..f448dcf 100644 --- a/opennebula-vm-etcd/etcd_wrapper.py +++ b/opennebula-vm-etcd/etcd_wrapper.py @@ -1,11 +1,9 @@ import etcd3 import json +import logging from functools import wraps -from uncloud import UncloudException -from uncloud.common import logger - class EtcdEntry: def __init__(self, meta_or_key, value, value_in_json=True): @@ -27,11 +25,11 @@ def readable_errors(func): try: return func(*args, **kwargs) except etcd3.exceptions.ConnectionFailedError: - raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') + raise etcd3.exceptions.ConnectionFailedError('Cannot connect to etcd: is etcd running as configured?') except etcd3.exceptions.ConnectionTimeoutError as err: raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err except Exception as err: - logger.exception('Some etcd error occured. See syslog for details.', err) + logging.exception('Some etcd error occured. See syslog for details.', err) return wrapper From e34abc449d5216715cb81dbe88f3eafc8c44522a Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 18:32:03 +0500 Subject: [PATCH 051/543] Removed config-and-secrets.conf --- .gitignore | 4 ++-- opennebula-vm-etcd/config-and-secrets.conf | 18 ------------------ 2 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 opennebula-vm-etcd/config-and-secrets.conf diff --git a/.gitignore b/.gitignore index c093faa..359653f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -opennebula-snapshot/config-and-secrets.conf +opennebula-vm-etcd/config-and-secrets.conf *.pyc .idea/ -.vscode/ \ No newline at end of file +.vscode/ diff --git a/opennebula-vm-etcd/config-and-secrets.conf b/opennebula-vm-etcd/config-and-secrets.conf deleted file mode 100644 index b7faa08..0000000 --- a/opennebula-vm-etcd/config-and-secrets.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Do not put single/double quotation mark for string as they are -# aslo considered as normal character. - -[oca] -client_secrets = ahmedbilal96@gmail.com:d00359fa33a74fcb5ea40bb088e299fd2ab85126 - -[etcd] -# url = localhost -# port = 2379 -# ca_cert -# cert_cert -# cert_key - -url = etcd1.ungleich.ch -port = 2379 -ca_cert = /home/meow/.cdist/files/etcd/ca.pem -cert_cert = /home/meow/.cdist/files/etcd/developer.pem -cert_key = /home/meow/.cdist/files/etcd/developer-key.pem \ No newline at end of file From 55a686800640af68f541fede1b3815e28289b586 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 23:28:00 +0500 Subject: [PATCH 052/543] Some optimizations to improve throughput --- opennebula-vm-etcd/put-vm-info-into-etcd.py | 47 +++++++++++++-------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py index 70a1b09..5b5495f 100644 --- a/opennebula-vm-etcd/put-vm-info-into-etcd.py +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -2,6 +2,7 @@ import pyone from enum import IntEnum from config import config, etcd_client +from functools import reduce # How to get client secrets? # 1. Login to OpenNebula @@ -14,24 +15,24 @@ one_client = pyone.OneServer( session=config['oca']['client_secrets'] ) +host_pool = { + host.NAME: { + 'name': host.NAME, + 'id': host.ID, + 'cluster': { + 'name': host.CLUSTER, + 'id': host.CLUSTER_ID + }, + 'vms': host.VMS.ID + } + for host in one_client.hostpool.info().HOST +} + def get_hostname_of_vm(vm_id): - host_pool = { - host.NAME: { - 'name': host.NAME, - 'id': host.ID, - 'cluster': { - 'name': host.CLUSTER, - 'id': host.CLUSTER_ID - }, - 'vms': host.VMS.ID - } - for host in one_client.hostpool.info().HOST - } for hostname, host in host_pool.items(): if vm_id in host['vms']: return host - return None @@ -115,11 +116,21 @@ def main(): START_ID = -1 # First id whatever it is END_ID = -1 # Last id whatever it is - for VM_STATE in VM_STATES: - vm_pool = one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, VM_STATE) - for i, vm in enumerate(vm_pool.VM): - vm = VM(vm) - etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + # Get VMs in all kind of states + + # vms is a list of lists + vms = [ + one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, vm_state).VM + for vm_state in VM_STATES + ] + # Take out elements from nested lists and put them into the original list + # forming a nice flat list + vms = list(reduce(lambda n, n_1: n + n_1, vms)) + print('Total VMs:', len(vms)) + for i, _vm in enumerate(vms): + vm = VM(_vm) + etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + print(i, end=' ') if __name__ == "__main__": From 78470501dd7c79d6a8094197cf6d9aa03e621591 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 31 Jan 2020 23:05:25 +0500 Subject: [PATCH 053/543] Directly use builtin XMLRPC client as opposed to pyone library --- .gitignore | 4 +- opennebula-vm-etcd/put-vm-info-into-etcd.py | 177 ++++++++------------ 2 files changed, 71 insertions(+), 110 deletions(-) diff --git a/.gitignore b/.gitignore index 359653f..f8835d9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ opennebula-vm-etcd/config-and-secrets.conf *.pyc -.idea/ -.vscode/ +.idea +.vscode diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py index 5b5495f..0ba2275 100644 --- a/opennebula-vm-etcd/put-vm-info-into-etcd.py +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -1,39 +1,17 @@ -import pyone +import json from enum import IntEnum +from xmlrpc.client import ServerProxy as RPCClient + +from xmltodict import parse + from config import config, etcd_client -from functools import reduce - -# How to get client secrets? -# 1. Login to OpenNebula -# 2. Go to Settings then Auth -# 3. Click on "Manage login tokens" button -# 4. Click on "Get a new token" button - -one_client = pyone.OneServer( - uri='https://opennebula.ungleich.ch:2634/RPC2', - session=config['oca']['client_secrets'] -) - -host_pool = { - host.NAME: { - 'name': host.NAME, - 'id': host.ID, - 'cluster': { - 'name': host.CLUSTER, - 'id': host.CLUSTER_ID - }, - 'vms': host.VMS.ID - } - for host in one_client.hostpool.info().HOST -} -def get_hostname_of_vm(vm_id): - for hostname, host in host_pool.items(): - if vm_id in host['vms']: - return host - return None +# Constants +ALL_VM_STATES = -1 +START_ID = -1 # First id whatever it is +END_ID = -1 # Last id whatever it is def put_under_list(obj): @@ -42,65 +20,19 @@ def put_under_list(obj): return obj -class Snapshot: - def __init__(self, disk_id, snapshot): - self.active = bool(snapshot.ACTIVE) - self.date = snapshot.DATE - self.id = snapshot.ID - self.name = snapshot.NAME - self.size = snapshot.SIZE - self.disk_id = disk_id - - def get_data(self): - return { - attr: getattr(self, attr) - for attr in dir(self) - if not attr.startswith('__') and not callable(getattr(self, attr)) - } - - -class VM: - def __init__(self, vm): - self.name = vm.get_NAME() - self.id = vm.get_ID() - self.owner = { - 'name': vm.get_UNAME(), - 'id': vm.get_UID(), - } - - template = vm.get_TEMPLATE() - host = get_hostname_of_vm(self.id) - - self.vcpu = template.get('VCPU', None) - self.memory = template.get('MEMORY', None) - self.disks = [dict(disk) for disk in put_under_list(template.get('DISK', []))] - self.graphics = [dict(graphics) for graphics in put_under_list(template.get('GRAPHICS', []))] - self.nics = [dict(nic) for nic in put_under_list(template.get('NIC', []))] - self.status = pyone.VM_STATE(vm.get_STATE()).name.lower() - self.snapshots = [] - - for disk in one_client.vm.info(self.id).SNAPSHOTS: - disk_id = disk.DISK_ID - for snapshot in disk.SNAPSHOT: - self.snapshots.append(Snapshot(disk_id, snapshot).get_data()) - - if host: - self.host = { - 'name': host['name'], - 'id': host['id'] - } - else: - self.host = host - - def get_data(self): - return { - attr: getattr(self, attr) - for attr in dir(self) - if not attr.startswith('__') and not callable(getattr(self, attr)) - } - - def __repr__(self): - return str(self.get_data()) +class VMState(IntEnum): + INIT = 0 + PENDING = 1 + HOLD = 2 + ACTIVE = 3 + STOPPED = 4 + SUSPENDED = 5 + DONE = 6 + FAILED = 7 + POWEROFF = 8 + UNDEPLOYED = 9 + CLONING = 10 + CLONING_FAILURE = 11 class VmFilterFlag(IntEnum): @@ -111,26 +43,55 @@ class VmFilterFlag(IntEnum): UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group +class VM: + def __init__(self, vm: dict): + self.id = vm.get('ID', None) + self.owner = { + 'id': vm.get('UID', None), + 'name': vm.get('UNAME', None), + 'gname': vm.get('GNAME', None) + } + self.name = vm.get('NAME', None) + self.status = vm.get('STATE', None) + if self.status: + self.status = VMState(int(self.status)).name.lower() + + template = vm['TEMPLATE'] + + self.disk = put_under_list(template.get('DISK', [])) + self.graphics = template.get('GRAPHICS', {}) + self.memory = template.get('MEMORY', None) + self.nic = put_under_list(template.get('NIC', [])) + self.vcpu = template.get('VCPU', None) + self.host = { + 'name': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None), + 'id': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HID', None), + } + self.snapshots = put_under_list(vm.get('SNAPSHOTS', [])) + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + def main(): - VM_STATES = list(pyone.VM_STATE) - START_ID = -1 # First id whatever it is - END_ID = -1 # Last id whatever it is + with RPCClient('https://opennebula.ungleich.ch:2634/RPC2') as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + config['oca']['client_secrets'], VmFilterFlag.AllResources.value, START_ID, END_ID, ALL_VM_STATES + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + for i, vm in enumerate(vms): + vm_id = vm['ID'] + etcd_client.put(f'/opennebula/vm/{vm_id}', vm) - # Get VMs in all kind of states - - # vms is a list of lists - vms = [ - one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, vm_state).VM - for vm_state in VM_STATES - ] - # Take out elements from nested lists and put them into the original list - # forming a nice flat list - vms = list(reduce(lambda n, n_1: n + n_1, vms)) - print('Total VMs:', len(vms)) - for i, _vm in enumerate(vms): - vm = VM(_vm) - etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) - print(i, end=' ') + parsed_vm = VM(vm) + etcd_client.put(f'/opennebula/parsed_vm/{parsed_vm.id}', parsed_vm.get_data()) + else: + print(response) if __name__ == "__main__": From 46fc35d3c3ad3e9028920e5257ee80b6d98af35c Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 31 Jan 2020 23:10:16 +0500 Subject: [PATCH 054/543] Fix vm-queries.py --- opennebula-vm-etcd/vm-queries.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/opennebula-vm-etcd/vm-queries.py b/opennebula-vm-etcd/vm-queries.py index e92ef14..e1da013 100644 --- a/opennebula-vm-etcd/vm-queries.py +++ b/opennebula-vm-etcd/vm-queries.py @@ -1,6 +1,6 @@ from pprint import pprint -from config import config, etcd_client +from config import etcd_client def get_vm_by_ip(vms, ip, status='active'): @@ -11,7 +11,7 @@ def get_vm_by_ip(vms, ip, status='active'): } for vm_id, vm in vms_by_status.items(): vm_ips = [] - for nic in vm.get('nics', []): + for nic in vm.get('nic', []): global_ipv6 = nic.get('IP6_GLOBAL', None) local_ipv6 = nic.get('IP6_LINK', None) ipv4 = nic.get('IP', None) @@ -23,7 +23,7 @@ def get_vm_by_ip(vms, ip, status='active'): def main(): - vm_prefix = '/opennebula/vm/' + vm_prefix = '/opennebula/parsed_vm/' vms = { int(vm.key.split('/')[-1]): vm.value @@ -39,10 +39,10 @@ def main(): print(vms.get(VM_ID).get('host').get('name')) # Get VNC Port of a VM - print(vms.get(VM_ID).get('graphics')[0].get('PORT')) + print(vms.get(VM_ID).get('graphics').get('PORT')) # Get all disks attached with VM - pprint(vms.get(VM_ID).get('disks')) + pprint(vms.get(VM_ID).get('disk')) # Who is owner of a VM? print(vms.get(VM_ID).get('owner').get('name')) @@ -51,5 +51,6 @@ def main(): search_ungleich_ch = get_vm_by_ip(vms, '2a0a:e5c0:0:5:0:78ff:fe11:d75f') pprint(search_ungleich_ch) + if __name__ == '__main__': main() From fb68ce7c1c07b422ab7362a93c936cdd50a8c2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 14 Feb 2020 15:23:54 +0100 Subject: [PATCH 055/543] Add OpenNebula image definition for Alpine Linux --- alpine-build-opennebula-image.sh | 179 +++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100755 alpine-build-opennebula-image.sh diff --git a/alpine-build-opennebula-image.sh b/alpine-build-opennebula-image.sh new file mode 100755 index 0000000..0a074b4 --- /dev/null +++ b/alpine-build-opennebula-image.sh @@ -0,0 +1,179 @@ +#!/bin/sh + +# This script generates Alpine images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=v3.11 +ARCH=x86_64 +IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 +APK_MIRROR=http://dl-2.alpinelinux.org/alpine/ # Mind the trailing / + +ONE_CONTEXT_APK_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-r1.apk" +ONE_CONTEXT_APK_PATH=/root/one-context.apk + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ "$(lsb_release --short --id)" != "Alpine" ]; then + echo "WARNING: this script has been designed to run on an Alpine system." >&2 + echo "WARNING: Not running Alpine. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +run_root setup-timezone -z UTC +if [ "$RELEASE" = "edge" ] +then + cat >/mnt/etc/apk/repositories </mnt/etc/apk/repositories <>/mnt/etc/fstab </mnt/boot/extlinux.conf < "/mnt$ONE_CONTEXT_APK_PATH" +run_root apk add --allow-untrusted "$ONE_CONTEXT_APK_PATH" +run_root rm "$ONE_CONTEXT_APK_PATH" + +# Remove resolvconf: handled by uncloud-init. +run_root rm /etc/resolv.conf + +# Make sure everything is written to disk before exiting. +sync From b6cf60b83b8aae1dacbca0d9feb33d99d2283bfa Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Tue, 3 Mar 2020 08:43:52 +0100 Subject: [PATCH 056/543] Add new directory for vnc_console --- vnc_console_connection/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 vnc_console_connection/.gitkeep diff --git a/vnc_console_connection/.gitkeep b/vnc_console_connection/.gitkeep new file mode 100644 index 0000000..e69de29 From 9774225f958eee50bb6edcd6d8cb1280a125b344 Mon Sep 17 00:00:00 2001 From: kjg Date: Tue, 3 Mar 2020 08:55:30 +0100 Subject: [PATCH 057/543] [file]update vnc_console --- vnc_console_connection/config.py | 5 ++ vnc_console_connection/db_export.py | 55 ++++++++++++++++++ vnc_console_connection/get_info.py | 88 +++++++++++++++++++++++++++++ vnc_console_connection/ldap_list.py | 30 ++++++++++ 4 files changed, 178 insertions(+) create mode 100755 vnc_console_connection/config.py create mode 100755 vnc_console_connection/db_export.py create mode 100755 vnc_console_connection/get_info.py create mode 100755 vnc_console_connection/ldap_list.py diff --git a/vnc_console_connection/config.py b/vnc_console_connection/config.py new file mode 100755 index 0000000..2a36c96 --- /dev/null +++ b/vnc_console_connection/config.py @@ -0,0 +1,5 @@ +import configparser + +config = configparser.ConfigParser(allow_no_value=True) +config.read('config-and-secrets.conf') + diff --git a/vnc_console_connection/db_export.py b/vnc_console_connection/db_export.py new file mode 100755 index 0000000..d283eb4 --- /dev/null +++ b/vnc_console_connection/db_export.py @@ -0,0 +1,55 @@ +import psycopg2 as pg2 +from config import config + +db_name = config['db']['db_name'] +db_user = config['db']['db_user'] +db_password = config['db']['db_password'] +db_port = config['db']['db_port'] + + +def setconn(u_id, vm_num, vm_port,vm_host): + conn = pg2.connect("host = localhost dbname={} user={} password={} port={}".format(db_name,db_user,db_password,db_port)) + conn.autocommit = True + cur = conn.cursor() + cur.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) + row = cur.fetchone() + if row == None: + cur.execute("INSERT INTO guacamole_entity (name, type) VALUES ('{}','USER')".format(u_id)) + cur.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) + row = cur.fetchone() + en_id = row[0] + cur.execute("INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('{}', '\x74657374', now())".format(en_id)) + print("create user : " , u_id) + else: + en_id = row[0] + cur.execute("SELECT password_hash FROM guacamole_user WHERE entity_id = '{}'".format(en_id)) + row = cur.fetchone() + if row == None: + cur.execute("INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('{}', '\x74657374', now())".format(en_id)) + print("user exsit") + cn = "{}{}".format(u_id,vm_num) + cur.execute("SELECT connection_id FROM guacamole_connection WHERE connection_name = '{}'".format(cn)) + row = cur.fetchone() + if row == None: + #create connection + cur.execute("INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('{}', 'vnc')".format(cn)) + cur.execute("SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '{}' AND parent_id IS NULL".format(cn)) + temp_cn_id = cur.fetchone() + cn_id = temp_cn_id[0] + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','hostname','{}')".format(cn_id, vm_host)) + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','port','{}')".format(cn_id,vm_port)) + #connection permission + cur.execute("INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('{}', '{}', 'READ')".format(en_id,cn_id)) + #clipboard-encoding + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','clipboard-encoding','UTF-8')".format(cn_id)) + print("create connection") + else: + cur.execute("SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '{}' AND parent_id IS NULL".format(cn)) + temp_cn_id = cur.fetchone() + cn_id = temp_cn_id[0] + cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='{}' where connection_id='{}' and parameter_name='hostname'".format(vm_host,cn_id)) + cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='{}' where connection_id='{}' and parameter_name='port'".format(vm_port,cn_id)) + #cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='UTF-8' where connection_id='{}' and parameter_name='clipboard-encoding'".format(cn_id)) + print("no connection") + conn.close() + return None \ No newline at end of file diff --git a/vnc_console_connection/get_info.py b/vnc_console_connection/get_info.py new file mode 100755 index 0000000..ac09633 --- /dev/null +++ b/vnc_console_connection/get_info.py @@ -0,0 +1,88 @@ +import json + +from enum import IntEnum +from xmlrpc.client import ServerProxy as RPCClient +from xmltodict import parse +from config import config +from ldap_list import vm_list +from db_export import setconn + +# Constants +ALL_VM_STATES = -1 +START_ID = -1 # First id whatever it is +END_ID = -1 # Last id whatever it is +session_string = config['oca']['client_secrets'] +opnserver = config['oca']['opn_server'] + +class VMState(IntEnum): + INIT = 0 + PENDING = 1 + HOLD = 2 + ACTIVE = 3 + STOPPED = 4 + SUSPENDED = 5 + DONE = 6 + FAILED = 7 + POWEROFF = 8 + UNDEPLOYED = 9 + CLONING = 10 + CLONING_FAILURE = 11 + + +class VmFilterFlag(IntEnum): + UIDUserResources = 0 # UID User’s Resources + UserAndItsGroupsResources = -1 # Resources belonging to the user and any of his groups + AllResources = -2 # All resources + UserResources = -3 # Resources belonging to the user + UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group + + +class VM: + def __init__(self, vm: dict): + self.id = vm.get('ID', None) + self.owner = { + 'id': vm.get('UID', None), + 'name': vm.get('UNAME', None), + 'gname': vm.get('GNAME', None) + } + self.name = vm.get('NAME', None) + self.status = vm.get('STATE', None) + if self.status: + self.status = VMState(int(self.status)).name.lower() + + template = vm['TEMPLATE'] + + self.graphics = template.get('GRAPHICS', {}) + self.memory = template.get('MEMORY', None) + self.vcpu = template.get('VCPU', None) + self.host = { + 'name': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None), + 'id': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HID', None), + } + + +def main(): + with RPCClient(opnserver) as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + session_string , VmFilterFlag.AllResources.value, START_ID, END_ID, VMState.ACTIVE.value + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + for entry in vm_list.entries: + temp_uname = entry.mail + for i, vm in enumerate(vms): + vm_user = vm['UNAME'] + vm_id = vm['ID'] + vm_port = vm['TEMPLATE']['GRAPHICS'].get('PORT') + vm_host = vm['HISTORY_RECORDS']['HISTORY']['HOSTNAME'] + if vm['UNAME'] == temp_uname: + #print(entry.uid, vm_id, vm_port, vm_host) + setconn(entry.uid, vm_id, vm_port, vm_host) + + else: + print(response) + + +if __name__ == "__main__": + main() + diff --git a/vnc_console_connection/ldap_list.py b/vnc_console_connection/ldap_list.py new file mode 100755 index 0000000..a9e322f --- /dev/null +++ b/vnc_console_connection/ldap_list.py @@ -0,0 +1,30 @@ +import ldap3 +import sys +from config import config +from ldap3 import Server, Connection, ObjectDef, Reader, ALL, SUBTREE, ALL_ATTRIBUTES +from ldap3.core import exceptions + + +LDAP_SERVER = config['ldap']['server'] +LDAP_PASSWORD = config['ldap']['admin_password'] +LDAP_USER = config['ldap']['admin_dn'] +LDAP_PORT = int(config['ldap']['ldap_port']) + +# Create the Server object with the given address. +server = Server(LDAP_SERVER, LDAP_PORT, get_info=ALL) +#Create a connection object, and bind with the given DN and password. +try: + conn = Connection(server, LDAP_USER, LDAP_PASSWORD, auto_bind=True) + print('LDAP Bind Successful.') + # Perform a search for a pre-defined criteria. + # Mention the search filter / filter type and attributes. + conn.search('ou=customer,dc=ungleich,dc=ch', '(&(!({}={})))'.format('mail','*@ungleich.ch') , attributes=['uid','mail']) + #conn.search('ou=customer,dc=ungleich,dc=ch', '(objectClass=*)' , attributes=['uid','mail']) + # Print the resulting entriesn. + #for entry in conn.entries: + #print(entry.uid, entry.mail) + vm_list = conn +except exceptions.LDAPException as err: + sys.exit(f'LDAP Error: {err}') + + From 873eca64673e075b37caacc1257d9d1d922a4ef2 Mon Sep 17 00:00:00 2001 From: "jinguk.kwon" Date: Fri, 20 Mar 2020 16:13:06 +0900 Subject: [PATCH 058/543] new file for vm map --- vm_map.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 vm_map.sh diff --git a/vm_map.sh b/vm_map.sh new file mode 100755 index 0000000..15c80dc --- /dev/null +++ b/vm_map.sh @@ -0,0 +1,5 @@ +vm_list=( $(virsh list | awk '{print $2}') ) + +for ((i=0; i<${#vm_list[@]}; i++)) do + ceph osd map hdd ${vm_list[i]} +done From ad8b7b2c6d5c30d1739d838320dcaf9b8589c2d7 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 8 Apr 2020 07:18:56 +0200 Subject: [PATCH 059/543] [tools]Update get_info.py for uid --- vnc_console_connection/get_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnc_console_connection/get_info.py b/vnc_console_connection/get_info.py index ac09633..e98ae72 100755 --- a/vnc_console_connection/get_info.py +++ b/vnc_console_connection/get_info.py @@ -69,7 +69,7 @@ def main(): if success: vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] for entry in vm_list.entries: - temp_uname = entry.mail + temp_uname = entry.uid for i, vm in enumerate(vms): vm_user = vm['UNAME'] vm_id = vm['ID'] From 4fc8995e074495f9fe9e495083fe0652749b8b97 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 15 Apr 2020 12:14:18 +0200 Subject: [PATCH 060/543] [ungleich-tool]Update config.py for path --- vnc_console_connection/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnc_console_connection/config.py b/vnc_console_connection/config.py index 2a36c96..7b7acc7 100755 --- a/vnc_console_connection/config.py +++ b/vnc_console_connection/config.py @@ -1,5 +1,5 @@ import configparser config = configparser.ConfigParser(allow_no_value=True) -config.read('config-and-secrets.conf') +config.read('/opt/ungleich-tools/vnc_console_connection/config-and-secrets.conf') From 6b51feb1fdfe2f0735c716e35bb3570dff37d669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 7 May 2020 08:25:00 +0200 Subject: [PATCH 061/543] Bump Fedora image (F31 -> F32) --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 45c1629..a84a960 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -18,7 +18,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=31 +RELEASE=32 ARCH=x86_64 IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G From 8970127c246a052d6aa24825f8db453719cbbca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 7 May 2020 08:30:08 +0200 Subject: [PATCH 062/543] Add haveged to Fedora image --- fedora-build-opennebula-image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index a84a960..b4bc5c6 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -132,6 +132,10 @@ run_root systemd-machine-id-setup run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service +# Install haveged due to lack of entropy in ONE environment. +run_root dnf -y install haveged +run_root systemctl enable haveged.service + # Install kernel and bootloader. # Note: linux-firmware is not required our environment and takes almost 200M # uncompressed but is a direct dependency of kernel-core... From 6ea31faa476f9b516b4bdc51c214e79b1b024f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:18:46 +0200 Subject: [PATCH 063/543] Move ONE image defitions to their own folder --- .../alpine-build-opennebula-image.sh | 0 .../fedora-build-opennebula-image.sh | 0 .../ubuntu-build-opennebula-image.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename alpine-build-opennebula-image.sh => opennebula-images/alpine-build-opennebula-image.sh (100%) rename fedora-build-opennebula-image.sh => opennebula-images/fedora-build-opennebula-image.sh (100%) rename ubuntu-build-opennebula-image.sh => opennebula-images/ubuntu-build-opennebula-image.sh (100%) diff --git a/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh similarity index 100% rename from alpine-build-opennebula-image.sh rename to opennebula-images/alpine-build-opennebula-image.sh diff --git a/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh similarity index 100% rename from fedora-build-opennebula-image.sh rename to opennebula-images/fedora-build-opennebula-image.sh diff --git a/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh similarity index 100% rename from ubuntu-build-opennebula-image.sh rename to opennebula-images/ubuntu-build-opennebula-image.sh From 43b52b264df75d8b223d40ed532cfb7c33a42857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:20:00 +0200 Subject: [PATCH 064/543] Add forgotten centos to opennebula-images dir, +x ubuntu-build-one-... --- .../centos-build-opennebula-image.sh | 0 opennebula-images/ubuntu-build-opennebula-image.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename centos-build-opennebula-image.sh => opennebula-images/centos-build-opennebula-image.sh (100%) mode change 100644 => 100755 opennebula-images/ubuntu-build-opennebula-image.sh diff --git a/centos-build-opennebula-image.sh b/opennebula-images/centos-build-opennebula-image.sh similarity index 100% rename from centos-build-opennebula-image.sh rename to opennebula-images/centos-build-opennebula-image.sh diff --git a/opennebula-images/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh old mode 100644 new mode 100755 From b3f79de311110baa532e81c9b3437db7093a05e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:51:26 +0200 Subject: [PATCH 065/543] Add debian image build definition --- .../debian-build-opennebula-image.sh | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 opennebula-images/debian-build-opennebula-image.sh diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh new file mode 100755 index 0000000..17dc1e3 --- /dev/null +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -0,0 +1,164 @@ +#!/bin/sh + +# This script generates Debian images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=buster # 10.X +ARCH=amd64 +IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Ubuntu" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat >/mnt/etc/apt/sources.list < "/mnt$ONE_CONTEXT_DEB_PATH" +run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" +run_root rm "$ONE_CONTEXT_DEB_PATH" + +# Manually install legacy network scripts used by one-context. +run_root apt-get -y install ifupdown + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and bootloader. Do not autoconfigure grub. +run_root 'echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections' +run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-image-amd64 grub-pc + +# Configure grub. +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root apt-get -y install openssh-server + +# Install haveged due to lack of entropy in ONE environment. +run_root apt-get -y install haveged +run_root systemctl enable haveged.service + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Mon, 11 May 2020 12:29:52 +0200 Subject: [PATCH 066/543] [OpenNebula-images] Add scripts to generate FreeBSD images. Tested for 12.1-RELEASE, 11.3-STABLE should be tested as well. Two scripts are added, FreeBSD-build-opennebula-image-generic.sh takes information from the environment and generates the specified image with either dualstack or IPv6-only kernel and base applications. FreeBSD-build-opennebula-image.sh has the versions supported in DCL and generates all neeeded images (combination of Version x Network support). This should run in a DCL VM with an OK amount of cores (4/8 minimum), 2-4G RAM, and storage of roughly 20G + 5G * #resulting_images. This is because there is the base system, a 'pristine chroot', and during the build there can be 2 copies of the resulting system written to the system. Since there are 4 combinations of images: {STABLE,RELEASE} x {dualstack, IPv6ONLY} That means we'll need to assign about 40G storage to be on the safe side. --- .../FreeBSD-build-opennebula-image-generic.sh | 243 ++++++++++++++++++ .../FreeBSD-build-opennebula-image.sh | 31 +++ 2 files changed, 274 insertions(+) create mode 100644 opennebula-images/FreeBSD-build-opennebula-image-generic.sh create mode 100755 opennebula-images/FreeBSD-build-opennebula-image.sh diff --git a/opennebula-images/FreeBSD-build-opennebula-image-generic.sh b/opennebula-images/FreeBSD-build-opennebula-image-generic.sh new file mode 100644 index 0000000..d251f56 --- /dev/null +++ b/opennebula-images/FreeBSD-build-opennebula-image-generic.sh @@ -0,0 +1,243 @@ +#!/bin/sh +# +# Copyright 2020 -- Evilham +# This is BSD licensed as it's based on BSD-licensed code +# +# We could have used e.g. something like: +# - https://git.sr.ht/~sircmpwn/builds.sr.ht/tree/master/images/freebsd/genimg +# +# But we actually do want to compile the kernel, so that the IPv6-only images +# are different and don't support INET. + +# Explode if something goes wrong +set -e + +# What are we building? +# These are the only configuration options. +# They default to current environment. +# RELEASE: should be 'CURRENT' for current or 'X.Y' Defaults to 'CURRENT'. +# ARCH: probably amd64 for DCL +# VMFORMATS: defaults to qcow2, can also be raw. See man mkimg. +# OPENNEBULA_CONTEXT_VERSION: For DCL's OpenNebula that'd be 5.10.0 (default) +# OPENNEBULA_CONTEXT_REVISION: Defaults to 1. +RELEASE=${RELEASE:-CURRENT} +if [ "${RELEASE}" == "CURRENT" ]; then + SRCBRANCH="master" +else + SRCBRANCH="releng/${RELEASE}" +fi +ARCH=${ARCH:-amd64} +VMFORMATS=${VMFORMATS:-qcow2} +OPENNEBULA_CONTEXT_VERSION=${OPENNEBULA_CONTEXT_VERSION:-5.10.0} +OPENNEBULA_CONTEXT_REVISION=${OPENNEBULA_CONTEXT_REVISION:-1} + +# Didn't see a need to make these configurable. +CHROOTDIR="/scratch" +SRCDIR="${CHROOTDIR}/usr/src" +OUR_DIR="$(realpath $(dirname "${0}"))" +OUR_SRCCONF="${SRCDIR}/release/src.conf" +OUR_RELEASE_CONF="${SRCDIR}/release/release.conf" +# Shorthand for the package file name. +OPENNEBULA_CONTEXT="one-context-${OPENNEBULA_CONTEXT_VERSION}_${OPENNEBULA_CONTEXT_REVISION}.txz" + +setup_sources() { + # Let's use git, we might need to install it + if ! which git 2>&1 > /dev/null; then + pkg install -y git + fi + + if [ ! -d "$(dirname ${SRCDIR})" ]; then + mkdir -p "$(dirname ${SRCDIR})" + fi + + # Checkout needed branch + if [ ! -d "${SRCDIR}" ]; then + git clone "https://github.com/freebsd/freebsd" \ + --branch "${SRCBRANCH}" "${SRCDIR}" + else + GIT_CMD="git -C ${SRCDIR}" + ${GIT_CMD} clean -df + ${GIT_CMD} reset --hard + ${GIT_CMD} fetch + ${GIT_CMD} checkout "${SRCBRANCH}" + ${GIT_CMD} pull + fi + + # Add settings for IPv6-only kernel + cat > "${SRCDIR}/sys/${ARCH}/conf/GENERIC-IPV6ONLY" << EOF +include GENERIC +ident GENERIC-IPV6ONLY +makeoptions MKMODULESENV+="WITHOUT_INET_SUPPORT=" +nooptions INET +nodevice gre +EOF + # Fix vmimage.subr to install custom package and fix other things + cat >> "${SRCDIR}/release/tools/vmimage.subr" << EOF +vm_extra_install_ports() { + # Make sure we install the opennbula context package + cp "/${OPENNEBULA_CONTEXT}" "\${DESTDIR}/tmp/${OPENNEBULA_CONTEXT}" + chroot \${DESTDIR} \${EMULATOR} env ASSUME_ALWAYS_YES=yes \\ + /usr/sbin/pkg add '/tmp/${OPENNEBULA_CONTEXT}' + + # Now make sure the system has better defaults + cat >> "\${DESTDIR}/etc/rc.conf" << eof +# Update to latest patch on first boot +firstboot_freebsd_update_enable="YES" +# Enable OpenNebula's service. +one_context_enable="YES" +# Enable SSH for customers +sshd_enable="YES" +# Clear tmp on boot +clear_tmp_enable="YES" +# Disable sendmail by default +sendmail_enable="NONE" +# Disable crash dumps +dumpdev="NO" +eof + # Enable root access with SSH key. + # It is user's responsibility to further secure their system. + sed -i '' -E \ + 's/(^#[ ]*|^)PermitRootLogin .*/PermitRootLogin without-password/' \ + "\${DESTDIR}/etc/ssh/sshd_config" +} +EOF + # Skip building iso images + rm "${SRCDIR}/release/${ARCH}/mkisoimages.sh" + # This is a hack to not build the memstick + cat > "${SRCDIR}/release/${ARCH}/make-memstick.sh" < \${CHROOTDIR}/etc/src-env.conf +} + +## Set the directory within which the release will be built. +CHROOTDIR="${CHROOTDIR}" + +## Set to override the default target architecture and kernel +TARGET="${ARCH}" +TARGET_ARCH="${ARCH}" +KERNEL="${KERNEL_CONFIG}" + +## Set to specify a custom make.conf and/or src.conf +SRC_CONF="${OUR_SRCCONF}" + +# Since these are VMs, users should add other components if they want to. +NODOC=YES +NOPORTS=YES +NOSRC=YES + +# We manage sources manually +SRC_UPDATE_SKIP=YES + +## Set to pass additional flags to make(1) for the build chroot setup, such +## as TARGET/TARGET_ARCH. +# This was necessary for "cross-compiling" +CHROOT_MAKEENV="MK_LLVM_TARGET_X86=yes" + +WITH_VMIMAGES=YES + +# VM image size, see man 1 truncate +VMSIZE="10G" + +# List of disk image formats, see man mkgimg. +VMFORMATS="${VMFORMATS}" + +# These variables have to be exported because they are needed in subprocesses. +export NOSWAP=YES +# Custom ports +# - firstboot-freebsd-update helps us not have to create an image for each +# patch level. We still will have to do it for each minor version update. +# - bash is apparently needed for one-context +export VM_EXTRA_PACKAGES="firstboot-freebsd-update bash" +EOF +} + +_do_run_release() { + . "${SRCDIR}/release/release.sh" +} +run_release() { + _do_run_release -c "${OUR_RELEASE_CONF}" +} + + +build_image() { + # Generate configuration + echo "${2}" > "${OUR_SRCCONF}" + KERNEL_CONFIG="${1}" + gen_releaseconf > "${OUR_RELEASE_CONF}" + # Be paranoid about files and stuff + sync + # Continue with the release script + run_release + # Be paranoid about files and stuff + sync + + mv "${CHROOTDIR}/R/vmimages" "${OUR_DIR}/FreeBSD-${RELEASE}-${1}" + + # Be paranoid about files and stuff + sync +} + +our_main() { + case "$1" in + --dualstack) + BUILD_DUALSTACK=yes + ;; + --ipv6only) + BUILD_IPV6ONLY=yes + ;; + *) + cat << EOF +Run with --dualstack or --ipv6only depending on the image you want. +EOF + exit 1 + ;; + esac + setup_sources + setup_our_env + # Fetch OpenNebula's context package + fetch "https://github.com/OpenNebula/addon-context-linux/releases/download/v${OPENNEBULA_CONTEXT_VERSION}/${OPENNEBULA_CONTEXT}" \ + -o "${CHROOTDIR}/${OPENNEBULA_CONTEXT}" + # Do run + if [ -n "${BUILD_DUALSTACK}" ]; then + build_image "GENERIC" + fi + if [ -n "${BUILD_IPV6ONLY}" ]; then + build_image "GENERIC-IPV6ONLY" "$(cat << EOF +WITHOUT_INET=yes +WITHOUT_INET_SUPPORT=yes +EOF +)" + fi + + cat << EOF + +*************** DONE *************** +You will find the images under "${OUR_DIR}". +************************************ +EOF +} + +our_main "${@}" diff --git a/opennebula-images/FreeBSD-build-opennebula-image.sh b/opennebula-images/FreeBSD-build-opennebula-image.sh new file mode 100755 index 0000000..c72a2b0 --- /dev/null +++ b/opennebula-images/FreeBSD-build-opennebula-image.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Copyright 2020 -- Evilham +# This is BSD licensed as it's based on BSD-licensed code +# +# +# This builds all needed FreeBSD images for ungleich's Data Center Light +# When there are new releases, they should be updated here and the script +# should run. +# 11.4 is scheduled end of June 2020 +# 12.2 is scheduled end of October 2020 +# + +SUPPORTED_RELEASES="11.3 12.1" + +# This should run in a DCL VM with an OK amount of cores (4/8 minimum), +# 4G RAM, and storage of roughly 20G + 5G * #resulting_images. +# +# This is because there is the base system, a 'pristine chroot', and during the +# build there can be 2 copies of the resulting system written to the system. +# Since there are 4 combinations of images: +# {STABLE,RELEASE} x {dualstack, IPv6ONLY} +# +# That means we'll need to assign about 40G storage to be on the safe side. + +for release in ${SUPPORTED_RELEASES}; do + for build in dualstack ipv6only; do + env RELEASE=${release} sh FreeBSD-build-opennebula-image-generic.sh --${build} \ + | tee "FreeBSD-${release}-${build}.log" + done +done From 4eb3bc55239e3439f126ba9cc7c0993abb15b1e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 May 2020 07:15:50 +0200 Subject: [PATCH 067/543] Add WIP archlinux image definition script --- .../arch-build-opennebula-image.sh | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100755 opennebula-images/arch-build-opennebula-image.sh diff --git a/opennebula-images/arch-build-opennebula-image.sh b/opennebula-images/arch-build-opennebula-image.sh new file mode 100755 index 0000000..00c454f --- /dev/null +++ b/opennebula-images/arch-build-opennebula-image.sh @@ -0,0 +1,154 @@ +#!/bin/sh + +# This script generates Debian images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +ARCH=amd64 +IMAGE_PATH=arch-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Arch" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat > /mnt/etc/pacman.d/mirrorlist << EOF +## +## Arch Linux repository mirrorlist +## Generated on 2020-05-12 +## + +## Switzerland +Server = http://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch +Server = https://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch +Server = http://mirror.init7.net/archlinux/$repo/os/$arch +Server = https://mirror.init7.net/archlinux/$repo/os/$arch +Server = http://mirror.puzzle.ch/archlinux/$repo/os/$arch +Server = https://mirror.puzzle.ch/archlinux/$repo/os/$arch +Server = https://mirror.ungleich.ch/mirror/packages/archlinux/$repo/os/$arch +EOF +run_root pacman --sync --refresh --upgrade + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and generate initramfs. +run_root pacman --sync linux mkinitcpio +run_rot mkinitcpio -P + +# Install and configure bootloader. +run_root pacman --sync grub +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root pacman --sync openssh-server + +# Install haveged due to lack of entropy in ONE environment. +run_root pacman --sync haveged +run_root systemctl enable haveged.service + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Wed, 13 May 2020 11:41:34 +0200 Subject: [PATCH 068/543] Rename freebsd build scripts for consistency --- ...image-generic.sh => freebsd-build-opennebula-image-generic.sh} | 0 ...uild-opennebula-image.sh => freebsd-build-opennebula-image.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename opennebula-images/{FreeBSD-build-opennebula-image-generic.sh => freebsd-build-opennebula-image-generic.sh} (100%) mode change 100644 => 100755 rename opennebula-images/{FreeBSD-build-opennebula-image.sh => freebsd-build-opennebula-image.sh} (100%) diff --git a/opennebula-images/FreeBSD-build-opennebula-image-generic.sh b/opennebula-images/freebsd-build-opennebula-image-generic.sh old mode 100644 new mode 100755 similarity index 100% rename from opennebula-images/FreeBSD-build-opennebula-image-generic.sh rename to opennebula-images/freebsd-build-opennebula-image-generic.sh diff --git a/opennebula-images/FreeBSD-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh similarity index 100% rename from opennebula-images/FreeBSD-build-opennebula-image.sh rename to opennebula-images/freebsd-build-opennebula-image.sh From 5af488b9d6a470f112740087529fffa97adce217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 13 May 2020 12:09:26 +0200 Subject: [PATCH 069/543] Fix script call in FreeBSD build following renaming --- opennebula-images/freebsd-build-opennebula-image.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index c72a2b0..266f639 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -23,9 +23,10 @@ SUPPORTED_RELEASES="11.3 12.1" # # That means we'll need to assign about 40G storage to be on the safe side. +date=$(date -I) for release in ${SUPPORTED_RELEASES}; do for build in dualstack ipv6only; do - env RELEASE=${release} sh FreeBSD-build-opennebula-image-generic.sh --${build} \ - | tee "FreeBSD-${release}-${build}.log" + env RELEASE=${release} sh freebsd-build-opennebula-image-generic.sh --${build} \ + | tee "freebsd-${release}-${build}-${date}.log" done done From 0e7aa75959cdc8673c6c62c7abf02d686770cd8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 13 May 2020 14:25:17 +0200 Subject: [PATCH 070/543] Fix locale issue in debian image --- opennebula-images/debian-build-opennebula-image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 17dc1e3..9a51f1e 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -144,6 +144,10 @@ run_root apt-get -y install openssh-server run_root apt-get -y install haveged run_root systemctl enable haveged.service +# Generate locales. +run_root sed -i 's/^# *\(en_GB.UTF-8\)/\1/' /etc/locale.gen +run_root locale-gen + # Generate fstab file. boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") From 5a2e01d0edfebc45e081067f259dc8fbe66d820d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 13 May 2020 14:27:58 +0200 Subject: [PATCH 071/543] s/Ubuntu/Debian/ in debian build script --- opennebula-images/debian-build-opennebula-image.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 9a51f1e..67714bf 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -43,9 +43,9 @@ if [ "$(whoami)" != 'root' ]; then exit 1 fi -if [ $(lsb_release --short --id) != "Ubuntu" ]; then - echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 - echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 +if [ $(lsb_release --short --id) != "Debian" ]; then + echo "WARNING: this script has been designed to run on an Debian system." >&2 + echo "WARNING: Not running Debian. Giving you 5 seconds to abort." >&2 sleep 5 fi From 1d3b786a0f635f533116c024efee9ca2664fac77 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 15 May 2020 20:51:28 +0200 Subject: [PATCH 072/543] add shell script to update server to ceph nautilus --- ceph-upgrade-server-to-nautilus.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ceph-upgrade-server-to-nautilus.sh diff --git a/ceph-upgrade-server-to-nautilus.sh b/ceph-upgrade-server-to-nautilus.sh new file mode 100644 index 0000000..058e40d --- /dev/null +++ b/ceph-upgrade-server-to-nautilus.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +rm -f /etc/apt/sources.list.d/ceph.list + +cat > /etc/apt/sources.list < /etc/apt/sources.list.d/backports.list +apt update +apt dist-upgrade -y +apt install -t buster-backports -y ceph From a2558df86804e403c1a2273fdf863fbad7349555 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 15 May 2020 20:52:45 +0200 Subject: [PATCH 073/543] Mode update --- ceph-upgrade-server-to-nautilus.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ceph-upgrade-server-to-nautilus.sh diff --git a/ceph-upgrade-server-to-nautilus.sh b/ceph-upgrade-server-to-nautilus.sh old mode 100644 new mode 100755 From b2d33e624ac99781af5395cc3b672fc3ba5693c0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 21 May 2020 17:09:39 +0200 Subject: [PATCH 074/543] mikrotik-update: add notes for crs326 --- mikrotik-update.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mikrotik-update.sh b/mikrotik-update.sh index b2c0fc6..934ea47 100755 --- a/mikrotik-update.sh +++ b/mikrotik-update.sh @@ -5,9 +5,15 @@ if [ $# -lt 2 ]; then echo "$0 router [router...]" cat < Date: Sun, 24 May 2020 08:26:18 +0200 Subject: [PATCH 075/543] Debian ONE image: set hostname --- opennebula-images/debian-build-opennebula-image.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 67714bf..577f573 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -14,6 +14,7 @@ ARCH=amd64 IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 +HOSTNAME=debian # TODO: find the package definition and built ourself, publish in some RPM repository. ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" @@ -94,11 +95,13 @@ cp /etc/resolv.conf /mnt/etc/resolv.conf # Initialize /etc/hosts. cat > /mnt/etc/hosts << EOF -127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 -::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +127.0.0.1 $HOSTNAME localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 $HOSTNAME localhost localhost.localdomain localhost6 localhost6.localdomain6 EOF +run_root hostnamectl set-hostname $HOSTNAME + # Configure package sources and update package index. cat >/mnt/etc/apt/sources.list < Date: Sun, 24 May 2020 08:33:59 +0200 Subject: [PATCH 076/543] Move EvilHam's FreeBSD scripts to legacy directory --- legacy/README.md | 2 ++ .../freebsd-build-opennebula-image-generic.sh | 0 {opennebula-images => legacy}/freebsd-build-opennebula-image.sh | 0 3 files changed, 2 insertions(+) create mode 100644 legacy/README.md rename {opennebula-images => legacy}/freebsd-build-opennebula-image-generic.sh (100%) rename {opennebula-images => legacy}/freebsd-build-opennebula-image.sh (100%) diff --git a/legacy/README.md b/legacy/README.md new file mode 100644 index 0000000..97eae65 --- /dev/null +++ b/legacy/README.md @@ -0,0 +1,2 @@ +This directory contains old scripts that are not used anymore but might still +be useful. diff --git a/opennebula-images/freebsd-build-opennebula-image-generic.sh b/legacy/freebsd-build-opennebula-image-generic.sh similarity index 100% rename from opennebula-images/freebsd-build-opennebula-image-generic.sh rename to legacy/freebsd-build-opennebula-image-generic.sh diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/legacy/freebsd-build-opennebula-image.sh similarity index 100% rename from opennebula-images/freebsd-build-opennebula-image.sh rename to legacy/freebsd-build-opennebula-image.sh From 0d554866c82ba9d6e3135265a59bfb4198a9f5f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 24 May 2020 08:49:41 +0200 Subject: [PATCH 077/543] Initial sourcehut-inspired freebsd image build script --- .../freebsd-build-opennebula-image.sh | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 opennebula-images/freebsd-build-opennebula-image.sh diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh new file mode 100755 index 0000000..18722ab --- /dev/null +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -0,0 +1,109 @@ +#!/bin/sh + +# This script generates FreeBSD images for OpenNebula, being heavily inspired +# from srht's FreeBSD build image definition. It assumes running on a FreeBSD host. + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=12.1-RELEASE +IMAGE_PATH=freebsd-$RELEASE-$(date +%+F).img.qcow2 +IMAGE_SIZE=10G + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + sync || true + umount /mnt/dev || true + umount /mnt || true + mdconfig -du md0 || true +} +trap cleanup EXIT + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +# Allocate and partition/format disk image. +truncate -s 6G disk.img +mdconfig -a -t vnode -f disk.img -u md0 +gpart create -s gpt /dev/md0 +gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 +gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md0 +gpart add -t freebsd-ufs -l rootfs -b 1M -s 5G md0 +newfs -U /dev/md0p2 + +# Mount allocated image. +mount /dev/md0p2 /mnt +mkdir -p /mnt/dev +mount -t devfs devfs /mnt/dev + +# Download and extract base system. +dist_files="kernel.txz base.txz" +dist_dir="/usr/freebsd-dist/$arch/$release" + +mkdir -p "$dist_dir" +for f in $dist_files +do + fetch -m -o "$dist_dir/$f" "$dist_base/$f" + tar -C /mnt -xJf "$dist_dir/$f" +done + +# Configure new system. +echo "/dev/gpt/rootfs / ufs rw,noatime 1 1" >/mnt/etc/fstab +touch /mnt/firstboot +echo 'autoboot_delay="-1"' >>/mnt/boot/loader.conf + +cat >>/mnt/etc/rc.conf < /mnt/etc/resolv.conf +tzsetup -s -C /mnt UTC + +cat >>/mnt/etc/ssh/sshd_config </mnt/usr/local/etc/pkg/repos/FreeBSD.conf < Date: Sun, 24 May 2020 08:59:19 +0200 Subject: [PATCH 078/543] FreeBSD image: fix environment, mktemp disk image --- .../freebsd-build-opennebula-image.sh | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 18722ab..ec64a56 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -8,9 +8,13 @@ set -x # XXX: Handle command-line arguments? RELEASE=12.1-RELEASE +ARCH=amd64 IMAGE_PATH=freebsd-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G +DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" +PORTS_BASE="https://download.freebsd.org/ftp/snapshots/$ARCH/12.1-STABLE" + # TODO: find the package definition and built ourself, publish in some RPM repository. ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm @@ -29,8 +33,9 @@ if [ "$(whoami)" != 'root' ]; then fi # Allocate and partition/format disk image. -truncate -s 6G disk.img -mdconfig -a -t vnode -f disk.img -u md0 +disk=$(mktemp) +truncate -s 6G $disk +mdconfig -a -t vnode -f $disk -u md0 gpart create -s gpt /dev/md0 gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md0 @@ -44,12 +49,12 @@ mount -t devfs devfs /mnt/dev # Download and extract base system. dist_files="kernel.txz base.txz" -dist_dir="/usr/freebsd-dist/$arch/$release" +dist_dir="/usr/freebsd-dist/$ARCH/$RELEASE" mkdir -p "$dist_dir" for f in $dist_files do - fetch -m -o "$dist_dir/$f" "$dist_base/$f" + fetch -m -o "$dist_dir/$f" "$DIST_BASE/$f" tar -C /mnt -xJf "$dist_dir/$f" done @@ -83,11 +88,11 @@ FreeBSD: { EOF # freebsd-update is only supported for RELEASE -if [ "${release%-RELEASE}" != "$release" ] +if [ "${release%-RELEASE}" != "$RELEASE" ] then env PAGER=true /usr/sbin/freebsd-update \ -b /mnt \ - --currently-running "$release" \ + --currently-running "$RELEASE" \ --not-running-from-cron -F \ fetch install fi @@ -95,15 +100,15 @@ fi env ASSUME_ALWAYS_YES=YES pkg -c /mnt bootstrap -f env ASSUME_ALWAYS_YES=YES pkg -c /mnt install bash -fetch -m -o "$dist_dir/ports.txz" "$ports_base/ports.txz" +fetch -m -o "$dist_dir/ports.txz" "$PORTS_BASE/ports.txz" tar -C /mnt -xJf "$dist_dir/ports.txz" cleanup trap : EXIT -mkdir -p "$arch" -qemu-img convert -f raw -O qcow2 disk.img "$arch"/root.img.qcow2 -rm disk.img +mkdir -p "$ARCH" +qemu-img convert -f raw -O qcow2 $disk "$ARCH"/root.img.qcow2 +rm $disk # Filesystem will be enlarged by growfs(7) on next startup qemu-img resize $IMAGE_PATH $IMAGE_SIZE From 9c17dad41ded4db2029ce0c39bfd9d1c85f2e3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 24 May 2020 09:22:29 +0200 Subject: [PATCH 079/543] Fix local generation for debian image --- opennebula-images/debian-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 577f573..d18e349 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -148,7 +148,7 @@ run_root apt-get -y install haveged run_root systemctl enable haveged.service # Generate locales. -run_root sed -i 's/^# *\(en_GB.UTF-8\)/\1/' /etc/locale.gen +run_root 'sed -i "s/^# *\(en_GB.UTF-8\)/\1/" etc/locale.gen' run_root locale-gen # Generate fstab file. From 95b1922a3663be95b09401929537ed7df3b7adea Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:15:58 +0200 Subject: [PATCH 080/543] Added WIP script for mapping osd to disk types, and helper script (megaclisas-status) that provides a nicely formatted output for megacli --- map-osd-to-disktype | 13 + megaclisas-status | 875 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 888 insertions(+) create mode 100644 map-osd-to-disktype create mode 100755 megaclisas-status diff --git a/map-osd-to-disktype b/map-osd-to-disktype new file mode 100644 index 0000000..f1a2ff8 --- /dev/null +++ b/map-osd-to-disktype @@ -0,0 +1,13 @@ +# Assumes, osd is up and + +# Tempfile +./megacli-status > /tmp/megaclisas-status.out + +# Gets osd numbers of a particular device class, and gets their mountpoints on the host, and puts them into a tempfile +for osd in $(ceph osd tree | grep $OSD | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep $osd| cut -c 24- >> /tmp/list_osd_mountpoint; done + +# Gets the Megacli mappings for the mountpoints +for MOUNT in $(cat /tmp/list_osd_mountpoint | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done + +# Gets the hardware types for the Megacli mappings +for i in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $i | cut -d '|' -f 3 >> /tmp/disk_types; done diff --git a/megaclisas-status b/megaclisas-status new file mode 100755 index 0000000..3495502 --- /dev/null +++ b/megaclisas-status @@ -0,0 +1,875 @@ +#!/usr/bin/python +# $Id: megaclisas-status,v 1.68 2016/10/21 14:38:56 root Exp root $ +# +# Written by Adam Cecile +# Modified by Vincent S. Cojot +# + +import os +import re +import sys +import pdb +if sys.platform == 'win32': + import ctypes + +def_megaclipath = "/opt/MegaRAID/MegaCli/MegaCli64" + +# Non-Nagios Mode defaults +nagiosmode = False +nagiosoutput='' +nagiosgoodarray = 0 +nagiosbadarray = 0 +nagiosgooddisk = 0 +nagiosbaddisk = 0 + +# Sane defaults +printarray = True +printcontroller = True +debugmode = False +notempmode = False +totaldrivenumber = 0 + +# Hardcode a max of 16 HBA and 128 LDs for now. LDTable must be initialized to accept populating list of LD's into each ctlr's list. +MaxNumHBA = 16 +MaxNumLD = 128 +LDTable = [ [] * MaxNumHBA for i in range(MaxNumLD) ] +NestedLDTable = [[False for i in range(MaxNumHBA)] for j in range(MaxNumLD)] + +# Outputs is a 'dict' of all MegaCLI outputs so we can re-use them during loops.. +Outputs = {} + +# Startup +def print_usage(): + print 'Usage: megaraid-status [--nagios|--debug|--notemp]' + +# We need root access to query +if __name__ == '__main__': + try: + root_or_admin = os.geteuid() == 0 + except AttributeError: + root_or_admin = ctypes.windll.shell32.IsUserAnAdmin() !=0 + if not root_or_admin: + print '# This script requires Administrator privileges' + sys.exit(5) + +# Check command line arguments to enable nagios or not +if len(sys.argv) > 2: + print_usage() + sys.exit(1) + +if len(sys.argv) > 1: + if sys.argv[1] == '--nagios': + nagiosmode = True + elif sys.argv[1] == '--debug': + debugmode = True + elif sys.argv[1] == '--notemp': + notempmode = True + else: + print_usage() + sys.exit(1) +# Functions +def dbgprint(msg): + if (debugmode): + sys.stderr.write ( str('# DEBUG : '+msg+'\n')) + +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +def which(program): + import os + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + # Add some defaults + os.environ["PATH"] += os.pathsep + '/opt/MegaRAID/MegaCli' + os.environ["PATH"] += os.pathsep + '/ms/dist/hwmgmt/bin' + os.environ["PATH"] += os.pathsep + os.path.dirname(os.path.realpath(sys.argv[0])) + for path in os.environ["PATH"].split(os.pathsep): + dbgprint ('Looking in PATH '+str(path)) + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + dbgprint ('Found "'+program+'" at '+exe_file) + return exe_file + return None + +# Find MegaCli +for megabin in "MegaCli64","MegaCli","megacli", "MegaCli.exe": + dbgprint ('Looking for '+str(megabin)+' in PATH next..') + megaclipath = which(megabin) + if (megaclipath != None): + dbgprint ('Will use MegaCLI from here: '+str(megaclipath)) + break + +# Check binary exists (and +x), if not print an error message +if (megaclipath != None): + if os.path.exists(megaclipath) and os.access(megaclipath, os.X_OK): + pass + else: + if nagiosmode: + print 'UNKNOWN - Cannot find '+megaclipath + else: + print 'Cannot find ' + megaclipath + 'in your PATH. Please install it.' + sys.exit(3) +else: + print 'Cannot find "MegaCli64","MegaCli" or "megacli" or "MegaCli.exe" in your PATH. Please install it.' + sys.exit(3) + + +#### pdb.set_trace() + +def returnWdthFromArrayCol(glarray,idx): + maxwdth = 0 + for glrow in glarray: + if ( len(glrow[idx]) > maxwdth): + maxwdth = len(glrow[idx]) + return maxwdth + +# Get command output +def getOutput(cmd): + lines = [] + if ( Outputs.has_key(cmd) ): + dbgprint ("Got Cached value: "+str(cmd)) + lines = Outputs[cmd] + else: + dbgprint ("Not a Cached value: "+str(cmd)) + output = os.popen(cmd) + for line in output: + if not re.match(r'^$',line.strip()): + lines.append(line.strip()) + Outputs[cmd] = lines + return lines + +def returnControllerNumber(output): + for line in output: + if re.match(r'^Controller Count.*$',line.strip()): + return int(line.split(':')[1].strip().strip('.')) + +def returnTotalDriveNumber(output): + for line in output: + if re.match(r'Number of Physical Drives on Adapter.*$',line.strip()): + return int(line.split(':')[1].strip()) + +def returnRebuildProgress(output): + percent = 0 + tmpstr = '' + for line in output: + if re.match(r'^Rebuild Progress on Device at Enclosure.*, Slot .* Completed ',line.strip()): + tmpstr = line.split('Completed')[1].strip() + percent = int(tmpstr.split('%')[0].strip()) + return percent + +def returnConfDriveNumber(output): + # Count the configured drives + confdrives = 0 + for line in output: + if re.match(r'.*Number of PDs:.*$',line.strip()): + confdrives += int(line.split(':')[2].strip()) + return int(confdrives) + +def returnUnConfDriveNumber(output): + # Count the configured drives + confdrives = 0 + for line in output: + if re.match(r'^Firmware state: Unconfigured.*$',line.strip()): + confdrives += 1 + return int(confdrives) + +def returnControllerModel(output): + for line in output: + if re.match(r'^Product Name.*$',line.strip()): + return line.split(':')[1].strip() + +def returnMemorySize(output): + for line in output: + if re.match(r'^Memory Size.*$',line.strip()): + return line.split(':')[1].strip() + +def returnFirmwareVersion(output): + for line in output: + if re.match(r'^FW Package Build.*$',line.strip()): + return line.split(':')[1].strip() + +def returnROCTemp(output): + ROCtemp = '' + tmpstr = '' + if (notempmode): + return str('N/A') + else: + for line in output: + if re.match(r'^ROC temperature :.*$',line.strip()): + tmpstr = line.split(':')[1].strip() + ROCtemp = re.sub(' +.*$', '', tmpstr) + if ( ROCtemp != '' ): + return str(str(ROCtemp)+'C') + else: + return str('N/A') + +def returnBBUPresence(output): + BBU = '' + tmpstr = '' + for line in output: + if re.match(r'^BBU +:.*$',line.strip()): + tmpstr = line.split(':')[1].strip() + BBU = re.sub(' +.*$', '', tmpstr) + break + if ( BBU != '' ): + return str(BBU) + else: + return str('N/A') + +def returnBBUStatus(output): + BBUStatus = '' + tmpstr = '' + for line in output: + if re.match(r'^ *Battery Replacement required +:.*$',line.strip()): + tmpstr = line.split(':')[1].strip() + BBUStatus = re.sub(' +.*$', '', tmpstr) + break + if ( BBUStatus == 'Yes' ): + return str('REPL') + else: + return str('Good') + +def returnArrayNumber(output): + i = 0 + for line in output: + if re.match(r'^(CacheCade )?Virtual Drive:.*$',line.strip()): + i += 1 + return i + +def returnHBAPCIInfo(output): + busprefix = '0000' + busid = '' + devid = '' + functionid = '' + pcipath = '' + for line in output: + if re.match(r'^Bus Number.*:.*$',line.strip()): + busid = str(line.strip().split(':')[1].strip()).zfill(2) + if re.match(r'^Device Number.*:.*$',line.strip()): + devid = str(line.strip().split(':')[1].strip()).zfill(2) + if re.match(r'^Function Number.*:.*$',line.strip()): + functionid = str(line.strip().split(':')[1].strip()).zfill(1) + if busid: + pcipath = str(busprefix + ':' + busid + ':' + devid + '.' + functionid) + dbgprint("Array PCI path : "+pcipath) + return str(pcipath) + else: + return None + +def returnHBAInfo(table,output,controllerid): + controllermodel = 'Unknown' + controllerram = 'Unknown' + controllerrev = 'Unknown' + controllertemp = '' + controllermodel = returnControllerModel(output) + controllerram = returnMemorySize(output) + controllerrev = returnFirmwareVersion(output) + controllertemp = returnROCTemp(output) + controllerbbu = returnBBUPresence(output) + if controllerbbu == 'Present': + cmd = '%s -AdpBbuCmd -GetBbuStatus -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + controllerbbu = returnBBUStatus(output) + + if controllermodel != 'Unknown': + table.append([ 'c'+str(controllerid), controllermodel, controllerram, str(controllertemp), str(controllerbbu), str('FW: '+controllerrev) ]) + +def returnArrayInfo(output,controllerid,arrayid,arrayindex): + id = 'c'+str(controllerid)+'u'+str(arrayid) + operationlinennumber = False + linenumber = 0 + targetid = '' + raidtype = '' + raidlvl = '' + size = '' + state = 'N/A' + strpsz = '' + dskcache = 'N/A' + properties = '' + spandepth = 0 + diskperspan = 0 + cachecade_info = 'None' + + for line in output: + if re.match(r'^(CacheCade )?Virtual Drive:.*(Target Id: [0-9]+).*$',line.strip()): + # Extract the SCSI Target ID + targetid = line.strip().split(':')[2].split(')')[0].strip() + elif re.match(r'^RAID Level.*?:.*$',line.strip()): + # Extract the primary raid type, decide on X0 RAID level later when we hit Span Depth + raidlvl = int(line.strip().split(':')[1].split(',')[0].split('-')[1].strip()) + elif re.match(r'^Size.*?:.*$',line.strip()): + # Size reported in MB + if re.match(r'^.*MB$',line.strip().split(':')[1]): + size = line.strip().split(':')[1].strip('MB').strip() + if ( float(size) > 1000): + size = str(int(round((float(size) / 1000))))+'G' + else: + size = str(int(round(float(size))))+'M' + # Size reported in TB + elif re.match(r'^.*TB$',line.strip().split(':')[1]): + size = line.strip().split(':')[1].strip('TB').strip() + size = str(int(round((float(size) * 1000))))+'G' + # Size reported in GB (default) + else: + size = line.strip().split(':')[1].strip('GB').strip() + size = str(int(round((float(size)))))+'G' + elif re.match(r'^Span Depth.*?:.*$',line.strip()): + # If Span Depth is greater than 1 chances are we have a RAID 10, 50 or 60 + spandepth = line.strip().split(':')[1].strip() + elif re.match(r'^State.*?:.*$',line.strip()): + state = line.strip().split(':')[1].strip() + elif re.match(r'^Strip Size.*?:.*$',line.strip()): + strpsz = line.strip().split(':')[1].strip() + elif re.match(r'^Number Of Drives per span.*:.*$',line.strip()): + diskperspan = int(line.strip().split(':')[1].strip()) + elif re.match(r'^Current Cache Policy.*?:.*$',line.strip()): + props = line.strip().split(':')[1].strip() + if re.search('ReadAdaptive', props): + properties += 'ADRA' + if re.search('ReadAhead', props): + properties += 'RA' + if re.match('ReadAheadNone', props): + properties += 'NORA' + if re.search('WriteBack', props): + properties += ',WB' + if re.match('WriteThrough', props): + properties += ',WT' + elif re.match(r'^Disk Cache Policy.*?:.*$',line.strip()): + props = line.strip().split(':')[1].strip() + if re.search('Disabled', props): + dskcache = 'Disabled' + if re.search('Disk.s Default', props): + dskcache = 'Default' + if re.search('Enabled', props): + dskcache = 'Enabled' + elif re.match(r'^Ongoing Progresses.*?:.*$',line.strip()): + operationlinennumber = linenumber + elif re.match(r'Cache Cade Type\s*:.*$', line): + cachecade_info = "Type : " + line.strip().split(':')[1].strip() + elif re.match(r'^Target Id of the Associated LDs\s*:.*$', line): + associated=[] + for array in line.split(':')[1].strip().split(','): + if array.isdigit(): + associated.append('c%du%d' % (controllerid, int(array))) + if len(associated) >= 1: + cachecade_info = "Associated : %s" %(', '.join(associated)) + linenumber += 1 + + # If there was an ongoing operation, find the relevant line in the previous output + if operationlinennumber: + inprogress = output[operationlinennumber + 1] + else: + inprogress = 'None' + + # Compute the RAID level + NestedLDTable[int(controllerid)][int(arrayindex)] = False + if raidlvl == '': + raidtype = str('N/A') + else: + if (int(spandepth) >= 2): + raidtype = str('RAID-' + str(raidlvl) + '0') + NestedLDTable[controllerid][int(arrayindex)] = True + else: + if(raidlvl == 1): + if(diskperspan > 2): + raidtype = str('RAID-10') + NestedLDTable[controllerid][int(arrayindex)] = True + else: + raidtype = str('RAID-' + str(raidlvl)) + else: + raidtype = str('RAID-' + str(raidlvl)) + + dbgprint('RAID Level: ' + str(raidlvl) + + ' Span Depth: ' + str(spandepth) + + ' Disk Per Span: ' + str(diskperspan) + + ' Raid Type: ' + str(raidtype)) + return [id,raidtype,size,strpsz,properties,dskcache,state,targetid,cachecade_info,inprogress] + +def returnDiskInfo(output,controllerid): + arrayid = False + arrayindex = -1 + sarrayid = 'Unknown' + diskid = False + oldenclid = False + enclid = False + spanid = False + slotid = False + lsidid = 'Unknown' + table = [] + fstate = 'Offline' + substate = 'Unknown' + model = 'Unknown' + speed = 'Unknown' + dsize = 'Unknown' + temp = 'Unk0C' + percent = 0 + for line in output: + if re.match(r'^Span: [0-9]+ - Number of PDs:',line.strip()): + spanid = line.split(':')[1].strip() + spanid = re.sub(' - Number of PDs.*', '', spanid) + elif re.match(r'Enclosure Device ID: .*$',line.strip()): + # We match here early in the analysis so reset the vars if this is a new disk we're reading.. + oldenclid = enclid + enclid = line.split(':')[1].strip() + if oldenclid != False: + fstate = 'Offline' + model = 'Unknown' + speed = 'Unknown' + temp = 'Unk0C' + slotid = False + lsidid = 'Unknown' + elif re.match(r'^Coerced Size: ',line.strip()): + dsize = line.split(':')[1].strip() + dsize = re.sub(' \[.*\.*$', '', dsize) + dsize = re.sub('[0-9][0-9] GB', ' Gb', dsize) + elif re.match(r'^(CacheCade )?Virtual (Disk|Drive): [0-9]+.*$',line.strip()): + arrayindex += 1 + arrayid = line.split('(')[0].split(':')[1].strip() + elif re.match(r'PD: [0-9]+ Information.*$',line.strip()): + diskid = line.split()[1].strip() + elif re.match(r'^Device Id: .*$',line.strip()): + lsidid = line.split(':')[1].strip() + elif re.match(r'Slot Number: .*$',line.strip()): + slotid = line.split(':')[1].strip() + elif re.match(r'Firmware state: .*$',line.strip()): + fstate = line.split(':')[1].strip() + subfstate = re.sub('\(.*', '', fstate) + dbgprint('Firmware State: '+str(fstate)+' '+str(subfstate)) + elif re.match(r'Inquiry Data: .*$',line.strip()): + model = line.split(':')[1].strip() + model = re.sub(' +', ' ', model) + # Sub code + manuf = re.sub(' .*', '', model) + dtype = re.sub(manuf+' ', '', model) + dtype = re.sub(' .*', '', dtype) + hwserial = re.sub('.*'+dtype+' *', '', model) + elif re.match(r'^Media Type: .*$',line.strip()): + mtype = line.split(':')[1].strip() + if mtype == 'Hard Disk Device': + mtype = 'HDD' + else: + if mtype == 'Solid State Device': + mtype = 'SSD' + else: + mtype = 'N/A' + elif re.match(r'Device Speed: .*$',line.strip()): + speed = line.split(':')[1].strip() + elif re.match(r'Drive Temperature :.*$',line.strip()): + if (notempmode): + temp = 'N/A' + else: + # Drive temp is amongst the last few lines matched, decide here if we add information to the table.. + temp = line.split(':')[1].strip() + temp = re.sub(' \(.*\)', '', temp) + if model != 'Unknown': + dbgprint('Disk Info: '+str(arrayid)+' '+str(diskid)+' '+str(oldenclid)) + if subfstate == 'Rebuild': + cmd = '%s pdrbld -showprog -physdrv\[%s:%s\] -a%d -NoLog' % (megaclipath, enclid, slotid, controllerid) + output = getOutput(cmd) + percent = returnRebuildProgress(output) + fstate = str('Rebuilding (%d%%)' % (percent)) + + if (( NestedLDTable[controllerid][int(arrayindex)] == True) and (spanid != False)): + sarrayid = str(arrayid)+"s"+spanid + else: + sarrayid = str(arrayid) + table.append([sarrayid, str(diskid), mtype, model, dsize, fstate , speed, temp, enclid, slotid, lsidid]) + return table + + +def returnUnconfDiskInfo(output,controllerid): + arrayid = False + diskid = False + olddiskid = False + enclid = False + slotid = False + lsidid = 'Unknown' + table = [] + fstate = 'Offline' + substate = 'Unknown' + model = 'Unknown' + speed = 'Unknown' + mtype = 'Unknown' + dsize = 'Unknown' + temp = 'Unk0C' + for line in output: + if re.match(r'Enclosure Device ID: .*$',line.strip()): + # We match here early in the analysis so reset the vars if this is a new disk we're reading.. + oldenclid = enclid + enclid = line.split(':')[1].strip() + if oldenclid != False: + arrayid = False + fstate = 'Offline' + model = 'Unknown' + speed = 'Unknown' + temp = 'Unk0C' + slotid = False + lsidid = 'Unknown' + + elif re.match(r'^Coerced Size: ',line.strip()): + dsize = line.split(':')[1].strip() + dsize = re.sub(' \[.*\.*$', '', dsize) + dsize = re.sub('[0-9][0-9] GB', ' Gb', dsize) + elif re.match(r'^Drive.s position: DiskGroup: [0-9]+,.*$',line.strip()): + arrayid = line.split(',')[1].split(':')[1].strip() + elif re.match(r'^Device Id: [0-9]+.*$',line.strip()): + diskid = line.split(':')[1].strip() + elif re.match(r'^Device Id: .*$',line.strip()): + lsidid = line.split(':')[1].strip() + elif re.match(r'Slot Number: .*$',line.strip()): + slotid = line.split(':')[1].strip() + elif re.match(r'Firmware state: .*$',line.strip()): + fstate = line.split(':')[1].strip() + subfstate = re.sub('\(.*', '', fstate) + dbgprint('Firmware State: '+str(fstate)+' '+str(subfstate)) + elif re.match(r'Inquiry Data: .*$',line.strip()): + model = line.split(':')[1].strip() + model = re.sub(' +', ' ', model) + manuf = re.sub(' .*', '', model) + dtype = re.sub(manuf+' ', '', model) + dtype = re.sub(' .*', '', dtype) + hwserial = re.sub('.*'+dtype+' *', '', model) + elif re.match(r'^Media Type: .*$',line.strip()): + mtype = line.split(':')[1].strip() + if mtype == 'Hard Disk Device': + mtype = 'HDD' + else: + if mtype == 'Solid State Device': + mtype = 'SSD' + else: + mtype = 'N/A' + elif re.match(r'Device Speed: .*$',line.strip()): + speed = line.split(':')[1].strip() + elif re.match(r'Drive Temperature :.*$',line.strip()): + temp = line.split(':')[1].strip() + temp = re.sub('\(.*\)', '', temp) + # Drive temp is amongst the last few lines matched, decide here if we add information to the table.. + if arrayid == False: + if subfstate == 'Unconfigured': + dbgprint('Unconfigured Disk: Arrayid: '+str(arrayid)+' DiskId: '+str(diskid)+' '+str(olddiskid)+' '+str(fstate)) + elif subfstate == 'Online, Spun Up': + dbgprint('Online Disk: Arrayid: '+str(arrayid)+' DiskId: '+str(diskid)+' '+str(olddiskid)+' '+str(fstate)) + table.append([ mtype, model, dsize, fstate, speed, temp, enclid, slotid, lsidid]) + return table + +cmd = '%s -adpCount -NoLog' % (megaclipath) +output = getOutput(cmd) +controllernumber = returnControllerNumber(output) + +bad = False + +# List available controller +if printcontroller: + if controllernumber: + if not nagiosmode: + print '-- Controller information --' + + i = 0 + controllerid = 0 + mlen = 0 + hbainfo = [] + while controllerid < controllernumber: + cmd = '%s -AdpAllInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + returnHBAInfo(hbainfo, output,controllerid) + controllerid += 1 + mlen = returnWdthFromArrayCol(hbainfo,1) + + controllerid = 0 + for hba in hbainfo: + hbafmt = str('%-5s | %-'+str(mlen)+'s | %-6s | %-4s | %-6s | %-12s ') + # Header + if ( i == 0 ): + if not nagiosmode: + print hbafmt % ("-- ID","H/W Model","RAM","Temp","BBU", "Firmware") + if not nagiosmode: + print hbafmt % ( + hba[0], + hba[1], + hba[2], + hba[3], + hba[4], + hba[5]) + i += 1 + if not nagiosmode: + print '' + else: + print "No MegaRAID or PERC adapter detected on your system!" + exit(1) + +if printarray: + if not nagiosmode: + print '-- Array information --' + + controllerid = 0 + pcipath = '' + diskpath = '' + i = 0 ; j = 0 + mlen = 0 ; rlen = 0 ; clen = 0 + while controllerid < controllernumber: + arrayindex = 0 + + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + # We need to explore each HBA to look for gaps in LD's + ldid = 0 ; ldcount = 0 + while ldcount < arraynumber: + cmd = '%s -LDInfo -l%d -a%d -NoLog' % (megaclipath, ldid, controllerid) + output = getOutput(cmd) + for line in output: + if re.match(r'^Adapter.*Virtual Drive .* Does not Exist',line.strip()): + ldid += 1 + elif re.match(r'^(CacheCade )?Virtual Drive:',line.strip()): + LDTable[controllerid].append ( ldid ) + #NestedLDTable[controllerid][int(arrayindex)] = False + ldcount += 1 + ldid += 1 + + while arrayindex < arraynumber: + ldid = LDTable[controllerid][arrayindex] + cmd = '%s -LDInfo -l%d -a%d -NoLog' % (megaclipath, ldid, controllerid) + output = getOutput(cmd) + arrayinfo = returnArrayInfo(output, controllerid, ldid, arrayindex) + if ( len(arrayinfo[1]) > rlen): + rlen = len(arrayinfo[1]) + if ( len(arrayinfo[4]) > mlen): + mlen = len(arrayinfo[4]) + if ( len(arrayinfo[8]) > clen): + clen = len(arrayinfo[8]) + arrayindex += 1 + controllerid += 1 + + controllerid = 0 + while controllerid < controllernumber: + arrayindex = 0 + + cmd = '%s -AdpGetPciInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + pcipath = returnHBAPCIInfo(output) + + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + while arrayindex < arraynumber: + ldid = LDTable[controllerid][arrayindex] + cmd = '%s -LDInfo -l%d -a%d -NoLog' % (megaclipath, ldid, controllerid) + output = getOutput(cmd) + arrayinfo = returnArrayInfo(output,controllerid, ldid, arrayindex) + + if pcipath: + diskprefix = str('/dev/disk/by-path/pci-' + pcipath + '-scsi-0:') + for j in range (8): + diskpath = diskprefix + str(j) + ':' + str(arrayinfo[7]) + ':0' + if os.path.exists(diskpath): + arrayinfo[7] = os.path.realpath(diskpath) + else: + arrayinfo[7] = 'N/A' + + # Pad the string length, just to make sure it's aligned with the headers... + if (rlen < len("Type")): + rlen = len("Type") + if (mlen < len("Flags")): + mlen = len("Flags") + if (clen < len("CacheCade")): + clen = len("CacheCade") + + ldfmt = str('%-5s | %-'+str(rlen)+'s | %7s | %7s | %'+str(mlen)+'s | %8s | %8s | %8s | %-'+str(clen)+'s |%-12s ') + # Header + if ( i == 0 ): + if not nagiosmode: + print ldfmt % ("-- ID", "Type", "Size", "Strpsz", "Flags", "DskCache", "Status", "OS Path", "CacheCade", "InProgress" ) + if not nagiosmode: + print ldfmt % ( + arrayinfo[0], + arrayinfo[1], + arrayinfo[2], + arrayinfo[3], + arrayinfo[4], + arrayinfo[5], + arrayinfo[6], + arrayinfo[7], + arrayinfo[8], + arrayinfo[9]) + dbgprint("Array state : "+arrayinfo[6]) + if arrayinfo[6] not in [ 'Optimal', 'N/A' ]: + bad = True + nagiosbadarray=nagiosbadarray+1 + else: + nagiosgoodarray=nagiosgoodarray+1 + arrayindex += 1 + i += 1 + controllerid += 1 + if not nagiosmode: + print '' + +controllerid = 0 +while controllerid < controllernumber: + cmd = '%s -PDGetNum -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + totaldrivenumber += returnTotalDriveNumber(output) + controllerid += 1 + +if totaldrivenumber: + if not nagiosmode: + print '-- Disk information --' + + i = 0 + dlen = 0 ; mlen = 0 ; flen = 0 + controllerid = 0 + while controllerid < controllernumber: + arrayid = 0 + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + #### BUG: -LdPdInfo shows all PD on the adapter, not just for said LD.. + #### while arrayid <= arraynumber: + cmd = '%s -LdPdInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraydisk = returnDiskInfo(output,controllerid) + for array in arraydisk: + dbgprint('Disk c'+str(controllerid)+'u'+array[0]+'p'+array[1] + ' status : ' + array[5]) + if array[5] not in [ 'Online', 'Online, Spun Up' ]: + bad = True + nagiosbaddisk=nagiosbaddisk+1 + else: + nagiosgooddisk=nagiosgooddisk+1 + + if ( returnWdthFromArrayCol(arraydisk,0) > dlen): + dlen = returnWdthFromArrayCol(arraydisk,0) + if ( returnWdthFromArrayCol(arraydisk,3) > mlen): + mlen = returnWdthFromArrayCol(arraydisk,3) + if ( returnWdthFromArrayCol(arraydisk,5) > flen): + flen = returnWdthFromArrayCol(arraydisk,5) + controllerid += 1 + + controllerid = 0 + while controllerid < controllernumber: + arrayid = 0 + + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + #### BUG: -LdPdInfo shows all PD on the adapter, not just for said LD.. + #### while arrayid <= arraynumber: + + cmd = '%s -LdPdInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraydisk = returnDiskInfo(output,controllerid) + + # Adjust print format with width computed above + drvfmt = "%-"+str(dlen+5)+"s | %-4s | %-"+str(mlen)+"s | %-8s | %-"+str(flen)+"s | %-8s | %-4s | %-8s | %-8s" + for array in arraydisk: + # Header + if ( i == 0 ): + if not nagiosmode: + print drvfmt % ( + "-- ID", "Type", "Drive Model", "Size", "Status", "Speed", "Temp", "Slot ID", "LSI Device ID") + # Drive information + if not nagiosmode: + print drvfmt % ( + str('c'+str(controllerid)+'u'+array[0]+'p'+array[1]), # c0p0 + array[2], # HDD/SDD + array[3], # Model Information (Variable len) + array[4], # Size + array[5], # Status (Variable len) + array[6], # Speed + array[7], # Temp + str('['+array[8]+':'+array[9]+']'), # Slot ID + array[10]) # LSI ID + i = i + 1 + controllerid += 1 + if not nagiosmode: + print '' + +controllerid = 0 +totalconfdrivenumber = 0 +totalunconfdrivenumber = 0 +totaldrivenumber = 0 +while controllerid < controllernumber: + cmd = '%s -LdPdInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + totalconfdrivenumber += returnConfDriveNumber(output) + + cmd = '%s -PDGetNum -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + totaldrivenumber += returnTotalDriveNumber(output) + + cmd = '%s -PDList -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + totalunconfdrivenumber += returnUnConfDriveNumber(output) + + controllerid += 1 + +dbgprint('Total Drives in system : ' + str(totaldrivenumber)) +dbgprint('Total Configured Drives : ' + str(totalconfdrivenumber)) +dbgprint('Total Unconfigured Drives : ' + str(totalunconfdrivenumber)) + +if totalunconfdrivenumber: + if not nagiosmode: + print '-- Unconfigured Disk information --' + + controllerid = 0 + while controllerid < controllernumber: + arrayid = 0 + + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + #### BUG: -LdPdInfo shows all PD on the adapter, not just for given LD.. + #### while arrayid <= arraynumber: + + cmd = '%s -PDList -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraydisk = returnUnconfDiskInfo(output,controllerid) + for array in arraydisk: + dbgprint('Disk c'+str(controllerid)+'uXpY status : ' + array[3]) + if array[3] not in [ 'Online', 'Unconfigured(good), Spun Up', 'Unconfigured(good), Spun down', 'JBOD','Hotspare, Spun Up','Hotspare, Spun down' ]: + bad = True + nagiosbaddisk=nagiosbaddisk+1 + else: + nagiosgooddisk=nagiosgooddisk+1 + + mlen = returnWdthFromArrayCol(arraydisk,1) + flen = returnWdthFromArrayCol(arraydisk,3) + + # Adjust print format with widths computed above + drvfmt = "%-7s | %-4s | %-"+str(mlen)+"s | %-8s | %-"+str(flen+2)+"s | %-8s | %-4s | %-8s | %-8s" + i = 0 + for array in arraydisk: + # Header + if ( i == 0 ): + if not nagiosmode: + print drvfmt % ( + "-- ID", "Type", "Drive Model", "Size", "Status", "Speed", "Temp", "Slot ID", "LSI Device ID") + # Drive information + if not nagiosmode: + print drvfmt % ( + str('c'+str(controllerid)+'uXpY'), # cXpY + array[0], # HDD/SDD + array[1], # Model Information (Variable len) + array[2], # Size + array[3], # Status (Variable len) + array[4], # Speed + array[5], # Temp + str('['+array[6]+':'+array[7]+']'), # Slot ID + array[8]) # LSI ID + i = i + 1 + controllerid += 1 + if not nagiosmode: + print '' + +if nagiosmode: + if bad: + print 'RAID ERROR - Arrays: OK:'+str(nagiosgoodarray)+' Bad:'+str(nagiosbadarray)+' - Disks: OK:'+str(nagiosgooddisk)+' Bad:'+str(nagiosbaddisk) + sys.exit(2) + else: + print 'RAID OK - Arrays: OK:'+str(nagiosgoodarray)+' Bad:'+str(nagiosbadarray)+' - Disks: OK:'+str(nagiosgooddisk)+' Bad:'+str(nagiosbaddisk) +else: + if bad: + print '\nThere is at least one disk/array in a NOT OPTIMAL state.' + sys.exit(1) From fbf4618afe503101db92665f4ac4e4f6248d5a8c Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:27:26 +0200 Subject: [PATCH 081/543] Fixing bugs in map disk types to osd script --- map-osd-to-disktype | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) mode change 100644 => 100755 map-osd-to-disktype diff --git a/map-osd-to-disktype b/map-osd-to-disktype old mode 100644 new mode 100755 index f1a2ff8..0a77dae --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -1,13 +1,17 @@ # Assumes, osd is up and +# +OSDTYPE=ssd + + # Tempfile -./megacli-status > /tmp/megaclisas-status.out +/opt/ungleich-tools/megaclisas-status > /tmp/megaclisas-status.out # Gets osd numbers of a particular device class, and gets their mountpoints on the host, and puts them into a tempfile -for osd in $(ceph osd tree | grep $OSD | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep $osd| cut -c 24- >> /tmp/list_osd_mountpoint; done +for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep $osd | cut -c 24- >> /tmp/list_osd_mountpoint.out; done # Gets the Megacli mappings for the mountpoints -for MOUNT in $(cat /tmp/list_osd_mountpoint | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done +for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done # Gets the hardware types for the Megacli mappings -for i in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $i | cut -d '|' -f 3 >> /tmp/disk_types; done +for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 3 >> /tmp/disk_types.out; done From 1b30ab71b626e5cd4446227ef7f5b91f81705c32 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:36:20 +0200 Subject: [PATCH 082/543] Fixing bugs in map disk types to osd script. Combining the outputs is needed, then it's complete --- map-osd-to-disktype | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 0a77dae..51f850e 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -8,7 +8,7 @@ OSDTYPE=ssd /opt/ungleich-tools/megaclisas-status > /tmp/megaclisas-status.out # Gets osd numbers of a particular device class, and gets their mountpoints on the host, and puts them into a tempfile -for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep $osd | cut -c 24- >> /tmp/list_osd_mountpoint.out; done +for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep "ceph-$osd " | cut -c 24- >> /tmp/list_osd_mountpoint.out; done # Gets the Megacli mappings for the mountpoints for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done From 6ecea763f9300dfb63e9b6824f14065f1308073b Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:52:02 +0200 Subject: [PATCH 083/543] map-osd-to-disktype v0.1, it works but haven't been tested extensively --- map-osd-to-disktype | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 51f850e..66e0f28 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -15,3 +15,13 @@ for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9 # Gets the hardware types for the Megacli mappings for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 3 >> /tmp/disk_types.out; done + +# Formatting, to get the local $OSDTYPE osd-s +cat /tmp/list_osd_mountpoint.out | awk '{print $1}' > /tmp/local_osds + +# Combine and display the outputs +paste /tmp/local_osds /tmp/disk_types.out + + +# Cleanup *.out files in the temp dir +rm /tmp/*.out From 9c567bb057fd7712fd1f9835cbe171dd0b4b87a0 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:56:25 +0200 Subject: [PATCH 084/543] map-osd-to-disktype v0.2, formatting output --- map-osd-to-disktype | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 66e0f28..552804a 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -17,10 +17,10 @@ for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9 for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 3 >> /tmp/disk_types.out; done # Formatting, to get the local $OSDTYPE osd-s -cat /tmp/list_osd_mountpoint.out | awk '{print $1}' > /tmp/local_osds +for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}'); do echo osd-$osd_num >> /tmp/local_osds.out; done # Combine and display the outputs -paste /tmp/local_osds /tmp/disk_types.out +paste /tmp/local_osds.out /tmp/disk_types.out # Cleanup *.out files in the temp dir From 058a1d0baa8f24dc1f1519c297be0653c51b952e Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 19:21:33 +0200 Subject: [PATCH 085/543] map-osd-to-disktype v1, changed from findmnt to lsblk, added more info to the output --- map-osd-to-disktype | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 552804a..4150bda 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -1,23 +1,24 @@ # Assumes, osd is up and # -OSDTYPE=ssd +#OSDTYPE=ssd # Tempfile /opt/ungleich-tools/megaclisas-status > /tmp/megaclisas-status.out # Gets osd numbers of a particular device class, and gets their mountpoints on the host, and puts them into a tempfile -for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep "ceph-$osd " | cut -c 24- >> /tmp/list_osd_mountpoint.out; done +#for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep "ceph-$osd " | cut -c 24- >> /tmp/list_osd_mountpoint.out; done +for osd in $(ceph osd tree | grep 'ssd\|hdd' | grep -v down | cut -b 1-3); do lsblk -p -o NAME,MOUNTPOINT | grep -w "/var/lib/ceph/osd/ceph-$osd" | cut -c 3- >> /tmp/list_osd_mountpoint.out ; done # Gets the Megacli mappings for the mountpoints -for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done +for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done # Gets the hardware types for the Megacli mappings -for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 3 >> /tmp/disk_types.out; done +for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 2-6,8 >> /tmp/disk_types.out; done # Formatting, to get the local $OSDTYPE osd-s -for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}'); do echo osd-$osd_num >> /tmp/local_osds.out; done +for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24- ); do echo osd-$osd_num >> /tmp/local_osds.out; done # Combine and display the outputs paste /tmp/local_osds.out /tmp/disk_types.out From 4309c1639d97532edb5431cca63364c6c3f34aef Mon Sep 17 00:00:00 2001 From: root Date: Wed, 27 May 2020 19:42:27 +0200 Subject: [PATCH 086/543] [map-osd-to-disktype] fixing bug: grep matched on c0u1 and c0u10 at the same time, causing an extra entry to appear, and messing up the output --- map-osd-to-disktype | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 4150bda..43a7a29 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -15,7 +15,7 @@ for osd in $(ceph osd tree | grep 'ssd\|hdd' | grep -v down | cut -b 1-3); do ls for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done # Gets the hardware types for the Megacli mappings -for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 2-6,8 >> /tmp/disk_types.out; done +for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep -w "$megacli_mappings"p0 | cut -d '|' -f 2-6,8 >> /tmp/disk_types.out; done # Formatting, to get the local $OSDTYPE osd-s for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24- ); do echo osd-$osd_num >> /tmp/local_osds.out; done From 97c1893dce92d8bf823aa58b37ec1a476b9ad302 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 20:13:39 +0200 Subject: [PATCH 087/543] [map-osd-to-disktype] improved formatting, printf fixed size, so the columns match every time! (probably will have a bug if we reach 99+ osds) --- map-osd-to-disktype | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 43a7a29..7bafb75 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -18,11 +18,11 @@ for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}' | sed 's/[0-9 for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep -w "$megacli_mappings"p0 | cut -d '|' -f 2-6,8 >> /tmp/disk_types.out; done # Formatting, to get the local $OSDTYPE osd-s -for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24- ); do echo osd-$osd_num >> /tmp/local_osds.out; done +for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24- ); do printf "%-7s%s\n" "osd-$osd_num" >> /tmp/local_osds.out; done # Combine and display the outputs -paste /tmp/local_osds.out /tmp/disk_types.out +paste /tmp/local_osds.out /tmp/disk_types.out -d '|' # Cleanup *.out files in the temp dir -rm /tmp/*.out +#rm /tmp/*.out From 2dfaedc85875f96bd5662a48bc9326c6edffe51d Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 20:16:58 +0200 Subject: [PATCH 088/543] [map-osd-to-disktype] bugfix: debug feature was added accidentally (no cleanup) --- map-osd-to-disktype | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 7bafb75..54d01fa 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -25,4 +25,4 @@ paste /tmp/local_osds.out /tmp/disk_types.out -d '|' # Cleanup *.out files in the temp dir -#rm /tmp/*.out +rm /tmp/*.out From a78649c6abdc95dce38cba2c7f2baa1d5853ee2b Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 20:41:21 +0200 Subject: [PATCH 089/543] [map-osd-to-disktype] added header, checked for bashism, should be posix compatible --- map-osd-to-disktype | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 54d01fa..c609c87 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -1,6 +1,15 @@ -# Assumes, osd is up and +#!/bin/sh +# This script will find the locally active osd-s and display their information with the help of Megacli tools. +# Assumes that you run it on a host which has at least 1 osd that matches hdd or ssd disk device class in ceph osd df tree output. +# +# An example for usage: +# for NUM in 14 2 3 4 6 ; do printf "server$NUM\n" >> /tmp/osd_infos; ssh root@server"$NUM".place6.ungleich.ch "/opt/ungleich-tools/map-osd-to-disktype" >> /tmp/osd_infos ; printf "\n \n" >> /tmp/osd_infos; done # +# llnu's most hacky/advanced script at the time of writing (2020-05-27) + + +# Future functionality (arguments, and filtering): #OSDTYPE=ssd @@ -23,6 +32,5 @@ for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24 # Combine and display the outputs paste /tmp/local_osds.out /tmp/disk_types.out -d '|' - # Cleanup *.out files in the temp dir rm /tmp/*.out From e4d3062d45c6240d227bda4c00c5b80864581f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 30 May 2020 07:50:27 +0200 Subject: [PATCH 090/543] Bump alpine script to 3.12 --- opennebula-images/alpine-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh index 0a074b4..1ec6f3f 100755 --- a/opennebula-images/alpine-build-opennebula-image.sh +++ b/opennebula-images/alpine-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=v3.11 +RELEASE=v3.12 ARCH=x86_64 IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G From b96c39ccb6fe6deb45003747025a2c8d61f4a2c4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 16:46:44 +0200 Subject: [PATCH 091/543] Setup hostname for debian/devuan --- debian-devuan-netboot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index d64de5e..5bc2d66 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -30,6 +30,7 @@ echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > $ chroot ${dir} apt update chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 +echo "unconfigured-host" > ${dir}/etc/hostname cp ${dir}/boot/vmlinuz-* ${kernel} @@ -67,7 +68,7 @@ ifup "\${dev}" exit 0 EOF -chmod a+rx ${dir}/etc/rc.local" +chmod a+rx "${dir}/etc/rc.local" # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init From 2a624e84422ebd51c3d2b49392156425e7e688a6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 17:20:12 +0200 Subject: [PATCH 092/543] Fix ssh dir and init link --- debian-devuan-netboot.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 5bc2d66..0dd9d97 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -42,6 +42,9 @@ for key in balazs dominique jinguk nico; do curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys done +# Fix possible permission issue from above +chown -R root:root ${dir}/root/ + ################################################################################ # networking @@ -73,7 +76,7 @@ chmod a+rx "${dir}/etc/rc.local" # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same -ln -s /sbin/init ${dir}/init +ln -fs /sbin/init ${dir}/init # Finally building the initramfs ( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From 2cfcca567959371ad6a0a51507dd276fa715b7c6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 18:31:06 +0200 Subject: [PATCH 093/543] cleanup the debian/devuan build script --- debian-devuan-netboot.sh | 43 ++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 0dd9d97..db56d64 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -2,7 +2,8 @@ # Nico Schottelius, 2019-12-09 # the ugly code is llnu -#this can only run in the ungleich-tools directory because of the cat magiccommand........ +set -e +set -x if [ $# -ne 2 ]; then echo $0 suite out-directory @@ -15,41 +16,45 @@ suite=$1; shift outdir=$1; shift date=$(date +%F) +mkdir -p ${outdir} basename=${suite}-${date} -dir=${outdir}/${basename} -kernel=${outdir}/kernel-${basename} -initramfs=${outdir}/initramfs-${basename} +abs_outdir=$(cd ${outdir} && pwd -P) + +chroot_dir=${abs_outdir}/${basename} +kernel=${abs_outdir}/kernel-${basename} +initramfs=${abs_outdir}/initramfs-${basename} + keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files -debootstrap "${suite}" "${dir}" +debootstrap "${suite}" "${chroot_dir}" # need non-free for firmware-bnx2 -echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${dir}/etc/apt/sources.list +echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${chroot_dir}/etc/apt/sources.list -chroot ${dir} apt update -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 +chroot ${chroot_dir} apt update +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 inetutils-ping -echo "unconfigured-host" > ${dir}/etc/hostname +echo "unconfigured-host" > ${chroot_dir}/etc/hostname -cp ${dir}/boot/vmlinuz-* ${kernel} +cp ${chroot_dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${chroot_dir}/etc/cron.d/ipv6addr -mkdir -p ${dir}/root/.ssh +mkdir -p ${chroot_dir}/root/.ssh for key in balazs dominique jinguk nico; do - curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys + curl -s ${keyurl}/${key}.pub >> ${chroot_dir}/root/.ssh/authorized_keys done # Fix possible permission issue from above -chown -R root:root ${dir}/root/ +chown -R root:root ${chroot_dir}/root/ ################################################################################ # networking # always lo -cat > ${dir}/etc/network/interfaces << EOF +cat > ${chroot_dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback @@ -57,7 +62,7 @@ source-directory /etc/network/interfaces.d EOF # find the boot interfaces at boot -cat > ${dir}/etc/rc.local < ${chroot_dir}/etc/rc.local < otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same -ln -fs /sbin/init ${dir}/init +ln -fs /sbin/init ${chroot_dir}/init # Finally building the initramfs -( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) +( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From e23de2ca3b603fdebb9752d168f959f1af25c97d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 18:31:24 +0200 Subject: [PATCH 094/543] ignore ipxe build directory --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f8835d9..f4be182 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ opennebula-vm-etcd/config-and-secrets.conf .idea .vscode + +ipxe/ From a7d53cae1bf14f6ccc55a7261f13e01eb4a701fc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 20:14:13 +0200 Subject: [PATCH 095/543] add script to revert to old style iptables --- debian-use-old-iptables | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 debian-use-old-iptables diff --git a/debian-use-old-iptables b/debian-use-old-iptables new file mode 100644 index 0000000..50d47c1 --- /dev/null +++ b/debian-use-old-iptables @@ -0,0 +1,9 @@ +#!/bin/sh + +# reverting for a running system that still needs access to old style +# rules + +update-alternatives --set iptables /usr/sbin/iptables-legacy +update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy +update-alternatives --set arptables /usr/sbin/arptables-legacy +update-alternatives --set ebtables /usr/sbin/ebtables-legacy From 4cad7eb28c4d15d0480d0124c4dd8ab76e3b6f20 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 20:14:28 +0200 Subject: [PATCH 096/543] + correct permissions --- debian-devuan-netboot.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index db56d64..5111f08 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -85,3 +85,7 @@ ln -fs /sbin/init ${chroot_dir}/init # Finally building the initramfs ( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) + +# Fix paranoid permissions +chmod a+rx ${abs_outdir} +chmod a+r ${kernel} ${initramfs} From 47d15a2349d150f1677d6d9dee3f6da93f0ce3a3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 22:44:50 +0200 Subject: [PATCH 097/543] do not install broken ping --- debian-devuan-netboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 5111f08..b703e60 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -33,7 +33,7 @@ debootstrap "${suite}" "${chroot_dir}" echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${chroot_dir}/etc/apt/sources.list chroot ${chroot_dir} apt update -chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 inetutils-ping +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 echo "unconfigured-host" > ${chroot_dir}/etc/hostname From 6a3750a6cf93e3c5fabcff0746731774201a70bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 22:44:57 +0200 Subject: [PATCH 098/543] + correct mode --- debian-use-old-iptables | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 debian-use-old-iptables diff --git a/debian-use-old-iptables b/debian-use-old-iptables old mode 100644 new mode 100755 From 8dfb2c95e62f407c4524008aff3497656fcadeab Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 13 Jun 2020 21:40:10 +0200 Subject: [PATCH 099/543] phase in script for setting up viirbs --- .gitignore | 2 + setup-viirb.sh | 213 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100755 setup-viirb.sh diff --git a/.gitignore b/.gitignore index f4be182..1de0e42 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ opennebula-vm-etcd/config-and-secrets.conf .vscode ipxe/ + +openwrt-*-*.bin diff --git a/setup-viirb.sh b/setup-viirb.sh new file mode 100755 index 0000000..79483c6 --- /dev/null +++ b/setup-viirb.sh @@ -0,0 +1,213 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 4 ]; then + echo "$0 interface viirb-id your-dot-cdist" + echo " interface to add the config ip address to" + echo " viirb-id: number in decimal format" + echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" + echo " owner-mail-reference: How to identify the owner" + echo "" + echo "Example to configure viirb02:" + echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342'" + exit 1 +fi + +set -x + +set -x +dev=$1; shift +id=$1; shift +dot_cdist=$1; shift +owner=$1; shift + +hex_id=$(printf "%0.2x\n" "$id") +viirb_hostname=viirb${hex_id} + +prefix_base=2a0a:e5c1:3 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +# openwrt +version=19.07.3 +filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin + +# IP address for setting it up initially +viirb_ip=192.168.61.1 + +# wireguard +private_key=$(wg genkey) +private_key=EL76tScnk84v8TGSSD3tPDhUjjYVPrfmNMBE3zbuRXg= +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch +vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= + +# cdist +dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files +peerfile=${dot_cdist_files}/${vpn_endpoint_host}.peer${hex_id} +vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} + +# Configure VPN server / update cdist +echo Updating VPNserver +cat < ${peerfile} +# ${viirb_hostname} ${owner} +[Peer] +PublicKey = ${public_key} +AllowedIPs = ${my_network} + +EOF + +# Generate real config +cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} +cd ${dot_cdist_files} +git add ${vpn_endpoint_host} +git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" +git pull +git push + +cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} + +exit 0 + +# System +cat </dev/null || true +sudo ip addr add 192.168.61.2/24 dev "$dev" + +# don't care about other/old known_host entries +ssh-keygen -R 192.168.61.1 + +ping -c2 ${viirb_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach any VIIRB - exiting" + exit 1 +fi + +set -e + + +# Don't re-download if we already have it +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} +scp ${filename} root@${viirb_ip}:/tmp +ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" + +wait=0 +found="" + +while [ $wait -lt 180 ]; do + ping -c1 ${viirb_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + fi +done + +if [ ! "$found" ]; then + echo "Did not find updated viirb - debug / restart it" + exit 1 +fi + +exit 0 From 4ca2a762fdcfcfe5c881794c7036625d6966722e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 00:07:47 +0200 Subject: [PATCH 100/543] update script to have stages --- setup-viirb.sh | 270 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 205 insertions(+), 65 deletions(-) diff --git a/setup-viirb.sh b/setup-viirb.sh index 79483c6..2f8ba82 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -2,18 +2,28 @@ # 2020-06-13, Nico Schottelius # See https://ungleich.ch/u/products/viirb-ipv6-box/ -if [ $# -ne 4 ]; then - echo "$0 interface viirb-id your-dot-cdist" +if [ $# -lt 4 ]; then + echo "$0 interface viirb-id your-dot-cdist [stages]" echo " interface to add the config ip address to" echo " viirb-id: number in decimal format" echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" echo " owner-mail-reference: How to identify the owner" + echo " stages: define which stages to execute" + echo "" + echo " stage1: setup your host, check connection to VIIRB" + echo " stage2: flash latest openwrt onto the VIIRB" + echo " stage3: configure the vpn endpoint" + echo " stage4: configure the VIIRB with wireguard + settings" + echo " stage5: Verify VIIRB on VPN, cleanup VIIRB" echo "" echo "Example to configure viirb02:" echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342'" + echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342' '1 3 4'" exit 1 fi +echo "FIXME: missing IPv6 announcements on LAN" + set -x set -x @@ -22,6 +32,12 @@ id=$1; shift dot_cdist=$1; shift owner=$1; shift +if [ $# -ge 1 ]; then + stages=$1; shift +else + stages="1 2 3 4 5" +fi + hex_id=$(printf "%0.2x\n" "$id") viirb_hostname=viirb${hex_id} @@ -37,12 +53,14 @@ my_wifi_ip=${my_prefix}:7ea::42 version=19.07.3 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin +# root password +root_password=$(pwgen -1 32) + # IP address for setting it up initially viirb_ip=192.168.61.1 # wireguard private_key=$(wg genkey) -private_key=EL76tScnk84v8TGSSD3tPDhUjjYVPrfmNMBE3zbuRXg= public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch @@ -50,12 +68,81 @@ vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= # cdist dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files -peerfile=${dot_cdist_files}/${vpn_endpoint_host}.peer${hex_id} +peerfilename=${vpn_endpoint_host}.peer${hex_id} +peerfile=${dot_cdist_files}/${peerfilename} vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} -# Configure VPN server / update cdist -echo Updating VPNserver -cat < ${peerfile} +################################################################################ +# Stage 1: test / connect to the new VIIRB +# +# We delete so that we can run idempotent +stage1() +{ + sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true + sudo ip addr add 192.168.61.2/24 dev "$dev" + + # don't care about other/old known_host entries + ssh-keygen -R ${viirb_ip} + + ping -c2 ${viirb_ip} + if [ $? -ne 0 ]; then + echo "Cannot reach any VIIRB - exiting" + exit 1 + fi + + cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_keys" +} + + + +################################################################################ +# Get latest OpenWRT & flash it +stage2() +{ + # Don't re-download if we already have it + wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} + scp ${filename} root@${viirb_ip}:/tmp + ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" + + # It still pings for some time - wait for the reboot to happen + echo "Waiting for VIIRB to disappear" + sleep 15 + + wait=0 + found="" + + while [ $wait -lt 180 ]; do + ping -c1 ${viirb_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + break + fi + + sleep 1 + wait=$((wait+1)) + done + + if [ ! "$found" ]; then + echo "Did not find updated viirb - debug / restart it" + exit 1 + fi + +} + + +################################################################################ +# Stage 3: prepare VPN endpoint +# + +stage3() +{ + + # Configure VPN server / update cdist + echo Updating VPNserver + cat < ${peerfile} # ${viirb_hostname} ${owner} [Peer] PublicKey = ${public_key} @@ -63,28 +150,32 @@ AllowedIPs = ${my_network} EOF -# Generate real config -cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} -cd ${dot_cdist_files} -git add ${vpn_endpoint_host} -git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" -git pull -git push + # Generate real config + cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} + cd ${dot_cdist_files} + git add ${vpn_endpoint_host} ${peerfilename} + git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" + git pull + git push -cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} + cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} +} -exit 0 +################################################################################ +# Stage 4: configure the VIIRB +# +stage4() +{ -# System -cat </dev/null || true -sudo ip addr add 192.168.61.2/24 dev "$dev" + while [ $wait -lt 180 ]; do + ping -c1 ${my_wireguard_ip} >/dev/null -# don't care about other/old known_host entries -ssh-keygen -R 192.168.61.1 + if [ $? -eq 0 ]; then + found=yes + break + fi + sleep 1 + wait=$((wait+1)) + done -ping -c2 ${viirb_ip} -if [ $? -ne 0 ]; then - echo "Cannot reach any VIIRB - exiting" - exit 1 -fi + if [ ! "$found" ]; then + echo "Cannot reach VIIRB via VPN - check manually" + exit 1 + fi -set -e + echo "Cleanup process." + echo "Set the root password when prompted to: ${root_password}" + # VPN works, remove artefacts, set correct DNS servers that support DNS64 + cat </dev/null +# Remove our ssh keys +rm -f /etc/dropbear/authorized_keys - if [ $? -eq 0 ]; then - found=yes - # wait for ssh to come up - sleep 10 +# Setup root password +printf "${root_password}\n${root_password}\n" | passwd +EOF + + echo "Submit to user the root password = ${root_password}" + +} + +for stage in $(seq 1 5);do + if echo $stages | grep -q $stage; then + eval stage${stage} fi done - -if [ ! "$found" ]; then - echo "Did not find updated viirb - debug / restart it" - exit 1 -fi - -exit 0 From 345905e70ee46dac7a4ad66f6df66d1d0181a547 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 00:52:07 +0200 Subject: [PATCH 101/543] [viirb] fix the dns problem --- setup-viirb.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup-viirb.sh b/setup-viirb.sh index 2f8ba82..b8d34ea 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -225,6 +225,14 @@ uci set dhcp.wifi.interface='wifi' uci set dhcp.wifi.ra='server' uci set dhcp.wifi.dynamicdhcp='0' +# Fix DNS: make dnsmasq NOT use a resolv.conf +# so that it only reads from our servers with DNS64 enabled +uci set dhcp.@dnsmasq[0].noresolv='1' + +# Fix DNS: make the OS use the locally provided DNS servers +# otherwise the VPN tunnel cannot be established +dhcp.@dnsmasq[0].localuse='0' + # VPN / Wireguard uci set network.wg0=interface From d7c1e5263972436f39de90810245466e50ced213 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 21:20:43 +0200 Subject: [PATCH 102/543] ++debian stuff --- debian-devuan-netboot.sh | 52 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index b703e60..eb0d381 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -33,7 +33,7 @@ debootstrap "${suite}" "${chroot_dir}" echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${chroot_dir}/etc/apt/sources.list chroot ${chroot_dir} apt update -chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 ifenslave vlan echo "unconfigured-host" > ${chroot_dir}/etc/hostname @@ -53,30 +53,52 @@ chown -R root:root ${chroot_dir}/root/ ################################################################################ # networking -# always lo +# echo bonding + cat > ${chroot_dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback -source-directory /etc/network/interfaces.d +auto bond0 +iface bond0 inet manual + bond-miimon 500 + bond-mode 4 + post-up /sbin/ip link set \$IFACE mtu 9000 + bond-slaves none + +auto eth0 +iface eth0 inet manual + bond-master bond0 + post-up /sbin/ip link set \$IFACE mtu 9000 + +auto eth1 +iface eth1 inet manual + bond-master bond0 + post-up /sbin/ip link set \$IFACE mtu 9000 + +# server network +auto bond0.11 +iface bond0.11 inet6 auto + post-up /sbin/ip link set \$IFACE mtu 9000 + vlan-raw-device bond0 EOF -# find the boot interfaces at boot -cat > ${chroot_dir}/etc/rc.local < ${chroot_dir}/etc/rc.local < /etc/network/interfaces.d/bootinterface << eof -auto \$dev -iface \$dev inet6 auto -eof +# cat > /etc/network/interfaces.d/bootinterface << eof +# auto \$dev +# iface \$dev inet6 auto +# eof -ifup "\${dev}" +# ifup "\${dev}" -exit 0 -EOF +# exit 0 +# EOF -chmod a+rx "${chroot_dir}/etc/rc.local" +# chmod a+rx "${chroot_dir}/etc/rc.local" # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init From 061d67377ad89544f8ac1abc7d483731cbe3cb24 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 21:21:01 +0200 Subject: [PATCH 103/543] virb: +dns --- setup-viirb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup-viirb.sh b/setup-viirb.sh index 2f8ba82..e6e2638 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -23,6 +23,7 @@ if [ $# -lt 4 ]; then fi echo "FIXME: missing IPv6 announcements on LAN" +echo "FIXME: DNS situation: upstream, non upstream, ungleich, how to resolve tunnel endpoint" set -x From 9b57f031ce050053019ed8d35fb84f1e71027fb5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 22:22:11 +0200 Subject: [PATCH 104/543] Make stage4 output the wireguard public key --- setup-viirb.sh | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/setup-viirb.sh b/setup-viirb.sh index b8e7ca6..3e60825 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -195,6 +195,12 @@ opkg update opkg install wireguard opkg install luci-app-wireguard +# The IPv6 lan configuration +uci set network.lanv6=interface +uci set network.lanv6.proto='static' +uci set network.lanv6.ip6addr='${my_lan_ip}/64' +uci set network.lanv6.ifname='br-lan' + # wifi ip address uci set network.wifi=interface uci set network.wifi.proto='static' @@ -226,6 +232,23 @@ uci set dhcp.wifi.interface='wifi' uci set dhcp.wifi.ra='server' uci set dhcp.wifi.dynamicdhcp='0' +# LAN / Router advertisements / DHCP +# DHCP: we are not authoratative +uci delete dhcp.@dnsmasq[0].authoritative +uci delete dhcp.lan.dhcpv6 +uci delete dhcp.lan.start +uci delete dhcp.lan.limit +uci delete dhcp.lan.leasetime + +# Do not announce ULA - we have GUA +uci delete network.globals.ula_prefix + +uci set dhcp.lan=dhcp +uci set dhcp.lan.interface='lanv6' +uci set dhcp.lan.ra='server' +uci set dhcp.lan.dynamicdhcp='0' + + # Fix DNS: make dnsmasq NOT use a resolv.conf # so that it only reads from our servers with DNS64 enabled uci set dhcp.@dnsmasq[0].noresolv='1' @@ -234,6 +257,10 @@ uci set dhcp.@dnsmasq[0].noresolv='1' # otherwise the VPN tunnel cannot be established dhcp.@dnsmasq[0].localuse='0' +# DNS upstream over VPN gives DNS64 +uci delete dhcp.@dnsmasq[0].server +uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:0:a::a' +uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:2:a::a' # VPN / Wireguard uci set network.wg0=interface @@ -292,7 +319,7 @@ uci set firewall.@rule[-1].target='ACCEPT' fi # Add interfaces to the right network zone -uci set firewall.@zone[0].network='lan lanv4 wifi' +uci set firewall.@zone[0].network='lan lanv6 wifi' uci set firewall.@zone[1].network='wg0' uci commit firewall @@ -301,6 +328,7 @@ uci commit firewall reboot EOF + echo "Wireguard public key: ${public_key}" } ################################################################################ @@ -334,14 +362,8 @@ stage5() # VPN works, remove artefacts, set correct DNS servers that support DNS64 cat < Date: Sun, 14 Jun 2020 22:31:03 +0200 Subject: [PATCH 105/543] ensure viirb has internet before updating pkg db --- setup-viirb.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup-viirb.sh b/setup-viirb.sh index 3e60825..d5337de 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -188,6 +188,10 @@ uci set network.lanv4temp.netmask='255.255.255.0' uci commit network /etc/init.d/network restart +# ensure internet is up and running +sleep 3 +ping -c5 ungleich.ch + # update the sources opkg update From 8921f6ac7d5002e9fb1cbcce73f09186bfe0bfd9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 22:39:49 +0200 Subject: [PATCH 106/543] add new single use script for the viirb --- setup-viirb.sh | 3 +++ viirb-cleanup-final-stage.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 viirb-cleanup-final-stage.sh diff --git a/setup-viirb.sh b/setup-viirb.sh index d5337de..63286cb 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -368,6 +368,9 @@ stage5() cat < Date: Sun, 14 Jun 2020 22:50:43 +0200 Subject: [PATCH 107/543] ++viirb --- viirb-cleanup-final-stage.sh | 0 viirb-configure-vpnendpoint.sh | 52 ++++++++++++++++++++++++++++ setup-viirb.sh => viirb-setup-all.sh | 0 3 files changed, 52 insertions(+) mode change 100644 => 100755 viirb-cleanup-final-stage.sh create mode 100755 viirb-configure-vpnendpoint.sh rename setup-viirb.sh => viirb-setup-all.sh (100%) diff --git a/viirb-cleanup-final-stage.sh b/viirb-cleanup-final-stage.sh old mode 100644 new mode 100755 diff --git a/viirb-configure-vpnendpoint.sh b/viirb-configure-vpnendpoint.sh new file mode 100755 index 0000000..96186fc --- /dev/null +++ b/viirb-configure-vpnendpoint.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 3 ]; then + echo "$0 viirb-id public-key your-dot-cdist [stages]" + echo " viirb-id: number in decimal format" + echo " wireguard public key" + echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" + exit 1 +fi + +set -x +id=$1; shift +public_key=$1; shift +dot_cdist=$1; shift + +hex_id=$(printf "%0.2x\n" "$id") +viirb_hostname=viirb${hex_id} + +prefix_base=2a0a:e5c1:3 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 + +vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch + +# cdist +dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files +peerfilename=${vpn_endpoint_host}.peer${hex_id} +peerfile=${dot_cdist_files}/${peerfilename} +vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} + + +# Configure VPN server / update cdist +echo Updating VPNserver +cat < ${peerfile} +# ${viirb_hostname}, $(date +%F) +[Peer] +PublicKey = ${public_key} +AllowedIPs = ${my_network} + +EOF + +# Generate real config +cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} +cd ${dot_cdist_files} +git add ${vpn_endpoint_host} ${peerfilename} +git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" +git pull +git push + +cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} diff --git a/setup-viirb.sh b/viirb-setup-all.sh similarity index 100% rename from setup-viirb.sh rename to viirb-setup-all.sh From c8036796101fd77e89da7a87bd220ff56093efe7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 22:54:40 +0200 Subject: [PATCH 108/543] ++viirb cleanup --- viirb-cleanup-final-stage.sh | 18 +++++++++++++++--- viirb-configure-vpnendpoint.sh | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/viirb-cleanup-final-stage.sh b/viirb-cleanup-final-stage.sh index ee7c0f6..a9a116a 100755 --- a/viirb-cleanup-final-stage.sh +++ b/viirb-cleanup-final-stage.sh @@ -5,9 +5,23 @@ set -e set -x +if [ $# -ne 2 ]; then + echo "$0 viirb-ip-address viirb-id" + echo " viirb-ip-address: where to find the viirb" + echo " viirb-id: number in decimal format" + exit 1 +fi + viirb_ip=$1; shift + +id=$1; shift +viirb_hostname=viirb${id} + root_password=$(pwgen -1 32) +# Save for sending to user +# FIXME: future make this more easy / better to transfer +echo $root_password > ${viirb_hostname}.rootpw cat < Date: Sun, 14 Jun 2020 23:01:16 +0200 Subject: [PATCH 109/543] viirb: flash latest firmware --- viirb-connect-flash-latest-openwrt.sh | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 viirb-connect-flash-latest-openwrt.sh diff --git a/viirb-connect-flash-latest-openwrt.sh b/viirb-connect-flash-latest-openwrt.sh new file mode 100755 index 0000000..e6f794e --- /dev/null +++ b/viirb-connect-flash-latest-openwrt.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 1 ]; then + echo "$0 interface" + echo " interface to add the config ip address to" + exit 1 +fi + +set -x +dev=$1; shift +id=$1; shift +dot_cdist=$1; shift +owner=$1; shift + +# openwrt +version=19.07.3 +filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin + +# IP address for setting it up initially +viirb_ip=192.168.61.1 + +sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true +sudo ip addr add 192.168.61.2/24 dev "$dev" + +# don't care about other/old known_host entries +ssh-keygen -R ${viirb_ip} + +ping -c2 ${viirb_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach any VIIRB - exiting" + exit 1 +fi + +cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_keys" + +# Don't re-download if we already have it +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} +scp ${filename} root@${viirb_ip}:/tmp +ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" + +# It still pings for some time - wait for the reboot to happen +echo "Waiting for VIIRB to disappear" +sleep 15 + +wait=0 +found="" + +while [ $wait -lt 180 ]; do + ping -c1 ${viirb_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + break + fi + + sleep 1 + wait=$((wait+1)) + done + +if [ ! "$found" ]; then + echo "Did not find updated viirb - debug / restart it" + exit 1 +fi + +echo "VIIRB successfully updated to ${version}" From a2ef38267af28c01ad733156c9115bbefbcacf84 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 23:01:59 +0200 Subject: [PATCH 110/543] remove unused vars --- viirb-connect-flash-latest-openwrt.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/viirb-connect-flash-latest-openwrt.sh b/viirb-connect-flash-latest-openwrt.sh index e6f794e..28ba121 100755 --- a/viirb-connect-flash-latest-openwrt.sh +++ b/viirb-connect-flash-latest-openwrt.sh @@ -10,9 +10,6 @@ fi set -x dev=$1; shift -id=$1; shift -dot_cdist=$1; shift -owner=$1; shift # openwrt version=19.07.3 From 53e827ba43aefbd4e9399c2a5900b88247b15f39 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 23:09:06 +0200 Subject: [PATCH 111/543] viirb: script to fully configure --- viirb-configure-fully-after-upgrade.sh | 199 +++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100755 viirb-configure-fully-after-upgrade.sh diff --git a/viirb-configure-fully-after-upgrade.sh b/viirb-configure-fully-after-upgrade.sh new file mode 100755 index 0000000..bd86c25 --- /dev/null +++ b/viirb-configure-fully-after-upgrade.sh @@ -0,0 +1,199 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 2 ]; then + echo "$0 viirb-ip-address viirb-id" + echo " viirb-ip-address: where to find the viirb" + echo " viirb-id: number in decimal format" + exit 1 +fi + + +set -x +viirb_ip=$1; shift +id=$1; shift +hex_id=$(printf "%0.2x\n" "$id") +viirb_hostname=viirb${id} + +prefix_base=2a0a:e5c1:3 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +# wireguard +private_key=$(wg genkey) +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch +vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= + +cat < ${viirb_hostname}.public_key From b7f84307b2b487c2ab6ac064a30ae3db225e11c6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 23:21:38 +0200 Subject: [PATCH 112/543] Change configuration to drop the conflicting ipv4 address --- viirb-configure-fully-after-upgrade.sh | 120 +++++++++++++------------ 1 file changed, 62 insertions(+), 58 deletions(-) diff --git a/viirb-configure-fully-after-upgrade.sh b/viirb-configure-fully-after-upgrade.sh index bd86c25..1868cd9 100755 --- a/viirb-configure-fully-after-upgrade.sh +++ b/viirb-configure-fully-after-upgrade.sh @@ -33,7 +33,6 @@ vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= cat < Date: Sun, 14 Jun 2020 23:32:38 +0200 Subject: [PATCH 113/543] add debug for reaching new v6 address --- ...latest-openwrt.sh => viirb-1-connect-flash-latest-openwrt.sh | 0 ...after-upgrade.sh => viirb-2-configure-fully-after-upgrade.sh | 2 ++ ...configure-vpnendpoint.sh => viirb-3-configure-vpnendpoint.sh | 0 3 files changed, 2 insertions(+) rename viirb-connect-flash-latest-openwrt.sh => viirb-1-connect-flash-latest-openwrt.sh (100%) rename viirb-configure-fully-after-upgrade.sh => viirb-2-configure-fully-after-upgrade.sh (99%) rename viirb-configure-vpnendpoint.sh => viirb-3-configure-vpnendpoint.sh (100%) diff --git a/viirb-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh similarity index 100% rename from viirb-connect-flash-latest-openwrt.sh rename to viirb-1-connect-flash-latest-openwrt.sh diff --git a/viirb-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh similarity index 99% rename from viirb-configure-fully-after-upgrade.sh rename to viirb-2-configure-fully-after-upgrade.sh index 1868cd9..3a4da33 100755 --- a/viirb-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -126,6 +126,8 @@ sleep 10 # change to ipv6 viirb_ip=${my_lan_ip} +ping -c5 ${viirb_ip} + cat < Date: Sun, 14 Jun 2020 23:43:20 +0200 Subject: [PATCH 114/543] [viirb] include reboot to reset resolv.conf --- viirb-2-configure-fully-after-upgrade.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 3a4da33..b96b967 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -62,7 +62,6 @@ uci set dhcp.lan.interface='lanv6' uci set dhcp.lan.ra='server' uci set dhcp.lan.dynamicdhcp='0' - # Fix DNS: make dnsmasq NOT use a resolv.conf # so that it only reads from our servers with DNS64 enabled uci set dhcp.@dnsmasq[0].noresolv='1' @@ -117,7 +116,10 @@ uci delete wireless.radio0.disabled uci commit -/etc/init.d/network restart +# Need to reboot to restore /etc/resolv.conf +reboot + +#/etc/init.d/network restart EOF # ensure network is up and running - bridge takes a bit of time @@ -129,7 +131,6 @@ viirb_ip=${my_lan_ip} ping -c5 ${viirb_ip} cat < Date: Sun, 14 Jun 2020 23:43:30 +0200 Subject: [PATCH 115/543] [viirb] ping ipv6 address in vpn config for verification --- viirb-3-configure-vpnendpoint.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-configure-vpnendpoint.sh index 4c45fdd..d03b90e 100755 --- a/viirb-3-configure-vpnendpoint.sh +++ b/viirb-3-configure-vpnendpoint.sh @@ -3,17 +3,17 @@ # See https://ungleich.ch/u/products/viirb-ipv6-box/ if [ $# -ne 3 ]; then - echo "$0 viirb-id public-key your-dot-cdist [stages]" + echo "$0 your-dot-cdist viirb-id public-key" + echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" echo " viirb-id: number in decimal format" echo " wireguard public key" - echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" exit 1 fi set -x +dot_cdist=$1; shift id=$1; shift public_key=$1; shift -dot_cdist=$1; shift hex_id=$(printf "%0.2x\n" "$id") viirb_hostname=viirb${id} @@ -21,6 +21,9 @@ viirb_hostname=viirb${id} prefix_base=2a0a:e5c1:3 my_prefix=${prefix_base}${hex_id} my_network=${my_prefix}::/48 +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch @@ -50,3 +53,10 @@ git pull git push cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} + +# Test that the VPN connection is established +sleep 10 + +ping -c3 ${my_wireguard_ip} +ping -c3 ${my_lan_ip} +ping -c3 ${my_wifi_ip} From 13257ad00b6e61aec166131a54a14b6c60bf5be3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 23:59:33 +0200 Subject: [PATCH 116/543] sleep longer for waiting for network and reboot --- viirb-2-configure-fully-after-upgrade.sh | 6 ++++-- ...cleanup-final-stage.sh => viirb-4-cleanup-final-stage.sh | 0 2 files changed, 4 insertions(+), 2 deletions(-) rename viirb-cleanup-final-stage.sh => viirb-4-cleanup-final-stage.sh (100%) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index b96b967..b190e94 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -122,8 +122,10 @@ reboot #/etc/init.d/network restart EOF -# ensure network is up and running - bridge takes a bit of time -sleep 10 +# ensure viirb is back +# ensure viirb network is up and running - bridge takes a bit of time +# ensure viirb gives out ipv6 address +sleep 30 # change to ipv6 viirb_ip=${my_lan_ip} diff --git a/viirb-cleanup-final-stage.sh b/viirb-4-cleanup-final-stage.sh similarity index 100% rename from viirb-cleanup-final-stage.sh rename to viirb-4-cleanup-final-stage.sh From 7472aeebf33c816615da51e6067e0b551de8bfb1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 15 Jun 2020 00:17:10 +0200 Subject: [PATCH 117/543] ++delay --- viirb-2-configure-fully-after-upgrade.sh | 3 ++- viirb-3-configure-vpnendpoint.sh | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index b190e94..c7dc10a 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -125,7 +125,8 @@ EOF # ensure viirb is back # ensure viirb network is up and running - bridge takes a bit of time # ensure viirb gives out ipv6 address -sleep 30 +# Found: 30s is not enough for full reconfiguration +sleep 60 # change to ipv6 viirb_ip=${my_lan_ip} diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-configure-vpnendpoint.sh index d03b90e..cd21131 100755 --- a/viirb-3-configure-vpnendpoint.sh +++ b/viirb-3-configure-vpnendpoint.sh @@ -55,8 +55,9 @@ git push cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} # Test that the VPN connection is established +# Might take longer due to reboot sleep 10 -ping -c3 ${my_wireguard_ip} -ping -c3 ${my_lan_ip} -ping -c3 ${my_wifi_ip} +ping -c10 ${my_wireguard_ip} +ping -c10 ${my_lan_ip} +ping -c10 ${my_wifi_ip} From 5b64fd461ee41b2a261dccb279b3a3a1d469d0ca Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 15 Jun 2020 00:27:53 +0200 Subject: [PATCH 118/543] Abort if config cannot find viirb --- viirb-2-configure-fully-after-upgrade.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index c7dc10a..161b39d 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -31,6 +31,12 @@ public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= +ping -c3 ${viirb_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach ${viirb_ip}, aborting" + exit 1 +fi + cat < Date: Mon, 22 Jun 2020 23:05:13 +0200 Subject: [PATCH 119/543] Allow to use ip address instead of device --- viirb-1-connect-flash-latest-openwrt.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index 28ba121..c551ef1 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -2,24 +2,33 @@ # 2020-06-13, Nico Schottelius # See https://ungleich.ch/u/products/viirb-ipv6-box/ -if [ $# -ne 1 ]; then - echo "$0 interface" +if [ $# -lt 1 ]; then + echo "$0 interface [address]" echo " interface to add the config ip address to" + echo " address: connect to this address, ignore the interface" exit 1 fi set -x dev=$1; shift +if [ $# -ge 1 ]; then + viirb_ip=$1; shift + dev="" +else + viirb_ip=192.168.61.1 +fi + # openwrt version=19.07.3 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin # IP address for setting it up initially -viirb_ip=192.168.61.1 -sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true -sudo ip addr add 192.168.61.2/24 dev "$dev" +if [ "$dev" ]; then + sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true + sudo ip addr add 192.168.61.2/24 dev "$dev" +fi # don't care about other/old known_host entries ssh-keygen -R ${viirb_ip} From 8e73c603c8f4ccf32a05594b990568262f221651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 24 Jun 2020 08:53:37 +0200 Subject: [PATCH 120/543] Add fnux's key to devuan netboot image --- debian-devuan-netboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index eb0d381..2858ae4 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -43,7 +43,7 @@ echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > mkdir -p ${chroot_dir}/root/.ssh -for key in balazs dominique jinguk nico; do +for key in fnux balazs dominique jinguk nico; do curl -s ${keyurl}/${key}.pub >> ${chroot_dir}/root/.ssh/authorized_keys done From afaa9ef712a536af0a314e28b0e71877c9fec85f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 29 Jun 2020 08:49:24 +0200 Subject: [PATCH 121/543] Create initial structure for LUKS-encrypted CentOS images --- .../centos-build-luks-opennebula-image.sh | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100755 opennebula-images/centos-build-luks-opennebula-image.sh diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh new file mode 100755 index 0000000..e438df8 --- /dev/null +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -0,0 +1,175 @@ +#!/bin/sh + +# This script generates CentOS images for OpenNebula. + +# Depends on the following packages (as of CentOS 8): +# qemu-img util-linux coreutils dnf curl e2fsprogs cryptsetup + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=7 +ARCH=x86_64 +IMAGE_PATH=centos-$RELEASE-$(date --iso-8601).img +IMAGE_SIZE=10G +LOOPBACK_DEVICE=/dev/loop0 +LUKS_PASSPHRASE=secretsecretsecret +LUKS_DEVICE_NAME=cryptroot +LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + losetup -d "$LOOPBACK_DEVICE" +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/centos-release' ]; then + echo "WARNING: this script has been designed to run on a CentOS system." >&2 + echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base RAW image (no LOOPBACK support in RHEL/CentOS). +qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" +losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +{ +sfdisk --no-reread "$LOOPBACK_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. +# network-scripts.x86_64 : Legacy scripts for manipulating of network devices +run_root dnf -y install network-scripts + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install and configure NTP client. +run_root dnf install -y chrony +run_root systemctl enable chronyd.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <>/mnt/etc/fstab < Date: Mon, 29 Jun 2020 12:25:39 +0200 Subject: [PATCH 122/543] Tune centos image for 7 --- .../centos-build-luks-opennebula-image.sh | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh index e438df8..fcd34c3 100755 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -6,23 +6,21 @@ # qemu-img util-linux coreutils dnf curl e2fsprogs cryptsetup # Run locally (without network) with: -# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=raw set -e set -x -# XXX: Handle command-line arguments? RELEASE=7 ARCH=x86_64 -IMAGE_PATH=centos-$RELEASE-$(date --iso-8601).img +IMAGE_PATH=centos-luks-$RELEASE-$(date --iso-8601).img IMAGE_SIZE=10G LOOPBACK_DEVICE=/dev/loop0 -LUKS_PASSPHRASE=secretsecretsecret +LUKS_PASSPHRASE=secret LUKS_DEVICE_NAME=cryptroot LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" -# TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el7.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm cleanup() { @@ -82,13 +80,18 @@ mount "${LUKS_DEVICE}" /mnt mkdir /mnt/boot mount "${LOOPBACK_DEVICE}p1" /mnt/boot +# AppStream landed in CentOS 8. +if [ $RELEASE >= 8 ]; then + enabled_repos="--enablerepo=BaseOS --enablerepo=AppStream --enablerepo=extras" +else + enabled_repos="--enablerepo=Base --enablerepo=extras" +fi + dnf -y \ --releasever=$RELEASE \ --installroot=/mnt \ --disablerepo='*' \ - --enablerepo=BaseOS \ - --enablerepo=AppStream \ - --enablerepo=extras \ + $enabled_repos \ --setopt=install_weak_deps=False install \ bash basesystem systemd systemd-udev dnf centos-release @@ -147,6 +150,8 @@ kernel_version=$(ls /mnt/boot | grep "vmlinuz.*.$ARCH" | cut -d- -f2-) run_root dracut --force --kver $kernel_version # Configure grub2. +luks_uuid=$(blkid --match-tag UUID --output value "${LOOPBACK_DEVICE}p2") +echo "GRUB_CMDLINE_LINUX='rd.luks.uuid=$luks_uuid'" >> /mnt/etc/default/grub run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg From 4682ed9a3aa35f714a876ae840ddf904fb01a922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 29 Jun 2020 17:46:48 +0200 Subject: [PATCH 123/543] More tuning to CentOS LUKS image --- .../centos-build-luks-opennebula-image.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh index fcd34c3..63effac 100755 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -20,7 +20,7 @@ LUKS_PASSPHRASE=secret LUKS_DEVICE_NAME=cryptroot LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el7.noarch.rpm" +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm cleanup() { @@ -81,10 +81,12 @@ mkdir /mnt/boot mount "${LOOPBACK_DEVICE}p1" /mnt/boot # AppStream landed in CentOS 8. -if [ $RELEASE >= 8 ]; then +if [ $RELEASE -ge 8 ]; then enabled_repos="--enablerepo=BaseOS --enablerepo=AppStream --enablerepo=extras" + release_specific_packages="systemd-udev" else - enabled_repos="--enablerepo=Base --enablerepo=extras" + enabled_repos="--enablerepo=base --enablerepo=extras" + release_specific_packages="dnf" fi dnf -y \ @@ -93,7 +95,7 @@ dnf -y \ --disablerepo='*' \ $enabled_repos \ --setopt=install_weak_deps=False install \ - bash basesystem systemd systemd-udev dnf centos-release + bash basesystem systemd dnf centos-release cryptsetup $release_specific_packages mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts @@ -119,7 +121,9 @@ EOF # See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. # network-scripts.x86_64 : Legacy scripts for manipulating of network devices -run_root dnf -y install network-scripts +if [ $RELEASE -ge 8 ]; then + run_root dnf -y install network-scripts +fi # Install (magic?) one-context RPM and hope things works as expected. curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" @@ -147,7 +151,7 @@ cat > /mnt/etc/dracut.conf.d/virtio-blk.conf < Date: Thu, 2 Jul 2020 21:42:46 +0200 Subject: [PATCH 124/543] adjust monit for broken init script / ceph osds --- monit-ceph-create-start | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/monit-ceph-create-start b/monit-ceph-create-start index 9b9cb58..afcac60 100755 --- a/monit-ceph-create-start +++ b/monit-ceph-create-start @@ -20,13 +20,26 @@ if echo $to_monitor | grep ^osd; then depends="${depends}, ${to_monitor}-whoami" osd="yes" osdid=$(echo $to_monitor | cut -d. -f2) -fi + cat > "$conf" < "$conf" < "$conf" <> "$conf" < Date: Thu, 2 Jul 2020 21:55:15 +0200 Subject: [PATCH 125/543] use pkill for ceph monitors --- monit-ceph-create-start | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monit-ceph-create-start b/monit-ceph-create-start index afcac60..a44efa0 100755 --- a/monit-ceph-create-start +++ b/monit-ceph-create-start @@ -24,7 +24,7 @@ if echo $to_monitor | grep ^osd; then # Generated by $0 check process ${to_monitor} with pidfile /var/run/ceph/${to_monitor}.pid start program = "/usr/bin/ceph-osd -i ${osdid} --pid-file /var/run/ceph/osd.${osdid}.pid -c /etc/ceph/ceph.conf --cluster ceph --setuser ceph --setgroup ceph" with timeout 3600 seconds - + stop program = "/usr/bin/pkill -f '/usr/bin/ceph-osd -i ${osdid}'" EOF else @@ -33,6 +33,7 @@ else # Generated by $0 check process ${to_monitor} with pidfile /var/run/ceph/${to_monitor}.pid start program = "/etc/init.d/ceph start ${to_monitor}" with timeout 60 seconds + stop program = "/etc/init.d/ceph stop ${to_monitor}" EOF @@ -40,7 +41,6 @@ fi # final clause same for both cat >> "$conf" < Date: Mon, 6 Jul 2020 09:35:47 +0200 Subject: [PATCH 126/543] CentOS luks image -> replace sfdisk by parted --- .../centos-build-luks-opennebula-image.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh index 63effac..d577e9f 100755 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -61,12 +61,10 @@ losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" trap cleanup EXIT # Create partition table, format partitions. -{ -sfdisk --no-reread "$LOOPBACK_DEVICE" < Date: Mon, 6 Jul 2020 09:36:10 +0200 Subject: [PATCH 127/543] centos luks image: use local repo dir, crypttab instead of kernel args --- .../centos-build-luks-opennebula-image.sh | 10 +++++----- .../rpm-repositories/centos-7-minus.repo | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 opennebula-images/rpm-repositories/centos-7-minus.repo diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh index d577e9f..f39d937 100755 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -93,6 +93,7 @@ dnf -y \ --disablerepo='*' \ $enabled_repos \ --setopt=install_weak_deps=False install \ + --setopt=reposdir=rpm-repositories \ bash basesystem systemd dnf centos-release cryptsetup $release_specific_packages mount --bind /dev /mnt/dev @@ -144,16 +145,15 @@ run_root systemctl enable chronyd.service # uncompressed but is a direct dependency of kernel-core... run_root dnf -y install kernel grub2 -# Add support for virtio block devices at boot time. +# Add support for virtio block devices at boot time, configure bootloader. cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <> /mnt/etc/default/grub +echo "cryptroot UUID=$luks_uuid luks,timeout=30" >> /mnt/etc/crypttab + +run_root dracut -v --force --kver $kernel_version run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg diff --git a/opennebula-images/rpm-repositories/centos-7-minus.repo b/opennebula-images/rpm-repositories/centos-7-minus.repo new file mode 100644 index 0000000..53bae51 --- /dev/null +++ b/opennebula-images/rpm-repositories/centos-7-minus.repo @@ -0,0 +1,16 @@ +[base] +name=CentOS-$releasever - Base +mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra +#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/ +gpgcheck=0 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#released updates +[updates] +name=CentOS-$releasever - Updates +mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra +#baseurl=http://mirror.centos.org/centos/$releasever/updates/$basearch/ +gpgcheck=0 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 From c9d1ab6e13521601c0d5ba807871f2b3dde86e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 09:44:41 +0200 Subject: [PATCH 128/543] Add missing centos-extras repo --- opennebula-images/rpm-repositories/centos-extras.repo | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 opennebula-images/rpm-repositories/centos-extras.repo diff --git a/opennebula-images/rpm-repositories/centos-extras.repo b/opennebula-images/rpm-repositories/centos-extras.repo new file mode 100644 index 0000000..c53d52a --- /dev/null +++ b/opennebula-images/rpm-repositories/centos-extras.repo @@ -0,0 +1,7 @@ +[extras] +name=CentOS-$releasever - Extras +mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra +#baseurl=http://mirror.centos.org/$contentdir/$releasever/extras/$basearch/os/ +gpgcheck=0 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial From 45f48f3e05ad7ab87c37c07706d723342d124cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 09:59:52 +0200 Subject: [PATCH 129/543] centos-build-luks -> centos7-build-luks, fix blkid calls --- .../centos7-build-luks-opennebula-image.sh | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100755 opennebula-images/centos7-build-luks-opennebula-image.sh diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh new file mode 100755 index 0000000..d5696fc --- /dev/null +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -0,0 +1,182 @@ +#!/bin/sh + +# This script generates CentOS images for OpenNebula. Expected to run on CentOS 7. + +# Depends on the following packages: +# qemu-img util-linux coreutils dnf curl e2fsprogs cryptsetup parted + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=raw + +set -e +set -x + +RELEASE=7 +ARCH=x86_64 +IMAGE_PATH=centos-luks-$RELEASE-$(date --iso-8601).img +IMAGE_SIZE=10G +LOOPBACK_DEVICE=/dev/loop2 +LUKS_PASSPHRASE=secret +LUKS_DEVICE_NAME=cryptroot2 +LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" + +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + losetup -d "$LOOPBACK_DEVICE" +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/centos-release' ]; then + echo "WARNING: this script has been designed to run on a CentOS system." >&2 + echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base RAW image (no LOOPBACK support in RHEL/CentOS). +qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" +losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +parted --script "$LOOPBACK_DEVICE" \ + mklabel msdos \ + mkpart primary ext4 1M 500M \ + mkpart primary ext4 500M 100% + +partprobe "$LOOPBACK_DEVICE" + +mkfs.ext4 "${LOOPBACK_DEVICE}p1" +echo -n "$LUKS_PASSPHRASE" | cryptsetup luksFormat -v -d - "${LOOPBACK_DEVICE}p2" +echo -n "$LUKS_PASSPHRASE" | cryptsetup open -v -d - "${LOOPBACK_DEVICE}p2" "$LUKS_DEVICE_NAME" +mkfs.ext4 "$LUKS_DEVICE" + +# Mount partitions, install base OS. +mount "${LUKS_DEVICE}" /mnt +mkdir /mnt/boot +mount "${LOOPBACK_DEVICE}p1" /mnt/boot + +# AppStream landed in CentOS 8. +if [ $RELEASE -ge 8 ]; then + enabled_repos="--enablerepo=BaseOS --enablerepo=AppStream --enablerepo=extras" + release_specific_packages="systemd-udev" +else + enabled_repos="--enablerepo=base --enablerepo=extras" + release_specific_packages="dnf" +fi + +dnf -y \ + --releasever=$RELEASE \ + --installroot=/mnt \ + --disablerepo='*' \ + $enabled_repos \ + --setopt=install_weak_deps=False install \ + --setopt=reposdir=rpm-repositories \ + bash basesystem systemd dnf centos-release cryptsetup $release_specific_packages + +mount --bind /dev /mnt/dev +mount --bind /dev/pts /mnt/dev/pts +mount --bind /dev/shm /mnt/dev/shm +mount --bind /proc /mnt/proc +mount --bind /run /mnt/run +mount --bind /sys /mnt/sys + +# Guest networking is to be handled by the one-context package. +# See https://github.com/OpenNebula/addon-context-linux for details. +# Note: as of writing, one-context does not support NetworkManager or +# systemd-networkd. + +# Required to resolve package mirror in chroot. +cp /etc/resolv.conf /mnt/etc/resolv.conf + +# Initialize /etc/hosts. +cat > /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. +# network-scripts.x86_64 : Legacy scripts for manipulating of network devices +if [ $RELEASE -ge 8 ]; then + run_root dnf -y install network-scripts +fi + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install and configure NTP client. +run_root dnf install -y chrony +run_root systemctl enable chronyd.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time, configure bootloader. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <> /mnt/etc/crypttab + +run_root dracut -v --force --kver $kernel_version +run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}" +run_root grub2-mkconfig -o /boot/grub2/grub.cfg + +# Install en configure SSH daemon. +run_root dnf -y install openssh-server +run_root systemctl enable sshd + +# Generate fstab file. +boot_uuid=$(blkid -o value "${LOOPBACK_DEVICE}p1" | head -n 1) +root_uuid=$(blkid -o value "$LUKS_DEVICE" | head -n 1) +cat >>/mnt/etc/fstab < Date: Mon, 6 Jul 2020 10:38:17 +0200 Subject: [PATCH 130/543] Remove incomplete centos-8 support centos-luks ONE image --- .../centos-build-luks-opennebula-image.sh | 182 ------------------ .../centos7-build-luks-opennebula-image.sh | 30 +-- 2 files changed, 9 insertions(+), 203 deletions(-) delete mode 100755 opennebula-images/centos-build-luks-opennebula-image.sh diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh deleted file mode 100755 index f39d937..0000000 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/sh - -# This script generates CentOS images for OpenNebula. - -# Depends on the following packages (as of CentOS 8): -# qemu-img util-linux coreutils dnf curl e2fsprogs cryptsetup - -# Run locally (without network) with: -# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=raw - -set -e -set -x - -RELEASE=7 -ARCH=x86_64 -IMAGE_PATH=centos-luks-$RELEASE-$(date --iso-8601).img -IMAGE_SIZE=10G -LOOPBACK_DEVICE=/dev/loop0 -LUKS_PASSPHRASE=secret -LUKS_DEVICE_NAME=cryptroot -LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" - -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" -ONE_CONTEXT_RPM_PATH=/root/one-context.rpm - -cleanup() { - # The order here is important. - umount /mnt/dev/pts 2>/dev/null || true - umount /mnt/dev/shm 2>/dev/null || true - umount /mnt/dev 2>/dev/null || true - umount /mnt/proc 2>/dev/null || true - umount /mnt/run 2>/dev/null || true - umount /mnt/sys 2>/dev/null || true - umount /mnt/boot 2>/dev/null || true - umount /mnt 2>/dev/null || true - losetup -d "$LOOPBACK_DEVICE" -} - -run_root() { - chroot /mnt /usr/bin/env \ - PATH=/sbin:/usr/sbin:/bin:/usr/bin \ - sh -c "$*" -} - -if [ "$(whoami)" != 'root' ]; then - echo "This script must be run as root." >&2 - exit 1 -fi - -if [ ! -f '/etc/centos-release' ]; then - echo "WARNING: this script has been designed to run on a CentOS system." >&2 - echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 - sleep 5 -fi - -# Create base RAW image (no LOOPBACK support in RHEL/CentOS). -qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" -losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" - -# Don't forget to cleanup, even if the script crash. -trap cleanup EXIT - -# Create partition table, format partitions. -parted --script "$LOOPBACK_DEVICE" \ - mklabel msdos \ - mkpart primary ext4 1M 500M \ - mkpart primary ext4 500M 100% - -partprobe "$LOOPBACK_DEVICE" - -mkfs.ext4 "${LOOPBACK_DEVICE}p1" -echo -n "$LUKS_PASSPHRASE" | cryptsetup luksFormat -v -d - "${LOOPBACK_DEVICE}p2" -echo -n "$LUKS_PASSPHRASE" | cryptsetup open -v -d - "${LOOPBACK_DEVICE}p2" "$LUKS_DEVICE_NAME" -mkfs.ext4 "$LUKS_DEVICE" - -# Mount partitions, install base OS. -mount "${LUKS_DEVICE}" /mnt -mkdir /mnt/boot -mount "${LOOPBACK_DEVICE}p1" /mnt/boot - -# AppStream landed in CentOS 8. -if [ $RELEASE -ge 8 ]; then - enabled_repos="--enablerepo=BaseOS --enablerepo=AppStream --enablerepo=extras" - release_specific_packages="systemd-udev" -else - enabled_repos="--enablerepo=base --enablerepo=extras" - release_specific_packages="dnf" -fi - -dnf -y \ - --releasever=$RELEASE \ - --installroot=/mnt \ - --disablerepo='*' \ - $enabled_repos \ - --setopt=install_weak_deps=False install \ - --setopt=reposdir=rpm-repositories \ - bash basesystem systemd dnf centos-release cryptsetup $release_specific_packages - -mount --bind /dev /mnt/dev -mount --bind /dev/pts /mnt/dev/pts -mount --bind /dev/shm /mnt/dev/shm -mount --bind /proc /mnt/proc -mount --bind /run /mnt/run -mount --bind /sys /mnt/sys - -# Guest networking is to be handled by the one-context package. -# See https://github.com/OpenNebula/addon-context-linux for details. -# Note: as of writing, one-context does not support NetworkManager or -# systemd-networkd. - -# Required to resolve package mirror in chroot. -cp /etc/resolv.conf /mnt/etc/resolv.conf - -# Initialize /etc/hosts. -cat > /mnt/etc/hosts << EOF -127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 -::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 - -EOF - -# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. -# network-scripts.x86_64 : Legacy scripts for manipulating of network devices -if [ $RELEASE -ge 8 ]; then - run_root dnf -y install network-scripts -fi - -# Install (magic?) one-context RPM and hope things works as expected. -curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" -run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" -run_root rm "$ONE_CONTEXT_RPM_PATH" - -# Install resize2fs, which is required to resize the root file-system. -run_root dnf -y install e2fsprogs - -# Initalize base services. -run_root systemd-machine-id-setup -run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime - -# Install and configure NTP client. -run_root dnf install -y chrony -run_root systemctl enable chronyd.service - -# Install kernel and bootloader. -# Note: linux-firmware is not required our environment and takes almost 200M -# uncompressed but is a direct dependency of kernel-core... -run_root dnf -y install kernel grub2 - -# Add support for virtio block devices at boot time, configure bootloader. -cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <> /mnt/etc/crypttab - -run_root dracut -v --force --kver $kernel_version -run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}" -run_root grub2-mkconfig -o /boot/grub2/grub.cfg - -# Install en configure SSH daemon. -run_root dnf -y install openssh-server -run_root systemctl enable sshd - -# Generate fstab file. -boot_uuid=$(blkid --match-tag UUID --output value "${LOOPBACK_DEVICE}p1") -root_uuid=$(blkid --match-tag UUID --output value "$LUKS_DEVICE") -cat >>/mnt/etc/fstab < /mnt/etc/hosts << EOF 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 - EOF -# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. -# network-scripts.x86_64 : Legacy scripts for manipulating of network devices -if [ $RELEASE -ge 8 ]; then - run_root dnf -y install network-scripts -fi - -# Install (magic?) one-context RPM and hope things works as expected. +# Install one-context RPM and hope things works as expected. curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" @@ -180,3 +165,6 @@ run_root dnf clean all # Make sure everything is written to disk before exiting. sync + +# Cleanup! +cleanup From c0f32f78e3059c9a0a9264e37f15a6e39d425c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 10:43:27 +0200 Subject: [PATCH 131/543] CentOS/luks: disable SSH/user+password ONE scripts --- opennebula-images/centos7-build-luks-opennebula-image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 6bc6eca..6ea75e9 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -19,6 +19,7 @@ LOOPBACK_DEVICE=/dev/loop0 LUKS_PASSPHRASE=secret LUKS_DEVICE_NAME=cryptroot LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" +DISABLED_ONE_SCRIPTS="loc-20-set-username-password loc-22-ssh_public_key" ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm @@ -113,6 +114,9 @@ EOF curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" +for script in "$DISABLED_ONE_SCRIPTS"; do + run_root rm "/etc/one-context.d/$script" +done # Install resize2fs, which is required to resize the root file-system. run_root dnf -y install e2fsprogs From 9ba788e75b4d47483c3495e64597bf8ad6df39e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 10:52:55 +0200 Subject: [PATCH 132/543] Centos 7 / LUKS: fix ONE script removal --- opennebula-images/centos7-build-luks-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 6ea75e9..ddb7189 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -114,7 +114,7 @@ EOF curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" -for script in "$DISABLED_ONE_SCRIPTS"; do +for script in $DISABLED_ONE_SCRIPTS; do run_root rm "/etc/one-context.d/$script" done From 1e39b0aa84d70a9264ba5e766c3c81c954998fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 11:36:09 +0200 Subject: [PATCH 133/543] Do not use custom reposdir for centos7/LUKS image --- opennebula-images/centos7-build-luks-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index ddb7189..367211e 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -79,6 +79,7 @@ mount "${LUKS_DEVICE}" /mnt mkdir /mnt/boot mount "${LOOPBACK_DEVICE}p1" /mnt/boot +# Add --setopt=reposdir=rpm-repositories if you do not run on CentOS 7. dnf -y \ --releasever=$RELEASE \ --installroot=/mnt \ @@ -86,7 +87,6 @@ dnf -y \ --enablerepo=base \ --enablerepo=extras \ --setopt=install_weak_deps=False install \ - --setopt=reposdir=rpm-repositories \ bash basesystem systemd dnf centos-release cryptsetup dnf mount --bind /dev /mnt/dev From 8db3786b5b291a002c780acc0ea6a74e161a5c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 12:37:18 +0200 Subject: [PATCH 134/543] centos7/LUKS script: take LUKS passphrase as command-line argument --- opennebula-images/centos7-build-luks-opennebula-image.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 367211e..3be4b61 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -16,7 +16,6 @@ ARCH=x86_64 IMAGE_PATH=centos-luks-$RELEASE-$(date --iso-8601).img IMAGE_SIZE=10G LOOPBACK_DEVICE=/dev/loop0 -LUKS_PASSPHRASE=secret LUKS_DEVICE_NAME=cryptroot LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" DISABLED_ONE_SCRIPTS="loc-20-set-username-password loc-22-ssh_public_key" @@ -24,6 +23,13 @@ DISABLED_ONE_SCRIPTS="loc-20-set-username-password loc-22-ssh_public_key" ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm +# Get LUKS passphrase. +if [ -z "$1" ]; then + echo "Usage: centos7-build-luks-opennebula-image.sh LUKS_PASSPHRASE" + exit 1 +fi +LUKS_PASSPHRASE="$1" + cleanup() { # The order here is important. umount /mnt/dev/pts 2>/dev/null || true From 3ff2b138cb1b2d36d6bd4c0f1ee96fd5adc071cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 7 Jul 2020 21:00:49 +0200 Subject: [PATCH 135/543] in viirb2, output the commandline for next command --- viirb-2-configure-fully-after-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 161b39d..284e783 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -211,5 +211,5 @@ uci commit reboot EOF -echo "Wireguard public key: ${public_key}" +echo "Wireguard public key and id: ${id} ${public_key}" echo ${public_key} > ${viirb_hostname}.public_key From 7e55e76b14d696724c3d9595ab05c4bc1ca5ff2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jul 2020 10:55:49 +0200 Subject: [PATCH 136/543] Fix FreeBSD ONE image build script --- .../freebsd-build-opennebula-image.sh | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index ec64a56..535d2a6 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -9,15 +9,13 @@ set -x # XXX: Handle command-line arguments? RELEASE=12.1-RELEASE ARCH=amd64 -IMAGE_PATH=freebsd-$RELEASE-$(date +%+F).img.qcow2 +IMAGE_PATH=freebsd-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" PORTS_BASE="https://download.freebsd.org/ftp/snapshots/$ARCH/12.1-STABLE" -# TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" -ONE_CONTEXT_RPM_PATH=/root/one-context.rpm +ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.12.0/one-context-5.12.0_1.txz" cleanup() { sync || true @@ -70,7 +68,7 @@ growfs_enable=YES hostname="freebsd" EOF -cp /etc/resolv.conf > /mnt/etc/resolv.conf +cp /etc/resolv.conf /mnt/etc/resolv.conf tzsetup -s -C /mnt UTC cat >>/mnt/etc/ssh/sshd_config < Date: Tue, 21 Jul 2020 12:08:38 +0200 Subject: [PATCH 137/543] Install requirements for plain centos7 to build centos7 VM --- opennebula-images/centos7-build-luks-opennebula-image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 3be4b61..f3d34c8 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -60,6 +60,9 @@ if [ ! -f '/etc/centos-release' ]; then sleep 5 fi +# Install requirements +yum install -y qemu cryptsetup + # Create base RAW image (no LOOPBACK support in RHEL/CentOS). qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" From 9e8e39b2ee5ea37b169a2418308eda3b15f8d521 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 21 Jul 2020 12:18:13 +0200 Subject: [PATCH 138/543] Also install dnf for centos --- opennebula-images/centos7-build-luks-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index f3d34c8..69a5c64 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -61,7 +61,7 @@ if [ ! -f '/etc/centos-release' ]; then fi # Install requirements -yum install -y qemu cryptsetup +yum install -y qemu cryptsetup dnf # Create base RAW image (no LOOPBACK support in RHEL/CentOS). qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" From a109bb16f8fbfd8ff16dcaeb464446f319416dbc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 28 Jul 2020 15:40:15 +0200 Subject: [PATCH 139/543] [mikrotik] + permissions +shebang --- mikrotik-setup.sh | 0 mikrotik-update.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 mikrotik-setup.sh diff --git a/mikrotik-setup.sh b/mikrotik-setup.sh old mode 100644 new mode 100755 diff --git a/mikrotik-update.sh b/mikrotik-update.sh index 934ea47..5d1c281 100755 --- a/mikrotik-update.sh +++ b/mikrotik-update.sh @@ -1,4 +1,4 @@ -#!/bin +#!/bin/sh # Nico Schottelius, 2019-12-02 # Update mikrotik routers to the latest package From 6d8c93b54cb5d7c1928331351ce2a2c84c8d743c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 12:26:17 +0200 Subject: [PATCH 140/543] iHmm? Signed-off-by: Nico Schottelius --- viirb-2-configure-fully-after-upgrade.sh | 4 + viirb-setup-all.sh | 391 ----------------------- 2 files changed, 4 insertions(+), 391 deletions(-) delete mode 100755 viirb-setup-all.sh diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 161b39d..f92a13c 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -63,7 +63,10 @@ uci delete dhcp.lan.leasetime # Do not announce ULA - we have GUA uci delete network.globals.ula_prefix +# This is configuring the dhcp IPv4 client uci set dhcp.lan=dhcp + +# Setup Router Advertisements uci set dhcp.lan.interface='lanv6' uci set dhcp.lan.ra='server' uci set dhcp.lan.dynamicdhcp='0' @@ -114,6 +117,7 @@ uci delete wireless.radio0.disabled # This is temporary - keeping it until the config process is through # Probably not needet - we can connect to the final IPv6 address! +# This code commented out == The address vanishes due to above reconfiguration # uci set network.lanv4temp=interface # uci set network.lanv4temp.proto='static' # uci set network.lanv4temp.ifname='br-lan' diff --git a/viirb-setup-all.sh b/viirb-setup-all.sh deleted file mode 100755 index 63286cb..0000000 --- a/viirb-setup-all.sh +++ /dev/null @@ -1,391 +0,0 @@ -#!/bin/sh -# 2020-06-13, Nico Schottelius -# See https://ungleich.ch/u/products/viirb-ipv6-box/ - -if [ $# -lt 4 ]; then - echo "$0 interface viirb-id your-dot-cdist [stages]" - echo " interface to add the config ip address to" - echo " viirb-id: number in decimal format" - echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" - echo " owner-mail-reference: How to identify the owner" - echo " stages: define which stages to execute" - echo "" - echo " stage1: setup your host, check connection to VIIRB" - echo " stage2: flash latest openwrt onto the VIIRB" - echo " stage3: configure the vpn endpoint" - echo " stage4: configure the VIIRB with wireguard + settings" - echo " stage5: Verify VIIRB on VPN, cleanup VIIRB" - echo "" - echo "Example to configure viirb02:" - echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342'" - echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342' '1 3 4'" - exit 1 -fi - -echo "FIXME: missing IPv6 announcements on LAN" -echo "FIXME: DNS situation: upstream, non upstream, ungleich, how to resolve tunnel endpoint" - -set -x - -set -x -dev=$1; shift -id=$1; shift -dot_cdist=$1; shift -owner=$1; shift - -if [ $# -ge 1 ]; then - stages=$1; shift -else - stages="1 2 3 4 5" -fi - -hex_id=$(printf "%0.2x\n" "$id") -viirb_hostname=viirb${hex_id} - -prefix_base=2a0a:e5c1:3 -my_prefix=${prefix_base}${hex_id} -my_network=${my_prefix}::/48 - -my_wireguard_ip=${my_prefix}::42 -my_lan_ip=${my_prefix}:cafe::42 -my_wifi_ip=${my_prefix}:7ea::42 - -# openwrt -version=19.07.3 -filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin - -# root password -root_password=$(pwgen -1 32) - -# IP address for setting it up initially -viirb_ip=192.168.61.1 - -# wireguard -private_key=$(wg genkey) -public_key=$(echo $private_key | wg pubkey) - -vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch -vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= - -# cdist -dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files -peerfilename=${vpn_endpoint_host}.peer${hex_id} -peerfile=${dot_cdist_files}/${peerfilename} -vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} - -################################################################################ -# Stage 1: test / connect to the new VIIRB -# -# We delete so that we can run idempotent -stage1() -{ - sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true - sudo ip addr add 192.168.61.2/24 dev "$dev" - - # don't care about other/old known_host entries - ssh-keygen -R ${viirb_ip} - - ping -c2 ${viirb_ip} - if [ $? -ne 0 ]; then - echo "Cannot reach any VIIRB - exiting" - exit 1 - fi - - cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_keys" -} - - - -################################################################################ -# Get latest OpenWRT & flash it -stage2() -{ - # Don't re-download if we already have it - wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} - scp ${filename} root@${viirb_ip}:/tmp - ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" - - # It still pings for some time - wait for the reboot to happen - echo "Waiting for VIIRB to disappear" - sleep 15 - - wait=0 - found="" - - while [ $wait -lt 180 ]; do - ping -c1 ${viirb_ip} >/dev/null - - if [ $? -eq 0 ]; then - found=yes - # wait for ssh to come up - sleep 10 - break - fi - - sleep 1 - wait=$((wait+1)) - done - - if [ ! "$found" ]; then - echo "Did not find updated viirb - debug / restart it" - exit 1 - fi - -} - - -################################################################################ -# Stage 3: prepare VPN endpoint -# - -stage3() -{ - - # Configure VPN server / update cdist - echo Updating VPNserver - cat < ${peerfile} -# ${viirb_hostname} ${owner} -[Peer] -PublicKey = ${public_key} -AllowedIPs = ${my_network} - -EOF - - # Generate real config - cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} - cd ${dot_cdist_files} - git add ${vpn_endpoint_host} ${peerfilename} - git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" - git pull - git push - - cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} -} - -################################################################################ -# Stage 4: configure the VIIRB -# -stage4() -{ - - cat </dev/null - - if [ $? -eq 0 ]; then - found=yes - break - fi - sleep 1 - wait=$((wait+1)) - done - - if [ ! "$found" ]; then - echo "Cannot reach VIIRB via VPN - check manually" - exit 1 - fi - - echo "Cleanup process." - echo "Set the root password when prompted to: ${root_password}" - - # VPN works, remove artefacts, set correct DNS servers that support DNS64 - cat < Date: Sat, 1 Aug 2020 12:37:51 +0200 Subject: [PATCH 141/543] [viirb] add script to do step1+2 together --- viirb1-2-together.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 viirb1-2-together.sh diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh new file mode 100644 index 0000000..b18c4d8 --- /dev/null +++ b/viirb1-2-together.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 viirb-id" + echo " viirb-id: number in decimal format" + exit 1 +fi + +id=$1; shift + +./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 +./viirb-2-configure-fully-after-upgrade.sh From b7649a60bca84babf8d22616f331f42cc9eafd29 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 12:38:27 +0200 Subject: [PATCH 142/543] +sleep between 1+2 --- viirb1-2-together.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh index b18c4d8..5d1d736 100644 --- a/viirb1-2-together.sh +++ b/viirb1-2-together.sh @@ -9,4 +9,6 @@ fi id=$1; shift ./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 +# reboot may take longer +sleep 60 ./viirb-2-configure-fully-after-upgrade.sh From 389e6417f1f2694d3389b7552995ddda4743af94 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 13:42:51 +0200 Subject: [PATCH 143/543] [viirb12] pass args to viirb2 --- viirb1-2-together.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh index 5d1d736..3edc0bf 100644 --- a/viirb1-2-together.sh +++ b/viirb1-2-together.sh @@ -11,4 +11,4 @@ id=$1; shift ./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 # reboot may take longer sleep 60 -./viirb-2-configure-fully-after-upgrade.sh +./viirb-2-configure-fully-after-upgrade.sh 192.168.61.1 "$id" From d68242713a703debbf88d841f8cd3663b77782ab Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 13:46:00 +0200 Subject: [PATCH 144/543] [viirb] + pubkey + args --- viirb-2-configure-fully-after-upgrade.sh | 1 + viirb1-2-together.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 16f704e..94fc628 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -136,6 +136,7 @@ EOF # ensure viirb network is up and running - bridge takes a bit of time # ensure viirb gives out ipv6 address # Found: 30s is not enough for full reconfiguration +echo "Public VPN key: ${id} ${public_key}" sleep 60 # change to ipv6 diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh index 3edc0bf..2d4378b 100644 --- a/viirb1-2-together.sh +++ b/viirb1-2-together.sh @@ -1,6 +1,6 @@ #!/bin/sh -if [ $# -ne 2 ]; then +if [ $# -ne 1 ]; then echo "$0 viirb-id" echo " viirb-id: number in decimal format" exit 1 From e0a313c7097b7f56c1a89cdbc071cd1b8d788163 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 14:00:49 +0200 Subject: [PATCH 145/543] increase timeout for flash writing --- viirb1-2-together.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh index 2d4378b..e1a4206 100644 --- a/viirb1-2-together.sh +++ b/viirb1-2-together.sh @@ -9,6 +9,6 @@ fi id=$1; shift ./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 -# reboot may take longer -sleep 60 +# reboot may take longer / rewriting the flash +sleep 120 ./viirb-2-configure-fully-after-upgrade.sh 192.168.61.1 "$id" From 7bbac98e3b6e28f32d6f56edc9136c9512fac621 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 16:21:33 +0200 Subject: [PATCH 146/543] [viirb] wait until it comes online --- viirb-1-connect-flash-latest-openwrt.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index c551ef1..f6b5a34 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -33,11 +33,10 @@ fi # don't care about other/old known_host entries ssh-keygen -R ${viirb_ip} -ping -c2 ${viirb_ip} -if [ $? -ne 0 ]; then - echo "Cannot reach any VIIRB - exiting" - exit 1 -fi +while ! ping -c1 ${viirb_ip}; do + echo "Cannot ping $viirb_ip yet - waiting" + sleep 1 +done cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_keys" From 40cb3be77683a288f494de501a092cfb0507f521 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 17:08:34 +0200 Subject: [PATCH 147/543] [viirb] ping it longer --- viirb-3-configure-vpnendpoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-configure-vpnendpoint.sh index cd21131..ac3b742 100755 --- a/viirb-3-configure-vpnendpoint.sh +++ b/viirb-3-configure-vpnendpoint.sh @@ -60,4 +60,4 @@ sleep 10 ping -c10 ${my_wireguard_ip} ping -c10 ${my_lan_ip} -ping -c10 ${my_wifi_ip} +ping -c90 ${my_wifi_ip} From 23139cfe9d4d328c5df7fe472a7e8bea056fcff7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 17:08:44 +0200 Subject: [PATCH 148/543] [viirb] update comment --- viirb-1-connect-flash-latest-openwrt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index f6b5a34..4cbb393 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -46,7 +46,7 @@ scp ${filename} root@${viirb_ip}:/tmp ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" # It still pings for some time - wait for the reboot to happen -echo "Waiting for VIIRB to disappear" +echo "Waiting for VIIRB to really disappear" sleep 15 wait=0 From 7b24804ac998ac3a0a81b910fca8c0c6bfd502e6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jul 2020 18:56:24 +0200 Subject: [PATCH 149/543] [mikrotik] update / minus bugs --- mikrotik-setup.sh | 7 ++++++- mikrotik-update.sh | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mikrotik-setup.sh b/mikrotik-setup.sh index dbe2c4f..e3654e5 100755 --- a/mikrotik-setup.sh +++ b/mikrotik-setup.sh @@ -16,7 +16,12 @@ conf() { ssh admin@${target} "$@" } copy() { - scp "$1" admin@${target}: + if echo ${target} | grep -q :; then + ltarget="[$target]" + else + ltarget="$target" + fi + scp "$1" admin@${ltarget}: } # store ssh key in the admin user! diff --git a/mikrotik-update.sh b/mikrotik-update.sh index 5d1c281..027bdba 100755 --- a/mikrotik-update.sh +++ b/mikrotik-update.sh @@ -29,7 +29,7 @@ cd "$tmp" wget "${url}" unzip "${file}" -pkg_list="dhcp ipv6 lcd lte multicast ppp routing security system user-manager wireless"! +pkg_list="dhcp ipv6 lcd lte multicast ppp routing security system user-manager wireless" while [ $# -ge 1 ]; do target=$1; shift From 91d6eac9c7ce4025185ce4a7c62e9a112ce4a114 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 3 Aug 2020 20:51:29 +0200 Subject: [PATCH 150/543] add script to "standard" configure a mikrotik-crs326 --- mikrotik-configure-crs326.sh | 103 +++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 mikrotik-configure-crs326.sh diff --git a/mikrotik-configure-crs326.sh b/mikrotik-configure-crs326.sh new file mode 100755 index 0000000..5f3b330 --- /dev/null +++ b/mikrotik-configure-crs326.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Nico Schottelius, 2020-08-03 +# Setup a standard crs326 + + + +if [ $# -ne 3 ]; then + echo "$0 " + echo "Example:" + echo "$0 fe80::764d:28ff:fe09:9355%eth1 mikrotik-crs326-8 $(pass place6-linthal/mikrotik)" + exit 1 +fi + +ip=$1; shift +hostname=$1; shift +password=$1; shift + +target=$ip +bridge=bridgevlans + +internal=10 +coworking=15 +server=11 +other="8 16 18 33 34" + +tagged="ether23 ether24 sfp-sfpplus1" + +net_internal=2a0a:e5c0:2::/64 + +conf() { + echo $@ + ssh admin@${target} "$@" +} + +commastring() { + echo $@ | sed 's/ /,/g' +} + +set -x + + +# do this out of band -- see mikrotik-setup.sh +#conf "/password new-password=$password confirm-new-password=$password old-password=\"\"" + +conf "/system identity set name=$hostname" +conf "/interface bridge add name=$bridge" + +################################################################################ +# MTU + +for i in $(seq 1 24); do + conf "/interface ethernet set ether$i mtu=9200 l2mtu=9204" +done + + +for i in $(seq 1 2); do + conf "/interface ethernet set sfp-sfpplus$i mtu=9200 l2mtu=9204" +done + + +################################################################################ +# VLANs + +# Internal ports 1-16 +ifaces="" +for i in $(seq 1 16); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$internal" + ifaces="ether$i ${ifaces}" +done + +# also tag the bridge for the vlan interface we need later +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged),$bridge untagged=$(commastring $ifaces) vlan-ids=$internal" + +# Coworking 17-18 +ifaces="" +for i in $(seq 17 18); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$coworking" + ifaces="ether$i ${ifaces}" +done +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) untagged=$(commastring $ifaces) vlan-ids=$coworking" + +# Server 19-20 +ifaces="" +for i in $(seq 19 20); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$server" + ifaces="ether$i ${ifaces}" +done +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) untagged=$(commastring $ifaces) vlan-ids=$server" + +# Not modified 21-22 + +# Tagged 23-24, sfp-sfpplus1 +for iface in ; do + conf "/interface bridge port add bridge=$bridge interface=$iface hw=yes" +done + +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) lan-ids=$(commastring $other)" +conf "/interface vlan add interface=$bridge vlan-id=$internal name=MGMT" +conf "/ipv6 address add eui-64=yes advertise=no address=$net_internal interface=MGMT" +conf "/interface bridge set $bridge vlan-filtering=yes" + +# Show neigh +conf "/interface bridge host print where !local" From 5e966b852d30ce9a7b2354033d7157810e84288a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Aug 2020 15:01:56 +0200 Subject: [PATCH 151/543] setup password in opennebula --- opennebula-images/centos7-build-luks-opennebula-image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 69a5c64..91df92b 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -119,6 +119,9 @@ cat > /mnt/etc/hosts << EOF ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 EOF +# Setup root password +run_root passwd + # Install one-context RPM and hope things works as expected. curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" From 8429201d0df3c9b5d52956bc25fdebef2cf955dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Aug 2020 15:03:09 +0200 Subject: [PATCH 152/543] + mikrotik scripts --- mikrotik-configure-crs326-dumb.sh | 59 ++++++++++++++ mikrotik-configure-crs326-with-vlans.sh | 103 ++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100755 mikrotik-configure-crs326-dumb.sh create mode 100755 mikrotik-configure-crs326-with-vlans.sh diff --git a/mikrotik-configure-crs326-dumb.sh b/mikrotik-configure-crs326-dumb.sh new file mode 100755 index 0000000..a2689d7 --- /dev/null +++ b/mikrotik-configure-crs326-dumb.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Nico Schottelius, 2020-08-03 +# Setup a standard crs326 + + + +if [ $# -ne 4 ]; then + echo "$0 " + echo "Example:" + echo "$0 fe80::764d:28ff:fe09:9355%eth1 2a0a:e5c0:2::/64 mikrotik-crs326-8 \$(pass ...)" + exit 1 +fi + +ip=$1; shift +newip=$1; shift +hostname=$1; shift +password=$1; shift + +target=$ip +bridge=bridge + +conf() { + echo $@ + ssh admin@${target} "$@" +} + +commastring() { + echo $@ | sed 's/ /,/g' +} + +conf "/system identity set name=$hostname" +conf "/interface bridge add name=$bridge" + +################################################################################ +# MTU + +for i in $(seq 1 24); do + conf "/interface ethernet set ether$i mtu=9200 l2mtu=9204" + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes" +done + + +for i in $(seq 1 2); do + conf "/interface ethernet set sfp-sfpplus$i mtu=9200 l2mtu=9204" + conf "/interface bridge port add bridge=$bridge interface=sfp-sfpplus$i hw=yes" +done + + +################################################################################ +# IPv6 address, password + +conf "/ipv6 address add eui-64=yes advertise=no address=$newip interface=$bridge" +conf "/ipv6 address print" +conf "/password old-password=\"\" new-password=$password confirm-new-password=$password" + +# Show neigh +conf "/interface bridge host print where !local" + +echo "do not forget to set a password" diff --git a/mikrotik-configure-crs326-with-vlans.sh b/mikrotik-configure-crs326-with-vlans.sh new file mode 100755 index 0000000..4ae1b62 --- /dev/null +++ b/mikrotik-configure-crs326-with-vlans.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Nico Schottelius, 2020-08-03 +# Setup a standard crs326 + + + +if [ $# -ne 2 ]; then + echo "$0 " + echo "Example:" + echo "$0 fe80::764d:28ff:fe09:9355%eth1 mikrotik-crs326-8" + exit 1 +fi + +ip=$1; shift +hostname=$1; shift +password=$1; shift + +target=$ip +bridge=bridgevlans + +internal=10 +coworking=15 +server=11 +other="8 16 18 33 34" + +tagged="ether23 ether24 sfp-sfpplus1" + +net_internal=2a0a:e5c0:2::/64 + +conf() { + echo $@ + ssh admin@${target} "$@" +} + +commastring() { + echo $@ | sed 's/ /,/g' +} + +#set -x + + +# do this out of band -- see mikrotik-setup.sh +#conf "/password new-password=$password confirm-new-password=$password old-password=\"\"" + +conf "/system identity set name=$hostname" +conf "/interface bridge add name=$bridge" + +################################################################################ +# MTU + +for i in $(seq 1 24); do + conf "/interface ethernet set ether$i mtu=9200 l2mtu=9204" +done + + +for i in $(seq 1 2); do + conf "/interface ethernet set sfp-sfpplus$i mtu=9200 l2mtu=9204" +done + + +################################################################################ +# VLANs + +# Internal ports 1-16 +ifaces="" +for i in $(seq 1 16); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$internal" + ifaces="ether$i ${ifaces}" +done + +# also tag the bridge for the vlan interface we need later +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged),$bridge untagged=$(commastring $ifaces) vlan-ids=$internal" + +# Coworking 17-18 +ifaces="" +for i in $(seq 17 18); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$coworking" + ifaces="ether$i ${ifaces}" +done +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) untagged=$(commastring $ifaces) vlan-ids=$coworking" + +# Server 19-20 +ifaces="" +for i in $(seq 19 20); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$server" + ifaces="ether$i ${ifaces}" +done +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) untagged=$(commastring $ifaces) vlan-ids=$server" + +# Not modified 21-22 + +# Tagged 23-24, sfp-sfpplus1 +for iface in $tagged; do + conf "/interface bridge port add bridge=$bridge interface=$iface hw=yes" +done + +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) vlan-ids=$(commastring $other)" +conf "/interface vlan add interface=$bridge vlan-id=$internal mtu=9200 name=internal" +conf "/ipv6 address add eui-64=yes advertise=no address=$net_internal interface=internal" +conf "/interface bridge set $bridge vlan-filtering=yes" + +# Show neigh +conf "/interface bridge host print where !local" From 7f07870d7f58d0342d052b5aadb66cd9f87e3156 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 24 Aug 2020 09:48:10 +0200 Subject: [PATCH 153/543] phase in pib setup --- pib-setup.sh | 168 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 pib-setup.sh diff --git a/pib-setup.sh b/pib-setup.sh new file mode 100755 index 0000000..b0e4438 --- /dev/null +++ b/pib-setup.sh @@ -0,0 +1,168 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 ip-address vpn-network" + echo " ip-address: where to find the PIB" + echo " network: 2a0a:e5c0:123::/48" + exit 1 +fi + + + +my_ip=$1; shift +my_network=$1; shift + +my_prefix=$(echo $my_network | sed 's,::/.*,,') +my_hostname=pib-$(echo ${my_prefix} | sed 's/:/-/g') + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_lanv6_ip=${my_prefix}:7ea::42 + +private_key=$(wg genkey) +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-2a0ae5c1.ungleich.ch +vpn_endpoint_pubkey=hi60lGP+xEUQ+kVnqA7PlJAO1SVqTS1W36g0LhFP0xQ= + +cat < ${my_hostname}.public_key + +exit 0 + + +# wget -O - https://downloads.openwrt.org/releases/19.07.3/targets/x86/64/openwrt-19.07.3-x86-64-rootfs-ext4.img.gz | gunzip > /dev/sda From 912dcabe9feb16230705d1cdb78210f948bd4824 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 24 Aug 2020 10:00:02 +0200 Subject: [PATCH 154/543] +passwd Signed-off-by: Nico Schottelius --- opennebula-images/centos7-build-luks-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 91df92b..6b7fe1d 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -96,7 +96,7 @@ dnf -y \ --enablerepo=base \ --enablerepo=extras \ --setopt=install_weak_deps=False install \ - bash basesystem systemd dnf centos-release cryptsetup dnf + bash basesystem systemd dnf centos-release cryptsetup dnf passwd mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts From 3470ca62df000f5013fa818c7a9071d5c835cfd3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 7 Sep 2020 16:20:36 +0200 Subject: [PATCH 155/543] [pib] update setup script --- pib-setup.sh | 94 +++++++++++++++++++++++++++------------------------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/pib-setup.sh b/pib-setup.sh index b0e4438..f95486e 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -1,25 +1,33 @@ #!/bin/sh +# Assumptions: +# - pib is factory reset +# - WAN port is connected with an active upstream +# - You are connected via LAN/can ssh into it + if [ $# -ne 2 ]; then - echo "$0 ip-address vpn-network" + echo "$0 ip-address vpn-network [wireguard-private-key]" echo " ip-address: where to find the PIB" echo " network: 2a0a:e5c0:123::/48" + echo " private-key: specify if you already have a private key" exit 1 fi - - my_ip=$1; shift my_network=$1; shift +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi + my_prefix=$(echo $my_network | sed 's,::/.*,,') my_hostname=pib-$(echo ${my_prefix} | sed 's/:/-/g') my_wireguard_ip=${my_prefix}::42 my_lan_ip=${my_prefix}:cafe::42 -my_lanv6_ip=${my_prefix}:7ea::42 -private_key=$(wg genkey) public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1.ungleich.ch @@ -28,7 +36,16 @@ vpn_endpoint_pubkey=hi60lGP+xEUQ+kVnqA7PlJAO1SVqTS1W36g0LhFP0xQ= cat < ${my_hostname}.public_key -exit 0 +cat < /dev/sda +- Remove your ssh key(s) from the device +- Setup a secure root password and forward it to the customer +EOF From caabcc2f47c88dfa38911cf0ab4e993a2b1ddb12 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 7 Sep 2020 16:52:14 +0200 Subject: [PATCH 156/543] [pib] cleanup / document the script --- pib-setup.sh | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pib-setup.sh b/pib-setup.sh index f95486e..39b7c0b 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -1,11 +1,14 @@ #!/bin/sh # Assumptions: -# - pib is factory reset -# - WAN port is connected with an active upstream -# - You are connected via LAN/can ssh into it +# - pib (APU) is factory reset OpenWRT +# - WAN port is connected with an active upstream (pib has internet connectivity via WAN port) +# - You are connected via LAN and you can ssh into it -if [ $# -ne 2 ]; then +# How it works +# + +if [ $# -lt 2 ]; then echo "$0 ip-address vpn-network [wireguard-private-key]" echo " ip-address: where to find the PIB" echo " network: 2a0a:e5c0:123::/48" @@ -44,8 +47,7 @@ opkg update opkg install libustream-openssl ca-bundle ca-certificates # install wireguard + gui -opkg install wireguard -opkg install luci-app-wireguard +opkg install wireguard luci-app-wireguard # We are never authoritative for IPv4 uci delete dhcp.@dnsmasq[0].authoritative @@ -53,6 +55,9 @@ uci delete dhcp.@dnsmasq[0].authoritative # Do not announce ULA - we have GUA uci delete network.globals.ula_prefix +# Setup hostname +uci set system.@system[0].hostname="${my_hostname}" + # Do not set/get? Was necessary, don't recall why uci set dhcp.@dnsmasq[0].noresolv='1' @@ -107,8 +112,6 @@ uci set network.@wireguard_wg0[0].endpoint_host='${vpn_endpoint_host}' uci set network.@wireguard_wg0[0].endpoint_port='51820' uci set network.@wireguard_wg0[0].route_allowed_ips='1' -uci set system.@system[0].hostname="${my_hostname}" - # Firewall configuration if ! uci show firewall | grep "name='Allow-SSH'"; then uci add firewall rule @@ -149,16 +152,19 @@ uci delete dhcp.@dnsmasq[0].server uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:0:a::a' uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:2:a::a' +# This is the save & apply button in LUCI (or just save button) uci commit + reboot EOF my_ip=$my_lan_ip + echo "Waiting for it to come back..." while ! ping -c1 ${my_ip}; do echo "Cannot ping $my_ip yet - waiting" - sleep 1 + sleep 2 done echo "Wireguard public key and id: ${id} ${public_key}" @@ -169,4 +175,5 @@ Open steps: - Remove your ssh key(s) from the device - Setup a secure root password and forward it to the customer + EOF From ddbc95f6f01f4de9ffaa01d34691244f9e7e2ef0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 8 Sep 2020 17:19:51 +0200 Subject: [PATCH 157/543] exit if ssh breaks --- pib-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pib-setup.sh b/pib-setup.sh index 39b7c0b..1bbdd71 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -36,7 +36,7 @@ public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1.ungleich.ch vpn_endpoint_pubkey=hi60lGP+xEUQ+kVnqA7PlJAO1SVqTS1W36g0LhFP0xQ= -cat < Date: Tue, 8 Sep 2020 17:22:52 +0200 Subject: [PATCH 158/543] [pib setup] ping6 alternatively before --- pib-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pib-setup.sh b/pib-setup.sh index 1bbdd71..13db223 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -40,7 +40,7 @@ cat < Date: Tue, 8 Sep 2020 17:36:32 +0200 Subject: [PATCH 159/543] ++comment phrasing --- pib-setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pib-setup.sh b/pib-setup.sh index 13db223..04efb70 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -173,7 +173,8 @@ echo ${public_key} > ${my_hostname}.public_key cat < Date: Tue, 8 Sep 2020 17:37:20 +0200 Subject: [PATCH 160/543] ++notes --- pib-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pib-setup.sh b/pib-setup.sh index 04efb70..649542a 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -176,5 +176,6 @@ Open steps: - Remove your ssh key(s) from the device (if any are present) - Setup a secure root password and forward it to the customer - Ensure that the VPN works +- Connect to the LAN port and surf in the Internet IPv6 only! EOF From 55f2843ce30b2644113e28adf9884d28bf5bd873 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 11 Sep 2020 13:23:14 +0200 Subject: [PATCH 161/543] [openwrt] add new script to configure LTE uplink --- openwrt-add-lte.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 openwrt-add-lte.sh diff --git a/openwrt-add-lte.sh b/openwrt-add-lte.sh new file mode 100644 index 0000000..5e3a2c7 --- /dev/null +++ b/openwrt-add-lte.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Based on work of Samuel Hailu, 2020-09-10 + +if [ $# -ne 2 ]; then + echo "$0 ip-address interface" + echo " ip-address: where to find the OpenWRT device" + echo " interface: which interface (eth3 for instance) is the LTE device" + exit 1 +fi + +my_ip=$1; shift +interface=$1; shift + +cat < Date: Thu, 24 Sep 2020 15:13:24 +0200 Subject: [PATCH 162/543] [openwrt] WIP: add new scripts for acme and camera --- openwrt-add-letsencrypt.sh | 6 ++++ openwrt-motion-config.sh | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 openwrt-add-letsencrypt.sh create mode 100644 openwrt-motion-config.sh diff --git a/openwrt-add-letsencrypt.sh b/openwrt-add-letsencrypt.sh new file mode 100644 index 0000000..43e3b4b --- /dev/null +++ b/openwrt-add-letsencrypt.sh @@ -0,0 +1,6 @@ + +opkg install luci-ssl-openssl acme luci-app-acme + +uci set uhttpd.main.redirect_https=1 +uci commit +/etc/init.d/uhttpd restart diff --git a/openwrt-motion-config.sh b/openwrt-motion-config.sh new file mode 100644 index 0000000..b4cb5cc --- /dev/null +++ b/openwrt-motion-config.sh @@ -0,0 +1,57 @@ +#!/bin/sh + + +if [ $# -ne 1 ]; then + echo "$0 ip-address interface" + echo " ip-address: where to find the OpenWRT device" + exit 1 +fi + +my_ip=$1; shift + +cat < Date: Thu, 24 Sep 2020 16:07:25 +0200 Subject: [PATCH 163/543] ++camera setting --- openwrt-motion-config.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openwrt-motion-config.sh b/openwrt-motion-config.sh index b4cb5cc..1648f90 100644 --- a/openwrt-motion-config.sh +++ b/openwrt-motion-config.sh @@ -55,3 +55,6 @@ webcontrol_parms 0 stream_port 8081 stream_localhost off + +# use native mjpeg +v4l2_palette 8 From 8d22809b454b4e50a72b8d560b5dda3c38a4acbf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 28 Sep 2020 00:07:24 +0200 Subject: [PATCH 164/543] update how to set the device class --- ceph-osd-create-start | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ceph-osd-create-start b/ceph-osd-create-start index f4d7bd0..a2c73f7 100755 --- a/ceph-osd-create-start +++ b/ceph-osd-create-start @@ -69,7 +69,9 @@ ln -s $dev_block "$mountpath/block" echo "$uuid_block" > "$mountpath/block_uuid" echo "$fsid" > "$mountpath/ceph_fsid" echo "$magic" > "$mountpath/magic" -echo "$CLASS" > "$mountpath/crush_device_class" + +# Not needed (anymore) - using ceph osd crush set-device-class below +#echo "$CLASS" > "$mountpath/crush_device_class" # Important, otherwise --mkfs later will try to create filestore echo bluestore > "$mountpath/type" @@ -92,6 +94,7 @@ else fi ceph osd crush add osd.${osd_id} ${WEIGHT} host=$(hostname) +ceph osd crush set-device-class $CLASS osd.${osd_id} echo "$metadata_dev /var/lib/ceph/osd/ceph-${osd_id} xfs noatime 0 0" >> /etc/fstab @@ -101,3 +104,5 @@ if [ -e /etc/monit ]; then else /etc/init.d/ceph start osd.${osd_id} fi + +#rm "$mountpath/crush_device_class" From 7b776e48a0953b80199c87ff32f0f1eeeb53e787 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 1 Oct 2020 23:04:50 +0200 Subject: [PATCH 165/543] [ceph] cleanup device class in case something old exists --- ceph-osd-create-start | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ceph-osd-create-start b/ceph-osd-create-start index a2c73f7..c2612b0 100755 --- a/ceph-osd-create-start +++ b/ceph-osd-create-start @@ -94,6 +94,9 @@ else fi ceph osd crush add osd.${osd_id} ${WEIGHT} host=$(hostname) + +# Ensure previous assigned class is gone +ceph osd crush rm-device-class osd.${osd_id} ceph osd crush set-device-class $CLASS osd.${osd_id} echo "$metadata_dev /var/lib/ceph/osd/ceph-${osd_id} xfs noatime 0 0" >> /etc/fstab From f30413d834f944e62eaabe8a5a1aaa10691b1946 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 1 Oct 2020 23:20:49 +0200 Subject: [PATCH 166/543] [ceph] fix block device permissions --- ceph-osd-create-start | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ceph-osd-create-start b/ceph-osd-create-start index c2612b0..917a530 100755 --- a/ceph-osd-create-start +++ b/ceph-osd-create-start @@ -85,6 +85,9 @@ touch "$mountpath/sysvinit" ceph-osd --cluster ceph -i "${osd_id}" --mkfs chown -R ceph:ceph "$mountpath" +# Also allow access to the blockdevices - via symlink +chown ceph:ceph "${mountpath}/block" + if [ $# -eq 1 ]; then WEIGHT=$1; shift else From 02cb81de7249ccc2265d436eb4b9b9d4715b00c0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 1 Oct 2020 23:52:39 +0200 Subject: [PATCH 167/543] [ceph] remove old parts --- ceph-osd-create-start | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ceph-osd-create-start b/ceph-osd-create-start index 917a530..106e363 100755 --- a/ceph-osd-create-start +++ b/ceph-osd-create-start @@ -29,7 +29,6 @@ fi export DEV=$1;shift export CLASS=$1; shift - uuid_metadata=$(uuidgen) uuid_block=$(uuidgen) @@ -65,14 +64,10 @@ mkdir -p "$mountpath" mount "$dev_metadata" "$mountpath" ln -s $dev_block "$mountpath/block" - echo "$uuid_block" > "$mountpath/block_uuid" echo "$fsid" > "$mountpath/ceph_fsid" echo "$magic" > "$mountpath/magic" -# Not needed (anymore) - using ceph osd crush set-device-class below -#echo "$CLASS" > "$mountpath/crush_device_class" - # Important, otherwise --mkfs later will try to create filestore echo bluestore > "$mountpath/type" @@ -85,8 +80,8 @@ touch "$mountpath/sysvinit" ceph-osd --cluster ceph -i "${osd_id}" --mkfs chown -R ceph:ceph "$mountpath" -# Also allow access to the blockdevices - via symlink -chown ceph:ceph "${mountpath}/block" +# Also allow access to the blockdevice +chown ceph:ceph ${dev_block} if [ $# -eq 1 ]; then WEIGHT=$1; shift @@ -96,9 +91,10 @@ else WEIGHT=$(lsblk -l -b | awk "/^$nodev/ { print \$4/(1024^4) }") fi +# Move into the correct position ceph osd crush add osd.${osd_id} ${WEIGHT} host=$(hostname) -# Ensure previous assigned class is gone +# Ensure previous assigned class is gone - if the osd id was used before ceph osd crush rm-device-class osd.${osd_id} ceph osd crush set-device-class $CLASS osd.${osd_id} @@ -110,5 +106,3 @@ if [ -e /etc/monit ]; then else /etc/init.d/ceph start osd.${osd_id} fi - -#rm "$mountpath/crush_device_class" From 3c07e54b11e5ac1de2fdf908ee67c0911dbb1a5e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Oct 2020 11:24:02 +0200 Subject: [PATCH 168/543] [viirb] update openwrt version --- viirb-1-connect-flash-latest-openwrt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index 4cbb393..2d43097 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -20,7 +20,7 @@ else fi # openwrt -version=19.07.3 +version=19.07.4 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin # IP address for setting it up initially From 35098485bbc9b627ec0281f2cbff4596a8295d6a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Oct 2020 11:42:40 +0200 Subject: [PATCH 169/543] fix viirb-1 to use ipv6 literal addresses --- viirb-1-connect-flash-latest-openwrt.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index 2d43097..1c69e93 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -12,6 +12,10 @@ fi set -x dev=$1; shift +# $# = 2 +# shift -> remove the first argument +# $# = 1 + if [ $# -ge 1 ]; then viirb_ip=$1; shift dev="" @@ -42,6 +46,13 @@ cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_key # Don't re-download if we already have it wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} + +if echo $viirb_ip | grep -q :; then + scp_ip="[$viirb_ip]" +else + scp_ip="$viirb_ip" +fi + scp ${filename} root@${viirb_ip}:/tmp ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" From 0bb0c4aea7e51d4b6e39db305ef293f717fe4be0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Oct 2020 11:43:05 +0200 Subject: [PATCH 170/543] [viirb] add script to get the firmware --- viirb-get-openwrt-firmware.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100755 viirb-get-openwrt-firmware.sh diff --git a/viirb-get-openwrt-firmware.sh b/viirb-get-openwrt-firmware.sh new file mode 100755 index 0000000..1682f7b --- /dev/null +++ b/viirb-get-openwrt-firmware.sh @@ -0,0 +1,14 @@ +#!/bin/sh +# 2020-10-08, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 1 ]; then + echo "$0 openwrt-version" + echo " openwrt-version: for instance 19.07.4" + exit 1 +fi + +version=$1 +filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin + +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} From bc6217fa0f32d2809328768bd84b27414e12a4a2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Oct 2020 11:52:31 +0200 Subject: [PATCH 171/543] Make use of the right variable --- viirb-1-connect-flash-latest-openwrt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index 1c69e93..4733acb 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -53,7 +53,7 @@ else scp_ip="$viirb_ip" fi -scp ${filename} root@${viirb_ip}:/tmp +scp ${filename} root@${scp_ip}:/tmp ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" # It still pings for some time - wait for the reboot to happen From 08753920987214a8aa2fa88256889b4fe1cac8cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Oct 2020 18:53:48 +0200 Subject: [PATCH 172/543] +mod Signed-off-by: Nico Schottelius --- alpine-rebuild-initramfs.sh | 87 ++++++++++++++++++++++--------------- mikrotik-update.sh | 13 +++++- openwrt-add-letsencrypt.sh | 0 openwrt-add-lte.sh | 0 openwrt-motion-config.sh | 0 5 files changed, 64 insertions(+), 36 deletions(-) mode change 100644 => 100755 openwrt-add-letsencrypt.sh mode change 100644 => 100755 openwrt-add-lte.sh mode change 100644 => 100755 openwrt-motion-config.sh diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index b56454b..8913924 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -1,90 +1,107 @@ #!/bin/sh +if [ $# -ne 1 ]; then + echo "$0 ssh-keyfile" + echo " ssh-keyfile: ssh keys to add into the image" + exit 1 +fi + set -e set -x -MAJOR_VERSION=3.11 -MINOR_VERSION=2 +SSH_KEYS=$1; shift + +MAJOR_VERSION=3.12 +MINOR_VERSION=0 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz -SSH_KEYS=$(cat ~/.ssh/id_rsa.pub) + RESOLVCONF=/etc/resolv.conf working_directory=$(pwd -P) rootfs_tmpdir=$(mktemp -d) +rootfs_tmpdir=alpine_${MAJOR_VERSION}-${MINOR_VERSION}-rootfs + +mkdir -p ${rootfs_tmpdir} + rootfs_url="http://dl-cdn.alpinelinux.org/alpine/v$MAJOR_VERSION/releases/x86_64/$IMAGE" run_root () { - chroot $rootfs_tmpdir /usr/bin/env \ + sudo chroot $rootfs_tmpdir /usr/bin/env \ PATH=/bin:/sbin \ /bin/sh -c "$*" } -if [ "$(whoami)" != 'root' ]; then - echo "This script must be run as root." >&2 - exit 1 -fi +wget -c "$rootfs_url" -O "$working_directory/$IMAGE" -# Download, extract inital rootfs. -curl "$rootfs_url" -o "$working_directory/$IMAGE" -tar xf $IMAGE -C $rootfs_tmpdir +# keep right permissions, use sudo +sudo tar xf $IMAGE -C $rootfs_tmpdir # Add SSH keys run_root mkdir -p root/.ssh -echo $SSH_KEYS > $rootfs_tmpdir/root/.ssh/authorized_keys +sudo cp $SSH_KEYS $rootfs_tmpdir/root/.ssh/authorized_keys +run_root chown root:root root/.ssh/authorized_keys run_root chmod 0600 root/.ssh/authorized_keys run_root chmod 0700 root/.ssh # Import local resolv.conf. -cat "$RESOLVCONF" > $rootfs_tmpdir/etc/resolv.conf +sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf # Make sure init is found by the kernel. -run_root ln -s /sbin/init /init +#run_root ln -sf /sbin/init /init + + +# Layer atop base rootfs. +run_root apk update +run_root apk upgrade +run_root apk add openssh linux-lts openrc udev ndisc6 + +run_root rc-update add udev +run_root rc-update add udev-trigger +run_root rc-update add sshd +run_root rc-update add networking +run_root rc-update add hostname +run_root rc-update add rdnssd + +# Generate iniramfs image +(cd $rootfs_tmpdir; sudo find . | sudo cpio -H newc -o | gzip -9 > "$working_directory/alpine-initramfs.gz") +cp "$rootfs_tmpdir/boot/vmlinuz-lts" "$working_directory/alpine-kernel" + # Servers have static addresses, disable the standard # alpine setting of using tempaddr = 2 -cat > "$rootfs_tmpdir/etc/sysctl.d/99-ipv6.conf" < "$rootfs_tmpdir/etc/network/interfaces" <> /etc/issue - post-up echo post post up >> /etc/issue EOF -cat > "$rootfs_tmpdir/etc/hostname" <> "$rootfs_tmpdir/etc/modules" +echo ipv6 | sudo tee -a "$rootfs_tmpdir/etc/modules" + + +# Upload to netboot server. - needs to be done outside sudo +echo "Use alpine-initramfs.gz alpine-kernel from $working_directory"! + +exit 0 -# Layer atop base rootfs. -run_root apk update -run_root apk upgrade -run_root apk add openssh linux-vanilla openrc udev -run_root rc-update add udev -run_root rc-update add udev-trigger -run_root rc-update add sshd -run_root rc-update add networking -run_root rc-update add hostname # FIXME: add / install rdnssd / ndisc6 / start it on boot # ndisc6 is only @testing -# Generate iniramfs image -(cd $rootfs_tmpdir; find . | cpio -H newc -o | gzip -9 > "$working_directory/alpine-initramfs.gz") -cp "$rootfs_tmpdir/boot/vmlinuz-vanilla" "$working_directory/alpine-kernel" - # Cleanup. #rm -r "$rootfs_tmpdir" - -# Upload to netboot server. - needs to be done outside sudo -echo "Use alpine-initramfs.gz alpine-kernel from $working_directory"! diff --git a/mikrotik-update.sh b/mikrotik-update.sh index 027bdba..1b4d252 100755 --- a/mikrotik-update.sh +++ b/mikrotik-update.sh @@ -2,6 +2,8 @@ # Nico Schottelius, 2019-12-02 # Update mikrotik routers to the latest package +set -e + if [ $# -lt 2 ]; then echo "$0 router [router...]" cat < Date: Mon, 12 Oct 2020 21:17:05 +0200 Subject: [PATCH 173/543] [viirb2] allow to inject private key --- viirb-2-configure-fully-after-upgrade.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 94fc628..ac84a76 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -3,9 +3,10 @@ # See https://ungleich.ch/u/products/viirb-ipv6-box/ if [ $# -ne 2 ]; then - echo "$0 viirb-ip-address viirb-id" + echo "$0 viirb-ip-address viirb-id [wgprivkey]" echo " viirb-ip-address: where to find the viirb" echo " viirb-id: number in decimal format" + echo " wgprivkey: if specified, use this private key" exit 1 fi @@ -25,7 +26,11 @@ my_lan_ip=${my_prefix}:cafe::42 my_wifi_ip=${my_prefix}:7ea::42 # wireguard -private_key=$(wg genkey) +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch From 45265b5e45a9c2d4b732e33a986380353d44103b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Oct 2020 21:22:06 +0200 Subject: [PATCH 174/543] fix argc chekch --- viirb-2-configure-fully-after-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index ac84a76..093d547 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -2,7 +2,7 @@ # 2020-06-13, Nico Schottelius # See https://ungleich.ch/u/products/viirb-ipv6-box/ -if [ $# -ne 2 ]; then +if [ $# -lt 2 ]; then echo "$0 viirb-ip-address viirb-id [wgprivkey]" echo " viirb-ip-address: where to find the viirb" echo " viirb-id: number in decimal format" From 1f0f798d3837e780d2edd1e11625f8ac4505cff1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Oct 2020 22:02:32 +0200 Subject: [PATCH 175/543] ++openwrt scripts --- openwrt-add-gps.sh | 5 +++++ openwrt-add-temper.sh | 8 ++++++++ 2 files changed, 13 insertions(+) create mode 100644 openwrt-add-gps.sh create mode 100644 openwrt-add-temper.sh diff --git a/openwrt-add-gps.sh b/openwrt-add-gps.sh new file mode 100644 index 0000000..772e1e8 --- /dev/null +++ b/openwrt-add-gps.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +opkg install kmod-usb-acm + +# ugps? diff --git a/openwrt-add-temper.sh b/openwrt-add-temper.sh new file mode 100644 index 0000000..866f3ee --- /dev/null +++ b/openwrt-add-temper.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +opkg install kmod-usb-hid usbtemper + +exit 0 + +[ 2089.624343] hid-generic 0003:413D:2107.0001: input,hidraw0: USB HID v1.11 Keyboard [HID 413d:2107] on usb-101c0000.ehci-1.2/input0 +[ 2089.826684] hid-generic 0003:413D:2107.0002: hiddev96,hidraw1: USB HID v1.10 Device [HID 413d:2107] on usb-101c0000.ehci-1.2/input1 From 76ff02656d1bac52ea7e976de10ebaf2efca179c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Oct 2020 23:06:24 +0200 Subject: [PATCH 176/543] [openwrt] ++scripts --- openwrt-add-camera.sh | 28 ++++++++++++++++++++++++++++ openwrt-add-temper.sh | 8 +++++++- 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 openwrt-add-camera.sh diff --git a/openwrt-add-camera.sh b/openwrt-add-camera.sh new file mode 100644 index 0000000..5688f0a --- /dev/null +++ b/openwrt-add-camera.sh @@ -0,0 +1,28 @@ +#!/bin/sh + +opkg install motion kmod-video-uvc + +uci set motion.general.enabled=1 + +cat > /etc/motion.conf < Date: Fri, 16 Oct 2020 19:12:40 +0200 Subject: [PATCH 177/543] [ipxe] adjust rebuild script for new ipxe source code --- rebuild-ipxe.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rebuild-ipxe.sh b/rebuild-ipxe.sh index 2d1ad5a..bdb09a8 100755 --- a/rebuild-ipxe.sh +++ b/rebuild-ipxe.sh @@ -14,7 +14,9 @@ fi cd ipxe/src -sed -i -e 's/^#undef.*NET_PROTO_IPV6/#define NET_PROTO_IPV6/' \ +sed -i \ + -e 's/^#undef.*NET_PROTO_IPV6/#define NET_PROTO_IPV6/' \ + -e 's,//^#define.*NET_PROTO_IPV6/#define NET_PROTO_IPV6/' \ -e 's/^#undef.*DOWNLOAD_PROTO_HTTPS/#define DOWNLOAD_PROTO_HTTPS/' \ -e 's,^//#define POWEROFF_CMD,#define POWEROFF_CMD,' \ -e 's,^//#define PING_CMD,#define PING_CMD,' \ From 9fa7f29d109c5be301c8bddfe9f5d311810b3be9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 16 Oct 2020 19:13:22 +0200 Subject: [PATCH 178/543] Fix ipxe^2 --- rebuild-ipxe.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebuild-ipxe.sh b/rebuild-ipxe.sh index bdb09a8..8ef978e 100755 --- a/rebuild-ipxe.sh +++ b/rebuild-ipxe.sh @@ -16,7 +16,7 @@ cd ipxe/src sed -i \ -e 's/^#undef.*NET_PROTO_IPV6/#define NET_PROTO_IPV6/' \ - -e 's,//^#define.*NET_PROTO_IPV6/#define NET_PROTO_IPV6/' \ + -e 's,//^#define.*NET_PROTO_IPV6,#define NET_PROTO_IPV6,' \ -e 's/^#undef.*DOWNLOAD_PROTO_HTTPS/#define DOWNLOAD_PROTO_HTTPS/' \ -e 's,^//#define POWEROFF_CMD,#define POWEROFF_CMD,' \ -e 's,^//#define PING_CMD,#define PING_CMD,' \ From 4d348ab724614d6d72119bbcdceab960464d3b54 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 18 Oct 2020 09:47:53 +0200 Subject: [PATCH 179/543] Update working script. --- alpine-rebuild-initramfs.sh | 47 +++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 8913924..668e0cb 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -19,7 +19,7 @@ RESOLVCONF=/etc/resolv.conf working_directory=$(pwd -P) rootfs_tmpdir=$(mktemp -d) -rootfs_tmpdir=alpine_${MAJOR_VERSION}-${MINOR_VERSION}-rootfs +#rootfs_tmpdir=alpine_${MAJOR_VERSION}-${MINOR_VERSION}-rootfs mkdir -p ${rootfs_tmpdir} @@ -31,7 +31,7 @@ run_root () { /bin/sh -c "$*" } -wget -c "$rootfs_url" -O "$working_directory/$IMAGE" +wget -c "$rootfs_url" -O "$IMAGE" # keep right permissions, use sudo sudo tar xf $IMAGE -C $rootfs_tmpdir @@ -47,25 +47,42 @@ run_root chmod 0700 root/.ssh sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf # Make sure init is found by the kernel. -#run_root ln -sf /sbin/init /init +run_root ln -sf /sbin/init /init - -# Layer atop base rootfs. run_root apk update -run_root apk upgrade -run_root apk add openssh linux-lts openrc udev ndisc6 - +run_root apk add linux-lts openrc udev openssh rdnssd run_root rc-update add udev run_root rc-update add udev-trigger run_root rc-update add sshd -run_root rc-update add networking -run_root rc-update add hostname run_root rc-update add rdnssd +# Layer atop base rootfs. +# run_root apk add openssh linux-lts openrc udev ndisc6 + + +# run_root rc-update add networking +# run_root rc-update add hostname +# run_root rc-update add rdnssd + + +sudo tee "$rootfs_tmpdir/etc/network/interfaces" < /proc/sys/net/ipv6/conf/eth0/accept_ra +EOF + # Generate iniramfs image (cd $rootfs_tmpdir; sudo find . | sudo cpio -H newc -o | gzip -9 > "$working_directory/alpine-initramfs.gz") cp "$rootfs_tmpdir/boot/vmlinuz-lts" "$working_directory/alpine-kernel" +echo rm -rf "$rootfs_tmpdir" + +exit 0 + # Servers have static addresses, disable the standard # alpine setting of using tempaddr = 2 @@ -76,16 +93,6 @@ net.ipv6.conf.all.use_tempaddr = 0 net.ipv6.conf.all.accept_ra = 1 EOF -sudo tee "$rootfs_tmpdir/etc/network/interfaces" <> /etc/issue -EOF sudo tee "$rootfs_tmpdir/etc/hostname" < Date: Sun, 18 Oct 2020 09:48:07 +0200 Subject: [PATCH 180/543] [alpine initramfs] tempaddr fix not necessary anymore --- alpine-rebuild-initramfs.sh | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 668e0cb..a5b8303 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -84,16 +84,6 @@ echo rm -rf "$rootfs_tmpdir" exit 0 -# Servers have static addresses, disable the standard -# alpine setting of using tempaddr = 2 -sudo tee "$rootfs_tmpdir/etc/sysctl.d/99-ipv6.conf" < Date: Sun, 18 Oct 2020 10:03:38 +0200 Subject: [PATCH 181/543] [alpine initramfs] disable rdnssd - does not yet exist --- alpine-rebuild-initramfs.sh | 36 ++++++++---------------------------- 1 file changed, 8 insertions(+), 28 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index a5b8303..89e9820 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -50,20 +50,14 @@ sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf run_root ln -sf /sbin/init /init run_root apk update -run_root apk add linux-lts openrc udev openssh rdnssd +run_root apk add linux-lts openrc udev openssh +# rdnssd run_root rc-update add udev run_root rc-update add udev-trigger run_root rc-update add sshd -run_root rc-update add rdnssd - -# Layer atop base rootfs. -# run_root apk add openssh linux-lts openrc udev ndisc6 - - -# run_root rc-update add networking -# run_root rc-update add hostname # run_root rc-update add rdnssd - +run_root rc-update add networking +run_root rc-update add hostname sudo tee "$rootfs_tmpdir/etc/network/interfaces" < /proc/sys/net/ipv6/conf/eth0/accept_ra EOF +sudo tee "$rootfs_tmpdir/etc/hostname" < "$working_directory/alpine-initramfs.gz") cp "$rootfs_tmpdir/boot/vmlinuz-lts" "$working_directory/alpine-kernel" echo rm -rf "$rootfs_tmpdir" -exit 0 - - -sudo tee "$rootfs_tmpdir/etc/hostname" < Date: Sat, 24 Oct 2020 13:15:23 +0200 Subject: [PATCH 182/543] Bump ubuntu image release --- opennebula-images/ubuntu-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh index 6535f66..01a4c3b 100755 --- a/opennebula-images/ubuntu-build-opennebula-image.sh +++ b/opennebula-images/ubuntu-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=eoan # 19.10 +RELEASE=groovy # 20.10 ARCH=amd64 IMAGE_PATH=ubuntu-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G From b2ab0959a3353928fd48d9e355425cc5d1e5c830 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Oct 2020 11:51:16 +0100 Subject: [PATCH 183/543] +exec Signed-off-by: Nico Schottelius --- openwrt-add-camera.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 openwrt-add-camera.sh diff --git a/openwrt-add-camera.sh b/openwrt-add-camera.sh old mode 100644 new mode 100755 From 9a7470d94a268c720f62802f5515f4d2b62584a5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 26 Oct 2020 11:52:49 +0100 Subject: [PATCH 184/543] import new camera script Signed-off-by: Nico Schottelius --- openwrt-add-camera.sh | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/openwrt-add-camera.sh b/openwrt-add-camera.sh index 5688f0a..33d5fd3 100755 --- a/openwrt-add-camera.sh +++ b/openwrt-add-camera.sh @@ -1,11 +1,33 @@ #!/bin/sh +if [ $# -lt 1 ]; then + echo "$0 ip-address" + echo " ip-address: where to find the device" + exit 1 +fi + + +set -x + +openwrt_ip=$1; shift + +ping -c3 ${openwrt_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach ${openwrt_ip}, aborting" + exit 1 +fi + +cat < /etc/motion.conf < /etc/motion.conf < Date: Tue, 3 Nov 2020 14:04:38 +0100 Subject: [PATCH 185/543] [openwrt/lte] add note which interface to use --- openwrt-add-lte.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openwrt-add-lte.sh b/openwrt-add-lte.sh index 5e3a2c7..0be94fe 100755 --- a/openwrt-add-lte.sh +++ b/openwrt-add-lte.sh @@ -4,7 +4,8 @@ if [ $# -ne 2 ]; then echo "$0 ip-address interface" echo " ip-address: where to find the OpenWRT device" - echo " interface: which interface (eth3 for instance) is the LTE device" + echo " interface: which interface is the LTE device" + echo " Usually eth1 on VIIRB, eth3 on PIB" exit 1 fi From 659066f8981e0ae262950e5e2a7e5222ad5ea80b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Nov 2020 10:55:48 +0100 Subject: [PATCH 186/543] [viirb] viirb3: less verbose cdist run --- viirb-3-configure-vpnendpoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-configure-vpnendpoint.sh index ac3b742..77d64f4 100755 --- a/viirb-3-configure-vpnendpoint.sh +++ b/viirb-3-configure-vpnendpoint.sh @@ -52,7 +52,7 @@ git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" git pull git push -cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} +cdist config -v -j8 ${vpn_endpoint_host} -c ${dot_cdist} # Test that the VPN connection is established # Might take longer due to reboot From c86635ed56995345eb6e1d4adb62ea79a67a17ff Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Nov 2020 10:57:44 +0100 Subject: [PATCH 187/543] [viirb] viirb3: stop if successful ping --- viirb-3-configure-vpnendpoint.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-configure-vpnendpoint.sh index 77d64f4..8387449 100755 --- a/viirb-3-configure-vpnendpoint.sh +++ b/viirb-3-configure-vpnendpoint.sh @@ -58,6 +58,20 @@ cdist config -v -j8 ${vpn_endpoint_host} -c ${dot_cdist} # Might take longer due to reboot sleep 10 -ping -c10 ${my_wireguard_ip} -ping -c10 ${my_lan_ip} -ping -c90 ${my_wifi_ip} +i=0 +while [ $i -lt 10 ]; do + ping -c1 ${my_wireguard_ip} && break + i=$((i+1)) +done + +i=0 +while [ $i -lt 10 ]; do + ping -c1 ${my_lan_ip} && break + i=$((i+1)) +done + +i=0 +while [ $i -lt 10 ]; do + ping -c1 ${my_wifi_ip} && break + i=$((i+1)) +done From 84944f25840c3f8d5d30209b59bad3362e1ba610 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Nov 2020 11:59:17 +0100 Subject: [PATCH 188/543] +add script to fix missing peers Signed-off-by: Nico Schottelius --- one-time-scripts/fix-vpn-peers.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 one-time-scripts/fix-vpn-peers.sh diff --git a/one-time-scripts/fix-vpn-peers.sh b/one-time-scripts/fix-vpn-peers.sh new file mode 100644 index 0000000..77ae2b5 --- /dev/null +++ b/one-time-scripts/fix-vpn-peers.sh @@ -0,0 +1,25 @@ +#!/bin/sh + +vpndir=/home/nico/vcs/ungleich-dot-cdist/type/__ungleich_wireguard/files + +for i in $(ls -1 viirb*public_key); do + viirb=${i%.public_key}; + num=${viirb#viirb}; + hex=$(printf "%0.2x" $num); + pubkey=$(cat $i); + network="2a0a:e5c1:3${hex}::/48"; + peerfilename=vpn-2a0ae5c1300.ungleich.ch.peer${hex} + + peerfile=${vpndir}/${peerfilename} + if [ ! -f "${peerfile}" ]; then + echo "VIIRB $num / peer $hex missing, recreating" + cat < "${peerfile}" +# viirb${num}, $(date +%F) +[Peer] +PublicKey = $pubkey +AllowedIPs = ${network} + +EOF + fi + +done From 5945e87f0f031602a6fb5eed3937401a01e6e90e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 4 Nov 2020 14:36:05 +0100 Subject: [PATCH 189/543] [openwrt] add script to enable wifi on hp notebooks --- openwrt-add-gps.sh | 0 openwrt-add-temper.sh | 0 openwrt-hp-250-g5-add-wifi.sh | 21 +++++++++++++++++++++ 3 files changed, 21 insertions(+) mode change 100644 => 100755 openwrt-add-gps.sh mode change 100644 => 100755 openwrt-add-temper.sh create mode 100755 openwrt-hp-250-g5-add-wifi.sh diff --git a/openwrt-add-gps.sh b/openwrt-add-gps.sh old mode 100644 new mode 100755 diff --git a/openwrt-add-temper.sh b/openwrt-add-temper.sh old mode 100644 new mode 100755 diff --git a/openwrt-hp-250-g5-add-wifi.sh b/openwrt-hp-250-g5-add-wifi.sh new file mode 100755 index 0000000..adb58fb --- /dev/null +++ b/openwrt-hp-250-g5-add-wifi.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +if [ $# -lt 1 ]; then + echo "$0 ip-address" + echo " ip-address: where to find the device" + exit 1 +fi + + +openwrt_ip=$1; shift + +ping -c3 ${openwrt_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach ${openwrt_ip}, aborting" + exit 1 +fi + +cat < Date: Wed, 4 Nov 2020 14:39:32 +0100 Subject: [PATCH 190/543] also add wpa-supplicant for wpa2 support --- openwrt-hp-250-g5-add-wifi.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt-hp-250-g5-add-wifi.sh b/openwrt-hp-250-g5-add-wifi.sh index adb58fb..cc26bbc 100755 --- a/openwrt-hp-250-g5-add-wifi.sh +++ b/openwrt-hp-250-g5-add-wifi.sh @@ -17,5 +17,5 @@ fi cat < Date: Thu, 19 Nov 2020 19:49:00 +0100 Subject: [PATCH 191/543] [openwrt] add script to only configure the wireguard vpn --- openwrt-add-wireguard.sh | 63 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100755 openwrt-add-wireguard.sh diff --git a/openwrt-add-wireguard.sh b/openwrt-add-wireguard.sh new file mode 100755 index 0000000..d44d9e9 --- /dev/null +++ b/openwrt-add-wireguard.sh @@ -0,0 +1,63 @@ +#!/bin/sh +# Nico Schottelius +# 2020-11-19 + +if [ $# -lt 4 ]; then + echo "$0 ip-address vpn-server ipv6-network [privatekey]" + echo " ip-address: where to find the OpenWRT device" + echo " vpn-server: where to connect to" + echo " vpn-server-pubkey: public key of the server" + echo " ipv6-network: which network to use for us (/48 expected)" + echo " private-key: specify wireguard key optionally" + exit 1 +fi + +my_ip=$1; shift +vpn_endpoint_host=$1; shift +vpn_endpoint_pubkey=$1; shift +network=$1; shift + +# wireguard +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi + +my_prefix=$(echo $network | sed 's,::/.*,,') +my_wireguard_ip=${my_prefix}::42 + +public_key=$(echo $private_key | wg pubkey) + +cat < Date: Thu, 19 Nov 2020 19:49:24 +0100 Subject: [PATCH 192/543] [openwrt] camera switch to mjpg --- openwrt-add-camera.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/openwrt-add-camera.sh b/openwrt-add-camera.sh index 33d5fd3..25d98a8 100755 --- a/openwrt-add-camera.sh +++ b/openwrt-add-camera.sh @@ -20,6 +20,27 @@ fi cat < Date: Thu, 19 Nov 2020 19:49:40 +0100 Subject: [PATCH 193/543] [openwrt] initial gps code --- openwrt-add-gps.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/openwrt-add-gps.sh b/openwrt-add-gps.sh index 772e1e8..ae65aa5 100755 --- a/openwrt-add-gps.sh +++ b/openwrt-add-gps.sh @@ -1,5 +1,31 @@ #!/bin/sh -opkg install kmod-usb-acm +if [ $# -lt 1 ]; then + echo "$0 ip-address" + echo " ip-address: where to find the device" + exit 1 +fi -# ugps? + +set -x + +openwrt_ip=$1; shift + +ping -c3 ${openwrt_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach ${openwrt_ip}, aborting" + exit 1 +fi + +cat < Date: Mon, 30 Nov 2020 10:34:53 +0100 Subject: [PATCH 194/543] ++lte/openwrt Signed-off-by: Nico Schottelius --- openwrt-add-qmi-lte.sh | 44 ++++++++++++++++++++ openwrt-add-lte.sh => openwrt-add-usb-lte.sh | 0 2 files changed, 44 insertions(+) create mode 100755 openwrt-add-qmi-lte.sh rename openwrt-add-lte.sh => openwrt-add-usb-lte.sh (100%) diff --git a/openwrt-add-qmi-lte.sh b/openwrt-add-qmi-lte.sh new file mode 100755 index 0000000..0f4caae --- /dev/null +++ b/openwrt-add-qmi-lte.sh @@ -0,0 +1,44 @@ +#!/bin/sh +# Based on work of Samuel Hailu, 2020-09-10 + +if [ $# -ne 1 ]; then + echo "$0 ip-address interface" + echo " ip-address: where to find the OpenWRT device" + exit 1 +fi + +my_ip=$1; shift +interface=$1; shift + +cat < Date: Wed, 2 Dec 2020 13:04:08 +0100 Subject: [PATCH 195/543] ++openwrt scripts --- ...> openwrt-add-camera-with-mjpg-streamer.sh | 0 openwrt-add-camera-with-motion.sh | 52 +++++++++++++++++++ openwrt-add-jool.sh | 32 ++++++++++++ 3 files changed, 84 insertions(+) rename openwrt-add-camera.sh => openwrt-add-camera-with-mjpg-streamer.sh (100%) create mode 100755 openwrt-add-camera-with-motion.sh create mode 100644 openwrt-add-jool.sh diff --git a/openwrt-add-camera.sh b/openwrt-add-camera-with-mjpg-streamer.sh similarity index 100% rename from openwrt-add-camera.sh rename to openwrt-add-camera-with-mjpg-streamer.sh diff --git a/openwrt-add-camera-with-motion.sh b/openwrt-add-camera-with-motion.sh new file mode 100755 index 0000000..33d5fd3 --- /dev/null +++ b/openwrt-add-camera-with-motion.sh @@ -0,0 +1,52 @@ +#!/bin/sh + +if [ $# -lt 1 ]; then + echo "$0 ip-address" + echo " ip-address: where to find the device" + exit 1 +fi + + +set -x + +openwrt_ip=$1; shift + +ping -c3 ${openwrt_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach ${openwrt_ip}, aborting" + exit 1 +fi + +cat < /etc/motion.conf < Date: Thu, 3 Dec 2020 20:03:23 +0100 Subject: [PATCH 196/543] [openwrt] update for temper support --- openwrt-add-temper.sh | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/openwrt-add-temper.sh b/openwrt-add-temper.sh index d2ddc4d..96d6635 100755 --- a/openwrt-add-temper.sh +++ b/openwrt-add-temper.sh @@ -1,10 +1,32 @@ #!/bin/sh -opkg install kmod-usb-hid -opkg install python3-pyserial +if [ $# -ne 1 ]; then + echo "$0 ip-address" + echo " ip-address: where to find the OpenWRT device" + exit 1 +fi + +my_ip=$1; shift + + + +cat < Date: Thu, 3 Dec 2020 20:11:33 +0100 Subject: [PATCH 197/543] update temper script --- openwrt-add-temper.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/openwrt-add-temper.sh b/openwrt-add-temper.sh index 96d6635..585f772 100755 --- a/openwrt-add-temper.sh +++ b/openwrt-add-temper.sh @@ -25,6 +25,10 @@ mv temper-py-0.0.1/temper.py /usr/bin rmdir temper-py-0.0.1 chmod a+rx /usr/bin/temper.py +if ! grep -q temper /etc/crontabs/root; then + echo "*/5 * * * * /usr/bin/temper.py awk '{ print $7 " " $8 }' > /www/temperature.txt" >> /etc/crontabs/root +fi + EOF exit 0 From 5914512604fa23a4257037eae2f82c3acb93e424 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 3 Dec 2020 20:12:50 +0100 Subject: [PATCH 198/543] +| --- openwrt-add-temper.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt-add-temper.sh b/openwrt-add-temper.sh index 585f772..3b66ac1 100755 --- a/openwrt-add-temper.sh +++ b/openwrt-add-temper.sh @@ -26,7 +26,7 @@ rmdir temper-py-0.0.1 chmod a+rx /usr/bin/temper.py if ! grep -q temper /etc/crontabs/root; then - echo "*/5 * * * * /usr/bin/temper.py awk '{ print $7 " " $8 }' > /www/temperature.txt" >> /etc/crontabs/root + echo "*/5 * * * * /usr/bin/temper.py | awk '{ print $7 " " $8 }' > /www/temperature.txt" >> /etc/crontabs/root fi EOF From 9ca5f9a96560c0451e79c41ca44d21e511d8833e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 3 Dec 2020 20:17:19 +0100 Subject: [PATCH 199/543] [openwrt] upgrade temper support to version 0.0.2 --- openwrt-add-temper.sh | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/openwrt-add-temper.sh b/openwrt-add-temper.sh index 3b66ac1..372fbcb 100755 --- a/openwrt-add-temper.sh +++ b/openwrt-add-temper.sh @@ -9,6 +9,8 @@ fi my_ip=$1; shift +temper_version=0.0.2 +temper_url=https://files.pythonhosted.org/packages/7e/1b/0579fef8482660fe194099d988a8ae82b5e5843a60070520039789b26a84/temper-py-0.0.2.tar.gz cat < Date: Thu, 3 Dec 2020 20:25:20 +0100 Subject: [PATCH 200/543] [openwrt/temper] Also restart cron --- openwrt-add-temper.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openwrt-add-temper.sh b/openwrt-add-temper.sh index 372fbcb..da44cf9 100755 --- a/openwrt-add-temper.sh +++ b/openwrt-add-temper.sh @@ -29,8 +29,10 @@ chmod a+rx /usr/bin/temper.py if ! grep -q temper /etc/crontabs/root; then echo "*/5 * * * * /usr/bin/temper.py | awk '{ print $7 " " $8 }' > /www/temperature.txt" >> /etc/crontabs/root + /etc/init.d/cron restart fi + EOF exit 0 From 568aab75e11efb917d0f71a5b17f9fc7849e1f00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 7 Dec 2020 07:57:57 +0100 Subject: [PATCH 201/543] Initial devuan ONE image build script --- .../devuan-build-opennebula-image.sh | 163 ++++++++++++++++++ 1 file changed, 163 insertions(+) create mode 100755 opennebula-images/devuan-build-opennebula-image.sh diff --git a/opennebula-images/devuan-build-opennebula-image.sh b/opennebula-images/devuan-build-opennebula-image.sh new file mode 100755 index 0000000..5f747b8 --- /dev/null +++ b/opennebula-images/devuan-build-opennebula-image.sh @@ -0,0 +1,163 @@ +#!/bin/sh + +# This script generates Devuan images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=beowulf +ARCH=amd64 +IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 +HOSTNAME=devuan +MIRROR=http://pkgmaster.devuan.org/merged + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Devuan" ]; then + echo "WARNING: this script has been designed to run on a Devuan system." >&2 + echo "WARNING: Not running Debian. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 $HOSTNAME localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 $HOSTNAME localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +run_root hostname $HOSTNAME + +# Configure package sources and update package index. +cat >/mnt/etc/apt/sources.list < "/mnt$ONE_CONTEXT_DEB_PATH" +run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" +run_root rm "$ONE_CONTEXT_DEB_PATH" + +# Manually install legacy network scripts used by one-context. +run_root apt-get -y install ifupdown + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install kernel and bootloader. Do not autoconfigure grub. +run_root 'echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections' +run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-image-amd64 grub-pc + +# Configure grub. +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root apt-get -y install openssh-server + +# Install haveged due to lack of entropy in ONE environment. +run_root apt-get -y install haveged +run_root update-rc.d haveged defaults + +# Generate locales. +run_root 'sed -i "s/^# *\(en_GB.UTF-8\)/\1/" etc/locale.gen' +run_root locale-gen + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Mon, 7 Dec 2020 08:40:01 +0100 Subject: [PATCH 202/543] Fix devuan IMAGE_PATH output --- opennebula-images/devuan-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/devuan-build-opennebula-image.sh b/opennebula-images/devuan-build-opennebula-image.sh index 5f747b8..ab9b598 100755 --- a/opennebula-images/devuan-build-opennebula-image.sh +++ b/opennebula-images/devuan-build-opennebula-image.sh @@ -11,7 +11,7 @@ set -x # XXX: Handle command-line arguments? RELEASE=beowulf ARCH=amd64 -IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_PATH=devuan-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 HOSTNAME=devuan From 496b354245dd9d5d431b8c2dc77f67210cd29d55 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 7 Dec 2020 13:43:10 +0100 Subject: [PATCH 203/543] [mystrom] add script to turn devices on --- mystrom-on.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100755 mystrom-on.sh diff --git a/mystrom-on.sh b/mystrom-on.sh new file mode 100755 index 0000000..7019f8f --- /dev/null +++ b/mystrom-on.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# 2020-12-07, Nico Schottelius + +if [ $# -lt 1 ]; then + echo "$0 hostname [hostname...]" + echo " hostname: which mystrom to turn on" + exit 1 +fi + +while [ $# -ge 1 ]; do + hostname=$1; shift + + curl --location \ + --request GET \ + "http://${hostname}/relay?state=1" + +done From a2894395acaf7c4d6191d4a5012db78b0bd9057a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 7 Dec 2020 13:47:34 +0100 Subject: [PATCH 204/543] [mystrom] add supporting scripts --- mystrom-off.sh | 17 +++++++++++++++++ mystrom-on.sh | 2 +- mystrom-status.sh | 15 +++++++++++++++ mystrom-temperature.sh | 15 +++++++++++++++ 4 files changed, 48 insertions(+), 1 deletion(-) create mode 100755 mystrom-off.sh create mode 100755 mystrom-status.sh create mode 100755 mystrom-temperature.sh diff --git a/mystrom-off.sh b/mystrom-off.sh new file mode 100755 index 0000000..7268e90 --- /dev/null +++ b/mystrom-off.sh @@ -0,0 +1,17 @@ +#!/bin/sh +# 2020-12-07, Nico Schottelius + +if [ $# -lt 1 ]; then + echo "$0 hostname [hostname...]" + echo " hostname: which mystrom to connect to" + exit 1 +fi + +while [ $# -ge 1 ]; do + hostname=$1; shift + + curl --location \ + --request GET \ + "http://${hostname}/relay?state=0" + +done diff --git a/mystrom-on.sh b/mystrom-on.sh index 7019f8f..37c439d 100755 --- a/mystrom-on.sh +++ b/mystrom-on.sh @@ -3,7 +3,7 @@ if [ $# -lt 1 ]; then echo "$0 hostname [hostname...]" - echo " hostname: which mystrom to turn on" + echo " hostname: which mystrom to connect to" exit 1 fi diff --git a/mystrom-status.sh b/mystrom-status.sh new file mode 100755 index 0000000..01f7366 --- /dev/null +++ b/mystrom-status.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# 2020-12-07, Nico Schottelius + +if [ $# -lt 1 ]; then + echo "$0 hostname [hostname...]" + echo " hostname: which mystrom to connect to" + exit 1 +fi + +while [ $# -ge 1 ]; do + hostname=$1; shift + + http --json "http://${hostname}/report" + +done diff --git a/mystrom-temperature.sh b/mystrom-temperature.sh new file mode 100755 index 0000000..a51ed03 --- /dev/null +++ b/mystrom-temperature.sh @@ -0,0 +1,15 @@ +#!/bin/sh +# 2020-12-07, Nico Schottelius + +if [ $# -lt 1 ]; then + echo "$0 hostname [hostname...]" + echo " hostname: which mystrom to connect to" + exit 1 +fi + +while [ $# -ge 1 ]; do + hostname=$1; shift + + http --json "http://${hostname}/temp" + +done From 4e7596a774b9ebeb21c513f9a5a689616c88329d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 9 Dec 2020 12:57:28 +0100 Subject: [PATCH 205/543] use GET for mystrom on http --- mystrom-status.sh | 2 +- mystrom-toggle.sh | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100755 mystrom-toggle.sh diff --git a/mystrom-status.sh b/mystrom-status.sh index 01f7366..d956cd1 100755 --- a/mystrom-status.sh +++ b/mystrom-status.sh @@ -10,6 +10,6 @@ fi while [ $# -ge 1 ]; do hostname=$1; shift - http --json "http://${hostname}/report" + http --json GET "http://${hostname}/report" done diff --git a/mystrom-toggle.sh b/mystrom-toggle.sh new file mode 100755 index 0000000..a397619 --- /dev/null +++ b/mystrom-toggle.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# 2020-12-07, Nico Schottelius + +set -x +exec >>/tmp/foo +exec 2>&1 + +if [ $# -lt 1 ]; then + echo "$0 hostname [hostname...]" + echo " hostname: which mystrom to connect to" + exit 1 +fi + +while [ $# -ge 1 ]; do + hostname=$1; shift + + http --json GET "http://${hostname}/toggle" +done From 7014e3cb821074d7154b948c46cb204ed56a42df Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 9 Dec 2020 12:57:48 +0100 Subject: [PATCH 206/543] fix regexp --- rebuild-ipxe.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebuild-ipxe.sh b/rebuild-ipxe.sh index 8ef978e..bd2fe65 100755 --- a/rebuild-ipxe.sh +++ b/rebuild-ipxe.sh @@ -16,7 +16,7 @@ cd ipxe/src sed -i \ -e 's/^#undef.*NET_PROTO_IPV6/#define NET_PROTO_IPV6/' \ - -e 's,//^#define.*NET_PROTO_IPV6,#define NET_PROTO_IPV6,' \ + -e 's,^//#define.*NET_PROTO_IPV6,#define NET_PROTO_IPV6,' \ -e 's/^#undef.*DOWNLOAD_PROTO_HTTPS/#define DOWNLOAD_PROTO_HTTPS/' \ -e 's,^//#define POWEROFF_CMD,#define POWEROFF_CMD,' \ -e 's,^//#define PING_CMD,#define PING_CMD,' \ From e8a9467784141cdb35279f0208d2955ef9066319 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 9 Dec 2020 12:59:48 +0100 Subject: [PATCH 207/543] -white --- mystrom-on.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/mystrom-on.sh b/mystrom-on.sh index 37c439d..c9aac5a 100755 --- a/mystrom-on.sh +++ b/mystrom-on.sh @@ -13,5 +13,4 @@ while [ $# -ge 1 ]; do curl --location \ --request GET \ "http://${hostname}/relay?state=1" - done From dbc41105303765b4ede2cce4903f5e6619569c6d Mon Sep 17 00:00:00 2001 From: samuel Date: Fri, 11 Dec 2020 16:05:29 +0100 Subject: [PATCH 208/543] Added new viwib script --- viwib-1-firmware-upgrade.sh | 85 +++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100755 viwib-1-firmware-upgrade.sh diff --git a/viwib-1-firmware-upgrade.sh b/viwib-1-firmware-upgrade.sh new file mode 100755 index 0000000..1aca868 --- /dev/null +++ b/viwib-1-firmware-upgrade.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -lt 1 ]; then + echo "$0 interface [address]" + echo " interface to add the config ip address to" + echo " address: connect to this address, ignore the interface" + exit 1 +fi + +set -x +dev=$1; shift + +# $# = 2 +# shift -> remove the first argument +# $# = 1 + +if [ $# -ge 1 ]; then + viwib_ip=$1; shift + dev="" +else + viwib_ip=192.168.8.1 +fi + +# openwrt +version=19.07.4 +filename=openwrt-${version}-ramips-mt76x8-gl-mt300n-v2-squashfs-sysupgrade.bin + +# IP address for setting it up initially + +if [ "$dev" ]; then + sudo ip addr del 192.168.8.2/24 dev "$dev" 2>/dev/null || true + sudo ip addr add 192.168.8.2/24 dev "$dev" +fi + +# don't care about other/old known_host entries +ssh-keygen -R ${viwib_ip} + +while ! ping -c1 ${viwib_ip}; do + echo "Cannot ping $viwib_ip yet - waiting" + sleep 1 +done + +cat ~/.ssh/id_rsa.pub | ssh root@${viwib_ip} "cat > /etc/dropbear/authorized_keys" + +# Don't re-download if we already have it +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} + +if echo $viwib_ip | grep -q :; then + scp_ip="[$viwib_ip]" +else + scp_ip="$viwib_ip" +fi + +scp ${filename} root@${scp_ip}:/tmp +ssh root@${viwib_ip} "sysupgrade -n /tmp/*.bin" + +# It still pings for some time - wait for the reboot to happen +echo "Waiting for viwib to really disappear" +sleep 15 + +wait=0 +found="" + +while [ $wait -lt 180 ]; do + ping -c1 ${viwib_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + break + fi + + sleep 1 + wait=$((wait+1)) + done + +if [ ! "$found" ]; then + echo "Did not find updated viwib - debug / restart it" + exit 1 +fi + +echo "viwib successfully updated to ${version}" From d82b18cdc21ce166c830fd1da0bb6290d88d1b58 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 11 Dec 2020 23:11:06 +0100 Subject: [PATCH 209/543] qmi update --- openwrt-add-qmi-lte.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt-add-qmi-lte.sh b/openwrt-add-qmi-lte.sh index 0f4caae..ecd0023 100755 --- a/openwrt-add-qmi-lte.sh +++ b/openwrt-add-qmi-lte.sh @@ -20,7 +20,7 @@ opkg install libustream-openssl ca-bundle ca-certificates # Install needed kernel module opkg install kmod-usb-net-qmi-wwan uqmi luci-proto-qmi -# opkg install usb-modeswitch kmod-mii kmod-usb-net kmod-usb-wdm kmod-usb-net-qmi-wwan uqmi +# opkg install usb-modeswitch kmod-mii kmod-usb-net # Create interface # uci set network.LTE=interface From 017d9613b43ccfe754b14251e91dd076daa2f025 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 11 Dec 2020 23:52:31 +0100 Subject: [PATCH 210/543] [viwib] phase in phase 3 (vpn config) --- ...configure-vpnendpoint.sh => viirb-3-vpn.sh | 0 viwib-3-vpn.sh | 76 +++++++++++++++++++ 2 files changed, 76 insertions(+) rename viirb-3-configure-vpnendpoint.sh => viirb-3-vpn.sh (100%) create mode 100755 viwib-3-vpn.sh diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-vpn.sh similarity index 100% rename from viirb-3-configure-vpnendpoint.sh rename to viirb-3-vpn.sh diff --git a/viwib-3-vpn.sh b/viwib-3-vpn.sh new file mode 100755 index 0000000..36ee375 --- /dev/null +++ b/viwib-3-vpn.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# 2020-12-11, Nico Schottelius + +if [ $# -ne 3 ]; then + echo "$0 your-dot-cdist viwib-id public-key" + echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" + echo " viwib-id: number in decimal format" + echo " wireguard public key" + exit 1 +fi + +set -x +dot_cdist=$1; shift +id=$1; shift +public_key=$1; shift + +hex_id=$(printf "%0.2x\n" "$id") +viwib_hostname=viwib${id} + +prefix_base=2a0a:e5c1:6 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +vpn_endpoint_host=vpn-2a0ae5c1600.ungleich.ch + +# cdist +dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files +peerfilename=${vpn_endpoint_host}.peer${hex_id} +peerfile=${dot_cdist_files}/${peerfilename} +vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} + + +# Configure VPN server / update cdist +echo Updating VPNserver +cat < ${peerfile} +# ${viwib_hostname}, $(date +%F) +[Peer] +PublicKey = ${public_key} +AllowedIPs = ${my_network} + +EOF + +# Generate real config +cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} +cd ${dot_cdist_files} +git add ${vpn_endpoint_host} ${peerfilename} +git commit -m "[vpn] Updated config for peer ${viwib_hostname} ${my_network}" +git pull +git push + +cdist config -v -j8 ${vpn_endpoint_host} -c ${dot_cdist} + +# Test that the VPN connection is established +# Might take longer due to reboot +sleep 10 + +i=0 +while [ $i -lt 10 ]; do + ping -c1 ${my_wireguard_ip} && break + i=$((i+1)) +done + +i=0 +while [ $i -lt 10 ]; do + ping -c1 ${my_lan_ip} && break + i=$((i+1)) +done + +i=0 +while [ $i -lt 10 ]; do + ping -c1 ${my_wifi_ip} && break + i=$((i+1)) +done From 21713d6e0bcf027a337291728adfec4bc1e1b7f3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 12 Dec 2020 13:27:49 +0100 Subject: [PATCH 211/543] Re-organise repo (subdirs per project) --- .../ceph-host-move-to-root | 0 .../ceph-keyring-create-initial | 0 .../ceph-mgr-create-start | 0 .../ceph-mgr-stop-delete | 0 .../ceph-mon-create-start | 0 .../ceph-mon-stop-delete | 0 ceph-osd-activate => ceph/ceph-osd-activate | 0 .../ceph-osd-activate-all | 0 .../ceph-osd-class-remove | 0 ceph-osd-class-set => ceph/ceph-osd-class-set | 0 .../ceph-osd-create-on-all-disks | 0 .../ceph-osd-create-start | 0 .../ceph-osd-create-start-alpine | 0 .../ceph-osd-stop-disable | 0 .../ceph-osd-stop-remove-permanently | 0 ceph-pool-create => ceph/ceph-pool-create | 0 .../ceph-upgrade-server-to-nautilus.sh | 0 .../mikrotik-configure-crs326-dumb.sh | 0 .../mikrotik-configure-crs326-with-vlans.sh | 0 .../mikrotik-configure-crs326.sh | 0 .../mikrotik-setup.sh | 0 .../mikrotik-update.sh | 0 mystrom-off.sh => mystrom/mystrom-off.sh | 0 mystrom-on.sh => mystrom/mystrom-on.sh | 0 .../mystrom-status.sh | 0 .../mystrom-temperature.sh | 0 .../mystrom-toggle.sh | 0 .../openwrt-add-camera-with-mjpg-streamer.sh | 0 .../openwrt-add-camera-with-motion.sh | 0 .../openwrt-add-gps.sh | 0 .../openwrt-add-jool.sh | 0 .../openwrt-add-letsencrypt.sh | 0 .../openwrt-add-qmi-lte.sh | 0 .../openwrt-add-temper.sh | 0 .../openwrt-add-usb-lte.sh | 0 .../openwrt-add-wireguard.sh | 0 .../openwrt-hp-250-g5-add-wifi.sh | 0 .../openwrt-motion-config.sh | 0 .../viirb-1-firmware-upgrade.sh | 0 .../viirb-2-configure.sh | 0 viirb-3-vpn.sh => openwrt/viirb-3-vpn.sh | 0 .../viirb-4-cleanup-final-stage.sh | 0 .../viirb-get-openwrt-firmware.sh | 0 .../viirb1-2-together.sh | 0 .../viwib-1-firmware-upgrade.sh | 0 openwrt/viwib-2-configure.sh | 225 ++++++++++++++++++ viwib-3-vpn.sh => openwrt/viwib-3-vpn.sh | 0 47 files changed, 225 insertions(+) rename ceph-host-move-to-root => ceph/ceph-host-move-to-root (100%) rename ceph-keyring-create-initial => ceph/ceph-keyring-create-initial (100%) rename ceph-mgr-create-start => ceph/ceph-mgr-create-start (100%) rename ceph-mgr-stop-delete => ceph/ceph-mgr-stop-delete (100%) rename ceph-mon-create-start => ceph/ceph-mon-create-start (100%) rename ceph-mon-stop-delete => ceph/ceph-mon-stop-delete (100%) rename ceph-osd-activate => ceph/ceph-osd-activate (100%) rename ceph-osd-activate-all => ceph/ceph-osd-activate-all (100%) rename ceph-osd-class-remove => ceph/ceph-osd-class-remove (100%) rename ceph-osd-class-set => ceph/ceph-osd-class-set (100%) rename ceph-osd-create-on-all-disks => ceph/ceph-osd-create-on-all-disks (100%) rename ceph-osd-create-start => ceph/ceph-osd-create-start (100%) rename ceph-osd-create-start-alpine => ceph/ceph-osd-create-start-alpine (100%) rename ceph-osd-stop-disable => ceph/ceph-osd-stop-disable (100%) rename ceph-osd-stop-remove-permanently => ceph/ceph-osd-stop-remove-permanently (100%) rename ceph-pool-create => ceph/ceph-pool-create (100%) rename ceph-upgrade-server-to-nautilus.sh => ceph/ceph-upgrade-server-to-nautilus.sh (100%) rename mikrotik-configure-crs326-dumb.sh => mikrotik/mikrotik-configure-crs326-dumb.sh (100%) rename mikrotik-configure-crs326-with-vlans.sh => mikrotik/mikrotik-configure-crs326-with-vlans.sh (100%) rename mikrotik-configure-crs326.sh => mikrotik/mikrotik-configure-crs326.sh (100%) rename mikrotik-setup.sh => mikrotik/mikrotik-setup.sh (100%) rename mikrotik-update.sh => mikrotik/mikrotik-update.sh (100%) rename mystrom-off.sh => mystrom/mystrom-off.sh (100%) rename mystrom-on.sh => mystrom/mystrom-on.sh (100%) rename mystrom-status.sh => mystrom/mystrom-status.sh (100%) rename mystrom-temperature.sh => mystrom/mystrom-temperature.sh (100%) rename mystrom-toggle.sh => mystrom/mystrom-toggle.sh (100%) rename openwrt-add-camera-with-mjpg-streamer.sh => openwrt/openwrt-add-camera-with-mjpg-streamer.sh (100%) rename openwrt-add-camera-with-motion.sh => openwrt/openwrt-add-camera-with-motion.sh (100%) rename openwrt-add-gps.sh => openwrt/openwrt-add-gps.sh (100%) rename openwrt-add-jool.sh => openwrt/openwrt-add-jool.sh (100%) rename openwrt-add-letsencrypt.sh => openwrt/openwrt-add-letsencrypt.sh (100%) rename openwrt-add-qmi-lte.sh => openwrt/openwrt-add-qmi-lte.sh (100%) rename openwrt-add-temper.sh => openwrt/openwrt-add-temper.sh (100%) rename openwrt-add-usb-lte.sh => openwrt/openwrt-add-usb-lte.sh (100%) rename openwrt-add-wireguard.sh => openwrt/openwrt-add-wireguard.sh (100%) rename openwrt-hp-250-g5-add-wifi.sh => openwrt/openwrt-hp-250-g5-add-wifi.sh (100%) rename openwrt-motion-config.sh => openwrt/openwrt-motion-config.sh (100%) rename viirb-1-connect-flash-latest-openwrt.sh => openwrt/viirb-1-firmware-upgrade.sh (100%) rename viirb-2-configure-fully-after-upgrade.sh => openwrt/viirb-2-configure.sh (100%) rename viirb-3-vpn.sh => openwrt/viirb-3-vpn.sh (100%) rename viirb-4-cleanup-final-stage.sh => openwrt/viirb-4-cleanup-final-stage.sh (100%) rename viirb-get-openwrt-firmware.sh => openwrt/viirb-get-openwrt-firmware.sh (100%) rename viirb1-2-together.sh => openwrt/viirb1-2-together.sh (100%) rename viwib-1-firmware-upgrade.sh => openwrt/viwib-1-firmware-upgrade.sh (100%) create mode 100755 openwrt/viwib-2-configure.sh rename viwib-3-vpn.sh => openwrt/viwib-3-vpn.sh (100%) diff --git a/ceph-host-move-to-root b/ceph/ceph-host-move-to-root similarity index 100% rename from ceph-host-move-to-root rename to ceph/ceph-host-move-to-root diff --git a/ceph-keyring-create-initial b/ceph/ceph-keyring-create-initial similarity index 100% rename from ceph-keyring-create-initial rename to ceph/ceph-keyring-create-initial diff --git a/ceph-mgr-create-start b/ceph/ceph-mgr-create-start similarity index 100% rename from ceph-mgr-create-start rename to ceph/ceph-mgr-create-start diff --git a/ceph-mgr-stop-delete b/ceph/ceph-mgr-stop-delete similarity index 100% rename from ceph-mgr-stop-delete rename to ceph/ceph-mgr-stop-delete diff --git a/ceph-mon-create-start b/ceph/ceph-mon-create-start similarity index 100% rename from ceph-mon-create-start rename to ceph/ceph-mon-create-start diff --git a/ceph-mon-stop-delete b/ceph/ceph-mon-stop-delete similarity index 100% rename from ceph-mon-stop-delete rename to ceph/ceph-mon-stop-delete diff --git a/ceph-osd-activate b/ceph/ceph-osd-activate similarity index 100% rename from ceph-osd-activate rename to ceph/ceph-osd-activate diff --git a/ceph-osd-activate-all b/ceph/ceph-osd-activate-all similarity index 100% rename from ceph-osd-activate-all rename to ceph/ceph-osd-activate-all diff --git a/ceph-osd-class-remove b/ceph/ceph-osd-class-remove similarity index 100% rename from ceph-osd-class-remove rename to ceph/ceph-osd-class-remove diff --git a/ceph-osd-class-set b/ceph/ceph-osd-class-set similarity index 100% rename from ceph-osd-class-set rename to ceph/ceph-osd-class-set diff --git a/ceph-osd-create-on-all-disks b/ceph/ceph-osd-create-on-all-disks similarity index 100% rename from ceph-osd-create-on-all-disks rename to ceph/ceph-osd-create-on-all-disks diff --git a/ceph-osd-create-start b/ceph/ceph-osd-create-start similarity index 100% rename from ceph-osd-create-start rename to ceph/ceph-osd-create-start diff --git a/ceph-osd-create-start-alpine b/ceph/ceph-osd-create-start-alpine similarity index 100% rename from ceph-osd-create-start-alpine rename to ceph/ceph-osd-create-start-alpine diff --git a/ceph-osd-stop-disable b/ceph/ceph-osd-stop-disable similarity index 100% rename from ceph-osd-stop-disable rename to ceph/ceph-osd-stop-disable diff --git a/ceph-osd-stop-remove-permanently b/ceph/ceph-osd-stop-remove-permanently similarity index 100% rename from ceph-osd-stop-remove-permanently rename to ceph/ceph-osd-stop-remove-permanently diff --git a/ceph-pool-create b/ceph/ceph-pool-create similarity index 100% rename from ceph-pool-create rename to ceph/ceph-pool-create diff --git a/ceph-upgrade-server-to-nautilus.sh b/ceph/ceph-upgrade-server-to-nautilus.sh similarity index 100% rename from ceph-upgrade-server-to-nautilus.sh rename to ceph/ceph-upgrade-server-to-nautilus.sh diff --git a/mikrotik-configure-crs326-dumb.sh b/mikrotik/mikrotik-configure-crs326-dumb.sh similarity index 100% rename from mikrotik-configure-crs326-dumb.sh rename to mikrotik/mikrotik-configure-crs326-dumb.sh diff --git a/mikrotik-configure-crs326-with-vlans.sh b/mikrotik/mikrotik-configure-crs326-with-vlans.sh similarity index 100% rename from mikrotik-configure-crs326-with-vlans.sh rename to mikrotik/mikrotik-configure-crs326-with-vlans.sh diff --git a/mikrotik-configure-crs326.sh b/mikrotik/mikrotik-configure-crs326.sh similarity index 100% rename from mikrotik-configure-crs326.sh rename to mikrotik/mikrotik-configure-crs326.sh diff --git a/mikrotik-setup.sh b/mikrotik/mikrotik-setup.sh similarity index 100% rename from mikrotik-setup.sh rename to mikrotik/mikrotik-setup.sh diff --git a/mikrotik-update.sh b/mikrotik/mikrotik-update.sh similarity index 100% rename from mikrotik-update.sh rename to mikrotik/mikrotik-update.sh diff --git a/mystrom-off.sh b/mystrom/mystrom-off.sh similarity index 100% rename from mystrom-off.sh rename to mystrom/mystrom-off.sh diff --git a/mystrom-on.sh b/mystrom/mystrom-on.sh similarity index 100% rename from mystrom-on.sh rename to mystrom/mystrom-on.sh diff --git a/mystrom-status.sh b/mystrom/mystrom-status.sh similarity index 100% rename from mystrom-status.sh rename to mystrom/mystrom-status.sh diff --git a/mystrom-temperature.sh b/mystrom/mystrom-temperature.sh similarity index 100% rename from mystrom-temperature.sh rename to mystrom/mystrom-temperature.sh diff --git a/mystrom-toggle.sh b/mystrom/mystrom-toggle.sh similarity index 100% rename from mystrom-toggle.sh rename to mystrom/mystrom-toggle.sh diff --git a/openwrt-add-camera-with-mjpg-streamer.sh b/openwrt/openwrt-add-camera-with-mjpg-streamer.sh similarity index 100% rename from openwrt-add-camera-with-mjpg-streamer.sh rename to openwrt/openwrt-add-camera-with-mjpg-streamer.sh diff --git a/openwrt-add-camera-with-motion.sh b/openwrt/openwrt-add-camera-with-motion.sh similarity index 100% rename from openwrt-add-camera-with-motion.sh rename to openwrt/openwrt-add-camera-with-motion.sh diff --git a/openwrt-add-gps.sh b/openwrt/openwrt-add-gps.sh similarity index 100% rename from openwrt-add-gps.sh rename to openwrt/openwrt-add-gps.sh diff --git a/openwrt-add-jool.sh b/openwrt/openwrt-add-jool.sh similarity index 100% rename from openwrt-add-jool.sh rename to openwrt/openwrt-add-jool.sh diff --git a/openwrt-add-letsencrypt.sh b/openwrt/openwrt-add-letsencrypt.sh similarity index 100% rename from openwrt-add-letsencrypt.sh rename to openwrt/openwrt-add-letsencrypt.sh diff --git a/openwrt-add-qmi-lte.sh b/openwrt/openwrt-add-qmi-lte.sh similarity index 100% rename from openwrt-add-qmi-lte.sh rename to openwrt/openwrt-add-qmi-lte.sh diff --git a/openwrt-add-temper.sh b/openwrt/openwrt-add-temper.sh similarity index 100% rename from openwrt-add-temper.sh rename to openwrt/openwrt-add-temper.sh diff --git a/openwrt-add-usb-lte.sh b/openwrt/openwrt-add-usb-lte.sh similarity index 100% rename from openwrt-add-usb-lte.sh rename to openwrt/openwrt-add-usb-lte.sh diff --git a/openwrt-add-wireguard.sh b/openwrt/openwrt-add-wireguard.sh similarity index 100% rename from openwrt-add-wireguard.sh rename to openwrt/openwrt-add-wireguard.sh diff --git a/openwrt-hp-250-g5-add-wifi.sh b/openwrt/openwrt-hp-250-g5-add-wifi.sh similarity index 100% rename from openwrt-hp-250-g5-add-wifi.sh rename to openwrt/openwrt-hp-250-g5-add-wifi.sh diff --git a/openwrt-motion-config.sh b/openwrt/openwrt-motion-config.sh similarity index 100% rename from openwrt-motion-config.sh rename to openwrt/openwrt-motion-config.sh diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/openwrt/viirb-1-firmware-upgrade.sh similarity index 100% rename from viirb-1-connect-flash-latest-openwrt.sh rename to openwrt/viirb-1-firmware-upgrade.sh diff --git a/viirb-2-configure-fully-after-upgrade.sh b/openwrt/viirb-2-configure.sh similarity index 100% rename from viirb-2-configure-fully-after-upgrade.sh rename to openwrt/viirb-2-configure.sh diff --git a/viirb-3-vpn.sh b/openwrt/viirb-3-vpn.sh similarity index 100% rename from viirb-3-vpn.sh rename to openwrt/viirb-3-vpn.sh diff --git a/viirb-4-cleanup-final-stage.sh b/openwrt/viirb-4-cleanup-final-stage.sh similarity index 100% rename from viirb-4-cleanup-final-stage.sh rename to openwrt/viirb-4-cleanup-final-stage.sh diff --git a/viirb-get-openwrt-firmware.sh b/openwrt/viirb-get-openwrt-firmware.sh similarity index 100% rename from viirb-get-openwrt-firmware.sh rename to openwrt/viirb-get-openwrt-firmware.sh diff --git a/viirb1-2-together.sh b/openwrt/viirb1-2-together.sh similarity index 100% rename from viirb1-2-together.sh rename to openwrt/viirb1-2-together.sh diff --git a/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh similarity index 100% rename from viwib-1-firmware-upgrade.sh rename to openwrt/viwib-1-firmware-upgrade.sh diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh new file mode 100755 index 0000000..bff5f03 --- /dev/null +++ b/openwrt/viwib-2-configure.sh @@ -0,0 +1,225 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viwib-wifi-ipv6-box/ + +if [ $# -lt 2 ]; then + echo "$0 viwib-ip-address viwib-id [wgprivkey]" + echo " viwib-ip-address: where to find the viwib" + echo " viwib-id: number in decimal format" + echo " wgprivkey: if specified, use this private key" + exit 1 +fi + + +set -x +viwib_ip=$1; shift +id=$1; shift +hex_id=$(printf "%0.2x\n" "$id") +viwib_hostname=viwib${id} + +prefix_base=2a0a:e5c1:3 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +# wireguard +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch +vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= + +ping -c3 ${viwib_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach ${viwib_ip}, aborting" + exit 1 +fi + +cat < ${viwib_hostname}.public_key diff --git a/viwib-3-vpn.sh b/openwrt/viwib-3-vpn.sh similarity index 100% rename from viwib-3-vpn.sh rename to openwrt/viwib-3-vpn.sh From e891c7160aff71795a122a3536c6a982a398d590 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 12 Dec 2020 13:40:56 +0100 Subject: [PATCH 212/543] [viirb,viwib] upgrade to openwrt 19.07.5 --- openwrt/viirb-1-firmware-upgrade.sh | 2 +- openwrt/viwib-1-firmware-upgrade.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openwrt/viirb-1-firmware-upgrade.sh b/openwrt/viirb-1-firmware-upgrade.sh index 4733acb..0c52ab3 100755 --- a/openwrt/viirb-1-firmware-upgrade.sh +++ b/openwrt/viirb-1-firmware-upgrade.sh @@ -24,7 +24,7 @@ else fi # openwrt -version=19.07.4 +version=19.07.5 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin # IP address for setting it up initially diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 1aca868..7a910f1 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -24,7 +24,7 @@ else fi # openwrt -version=19.07.4 +version=19.07.5 filename=openwrt-${version}-ramips-mt76x8-gl-mt300n-v2-squashfs-sysupgrade.bin # IP address for setting it up initially From cfcd2d9b19fd52ce71fa506784d91665e0de02f7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 15 Dec 2020 22:51:28 +0100 Subject: [PATCH 213/543] update viwib configs --- openwrt/viirb-2-configure.sh | 2 +- openwrt/viwib-2-configure.sh | 84 +++++++++++++----------------------- 2 files changed, 30 insertions(+), 56 deletions(-) diff --git a/openwrt/viirb-2-configure.sh b/openwrt/viirb-2-configure.sh index 093d547..c9ada76 100755 --- a/openwrt/viirb-2-configure.sh +++ b/openwrt/viirb-2-configure.sh @@ -17,7 +17,7 @@ id=$1; shift hex_id=$(printf "%0.2x\n" "$id") viirb_hostname=viirb${id} -prefix_base=2a0a:e5c1:3 +prefix_base=2a0a:e5c1:6 my_prefix=${prefix_base}${hex_id} my_network=${my_prefix}::/48 diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index bff5f03..c1d66da 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -17,7 +17,7 @@ id=$1; shift hex_id=$(printf "%0.2x\n" "$id") viwib_hostname=viwib${id} -prefix_base=2a0a:e5c1:3 +prefix_base=2a0a:e5c1:6 my_prefix=${prefix_base}${hex_id} my_network=${my_prefix}::/48 @@ -33,8 +33,8 @@ else fi public_key=$(echo $private_key | wg pubkey) -vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch -vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= +vpn_endpoint_host=vpn-2a0ae5c1600.ungleich.ch +vpn_endpoint_pubkey=ygZQW3OSiMJl/RpKyaJVE0GSt6bjEDnoxdMJsNiloRE= ping -c3 ${viwib_ip} if [ $? -ne 0 ]; then @@ -46,36 +46,27 @@ cat < Date: Tue, 15 Dec 2020 23:01:30 +0100 Subject: [PATCH 214/543] fix --- openwrt/viwib-2-configure.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index c1d66da..3649edb 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -73,13 +73,15 @@ uci set dhcp.@dnsmasq[0].noresolv='1' # Fix DNS: make the OS use the locally provided DNS servers # otherwise the VPN tunnel cannot be established -dhcp.@dnsmasq[0].localuse='0' +uci set dhcp.@dnsmasq[0].localuse='0' # DNS upstream over VPN gives DNS64 uci delete dhcp.@dnsmasq[0].server uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:0:a::a' uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:2:a::a' +uci set system.@system[0].hostname="${viwib_hostname}" + # wifi ip address # uci set network.wifi=interface # uci set network.wifi.proto='static' @@ -154,8 +156,6 @@ uci set network.@wireguard_wg0[0].endpoint_host="${vpn_endpoint_host}" uci set network.@wireguard_wg0[0].endpoint_port='51820' uci set network.@wireguard_wg0[0].route_allowed_ips='1' -uci set system.@system[0].hostname="${viwib_hostname}" - # Firewall configuration if ! uci show firewall | grep "name='Allow-SSH'"; then uci add firewall rule From 8f161e2137bbef550d33402a488e0d9741a46aa3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 15 Dec 2020 23:16:50 +0100 Subject: [PATCH 215/543] viviwb2 -> all in one step --- openwrt/viwib-2-configure.sh | 27 ++++++++------------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 3649edb..688727b 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -108,25 +108,6 @@ uci delete wireless.radio0.disabled uci commit -# Need to reboot to restore /etc/resolv.conf -reboot - -EOF - -# ensure viwib is back -# ensure viwib network is up and running - bridge takes a bit of time -# ensure viwib gives out ipv6 address -# Found: 30s is not enough for full reconfiguration -echo "Public VPN key: ${id} ${public_key}" - -sleep 60 - -# change to ipv6 -viwib_ip=${my_lan_ip} - -ping -c5 ${viwib_ip} - -cat < ${viwib_hostname}.public_key + +# change to ipv6 +viwib_ip=${my_lan_ip} + +while ! ping -c5 ${viwib_ip}; do + echo "Waiting for viwib ${id}" + sleep 2 +done From 1b0d8220e65ec5b996757cfcde7fc6e0a9073a2d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 15 Dec 2020 23:43:20 +0100 Subject: [PATCH 216/543] viwib: all in stage2 --- ...-cleanup-final-stage.sh => viirb-4-cleanup.sh} | 0 openwrt/viwib-2-configure.sh | 15 +++++++++++++++ 2 files changed, 15 insertions(+) rename openwrt/{viirb-4-cleanup-final-stage.sh => viirb-4-cleanup.sh} (100%) diff --git a/openwrt/viirb-4-cleanup-final-stage.sh b/openwrt/viirb-4-cleanup.sh similarity index 100% rename from openwrt/viirb-4-cleanup-final-stage.sh rename to openwrt/viirb-4-cleanup.sh diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 688727b..cc4ef33 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -42,6 +42,10 @@ if [ $? -ne 0 ]; then exit 1 fi +root_password=$(pwgen -1 32) +echo $root_password > ${viwib_hostname}.rootpw + + cat < ${viwib_hostname}.public_key # change to ipv6 viwib_ip=${my_lan_ip} +sleep 15 + while ! ping -c5 ${viwib_ip}; do echo "Waiting for viwib ${id}" sleep 2 From 02ea7054b1f9bbb857b686010c918147c53d367f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 16 Dec 2020 10:42:12 +0100 Subject: [PATCH 217/543] [viwib] remove 192.168.1.1 (runtime) --- openwrt/viwib-2-configure.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index cc4ef33..d1f28be 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -112,6 +112,9 @@ uci delete wireless.radio0.disabled uci commit +# ensure no overlapping networks are there +ip addr del 192.168.1.1/24 dev br-lan + ping -c5 ungleich.ch || exit 1 # update the sources @@ -177,14 +180,12 @@ uci set firewall.@zone[1].network='wan wan6 wg0' uci commit - # Remove our ssh keys rm -f /etc/dropbear/authorized_keys # Setup root password printf "${root_password}\n${root_password}\n" | passwd - reboot EOF From 5eb3a5d1fec239526af91871d3aa61e0d4614786 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 17 Dec 2020 09:39:15 +0100 Subject: [PATCH 218/543] [viwib] updates --- openwrt/viwib-2-configure.sh | 10 ++++----- openwrt/viwib-3-vpn.sh | 8 +------ openwrt/viwib-4-cleanup.sh | 42 ++++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 13 deletions(-) create mode 100755 openwrt/viwib-4-cleanup.sh diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index d1f28be..59eb7e5 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -180,12 +180,6 @@ uci set firewall.@zone[1].network='wan wan6 wg0' uci commit -# Remove our ssh keys -rm -f /etc/dropbear/authorized_keys - -# Setup root password -printf "${root_password}\n${root_password}\n" | passwd - reboot EOF @@ -202,3 +196,7 @@ while ! ping -c5 ${viwib_ip}; do echo "Waiting for viwib ${id}" sleep 2 done + +echo "Wireguard public key and id: ${id} ${public_key}" +echo "Root password: ${root_password}" +echo ${public_key} > ${viwib_hostname}.public_key diff --git a/openwrt/viwib-3-vpn.sh b/openwrt/viwib-3-vpn.sh index 36ee375..7e2d4a8 100755 --- a/openwrt/viwib-3-vpn.sh +++ b/openwrt/viwib-3-vpn.sh @@ -64,13 +64,7 @@ while [ $i -lt 10 ]; do done i=0 -while [ $i -lt 10 ]; do +while [ $i -lt 30 ]; do ping -c1 ${my_lan_ip} && break i=$((i+1)) done - -i=0 -while [ $i -lt 10 ]; do - ping -c1 ${my_wifi_ip} && break - i=$((i+1)) -done diff --git a/openwrt/viwib-4-cleanup.sh b/openwrt/viwib-4-cleanup.sh new file mode 100755 index 0000000..5ed6c95 --- /dev/null +++ b/openwrt/viwib-4-cleanup.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Nico Schottelius +# 2020-06-14 + +set -e +set -x + +if [ $# -ne 2 ]; then + echo "$0 viwib-ip-address viwib-id" + echo " viwib-ip-address: where to find the viwib" + echo " viwib-id: number in decimal format" + exit 1 +fi + +viwib_ip=$1; shift + +id=$1; shift +viwib_hostname=viwib${id} + +root_password=$(pwgen -1 32) + +# Save for sending to user +# FIXME: future make this more easy / better to transfer +echo $root_password > ${viwib_hostname}.rootpw + +cat < Date: Thu, 17 Dec 2020 10:01:51 +0100 Subject: [PATCH 219/543] ++viwib2/no pw --- openwrt/viwib-2-configure.sh | 6 ------ 1 file changed, 6 deletions(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 59eb7e5..1c582d2 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -42,9 +42,6 @@ if [ $? -ne 0 ]; then exit 1 fi -root_password=$(pwgen -1 32) -echo $root_password > ${viwib_hostname}.rootpw - cat < ${viwib_hostname}.public_key # change to ipv6 @@ -198,5 +194,3 @@ while ! ping -c5 ${viwib_ip}; do done echo "Wireguard public key and id: ${id} ${public_key}" -echo "Root password: ${root_password}" -echo ${public_key} > ${viwib_hostname}.public_key From 0a07a197bb79ec3ae6caca7e55cc1ce17b1aaba0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 21 Dec 2020 14:34:26 +0100 Subject: [PATCH 220/543] viwib2: wait up to 30 seconds trying to reach the viwib --- openwrt/viwib-2-configure.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 1c582d2..ce2d83c 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -36,11 +36,12 @@ public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1600.ungleich.ch vpn_endpoint_pubkey=ygZQW3OSiMJl/RpKyaJVE0GSt6bjEDnoxdMJsNiloRE= -ping -c3 ${viwib_ip} -if [ $? -ne 0 ]; then - echo "Cannot reach ${viwib_ip}, aborting" - exit 1 -fi +i=0 +while [ $i -lt 30 ]; do + echo "Trying to reach ${viwib_ip} ($i)" + ping -c1 ${viwib_ip} >/dev/null && break + i=$((i+1)) +done cat < Date: Mon, 21 Dec 2020 14:36:40 +0100 Subject: [PATCH 221/543] if not found, exit viwib2 --- openwrt/viwib-2-configure.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index ce2d83c..cb16a40 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -37,12 +37,21 @@ vpn_endpoint_host=vpn-2a0ae5c1600.ungleich.ch vpn_endpoint_pubkey=ygZQW3OSiMJl/RpKyaJVE0GSt6bjEDnoxdMJsNiloRE= i=0 +found="" while [ $i -lt 30 ]; do echo "Trying to reach ${viwib_ip} ($i)" - ping -c1 ${viwib_ip} >/dev/null && break + if ping -c1 ${viwib_ip} >/dev/null; then + found="yes" + break + fi i=$((i+1)) done +if [ -z "$found" ]; then + echo "Unable to contact viwib. Exiting" + exit 1 +fi + cat < Date: Mon, 21 Dec 2020 19:55:04 +0100 Subject: [PATCH 222/543] [vigir] add firmware update script --- openwrt/vigir-1-firmware-upgrade.sh | 85 +++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100755 openwrt/vigir-1-firmware-upgrade.sh diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh new file mode 100755 index 0000000..e192bc9 --- /dev/null +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -0,0 +1,85 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -lt 1 ]; then + echo "$0 interface [address]" + echo " interface to add the config ip address to" + echo " address: connect to this address, ignore the interface" + exit 1 +fi + +set -x +dev=$1; shift + +# $# = 2 +# shift -> remove the first argument +# $# = 1 + +if [ $# -ge 1 ]; then + vigir_ip=$1; shift + dev="" +else + vigir_ip=192.168.8.1 +fi + +# openwrt +version=19.07.5 +filename=openwrt-${version}-ramips-mt7621-zbt-wg3526-16M-squashfs-sysupgrade.bin + +# IP address for setting it up initially + +if [ "$dev" ]; then + sudo ip addr del 192.168.8.2/24 dev "$dev" 2>/dev/null || true + sudo ip addr add 192.168.8.2/24 dev "$dev" +fi + +# don't care about other/old known_host entries +ssh-keygen -R ${vigir_ip} + +while ! ping -c1 ${vigir_ip}; do + echo "Cannot ping $vigir_ip yet - waiting" + sleep 1 +done + +cat ~/.ssh/id_rsa.pub | ssh root@${vigir_ip} "cat > /etc/dropbear/authorized_keys" + +# Don't re-download if we already have it +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt7621/${filename} + +if echo $vigir_ip | grep -q :; then + scp_ip="[$vigir_ip]" +else + scp_ip="$vigir_ip" +fi + +scp ${filename} root@${scp_ip}:/tmp +ssh root@${vigir_ip} "sysupgrade -n /tmp/*.bin" + +# It still pings for some time - wait for the reboot to happen +echo "Waiting for vigir to really disappear" +sleep 15 + +wait=0 +found="" + +while [ $wait -lt 180 ]; do + ping -c1 ${vigir_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + break + fi + + sleep 1 + wait=$((wait+1)) + done + +if [ ! "$found" ]; then + echo "Did not find updated vigir - debug / restart it" + exit 1 +fi + +echo "vigir successfully updated to ${version}" From 587599a74a8076fb23568b55fad0e7b9b53f5d45 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 21 Dec 2020 20:24:52 +0100 Subject: [PATCH 223/543] [vigir] phase in 2/3/4 scripts --- openwrt/vigir-2-configure.sh | 206 +++++++++++++++++++++++++++++++++++ openwrt/vigir-3-vpn.sh | 70 ++++++++++++ openwrt/vigir-4-cleanup.sh | 42 +++++++ 3 files changed, 318 insertions(+) create mode 100755 openwrt/vigir-2-configure.sh create mode 100755 openwrt/vigir-3-vpn.sh create mode 100755 openwrt/vigir-4-cleanup.sh diff --git a/openwrt/vigir-2-configure.sh b/openwrt/vigir-2-configure.sh new file mode 100755 index 0000000..d692f01 --- /dev/null +++ b/openwrt/vigir-2-configure.sh @@ -0,0 +1,206 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/vigir/ + +if [ $# -lt 2 ]; then + echo "$0 vigir-ip-address vigir-id [wgprivkey]" + echo " vigir-ip-address: where to find the vigir" + echo " vigir-id: number in decimal format" + echo " wgprivkey: if specified, use this private key" + exit 1 +fi + + +set -x +vigir_ip=$1; shift +id=$1; shift +hex_id=$(printf "%0.2x\n" "$id") +vigir_hostname=vigir${id} + +prefix_base=2a0a:e5c1:5 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +# wireguard +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-2a0ae5c1500.ungleich.ch +vpn_endpoint_pubkey=ygZQW3OSiMJl/RpKyaJVE0GSt6bjEDnoxdMJsNiloRE= + +i=0 +found="" +while [ $i -lt 30 ]; do + echo "Trying to reach ${vigir_ip} ($i)" + if ping -c1 ${vigir_ip} >/dev/null; then + found="yes" + break + fi + i=$((i+1)) +done + +if [ -z "$found" ]; then + echo "Unable to contact vigir. Exiting" + exit 1 +fi + + +cat < ${vigir_hostname}.public_key + +# change to ipv6 +vigir_ip=${my_lan_ip} + +sleep 15 + +while ! ping -c5 ${vigir_ip}; do + echo "Waiting for vigir ${id}" + sleep 2 +done + +echo "Wireguard public key and id: ${id} ${public_key}" diff --git a/openwrt/vigir-3-vpn.sh b/openwrt/vigir-3-vpn.sh new file mode 100755 index 0000000..615594a --- /dev/null +++ b/openwrt/vigir-3-vpn.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# 2020-12-21, Nico Schottelius + +if [ $# -ne 3 ]; then + echo "$0 your-dot-cdist vigir-id public-key" + echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" + echo " vigir-id: number in decimal format" + echo " wireguard public key" + exit 1 +fi + +set -x +dot_cdist=$1; shift +id=$1; shift +public_key=$1; shift + +hex_id=$(printf "%0.2x\n" "$id") +vigir_hostname=vigir${id} + +prefix_base=2a0a:e5c1:5 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +vpn_endpoint_host=vpn-2a0ae5c1500.ungleich.ch + +# cdist +dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files +peerfilename=${vpn_endpoint_host}.peer${hex_id} +peerfile=${dot_cdist_files}/${peerfilename} +vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} + + +# Configure VPN server / update cdist +echo Updating VPNserver +cat < ${peerfile} +# ${vigir_hostname}, $(date +%F) +[Peer] +PublicKey = ${public_key} +AllowedIPs = ${my_network} + +EOF + +# Generate real config +cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} +cd ${dot_cdist_files} +git add ${vpn_endpoint_host} ${peerfilename} +git commit -m "[vpn] Updated config for peer ${vigir_hostname} ${my_network}" +git pull +git push + +cdist config -v -j8 ${vpn_endpoint_host} -c ${dot_cdist} + +# Test that the VPN connection is established +# Might take longer due to reboot +sleep 10 + +i=0 +while [ $i -lt 10 ]; do + ping -c1 ${my_wireguard_ip} && break + i=$((i+1)) +done + +i=0 +while [ $i -lt 30 ]; do + ping -c1 ${my_lan_ip} && break + i=$((i+1)) +done diff --git a/openwrt/vigir-4-cleanup.sh b/openwrt/vigir-4-cleanup.sh new file mode 100755 index 0000000..8b85106 --- /dev/null +++ b/openwrt/vigir-4-cleanup.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Nico Schottelius +# 2020-12-21 + +set -e +set -x + +if [ $# -ne 2 ]; then + echo "$0 vigir-ip-address vigir-id" + echo " vigir-ip-address: where to find the vigir" + echo " vigir-id: number in decimal format" + exit 1 +fi + +vigir_ip=$1; shift + +id=$1; shift +vigir_hostname=vigir${id} + +root_password=$(pwgen -1 32) + +# Save for sending to user +# FIXME: future make this more easy / better to transfer +echo $root_password > ${vigir_hostname}.rootpw + +cat < Date: Mon, 21 Dec 2020 20:32:25 +0100 Subject: [PATCH 224/543] remove ip removal --- openwrt/vigir-2-configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/vigir-2-configure.sh b/openwrt/vigir-2-configure.sh index d692f01..9688c13 100755 --- a/openwrt/vigir-2-configure.sh +++ b/openwrt/vigir-2-configure.sh @@ -120,7 +120,7 @@ uci delete wireless.radio0.disabled uci commit # ensure no overlapping networks are there -ip addr del 192.168.1.1/24 dev br-lan +#ip addr del 192.168.1.1/24 dev br-lan ping -c5 ungleich.ch || exit 1 From b0be0cc7ebeba2382ef2e1e454692c535107858f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 22 Dec 2020 08:54:49 +0100 Subject: [PATCH 225/543] [vigir] adjust scripts for two radios --- openwrt/vigir-2-configure.sh | 30 +++++++++--------------------- openwrt/vigir-4-cleanup.sh | 2 ++ 2 files changed, 11 insertions(+), 21 deletions(-) diff --git a/openwrt/vigir-2-configure.sh b/openwrt/vigir-2-configure.sh index 9688c13..5a4ba21 100755 --- a/openwrt/vigir-2-configure.sh +++ b/openwrt/vigir-2-configure.sh @@ -34,7 +34,7 @@ fi public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1500.ungleich.ch -vpn_endpoint_pubkey=ygZQW3OSiMJl/RpKyaJVE0GSt6bjEDnoxdMJsNiloRE= +vpn_endpoint_pubkey=oaFiIVV1NjvDcfdtwJqR4F3k2XIC07npNgj0YjIEem4= i=0 found="" @@ -55,11 +55,8 @@ fi cat < Date: Sat, 26 Dec 2020 17:22:47 +0100 Subject: [PATCH 226/543] [mystrom] turning on/off can be quiet --- mystrom/mystrom-off.sh | 2 +- mystrom/mystrom-on.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mystrom/mystrom-off.sh b/mystrom/mystrom-off.sh index 7268e90..9193ac4 100755 --- a/mystrom/mystrom-off.sh +++ b/mystrom/mystrom-off.sh @@ -10,7 +10,7 @@ fi while [ $# -ge 1 ]; do hostname=$1; shift - curl --location \ + curl -s --location \ --request GET \ "http://${hostname}/relay?state=0" diff --git a/mystrom/mystrom-on.sh b/mystrom/mystrom-on.sh index c9aac5a..f5387c8 100755 --- a/mystrom/mystrom-on.sh +++ b/mystrom/mystrom-on.sh @@ -10,7 +10,7 @@ fi while [ $# -ge 1 ]; do hostname=$1; shift - curl --location \ + curl -s --location \ --request GET \ "http://${hostname}/relay?state=1" done From e0623f7938625c69cb1d3cad8bda5131392b01b5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 29 Dec 2020 11:39:26 +0100 Subject: [PATCH 227/543] cleanup openwrt dir --- openwrt/openwrt-add-qmi-lte.sh | 18 ++++++++++++++++-- openwrt/viirb-get-openwrt-firmware.sh | 14 -------------- openwrt/viirb1-2-together.sh | 14 -------------- 3 files changed, 16 insertions(+), 30 deletions(-) delete mode 100755 openwrt/viirb-get-openwrt-firmware.sh delete mode 100644 openwrt/viirb1-2-together.sh diff --git a/openwrt/openwrt-add-qmi-lte.sh b/openwrt/openwrt-add-qmi-lte.sh index ecd0023..0a232b1 100755 --- a/openwrt/openwrt-add-qmi-lte.sh +++ b/openwrt/openwrt-add-qmi-lte.sh @@ -20,12 +20,14 @@ opkg install libustream-openssl ca-bundle ca-certificates # Install needed kernel module opkg install kmod-usb-net-qmi-wwan uqmi luci-proto-qmi + + # opkg install usb-modeswitch kmod-mii kmod-usb-net # Create interface -# uci set network.LTE=interface +# uci set network.lte=interface # uci set network.LTE.ifname='${interface}' -# uci set network.LTE.proto='dhcp' +# uci set network.lte.proto='qmi' # add to correct firewall zone # current_networks=\$(uci get firewall.@zone[1].network) @@ -42,3 +44,15 @@ uci commit reboot EOF + +exit 0 + +config interface 'lte' + option proto 'qmi' + option device '/dev/cdc-wdm0' + option apn 'internet' + option auth 'both' + option modes 'lte' + option pdptype 'ipv4' + option username 'any' + option password 'any' diff --git a/openwrt/viirb-get-openwrt-firmware.sh b/openwrt/viirb-get-openwrt-firmware.sh deleted file mode 100755 index 1682f7b..0000000 --- a/openwrt/viirb-get-openwrt-firmware.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh -# 2020-10-08, Nico Schottelius -# See https://ungleich.ch/u/products/viirb-ipv6-box/ - -if [ $# -ne 1 ]; then - echo "$0 openwrt-version" - echo " openwrt-version: for instance 19.07.4" - exit 1 -fi - -version=$1 -filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin - -wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} diff --git a/openwrt/viirb1-2-together.sh b/openwrt/viirb1-2-together.sh deleted file mode 100644 index e1a4206..0000000 --- a/openwrt/viirb1-2-together.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/sh - -if [ $# -ne 1 ]; then - echo "$0 viirb-id" - echo " viirb-id: number in decimal format" - exit 1 -fi - -id=$1; shift - -./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 -# reboot may take longer / rewriting the flash -sleep 120 -./viirb-2-configure-fully-after-upgrade.sh 192.168.61.1 "$id" From 4405d4025e881e9d810ccd402914a7ac98973954 Mon Sep 17 00:00:00 2001 From: samuel hailu Date: Tue, 29 Dec 2020 15:24:59 +0100 Subject: [PATCH 228/543] Added ping6 to ping viwib-2-script --- openwrt/viwib-2-configure.sh | 4 ++-- openwrt/viwib1.public_key | 1 + openwrt/viwib4.public_key | 1 + openwrt/viwib4.rootpw | 1 + 4 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 openwrt/viwib1.public_key create mode 100644 openwrt/viwib4.public_key create mode 100644 openwrt/viwib4.rootpw diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index cb16a40..d720d6d 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -122,7 +122,7 @@ uci commit # ensure no overlapping networks are there ip addr del 192.168.1.1/24 dev br-lan -ping -c5 ungleich.ch || exit 1 +ping6 -c5 ungleich.ch || exit 1 # update the sources opkg update @@ -198,7 +198,7 @@ viwib_ip=${my_lan_ip} sleep 15 -while ! ping -c5 ${viwib_ip}; do +while ! ping6 -c5 ${viwib_ip}; do echo "Waiting for viwib ${id}" sleep 2 done diff --git a/openwrt/viwib1.public_key b/openwrt/viwib1.public_key new file mode 100644 index 0000000..fcfff40 --- /dev/null +++ b/openwrt/viwib1.public_key @@ -0,0 +1 @@ +/m35eP83YNduvxCXUYpjVrMKMW+cyi65CXrtPscWAAY= diff --git a/openwrt/viwib4.public_key b/openwrt/viwib4.public_key new file mode 100644 index 0000000..27e3640 --- /dev/null +++ b/openwrt/viwib4.public_key @@ -0,0 +1 @@ +r/MGL7pNWaF8qL/g8v0e/1wlQqgCbdeF15pMwEFvMW0= diff --git a/openwrt/viwib4.rootpw b/openwrt/viwib4.rootpw new file mode 100644 index 0000000..9a8673d --- /dev/null +++ b/openwrt/viwib4.rootpw @@ -0,0 +1 @@ +ta9oc1eecahP1iutho3eixeel8eich2u From 2174a54f1a52cb9f4e0cb19864261678af2775a0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 29 Dec 2020 18:13:05 +0100 Subject: [PATCH 229/543] [qmi/lte] fix parameter shifting --- openwrt/openwrt-add-qmi-lte.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/openwrt/openwrt-add-qmi-lte.sh b/openwrt/openwrt-add-qmi-lte.sh index 0a232b1..b8089eb 100755 --- a/openwrt/openwrt-add-qmi-lte.sh +++ b/openwrt/openwrt-add-qmi-lte.sh @@ -8,7 +8,6 @@ if [ $# -ne 1 ]; then fi my_ip=$1; shift -interface=$1; shift cat < Date: Thu, 31 Dec 2020 10:15:51 +0100 Subject: [PATCH 230/543] Add new directory --- matrix/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 matrix/.gitkeep diff --git a/matrix/.gitkeep b/matrix/.gitkeep new file mode 100644 index 0000000..e69de29 From 5866e65cc335c47ad40bfea698b4c629125831cd Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 31 Dec 2020 10:17:24 +0100 Subject: [PATCH 231/543] [tools]Add new file for matrix --- matrix/create_user | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 matrix/create_user diff --git a/matrix/create_user b/matrix/create_user new file mode 100644 index 0000000..a972b0b --- /dev/null +++ b/matrix/create_user @@ -0,0 +1,11 @@ +while read A B C D E +do + pw=$E + name=$A" "$B + ad=$C + id="@"$D + data='{"password":"'${pw}'", "displayname": "'${name}'", "threepids": [ { "medium": "email", "address": "'${ad}'" }], "admin": false, "deactivated": false, "avatar_url": null }' + h='Authorization: Bearer ' + curl -v -X PUT -H "$h" -d "$data" http://localhost:8008/_synapse/admin/v2/users/$id:politikergespraeche.germanzero.org + sleep 2 +done < info.txt \ No newline at end of file From 1fc6a1043910d22e918b54fadd96b909b596d4e1 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 31 Dec 2020 10:18:26 +0100 Subject: [PATCH 232/543] Update create_user --- matrix/create_user | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/create_user b/matrix/create_user index a972b0b..d786576 100644 --- a/matrix/create_user +++ b/matrix/create_user @@ -6,6 +6,6 @@ do id="@"$D data='{"password":"'${pw}'", "displayname": "'${name}'", "threepids": [ { "medium": "email", "address": "'${ad}'" }], "admin": false, "deactivated": false, "avatar_url": null }' h='Authorization: Bearer ' - curl -v -X PUT -H "$h" -d "$data" http://localhost:8008/_synapse/admin/v2/users/$id:politikergespraeche.germanzero.org + curl -v -X PUT -H "$h" -d "$data" http://localhost:8008/_synapse/admin/v2/users/$id: sleep 2 done < info.txt \ No newline at end of file From 16effab79e4d0895efd6e7c22f015497642f00b5 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 31 Dec 2020 10:22:19 +0100 Subject: [PATCH 233/543] [tools]Add new file for matrix --- matrix/invite_rooms | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 matrix/invite_rooms diff --git a/matrix/invite_rooms b/matrix/invite_rooms new file mode 100644 index 0000000..8170d04 --- /dev/null +++ b/matrix/invite_rooms @@ -0,0 +1,35 @@ +matrixserver="" + +generate_post_data() +{ + cat </invite?access_token=") + echo $res + #avoid error { "errcode": "M_LIMIT_EXCEEDED", "error": "Too Many Requests", "retry_after_ms": 2895 } + if [[ $res =~ $errcode ]];then + sleep 5 + res2=$(curl -XPOST -d "$(generate_post_data)" "http://localhost:8008/_matrix/client/r0/rooms/%21$rm_id%3A/invite?access_token=") + fi + + if [[ $res2 =~ $errcode ]];then + echo "===error===" + echo $res2 + exit 1 + fi + sleep 1 + done < info.txt +done \ No newline at end of file From 085f386168c7773934e383c4efcf1dcdfde72fa1 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 31 Dec 2020 10:23:12 +0100 Subject: [PATCH 234/543] [tools]Add new file for matrix --- matrix/info.txt | 1 + 1 file changed, 1 insertion(+) create mode 100644 matrix/info.txt diff --git a/matrix/info.txt b/matrix/info.txt new file mode 100644 index 0000000..9d64372 --- /dev/null +++ b/matrix/info.txt @@ -0,0 +1 @@ +FirstName LastName Email UserID PW \ No newline at end of file From 25f67f254161a2ad33f8019cca241386734838a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 4 Jan 2021 08:47:16 +0100 Subject: [PATCH 235/543] Bump Fedora image to F33 --- opennebula-images/fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh index b4bc5c6..d7e4ddc 100755 --- a/opennebula-images/fedora-build-opennebula-image.sh +++ b/opennebula-images/fedora-build-opennebula-image.sh @@ -18,7 +18,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=32 +RELEASE=33 ARCH=x86_64 IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G From 143917ca8618de8c002ad41376472b126b64f602 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 4 Jan 2021 16:32:34 +0100 Subject: [PATCH 236/543] ++mode --- openwrt/openwrt-add-jool.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 openwrt/openwrt-add-jool.sh diff --git a/openwrt/openwrt-add-jool.sh b/openwrt/openwrt-add-jool.sh old mode 100644 new mode 100755 From 00719a5e59f3f84a3b2e7a15e1dcb85066565cc8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 4 Jan 2021 16:40:38 +0100 Subject: [PATCH 237/543] [openwrt] add script to create ipv4 islands --- openwrt/openwrt-add-jool-ipv4island.sh | 31 ++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 openwrt/openwrt-add-jool-ipv4island.sh diff --git a/openwrt/openwrt-add-jool-ipv4island.sh b/openwrt/openwrt-add-jool-ipv4island.sh new file mode 100755 index 0000000..2b38865 --- /dev/null +++ b/openwrt/openwrt-add-jool-ipv4island.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +if [ $# -ne 1 ]; then + echo "$0 ip-address nat64-prefix" + echo " ip-address: where to find the OpenWRT device" + echo " nat64-prefix: which network to use for INCOMING NAT64 (often 2a0a:e5c1:XXX:b00::/96" + echo " ip4-address: where to map ipv6 traffic to (often: 192.168.1.42)" + exit 1 +fi + +my_ip=$1; shift +nat64_prefix=$1; shift +v4addr=$1; shift + +cat < /etc/rc.local << EO2 + +modprobe jool + +jool -4 -a ${v4addr} +jool -6 ${nat64_prefix} +EO2 + +sh /etc/rc.local + +EOF From c2e40ac71338eb72e01f4ce0aca7da017dc9fb6b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 4 Jan 2021 16:55:16 +0100 Subject: [PATCH 238/543] [viwib] remove address deletion --- openwrt/viwib-2-configure.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index d720d6d..71db5ba 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -119,9 +119,6 @@ uci delete wireless.radio0.disabled uci commit -# ensure no overlapping networks are there -ip addr del 192.168.1.1/24 dev br-lan - ping6 -c5 ungleich.ch || exit 1 # update the sources From 41ac7d2452cc21c3f611518a172472f01785123a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 4 Jan 2021 16:55:39 +0100 Subject: [PATCH 239/543] [vigir] re-order (cosmetic) --- openwrt/vigir-2-configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/vigir-2-configure.sh b/openwrt/vigir-2-configure.sh index 5a4ba21..732ec13 100755 --- a/openwrt/vigir-2-configure.sh +++ b/openwrt/vigir-2-configure.sh @@ -85,9 +85,9 @@ uci set system.@system[0].hostname="${vigir_hostname}" # Wifi configuration uci set wireless.radio0=wifi-device uci set wireless.radio0.htmode='HT40' -uci set wireless.radio0.country='CH' uci set wireless.radio0.channel='6' +uci set wireless.radio0.country='CH' uci set wireless.default_radio0.encryption='psk2' uci set wireless.default_radio0.key='iloveipv6' uci set wireless.default_radio0.ssid='IPv6 everywhere ${vigir_hostname}' From 7849d12baf91c2608c8d7f212bc7ecfdb3bad3c9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 5 Jan 2021 13:40:04 +0100 Subject: [PATCH 240/543] [jool/openwrt] require 3 arguments --- openwrt/openwrt-add-jool-ipv4island.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-add-jool-ipv4island.sh b/openwrt/openwrt-add-jool-ipv4island.sh index 2b38865..0f1d75b 100755 --- a/openwrt/openwrt-add-jool-ipv4island.sh +++ b/openwrt/openwrt-add-jool-ipv4island.sh @@ -1,6 +1,6 @@ #!/bin/sh -if [ $# -ne 1 ]; then +if [ $# -ne 3 ]; then echo "$0 ip-address nat64-prefix" echo " ip-address: where to find the OpenWRT device" echo " nat64-prefix: which network to use for INCOMING NAT64 (often 2a0a:e5c1:XXX:b00::/96" From 41b6f55131444bc7d9834331f4fe7b3c4151ad36 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 5 Jan 2021 14:05:16 +0100 Subject: [PATCH 241/543] [viwib] remove dev argument for viwib-1 script --- openwrt/viwib-1-firmware-upgrade.sh | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 7a910f1..88c8748 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -2,38 +2,20 @@ # 2020-06-13, Nico Schottelius # See https://ungleich.ch/u/products/viirb-ipv6-box/ -if [ $# -lt 1 ]; then - echo "$0 interface [address]" - echo " interface to add the config ip address to" +if [ $# -ne 1 ]; then + echo "$0 address" echo " address: connect to this address, ignore the interface" exit 1 fi set -x -dev=$1; shift -# $# = 2 -# shift -> remove the first argument -# $# = 1 - -if [ $# -ge 1 ]; then - viwib_ip=$1; shift - dev="" -else - viwib_ip=192.168.8.1 -fi +viwib_ip=$1; shift # openwrt version=19.07.5 filename=openwrt-${version}-ramips-mt76x8-gl-mt300n-v2-squashfs-sysupgrade.bin -# IP address for setting it up initially - -if [ "$dev" ]; then - sudo ip addr del 192.168.8.2/24 dev "$dev" 2>/dev/null || true - sudo ip addr add 192.168.8.2/24 dev "$dev" -fi - # don't care about other/old known_host entries ssh-keygen -R ${viwib_ip} From a7d993a2c3c41c35f13efc5e214e975ab74877e9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 5 Jan 2021 20:00:46 +0100 Subject: [PATCH 242/543] [vigir] enable 2nd radio --- openwrt/vigir-2-configure.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/openwrt/vigir-2-configure.sh b/openwrt/vigir-2-configure.sh index 732ec13..a6d0189 100755 --- a/openwrt/vigir-2-configure.sh +++ b/openwrt/vigir-2-configure.sh @@ -100,6 +100,7 @@ uci set wireless.default_radio1.ssid='IPv6 everywhere ${vigir_hostname} 5Ghz' # Ensure it is not disabled uci delete wireless.radio0.disabled +uci delete wireless.radio1.disabled # Create temporary IPV4 CLIENT on LAN # This way we get Internet/upstream from the LAN port From 8fbb6c8eddb518dc9daf83fb31efab75055cf94c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 5 Jan 2021 20:00:59 +0100 Subject: [PATCH 243/543] use ping instead of ping6 --- openwrt/viwib-2-configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 71db5ba..65ae4fc 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -119,7 +119,7 @@ uci delete wireless.radio0.disabled uci commit -ping6 -c5 ungleich.ch || exit 1 +ping -c5 ungleich.ch || exit 1 # update the sources opkg update From 119f8533ad0e5fc3b4ffa794041a0adaf786e188 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 6 Jan 2021 18:21:17 +0100 Subject: [PATCH 244/543] [vigir/viirb] firmware upgrade needs ip address --- openwrt/vigir-1-firmware-upgrade.sh | 23 ++--------------------- openwrt/viirb-1-firmware-upgrade.sh | 22 ++-------------------- 2 files changed, 4 insertions(+), 41 deletions(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index e192bc9..9f988f4 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -2,38 +2,19 @@ # 2020-06-13, Nico Schottelius # See https://ungleich.ch/u/products/viirb-ipv6-box/ -if [ $# -lt 1 ]; then +if [ $# -ne 1 ]; then echo "$0 interface [address]" - echo " interface to add the config ip address to" echo " address: connect to this address, ignore the interface" exit 1 fi set -x -dev=$1; shift - -# $# = 2 -# shift -> remove the first argument -# $# = 1 - -if [ $# -ge 1 ]; then - vigir_ip=$1; shift - dev="" -else - vigir_ip=192.168.8.1 -fi +vigir_ip=$1; shift # openwrt version=19.07.5 filename=openwrt-${version}-ramips-mt7621-zbt-wg3526-16M-squashfs-sysupgrade.bin -# IP address for setting it up initially - -if [ "$dev" ]; then - sudo ip addr del 192.168.8.2/24 dev "$dev" 2>/dev/null || true - sudo ip addr add 192.168.8.2/24 dev "$dev" -fi - # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} diff --git a/openwrt/viirb-1-firmware-upgrade.sh b/openwrt/viirb-1-firmware-upgrade.sh index 0c52ab3..4cdace7 100755 --- a/openwrt/viirb-1-firmware-upgrade.sh +++ b/openwrt/viirb-1-firmware-upgrade.sh @@ -2,38 +2,20 @@ # 2020-06-13, Nico Schottelius # See https://ungleich.ch/u/products/viirb-ipv6-box/ -if [ $# -lt 1 ]; then +if [ $# -ne 1 ]; then echo "$0 interface [address]" - echo " interface to add the config ip address to" echo " address: connect to this address, ignore the interface" exit 1 fi set -x -dev=$1; shift -# $# = 2 -# shift -> remove the first argument -# $# = 1 - -if [ $# -ge 1 ]; then - viirb_ip=$1; shift - dev="" -else - viirb_ip=192.168.61.1 -fi +viirb_ip=$1; shift # openwrt version=19.07.5 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin -# IP address for setting it up initially - -if [ "$dev" ]; then - sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true - sudo ip addr add 192.168.61.2/24 dev "$dev" -fi - # don't care about other/old known_host entries ssh-keygen -R ${viirb_ip} From 40fd7a43ab6732d48623ca7fd3d8880667af0095 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 6 Jan 2021 18:22:11 +0100 Subject: [PATCH 245/543] [viirb] firmware upgrade: clean configuration --- openwrt/viirb-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viirb-1-firmware-upgrade.sh b/openwrt/viirb-1-firmware-upgrade.sh index 4cdace7..0f91c70 100755 --- a/openwrt/viirb-1-firmware-upgrade.sh +++ b/openwrt/viirb-1-firmware-upgrade.sh @@ -36,7 +36,7 @@ else fi scp ${filename} root@${scp_ip}:/tmp -ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" +ssh root@${viirb_ip} "sysupgrade -n /tmp/*.bin" # It still pings for some time - wait for the reboot to happen echo "Waiting for VIIRB to really disappear" From fad70bccc532c3d3d54907f10b9115591dbe8077 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 8 Jan 2021 13:23:55 +0100 Subject: [PATCH 246/543] No need for explicit v4 mapping in jool -> actually helps us! --- openwrt/openwrt-add-jool-ipv4island.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/openwrt/openwrt-add-jool-ipv4island.sh b/openwrt/openwrt-add-jool-ipv4island.sh index 0f1d75b..cac2a24 100755 --- a/openwrt/openwrt-add-jool-ipv4island.sh +++ b/openwrt/openwrt-add-jool-ipv4island.sh @@ -4,7 +4,6 @@ if [ $# -ne 3 ]; then echo "$0 ip-address nat64-prefix" echo " ip-address: where to find the OpenWRT device" echo " nat64-prefix: which network to use for INCOMING NAT64 (often 2a0a:e5c1:XXX:b00::/96" - echo " ip4-address: where to map ipv6 traffic to (often: 192.168.1.42)" exit 1 fi @@ -22,7 +21,6 @@ cat > /etc/rc.local << EO2 modprobe jool -jool -4 -a ${v4addr} jool -6 ${nat64_prefix} EO2 From fae82faa80a97ad39588fe3c255e86d7092073b8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 8 Jan 2021 13:28:36 +0100 Subject: [PATCH 247/543] fix args --- openwrt/openwrt-add-jool-ipv4island.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-add-jool-ipv4island.sh b/openwrt/openwrt-add-jool-ipv4island.sh index cac2a24..3b1c7d6 100755 --- a/openwrt/openwrt-add-jool-ipv4island.sh +++ b/openwrt/openwrt-add-jool-ipv4island.sh @@ -1,6 +1,6 @@ #!/bin/sh -if [ $# -ne 3 ]; then +if [ $# -ne 2 ]; then echo "$0 ip-address nat64-prefix" echo " ip-address: where to find the OpenWRT device" echo " nat64-prefix: which network to use for INCOMING NAT64 (often 2a0a:e5c1:XXX:b00::/96" From a105e0c45d92b10e3f069884c5001ea9cc448561 Mon Sep 17 00:00:00 2001 From: samuel hailu Date: Wed, 27 Jan 2021 13:53:38 +0100 Subject: [PATCH 248/543] ADDED PING6 TO SCRIPT --- openwrt/vigir-2-configure.sh | 4 ++-- openwrt/vigir15.public_key | 1 + openwrt/vigir5.public_key | 1 + openwrt/viwib-2-configure.sh | 2 +- openwrt/viwib1.public_key | 2 +- openwrt/viwib10.public_key | 1 + openwrt/viwib2.public_key | 1 + openwrt/viwib21.public_key | 1 + 8 files changed, 9 insertions(+), 4 deletions(-) create mode 100644 openwrt/vigir15.public_key create mode 100644 openwrt/vigir5.public_key create mode 100644 openwrt/viwib10.public_key create mode 100644 openwrt/viwib2.public_key create mode 100644 openwrt/viwib21.public_key diff --git a/openwrt/vigir-2-configure.sh b/openwrt/vigir-2-configure.sh index a6d0189..e071265 100755 --- a/openwrt/vigir-2-configure.sh +++ b/openwrt/vigir-2-configure.sh @@ -111,7 +111,7 @@ uci delete wireless.radio1.disabled uci commit -ping -c5 ungleich.ch || exit 1 +ping6 -c5 ungleich.ch || exit 1 # update the sources opkg update @@ -187,7 +187,7 @@ vigir_ip=${my_lan_ip} sleep 15 -while ! ping -c5 ${vigir_ip}; do +while ! ping6 -c5 ${vigir_ip}; do echo "Waiting for vigir ${id}" sleep 2 done diff --git a/openwrt/vigir15.public_key b/openwrt/vigir15.public_key new file mode 100644 index 0000000..3e138c0 --- /dev/null +++ b/openwrt/vigir15.public_key @@ -0,0 +1 @@ +/QvWXp3v5n6cVdQ78sBpNczY8jI4nzE6uUlrav65N00= diff --git a/openwrt/vigir5.public_key b/openwrt/vigir5.public_key new file mode 100644 index 0000000..ad03e4e --- /dev/null +++ b/openwrt/vigir5.public_key @@ -0,0 +1 @@ +020sjVDopRMMLSIgbyviDhSZ2FACa7CQ531DJNDVBRY= diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 65ae4fc..71db5ba 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -119,7 +119,7 @@ uci delete wireless.radio0.disabled uci commit -ping -c5 ungleich.ch || exit 1 +ping6 -c5 ungleich.ch || exit 1 # update the sources opkg update diff --git a/openwrt/viwib1.public_key b/openwrt/viwib1.public_key index fcfff40..58596f5 100644 --- a/openwrt/viwib1.public_key +++ b/openwrt/viwib1.public_key @@ -1 +1 @@ -/m35eP83YNduvxCXUYpjVrMKMW+cyi65CXrtPscWAAY= +bKobh51z1mg2p3b2zPyj+gu8vTq5RtBpxvKo5+XSCic= diff --git a/openwrt/viwib10.public_key b/openwrt/viwib10.public_key new file mode 100644 index 0000000..19c544b --- /dev/null +++ b/openwrt/viwib10.public_key @@ -0,0 +1 @@ +1GdDCQM3wxRzSLSSXy7OiXavD8a+XAK4UbYIa8och2g= diff --git a/openwrt/viwib2.public_key b/openwrt/viwib2.public_key new file mode 100644 index 0000000..b24c4b5 --- /dev/null +++ b/openwrt/viwib2.public_key @@ -0,0 +1 @@ +ifKKyAHG0T7mT7D0QmMk+t8btvaTpQIpgYpD9kfXhko= diff --git a/openwrt/viwib21.public_key b/openwrt/viwib21.public_key new file mode 100644 index 0000000..7539b33 --- /dev/null +++ b/openwrt/viwib21.public_key @@ -0,0 +1 @@ +iKTwnViboBnM1/7eBUPhNWpqYPWKbAs07d4/Z7Qhggg= From 721bbdf0d619bc5c326858b428ca8451613f105b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 29 Jan 2021 22:42:17 +0100 Subject: [PATCH 249/543] [openwrt] upgrade to 19.07.6 --- openwrt/vigir-1-firmware-upgrade.sh | 2 +- openwrt/viirb-1-firmware-upgrade.sh | 2 +- openwrt/viwib-1-firmware-upgrade.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index 9f988f4..4cc4d19 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,7 +12,7 @@ set -x vigir_ip=$1; shift # openwrt -version=19.07.5 +version=19.07.6 filename=openwrt-${version}-ramips-mt7621-zbt-wg3526-16M-squashfs-sysupgrade.bin # don't care about other/old known_host entries diff --git a/openwrt/viirb-1-firmware-upgrade.sh b/openwrt/viirb-1-firmware-upgrade.sh index 0f91c70..786e6da 100755 --- a/openwrt/viirb-1-firmware-upgrade.sh +++ b/openwrt/viirb-1-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viirb_ip=$1; shift # openwrt -version=19.07.5 +version=19.07.6 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin # don't care about other/old known_host entries diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 88c8748..35d1656 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=19.07.5 +version=19.07.6 filename=openwrt-${version}-ramips-mt76x8-gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries From 095b184be999aaf409ae6422079be2ce21123c77 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 6 Feb 2021 22:17:13 +0100 Subject: [PATCH 250/543] [alpine/opennebula] update to v3.13 --- opennebula-images/alpine-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh index 1ec6f3f..e6120d3 100755 --- a/opennebula-images/alpine-build-opennebula-image.sh +++ b/opennebula-images/alpine-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=v3.12 +RELEASE=v3.13 ARCH=x86_64 IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G From 3c3c53de2e1171b98813f03938a6c987608e86de Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 7 Feb 2021 12:37:53 +0100 Subject: [PATCH 251/543] [ceph] phase in new script for creating monitor --- ceph/ceph-mon-create-start | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/ceph/ceph-mon-create-start b/ceph/ceph-mon-create-start index ab4d7a8..9fd4444 100755 --- a/ceph/ceph-mon-create-start +++ b/ceph/ceph-mon-create-start @@ -5,12 +5,17 @@ if [ $# -ne 1 ]; then exit 1 fi -fname=$1 +monkey=$(mktemp) +monmap=$(mktemp) -ceph-mon --mkfs -i $(hostname) --keyring "$fname" --setuser ceph --setgroup ceph -touch /var/lib/ceph/mon/ceph-$(hostname)/sysvinit +ceph auth get mon. -o $monkey +ceph mon getmap -o $monmap -# Fix broken permissions +mkdir /var/lib/ceph/mon/ceph-$(hostname) +ceph-mon -i $(hostname) --mkfs --monmap $monmap --keyring $monkey +chown -R ceph:ceph /var/lib/ceph/mon/ceph-$(hostname) + +# Fix broken permissions on Debian chown ceph:ceph /var/run/ceph/ # Starting with monit, if available From dd27b12052a0be1d93922bbb476e0889e789953c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 7 Feb 2021 12:42:50 +0100 Subject: [PATCH 252/543] [ceph/mon] no args --- ceph/ceph-mon-create-start | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ceph/ceph-mon-create-start b/ceph/ceph-mon-create-start index 9fd4444..1a801a8 100755 --- a/ceph/ceph-mon-create-start +++ b/ceph/ceph-mon-create-start @@ -1,10 +1,5 @@ #!/bin/sh -if [ $# -ne 1 ]; then - echo "$0 initial-key-file" - exit 1 -fi - monkey=$(mktemp) monmap=$(mktemp) From 6bcb2b6734b9ca3842791c082a7106d0b34373bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 7 Feb 2021 13:01:48 +0100 Subject: [PATCH 253/543] [monit/ceph] always use monit -> same for all OS --- ceph/ceph-mgr-create-start | 12 ++++------ ceph/ceph-mon-create-start | 8 ++----- ceph/ceph-osd-create-start | 8 ++----- monit-ceph-create-start | 45 +++++++++++++++----------------------- 4 files changed, 26 insertions(+), 47 deletions(-) diff --git a/ceph/ceph-mgr-create-start b/ceph/ceph-mgr-create-start index e9a8842..e47cf4b 100755 --- a/ceph/ceph-mgr-create-start +++ b/ceph/ceph-mgr-create-start @@ -3,7 +3,7 @@ name=$(hostname) CEPH_PATH=/var/lib/ceph -MGR_PATH=$CEPH_PATH/mgr/ceph-$name +MGR_PATH=$CEPH_PATH/mgr/ceph-${name} if [ -e "$MGR_PATH" ]; then echo "$MGR_PATH exists - aborting" @@ -14,14 +14,10 @@ mkdir "$MGR_PATH" chown ceph:ceph "$MGR_PATH" touch "$MGR_PATH/sysvinit" -ceph auth get-or-create mgr.$name \ +ceph auth get-or-create mgr.${name} \ mon 'allow profile mgr' \ osd 'allow *' \ mds 'allow *' > "$MGR_PATH/keyring" -# Starting with monit, if available -if [ -e /etc/monit ]; then - /opt/ungleich-tools/monit-ceph-create-start mgr.${name} -else - /etc/init.d/ceph start mgr.${name} -fi +# Starting with monit - same on every os +/opt/ungleich-tools/monit-ceph-create-start mgr.${name} diff --git a/ceph/ceph-mon-create-start b/ceph/ceph-mon-create-start index 1a801a8..239e221 100755 --- a/ceph/ceph-mon-create-start +++ b/ceph/ceph-mon-create-start @@ -13,9 +13,5 @@ chown -R ceph:ceph /var/lib/ceph/mon/ceph-$(hostname) # Fix broken permissions on Debian chown ceph:ceph /var/run/ceph/ -# Starting with monit, if available -if [ -e /etc/monit ]; then - /opt/ungleich-tools/monit-ceph-create-start mon.$(hostname) -else - /etc/init.d/ceph start mon.$(hostname) -fi +# Starting with monit +/opt/ungleich-tools/monit-ceph-create-start mon.$(hostname) diff --git a/ceph/ceph-osd-create-start b/ceph/ceph-osd-create-start index 106e363..e1b350e 100755 --- a/ceph/ceph-osd-create-start +++ b/ceph/ceph-osd-create-start @@ -100,9 +100,5 @@ ceph osd crush set-device-class $CLASS osd.${osd_id} echo "$metadata_dev /var/lib/ceph/osd/ceph-${osd_id} xfs noatime 0 0" >> /etc/fstab -# Starting with monit, if available -if [ -e /etc/monit ]; then - /opt/ungleich-tools/monit-ceph-create-start osd.${osd_id} -else - /etc/init.d/ceph start osd.${osd_id} -fi +# Start it +/opt/ungleich-tools/monit-ceph-create-start osd.${osd_id} diff --git a/monit-ceph-create-start b/monit-ceph-create-start index a44efa0..2c69b4c 100755 --- a/monit-ceph-create-start +++ b/monit-ceph-create-start @@ -13,44 +13,35 @@ to_monitor=$1 set -e depends="cephrundir" -osd="" conf="/etc/monit/conf.d/$to_monitor" -if echo $to_monitor | grep ^osd; then - depends="${depends}, ${to_monitor}-whoami" - osd="yes" - osdid=$(echo $to_monitor | cut -d. -f2) - cat > "$conf" <> "$conf" < "$conf" <> "$conf" <> "$conf" < Date: Sun, 7 Feb 2021 13:03:05 +0100 Subject: [PATCH 254/543] [ceph/mgr] create mgr directory with -p --- ceph/ceph-mgr-create-start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceph/ceph-mgr-create-start b/ceph/ceph-mgr-create-start index e47cf4b..011f754 100755 --- a/ceph/ceph-mgr-create-start +++ b/ceph/ceph-mgr-create-start @@ -10,7 +10,7 @@ if [ -e "$MGR_PATH" ]; then exit 1 fi -mkdir "$MGR_PATH" +mkdir -p "$MGR_PATH" chown ceph:ceph "$MGR_PATH" touch "$MGR_PATH/sysvinit" From 48202b533d5e7b8877a57ea19e8b7e6809513cdb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 7 Feb 2021 13:04:21 +0100 Subject: [PATCH 255/543] --syntax errors --- monit-ceph-create-start | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/monit-ceph-create-start b/monit-ceph-create-start index 2c69b4c..391d1e6 100755 --- a/monit-ceph-create-start +++ b/monit-ceph-create-start @@ -28,10 +28,7 @@ check file ${to_monitor}-whoami with path /var/lib/ceph/osd/ceph-${id}/whoami EOF ;; - mon|mgr) - : - ;; -EOF +esac cat >> "$conf" < Date: Sun, 7 Feb 2021 13:27:51 +0100 Subject: [PATCH 256/543] [monit] fix ceph daemon pidfile --- monit-ceph-create-start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monit-ceph-create-start b/monit-ceph-create-start index 391d1e6..a11dbb7 100755 --- a/monit-ceph-create-start +++ b/monit-ceph-create-start @@ -33,7 +33,7 @@ esac cat >> "$conf" < Date: Sun, 7 Feb 2021 14:20:34 +0100 Subject: [PATCH 257/543] [ceph] remove key/monmap --- ceph/ceph-mon-create-start | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ceph/ceph-mon-create-start b/ceph/ceph-mon-create-start index 239e221..87f3a53 100755 --- a/ceph/ceph-mon-create-start +++ b/ceph/ceph-mon-create-start @@ -1,5 +1,7 @@ #!/bin/sh +set -x + monkey=$(mktemp) monmap=$(mktemp) @@ -15,3 +17,5 @@ chown ceph:ceph /var/run/ceph/ # Starting with monit /opt/ungleich-tools/monit-ceph-create-start mon.$(hostname) + +rm -f ${monkey} ${monmap} From 595f402ced4863179fc9c1407ad3dd3dae9291bf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 8 Feb 2021 22:53:15 +0100 Subject: [PATCH 258/543] [alpine builder] upgrade to 3.13.1 --- alpine-rebuild-initramfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 89e9820..309e68a 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -11,8 +11,8 @@ set -x SSH_KEYS=$1; shift -MAJOR_VERSION=3.12 -MINOR_VERSION=0 +MAJOR_VERSION=3.13 +MINOR_VERSION=1 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz RESOLVCONF=/etc/resolv.conf From 1b720cfab6c4eb3b622dbdc5516fd07045d6a7f1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 9 Feb 2021 12:50:55 +0100 Subject: [PATCH 259/543] [alpine buildr] unlock root, add rdnssd --- alpine-rebuild-initramfs.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 309e68a..30923d1 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -18,8 +18,8 @@ IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz RESOLVCONF=/etc/resolv.conf working_directory=$(pwd -P) -rootfs_tmpdir=$(mktemp -d) -#rootfs_tmpdir=alpine_${MAJOR_VERSION}-${MINOR_VERSION}-rootfs +#rootfs_tmpdir=$(mktemp -d) +rootfs_tmpdir=alpine_${MAJOR_VERSION}-${MINOR_VERSION}-rootfs mkdir -p ${rootfs_tmpdir} @@ -39,9 +39,9 @@ sudo tar xf $IMAGE -C $rootfs_tmpdir # Add SSH keys run_root mkdir -p root/.ssh sudo cp $SSH_KEYS $rootfs_tmpdir/root/.ssh/authorized_keys -run_root chown root:root root/.ssh/authorized_keys -run_root chmod 0600 root/.ssh/authorized_keys -run_root chmod 0700 root/.ssh +run_root chown root:root /root/.ssh/authorized_keys +run_root chmod 0600 /root/.ssh/authorized_keys +run_root chmod 0700 /root/.ssh # Import local resolv.conf. sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf @@ -50,14 +50,15 @@ sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf run_root ln -sf /sbin/init /init run_root apk update -run_root apk add linux-lts openrc udev openssh +run_root apk add linux-lts openrc udev openssh rdnssd # rdnssd run_root rc-update add udev run_root rc-update add udev-trigger run_root rc-update add sshd -# run_root rc-update add rdnssd +run_root rc-update add rdnssd run_root rc-update add networking run_root rc-update add hostname +run_root sed -i 's/root:!::0:::::/root:::0:::::/' /etc/shadow sudo tee "$rootfs_tmpdir/etc/network/interfaces" < /proc/sys/net/ipv6/conf/eth0/accept_ra EOF sudo tee "$rootfs_tmpdir/etc/hostname" < Date: Tue, 9 Feb 2021 14:29:52 +0100 Subject: [PATCH 260/543] do not use the empty password for alpine --- alpine-rebuild-initramfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 30923d1..daa32d7 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -58,7 +58,7 @@ run_root rc-update add sshd run_root rc-update add rdnssd run_root rc-update add networking run_root rc-update add hostname -run_root sed -i 's/root:!::0:::::/root:::0:::::/' /etc/shadow +run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow sudo tee "$rootfs_tmpdir/etc/network/interfaces" < Date: Thu, 11 Mar 2021 10:56:06 +0100 Subject: [PATCH 261/543] Cleaning and tweaking (missing one-context?) of arch ONE script --- .../arch-build-opennebula-image.sh | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/opennebula-images/arch-build-opennebula-image.sh b/opennebula-images/arch-build-opennebula-image.sh index 00c454f..f809de5 100755 --- a/opennebula-images/arch-build-opennebula-image.sh +++ b/opennebula-images/arch-build-opennebula-image.sh @@ -9,14 +9,12 @@ set -e set -x # XXX: Handle command-line arguments? -ARCH=amd64 IMAGE_PATH=arch-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 -# TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" -ONE_CONTEXT_DEB_PATH=/root/one-context.deb +ONE_CONTEXT_VERSION=5.12.0.2 +ONE_CONTEXT_SOURCE_ARCHIVE="https://github.com/OpenNebula/addon-context-linux/archive/v$ONE_CONTEXT_VERSION.tar.gz" cleanup() { # The order here is important. @@ -42,9 +40,9 @@ if [ "$(whoami)" != 'root' ]; then exit 1 fi -if [ $(lsb_release --short --id) != "Arch" ]; then - echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 - echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 +if [ "$(lsb_release --short --id)" != "Arch" ]; then + echo "WARNING: this script has been designed to run on Arch Linux." >&2 + echo "WARNING: Not running Arch. Giving you 5 seconds to abort." >&2 sleep 5 fi @@ -75,7 +73,7 @@ mkdir /mnt/boot mount "${NBD_DEVICE}p1" /mnt/boot # Install base system. -pacstrap /mnt nbase base-devel openssh +pacstrap /mnt base base-devel openssh mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts @@ -84,9 +82,6 @@ mount --bind /proc /mnt/proc mount --bind /run /mnt/run mount --bind /sys /mnt/sys -# Guest networking is to be handled by the one-context package. -# See https://github.com/OpenNebula/addon-context-linux for details. - # Required to resolve package mirror in chroot. cp /etc/resolv.conf /mnt/etc/resolv.conf @@ -105,16 +100,25 @@ cat > /mnt/etc/pacman.d/mirrorlist << EOF ## ## Switzerland -Server = http://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch -Server = https://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch -Server = http://mirror.init7.net/archlinux/$repo/os/$arch -Server = https://mirror.init7.net/archlinux/$repo/os/$arch -Server = http://mirror.puzzle.ch/archlinux/$repo/os/$arch -Server = https://mirror.puzzle.ch/archlinux/$repo/os/$arch -Server = https://mirror.ungleich.ch/mirror/packages/archlinux/$repo/os/$arch +Server = http://pkg.adfinis-sygroup.ch/archlinux/\$repo/os/\$arch +Server = https://pkg.adfinis-sygroup.ch/archlinux/\$repo/os/\$arch +Server = http://mirror.init7.net/archlinux/\$repo/os/\$arch +Server = https://mirror.init7.net/archlinux/\$repo/os/\$arch +Server = http://mirror.puzzle.ch/archlinux/\$repo/os/\$arch +Server = https://mirror.puzzle.ch/archlinux/\$repo/os/\$arch +Server = https://mirror.ungleich.ch/mirror/packages/archlinux/\$repo/os/\$arch EOF run_root pacman --sync --refresh --upgrade +# Guest networking is to be handled by the one-context package. +# See https://github.com/OpenNebula/addon-context-linux for details. +run_root pacman --sync curl tar +run_root curl -L "$ONE_CONTEXT_SOURCE_ARCHIVE" -o one-context.tar.gz +run_root tar xf one-context.tar.gz +run_root cp -rT addon-context-linux-${ONE_CONTEXT_VERSION}/src/ / +run_root rm -r addon-context-linux-${ONE_CONTEXT_VERSION} +run_root systemctl enable one-context.service + # Initalize base services. run_root systemd-machine-id-setup From d979bd52ceed5a2e9e8f0ea1b3ddf38c6f915f3e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 12 Mar 2021 11:52:27 +0100 Subject: [PATCH 262/543] [netboot] use key.wf, re-introduce fixing network interfaces --- debian-devuan-netboot.sh | 73 ++++++++++++++++++++++------------------ 1 file changed, 40 insertions(+), 33 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 2858ae4..c83b9cf 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -25,7 +25,8 @@ chroot_dir=${abs_outdir}/${basename} kernel=${abs_outdir}/kernel-${basename} initramfs=${abs_outdir}/initramfs-${basename} -keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files +#keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files +keyurl=https://key.wf debootstrap "${suite}" "${chroot_dir}" @@ -43,8 +44,8 @@ echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > mkdir -p ${chroot_dir}/root/.ssh -for key in fnux balazs dominique jinguk nico; do - curl -s ${keyurl}/${key}.pub >> ${chroot_dir}/root/.ssh/authorized_keys +for key in sami dominique jinguk nico; do + curl -s ${keyurl}/${key} >> ${chroot_dir}/root/.ssh/authorized_keys done # Fix possible permission issue from above @@ -59,6 +60,42 @@ cat > ${chroot_dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback +EOF + +# find the boot interfaces at boot: HP servers still have ifnames=1 +cat > ${chroot_dir}/etc/rc.local <> /etc/network/interfaces << eof +auto \$dev +iface \$dev inet6 auto +eof + +ifup "\${dev}" + +exit 0 +EOF + +chmod a+rx "${chroot_dir}/etc/rc.local" + +# ensure there is /init in the initramfs -> otherwise there is a kernel panic +# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init +# in our case, they are just the same +ln -fs /sbin/init ${chroot_dir}/init + +# Finally building the initramfs +( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) + +# Fix paranoid permissions +chmod a+rx ${abs_outdir} +chmod a+r ${kernel} ${initramfs} + + +exit 0 + +# I would like to have a generic block like this below +# But as long as interface auto bond0 iface bond0 inet manual bond-miimon 500 @@ -81,33 +118,3 @@ auto bond0.11 iface bond0.11 inet6 auto post-up /sbin/ip link set \$IFACE mtu 9000 vlan-raw-device bond0 -EOF - -# find the boot interfaces at boot - not needed, always eth0/eth1 -# cat > ${chroot_dir}/etc/rc.local < /etc/network/interfaces.d/bootinterface << eof -# auto \$dev -# iface \$dev inet6 auto -# eof - -# ifup "\${dev}" - -# exit 0 -# EOF - -# chmod a+rx "${chroot_dir}/etc/rc.local" - -# ensure there is /init in the initramfs -> otherwise there is a kernel panic -# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init -# in our case, they are just the same -ln -fs /sbin/init ${chroot_dir}/init - -# Finally building the initramfs -( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) - -# Fix paranoid permissions -chmod a+rx ${abs_outdir} -chmod a+r ${kernel} ${initramfs} From eb9cbbdc1f29e497f735cb0e67301e23f5f6e025 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 12 Mar 2021 14:07:15 +0100 Subject: [PATCH 263/543] fix viirb script --- openwrt/viirb-2-configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viirb-2-configure.sh b/openwrt/viirb-2-configure.sh index c9ada76..093d547 100755 --- a/openwrt/viirb-2-configure.sh +++ b/openwrt/viirb-2-configure.sh @@ -17,7 +17,7 @@ id=$1; shift hex_id=$(printf "%0.2x\n" "$id") viirb_hostname=viirb${id} -prefix_base=2a0a:e5c1:6 +prefix_base=2a0a:e5c1:3 my_prefix=${prefix_base}${hex_id} my_network=${my_prefix}::/48 From 6a6f9850b61e2cbe1683bfbc48243d5f7825d4d1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 12 Mar 2021 15:24:32 +0100 Subject: [PATCH 264/543] NO MORE FIXES! all severs boot from fiber --- debian-devuan-netboot.sh | 68 ++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index c83b9cf..a613903 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -60,40 +60,6 @@ cat > ${chroot_dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback -EOF - -# find the boot interfaces at boot: HP servers still have ifnames=1 -cat > ${chroot_dir}/etc/rc.local <> /etc/network/interfaces << eof -auto \$dev -iface \$dev inet6 auto -eof - -ifup "\${dev}" - -exit 0 -EOF - -chmod a+rx "${chroot_dir}/etc/rc.local" - -# ensure there is /init in the initramfs -> otherwise there is a kernel panic -# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init -# in our case, they are just the same -ln -fs /sbin/init ${chroot_dir}/init - -# Finally building the initramfs -( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) - -# Fix paranoid permissions -chmod a+rx ${abs_outdir} -chmod a+r ${kernel} ${initramfs} - - -exit 0 - # I would like to have a generic block like this below # But as long as interface auto bond0 @@ -118,3 +84,37 @@ auto bond0.11 iface bond0.11 inet6 auto post-up /sbin/ip link set \$IFACE mtu 9000 vlan-raw-device bond0 + +EOF + +# # find the boot interfaces at boot: HP servers still have ifnames=1 +# cat > ${chroot_dir}/etc/rc.local <> /etc/network/interfaces << eof +# auto \$dev +# iface \$dev inet6 auto +# eof + +# ifup "\${dev}" + +# exit 0 +# EOF + +# chmod a+rx "${chroot_dir}/etc/rc.local" + +# ensure there is /init in the initramfs -> otherwise there is a kernel panic +# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init +# in our case, they are just the same +ln -fs /sbin/init ${chroot_dir}/init + +# Finally building the initramfs +( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) + +# Fix paranoid permissions +chmod a+rx ${abs_outdir} +chmod a+r ${kernel} ${initramfs} + + +exit 0 From 6891dfe41a1958a78b47eb5dce297884c805934d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Mar 2021 17:06:28 +0100 Subject: [PATCH 265/543] [alpine] setup bond0 by default --- alpine-rebuild-initramfs.sh | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index daa32d7..d00df16 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -12,7 +12,7 @@ set -x SSH_KEYS=$1; shift MAJOR_VERSION=3.13 -MINOR_VERSION=1 +MINOR_VERSION=2 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz RESOLVCONF=/etc/resolv.conf @@ -21,6 +21,9 @@ working_directory=$(pwd -P) #rootfs_tmpdir=$(mktemp -d) rootfs_tmpdir=alpine_${MAJOR_VERSION}-${MINOR_VERSION}-rootfs +initramfs="$working_directory/initramfs-alpine-${MAJOR_VERSION}.${MINOR_VERSION}" +kernel="$working_directory/kernel-alpine-${MAJOR_VERSION}.${MINOR_VERSION}" + mkdir -p ${rootfs_tmpdir} rootfs_url="http://dl-cdn.alpinelinux.org/alpine/v$MAJOR_VERSION/releases/x86_64/$IMAGE" @@ -66,19 +69,39 @@ iface lo inet loopback auto eth0 iface eth0 inet6 manual - pre-up ip link set eth0 up + bond-master bond0 + post-up /sbin/ip link set \$IFACE up + post-up /sbin/ip link set \$IFACE mtu 9000 + +auto eth1 +iface eth1 inet6 manual + bond-master bond0 + post-up /sbin/ip link set \$IFACE up + post-up /sbin/ip link set \$IFACE mtu 9000 + +auto bond0 +iface bond0 inet6 manual + bond-miimon 500 + bond-mode 4 + post-up /sbin/ip link set \$IFACE up + post-up /sbin/ip link set \$IFACE mtu 9000 + bond-slaves none + EOF sudo tee "$rootfs_tmpdir/etc/hostname" < "$working_directory/alpine-initramfs.gz") -cp "$rootfs_tmpdir/boot/vmlinuz-lts" "$working_directory/alpine-kernel" +(cd $rootfs_tmpdir; sudo find . | sudo cpio -H newc -o | gzip -9 > ${initramfs}) +cp "$rootfs_tmpdir/boot/vmlinuz-lts" "${kernel}" echo rm -rf "$rootfs_tmpdir" -echo "Use alpine-initramfs.gz alpine-kernel from $working_directory"! +echo "Use ${initramfs} and ${kernel} from $working_directory"! exit 0 From c225cace2de0fb7c4f327bd17b191397a44dc31d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Mar 2021 17:16:07 +0100 Subject: [PATCH 266/543] [initramfs/alpine] add required bonding/vlan packages --- alpine-rebuild-initramfs.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index d00df16..efb6cb0 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -53,7 +53,7 @@ sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf run_root ln -sf /sbin/init /init run_root apk update -run_root apk add linux-lts openrc udev openssh rdnssd +run_root apk add linux-lts openrc udev openssh rdnssd bonding vlan # rdnssd run_root rc-update add udev run_root rc-update add udev-trigger From 385ee49a2edf2dc910979332179c7bcbc8af9356 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 2 Apr 2021 09:03:15 -0400 Subject: [PATCH 267/543] [tools]create almalinux-script --- .../almalinux-build-opennebula-image.sh | 170 ++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100755 opennebula-images/almalinux-build-opennebula-image.sh diff --git a/opennebula-images/almalinux-build-opennebula-image.sh b/opennebula-images/almalinux-build-opennebula-image.sh new file mode 100755 index 0000000..079e5d7 --- /dev/null +++ b/opennebula-images/almalinux-build-opennebula-image.sh @@ -0,0 +1,170 @@ +#!/bin/sh + +# This script generates almalinux images for OpenNebula. + +# Depends on the following packages (as of Almalinux 8.3): +# qemu-img util-linux coreutils dnf curl e2fsprogs + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=8.3 +ARCH=x86_64 +IMAGE_PATH=almalinux-$RELEASE-$(date --iso-8601).img +IMAGE_SIZE=10G +LOOPBACK_DEVICE=/dev/loop0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + losetup -d "$LOOPBACK_DEVICE" +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/almalinux-release' ]; then + echo "WARNING: this script has been designed to run on a AlmaLinux system." >&2 + echo "WARNING: Not running AlmaLinux. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base RAW image (no LOOPBACK support in RHEL/AlmaLinux). +qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" +losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +{ +sfdisk --no-reread "$LOOPBACK_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. +# network-scripts.x86_64 : Legacy scripts for manipulating of network devices +run_root dnf -y install network-scripts + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install and configure NTP client. +run_root dnf install -y chrony +run_root systemctl enable chronyd.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <>/mnt/etc/fstab < Date: Sun, 18 Apr 2021 15:06:51 +0200 Subject: [PATCH 268/543] [viwi] upgrade to 19.07.7 --- openwrt/viwib-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 35d1656..245f6ec 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=19.07.6 +version=19.07.7 filename=openwrt-${version}-ramips-mt76x8-gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries From 9b594fda091212ec38c0163ea67a91d907d23ac0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 18 Apr 2021 15:43:53 +0200 Subject: [PATCH 269/543] [openwrt/lte] adding drivers for rndis --- openwrt/openwrt-add-usb-lte.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-add-usb-lte.sh b/openwrt/openwrt-add-usb-lte.sh index 0be94fe..d666fe7 100755 --- a/openwrt/openwrt-add-usb-lte.sh +++ b/openwrt/openwrt-add-usb-lte.sh @@ -20,7 +20,7 @@ opkg update opkg install libustream-openssl ca-bundle ca-certificates # Install needed kernel module -opkg install kmod-usb-net-cdc-ether usb-modeswitch +opkg install kmod-usb-net-cdc-ether usb-modeswitch kmod-usb-net-rndis # Create interface uci set network.LTE=interface From 88f824dc3b168ac1c2df9ea2b661f1f4495111a1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 18 Apr 2021 15:47:54 +0200 Subject: [PATCH 270/543] [ceph] also delete the authentication token when deleting osd This helps better use of ceph-volume --- ceph/ceph-osd-stop-remove-permanently | 1 + 1 file changed, 1 insertion(+) diff --git a/ceph/ceph-osd-stop-remove-permanently b/ceph/ceph-osd-stop-remove-permanently index f35bdde..dbb6f65 100755 --- a/ceph/ceph-osd-stop-remove-permanently +++ b/ceph/ceph-osd-stop-remove-permanently @@ -25,6 +25,7 @@ fi ceph osd crush remove $osd_name ceph osd rm $osd_name +ceph auth del $osd_name echo "Mount path before umounting: " mount | grep "$mountpath" From 8e853e65de7f887b91794a0a97214b264120dcc2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Apr 2021 16:51:45 +0200 Subject: [PATCH 271/543] [ceph] phase in ceph-volume --- ceph/ceph-osd-activate-all | 1 + ceph/ceph-osd-create-start | 105 ++++++------------------------------- 2 files changed, 17 insertions(+), 89 deletions(-) diff --git a/ceph/ceph-osd-activate-all b/ceph/ceph-osd-activate-all index 8242d57..95ef7bd 100755 --- a/ceph/ceph-osd-activate-all +++ b/ceph/ceph-osd-activate-all @@ -7,6 +7,7 @@ set -x tmpdir=$(mktemp -d) +# XFS based partition scheme for dev in $(fdisk -l | awk '$6 ~/Ceph/ { print $1 }'); do mount "$dev" "$tmpdir" id=$(cat "${tmpdir}/whoami") diff --git a/ceph/ceph-osd-create-start b/ceph/ceph-osd-create-start index e1b350e..0c752af 100755 --- a/ceph/ceph-osd-create-start +++ b/ceph/ceph-osd-create-start @@ -2,103 +2,30 @@ # 17:19, 2018-02-09 # Nico Schottelius -# Based on ceph-disk -v prepare --bluestore /dev/sdc --osd-id ${ID} --osd-uuid $(uuidgen) --crush-device-class "ssd" - -# Create: -# - block -> link to partuuid -# - block_uuid -e> uuid if the block -# - ceph_fsid -> get from ceph-conf -# crush_device_class -> ssd, hdd -# fsid -> uuidgen! -# magic -> string "ceph osd volume v026" -# type -> bluestore - -fsid=$(ceph-conf --cluster=ceph --name=osd. --lookup fsid) -fs_uuid=$(uuidgen) -magic="ceph osd volume v026" - -set -x -set -e - if [ $# -lt 2 ]; then - echo "$0 disk class [osdweight]" + echo "$0 disk class [nostart]" echo "class = hdd or ssd" + echo "If specifying anything after the class, monit will not be created" exit 1 fi export DEV=$1;shift export CLASS=$1; shift -uuid_metadata=$(uuidgen) -uuid_block=$(uuidgen) - -osd_id=$(ceph osd create) - -dev_metadata="/dev/disk/by-partuuid/$uuid_metadata" -dev_block="/dev/disk/by-partuuid/$uuid_block" - -/sbin/sgdisk --new=0:0:+100M --change-name="0:ceph data" \ - --partition-guid="0:$uuid_metadata" \ - --typecode=0:4fbd7e29-9d25-41b8-afd0-062c0ceff05d \ - --mbrtogpt -- $DEV -/sbin/udevadm settle --timeout=600 - -# Using gdisk --largest-new does not change the name or set guid; -# So use 2 steps instead -/sbin/sgdisk --largest-new=0 --mbrtogpt -- $DEV -/sbin/udevadm settle --timeout=600 - - -lastpart=$(gdisk -l $DEV | tail -n1 | awk '{ print $1 }') -/sbin/sgdisk --change-name="${lastpart}:ceph block" \ - --partition-guid="${lastpart}:$uuid_block" \ - --typecode="${lastpart}:cafecafe-9b03-4f30-b4c6-b4b80ceff106" \ - --mbrtogpt -- $DEV -/sbin/udevadm settle --timeout=600 - -/sbin/mkfs -t xfs -f -i size=2048 -- "$dev_metadata" - -mountpath=/var/lib/ceph/osd/ceph-${osd_id} - -mkdir -p "$mountpath" -mount "$dev_metadata" "$mountpath" - -ln -s $dev_block "$mountpath/block" -echo "$uuid_block" > "$mountpath/block_uuid" -echo "$fsid" > "$mountpath/ceph_fsid" -echo "$magic" > "$mountpath/magic" - -# Important, otherwise --mkfs later will try to create filestore -echo bluestore > "$mountpath/type" - -ceph auth get-or-create "osd.${osd_id}" osd \ - 'allow *' mon 'allow profile osd' > $mountpath/keyring - -echo ${osd_id} > "$mountpath/whoami" -touch "$mountpath/sysvinit" - -ceph-osd --cluster ceph -i "${osd_id}" --mkfs -chown -R ceph:ceph "$mountpath" - -# Also allow access to the blockdevice -chown ceph:ceph ${dev_block} - -if [ $# -eq 1 ]; then - WEIGHT=$1; shift -else - devname=$(readlink -f $dev_block) - nodev=$(echo $devname | sed 's,/dev/,,') - WEIGHT=$(lsblk -l -b | awk "/^$nodev/ { print \$4/(1024^4) }") +# Ensure ceph-volume has all pre-requisites +if [ ! -f /var/lib/ceph/bootstrap-osd/ceph.keyring ]; then + mkdir -p /var/lib/ceph/bootstrap-osd + ceph auth get client.bootstrap-osd > /var/lib/ceph/bootstrap-osd/ceph.keyring +fi +if [ ! -f /etc/ceph/ceph.client.bootstrap-osd.keyring ]; then + ceph auth get client.bootstrap-osd > /etc/ceph/ceph.client.bootstrap-osd.keyring fi -# Move into the correct position -ceph osd crush add osd.${osd_id} ${WEIGHT} host=$(hostname) +ceph-volume lvm prepare --data $DEV --crush-device-class $CLASS -# Ensure previous assigned class is gone - if the osd id was used before -ceph osd crush rm-device-class osd.${osd_id} -ceph osd crush set-device-class $CLASS osd.${osd_id} - -echo "$metadata_dev /var/lib/ceph/osd/ceph-${osd_id} xfs noatime 0 0" >> /etc/fstab - -# Start it -/opt/ungleich-tools/monit-ceph-create-start osd.${osd_id} +if [ $# -eq 1 ]; then + echo "Not executing: /opt/ungleich-tools/monit-ceph-create-start osd.${osd_id}" +else + # Start it + /opt/ungleich-tools/monit-ceph-create-start osd.${osd_id} +fi From 0031c67d6578b848f917184cfb9d91a06db0ba19 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 20 Apr 2021 11:38:49 +0200 Subject: [PATCH 272/543] [ceph] find osd_id when using ceph-volume --- ceph/ceph-osd-create-start | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ceph/ceph-osd-create-start b/ceph/ceph-osd-create-start index 0c752af..6026860 100755 --- a/ceph/ceph-osd-create-start +++ b/ceph/ceph-osd-create-start @@ -12,6 +12,8 @@ fi export DEV=$1;shift export CLASS=$1; shift +set -e + # Ensure ceph-volume has all pre-requisites if [ ! -f /var/lib/ceph/bootstrap-osd/ceph.keyring ]; then mkdir -p /var/lib/ceph/bootstrap-osd @@ -21,7 +23,13 @@ if [ ! -f /etc/ceph/ceph.client.bootstrap-osd.keyring ]; then ceph auth get client.bootstrap-osd > /etc/ceph/ceph.client.bootstrap-osd.keyring fi -ceph-volume lvm prepare --data $DEV --crush-device-class $CLASS +# We are redirecting to a tempfile so that the output is visible for debugging, +# but we can still easily filter for the osd id +tmp=$(mktemp) + +ceph-volume lvm prepare --data $DEV --crush-device-class $CLASS 2>&1 | tee ${tmp} +osd_id=$(grep /var/lib/ceph/osd/ceph- ${tmp} | sed -e 's/.*ceph-//' -e 's,/.*,,' | head -n1) +rm -f ${tmp} if [ $# -eq 1 ]; then echo "Not executing: /opt/ungleich-tools/monit-ceph-create-start osd.${osd_id}" From 97598ef9c6756ba8b7d8a59152a02629f72ae388 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 27 Apr 2021 11:13:20 +0200 Subject: [PATCH 273/543] [openwrt/qmi] create interface, correct firewall parsing --- openwrt/openwrt-add-qmi-lte.sh | 40 +++++++++++----------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/openwrt/openwrt-add-qmi-lte.sh b/openwrt/openwrt-add-qmi-lte.sh index b8089eb..715e229 100755 --- a/openwrt/openwrt-add-qmi-lte.sh +++ b/openwrt/openwrt-add-qmi-lte.sh @@ -12,45 +12,31 @@ my_ip=$1; shift cat < Date: Tue, 27 Apr 2021 12:00:16 +0200 Subject: [PATCH 274/543] ++script for creating ipv4 vpn --- openwrt/openwrt-add-ipv4-vpn.sh | 70 +++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100755 openwrt/openwrt-add-ipv4-vpn.sh diff --git a/openwrt/openwrt-add-ipv4-vpn.sh b/openwrt/openwrt-add-ipv4-vpn.sh new file mode 100755 index 0000000..9a824e3 --- /dev/null +++ b/openwrt/openwrt-add-ipv4-vpn.sh @@ -0,0 +1,70 @@ +#!/bin/sh +# 2021-04-27 + + +if [ $# -lt 2 ]; then + echo "$0 host ipv4-address interface [private-key]" + echo " host: where to find the OpenWRT device" + echo " ipv4-address: which ipv4 address to use" + echo " private-key: Use this wireguard key instead of generating one" + exit 1 +fi + +my_ip=$1; shift +my_wireguard_ip=$1; shift + +interface=ungleichipv4 +vpn_endpoint_host=vpn-18515529.ungleich.ch + +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-18515529.ungleich.ch +vpn_endpoint_pubkey=6BRnQ+dmeFzVCH9RbM1pbJ7u3y3qrl+zUzzYCmC88kE= + + +cat < Date: Sun, 16 May 2021 18:15:45 +0200 Subject: [PATCH 275/543] [alpine] build with eth0 only --- alpine-rebuild-initramfs.sh | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index efb6cb0..848f106 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -61,7 +61,8 @@ run_root rc-update add sshd run_root rc-update add rdnssd run_root rc-update add networking run_root rc-update add hostname -run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow +#run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow +run_root sed -i 's/root:!::0:::::/root:::0:::::/' /etc/shadow sudo tee "$rootfs_tmpdir/etc/network/interfaces" < Date: Sun, 16 May 2021 18:53:38 +0200 Subject: [PATCH 276/543] Finally fix bonding --- alpine-rebuild-initramfs.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 848f106..33f45f1 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -61,8 +61,7 @@ run_root rc-update add sshd run_root rc-update add rdnssd run_root rc-update add networking run_root rc-update add hostname -#run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow -run_root sed -i 's/root:!::0:::::/root:::0:::::/' /etc/shadow +run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow sudo tee "$rootfs_tmpdir/etc/network/interfaces" < Date: Sun, 16 May 2021 19:26:57 +0200 Subject: [PATCH 277/543] [alpine/netboot] fix rdnssd until -r5 of rdnssd is installed --- alpine-rebuild-initramfs.sh | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 33f45f1..2d44df7 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -12,7 +12,7 @@ set -x SSH_KEYS=$1; shift MAJOR_VERSION=3.13 -MINOR_VERSION=2 +MINOR_VERSION=5 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz RESOLVCONF=/etc/resolv.conf @@ -63,6 +63,21 @@ run_root rc-update add networking run_root rc-update add hostname run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow +# Fix not yet updated initscript for rdnssd +sudo tee "$rootfs_tmpdir/etc/init.d/rdnssd" < Date: Sat, 29 May 2021 19:53:40 +0200 Subject: [PATCH 278/543] Add script to install alpine to a usb disk --- alpine-install-on-disk.sh | 102 ++++++++++++++++++++++++++++++++++++ alpine-rebuild-initramfs.sh | 2 +- 2 files changed, 103 insertions(+), 1 deletion(-) create mode 100755 alpine-install-on-disk.sh diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh new file mode 100755 index 0000000..2e1c5e2 --- /dev/null +++ b/alpine-install-on-disk.sh @@ -0,0 +1,102 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 disk ssh-keyfile" + echo " disk: which disk to install to" + echo " ssh-keyfile: ssh keys to add into the image" + exit 1 +fi + +set -e +set -x + +DISK=$1; shift +SSH_KEYS=$1; shift + +MAJOR_VERSION=3.13 +MINOR_VERSION=5 +IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz + +RESOLVCONF=/etc/resolv.conf + +working_directory=$(pwd -P) +rootfs_tmpdir=$(mktemp -d) + +rootfs_url="http://dl-cdn.alpinelinux.org/alpine/v$MAJOR_VERSION/releases/x86_64/$IMAGE" + +run_root () { + sudo chroot $rootfs_tmpdir /usr/bin/env \ + PATH=/bin:/sbin \ + /bin/sh -c "$*" +} + +wget -c "$rootfs_url" -O "$IMAGE" + +# Partition disk with 1 Linux partition +sudo sfdisk "$DISK" < Date: Sat, 29 May 2021 20:05:42 +0200 Subject: [PATCH 279/543] Alpine install: choose --target for grub and cleanup the gap To avoid this error: nb2:~nico/vcs/ungleich-tools# sudo chroot /tmp/tmp.U3RiZH2Nu9 /bin/sh -c 'grub-install --target=i386-pc /dev/sda' Installing for i386-pc platform. grub-install: warning: Attempting to install GRUB to a disk with multiple partition labels. This is not supported yet.. grub-install: warning: Embedding is not possible. GRUB can only be installed in this setup by using blocklists. However, blocklists are UNRELIABLE and their use is discouraged.. grub-install: error: will not proceed with blocklists. See also: https://askubuntu.com/questions/666527/install-grub-claims-that-i-have-multiple-partiton-labels-and-that-embedding-is --- alpine-install-on-disk.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 2e1c5e2..ddaf511 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -32,6 +32,10 @@ run_root () { wget -c "$rootfs_url" -O "$IMAGE" +# Clean the first 2M - getting rid of old things +# in the gap and also the paritition table +dd if=/dev/zero of=${DISK} bs=1M count=2 + # Partition disk with 1 Linux partition sudo sfdisk "$DISK" < Date: Sat, 29 May 2021 20:10:46 +0200 Subject: [PATCH 280/543] Remove tmpdir --- alpine-install-on-disk.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index ddaf511..4869b5e 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -25,8 +25,8 @@ rootfs_tmpdir=$(mktemp -d) rootfs_url="http://dl-cdn.alpinelinux.org/alpine/v$MAJOR_VERSION/releases/x86_64/$IMAGE" run_root () { - sudo chroot $rootfs_tmpdir /usr/bin/env \ - PATH=/bin:/sbin \ + sudo chroot $rootfs_tmpdir + PATH=/sbin:/bin:/usr/sbin:/usr/bin \ /bin/sh -c "$*" } @@ -100,6 +100,7 @@ done sudo umount $rootfs_tmpdir sync +rmdir ${rootfs_tmpdir} echo "${DISK} has been setup with Alpine Linux" From 838a7171ba0f8ef7845af9c37650acc5e6ffc5d7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 29 May 2021 20:22:25 +0200 Subject: [PATCH 281/543] Umount, not mount a 2nd time ... --- alpine-install-on-disk.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 4869b5e..7068401 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -25,7 +25,7 @@ rootfs_tmpdir=$(mktemp -d) rootfs_url="http://dl-cdn.alpinelinux.org/alpine/v$MAJOR_VERSION/releases/x86_64/$IMAGE" run_root () { - sudo chroot $rootfs_tmpdir + sudo chroot $rootfs_tmpdir /usr/bin/env \ PATH=/sbin:/bin:/usr/sbin:/usr/bin \ /bin/sh -c "$*" } @@ -95,7 +95,7 @@ run_root grub-install --target=i386-pc ${DISK} # Cleanup run_root rm -f /etc/resolv.conf for dir in dev proc sys; do - sudo mount --bind /${dir} ${rootfs_tmpdir}/${dir} + sudo umount ${rootfs_tmpdir}/${dir} done sudo umount $rootfs_tmpdir From 7452236f2ba93a93eadc3647c9ea77aa1b58961b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 29 May 2021 21:07:58 +0200 Subject: [PATCH 282/543] modify the fstab --- alpine-install-on-disk.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 7068401..1747430 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -60,6 +60,15 @@ run_root chmod 0700 /root/.ssh # Import local resolv.conf. sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf +# Generate fstab which is later included in the initramfs + +# Add filesystem to fstab, because busybox mount does not work +# without -t ext4 for mounting and returns "No such file or directory" +# nb2:~# blkid| grep ^${DISK}1 | awk '{ print $2 }' +# UUID="fecf4182-f6dd-4d2c-9af7-8f36444ee25c" +eval $(blkid | grep ^${DISK}1 | awk '{ print $2 }') +echo "UUID=$UUID / ext4 0 1" >> ${rootfs_tmpdir}/etc/fstab + run_root apk update run_root apk add linux-lts openrc udev openssh From 913598dc92c56580b903c524549e58a46c4a67b1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 29 May 2021 21:23:05 +0200 Subject: [PATCH 283/543] add defaults entry to fstab --- alpine-install-on-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 1747430..320499e 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -67,7 +67,7 @@ sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf # nb2:~# blkid| grep ^${DISK}1 | awk '{ print $2 }' # UUID="fecf4182-f6dd-4d2c-9af7-8f36444ee25c" eval $(blkid | grep ^${DISK}1 | awk '{ print $2 }') -echo "UUID=$UUID / ext4 0 1" >> ${rootfs_tmpdir}/etc/fstab +echo "UUID=$UUID / ext4 defaults 0 1" >> ${rootfs_tmpdir}/etc/fstab run_root apk update run_root apk add linux-lts openrc udev openssh From 1a7c370fe36dda70531470d5fb44cf29d97e15bd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 29 May 2021 22:24:10 +0200 Subject: [PATCH 284/543] add grub boot params --- alpine-install-on-disk.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 320499e..5fc48ff 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -50,6 +50,12 @@ sudo mount ${DISK}1 $rootfs_tmpdir # keep right permissions, use sudo sudo tar xf $IMAGE -C $rootfs_tmpdir +# These are required by grub-install +# And also for generating grub config that contains rootfstype +for dir in dev proc sys; do + sudo mount --bind /${dir} ${rootfs_tmpdir}/${dir} +done + # Add SSH keys run_root mkdir -p root/.ssh sudo cp $SSH_KEYS $rootfs_tmpdir/root/.ssh/authorized_keys @@ -94,11 +100,10 @@ alpine-unconfigured EOF # Setup bootloader -for dir in dev proc sys; do - sudo mount --bind /${dir} ${rootfs_tmpdir}/${dir} -done run_root apk add grub-bios +echo 'GRUB_CMDLINE_LINUX_DEFAULT="quiet rootfstype=ext4"' >> ${rootfs_tmpdir}/etc/default/grub +run_root grub-mkconfig -o /boot/grub/grub.cfg run_root grub-install --target=i386-pc ${DISK} # Cleanup From 9ff028147fdac1741f3dd15edaede76fe9ce907c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 29 May 2021 22:44:43 +0200 Subject: [PATCH 285/543] ++e2fsprogs --- alpine-install-on-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 5fc48ff..a0b7cd3 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -76,7 +76,7 @@ eval $(blkid | grep ^${DISK}1 | awk '{ print $2 }') echo "UUID=$UUID / ext4 defaults 0 1" >> ${rootfs_tmpdir}/etc/fstab run_root apk update -run_root apk add linux-lts openrc udev openssh +run_root apk add linux-lts openrc udev openssh e2fsprogs run_root rc-update add udev run_root rc-update add udev-trigger From 2dae22a0d716da7ac4437cb8878f8286f2545296 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 7 Jun 2021 19:15:35 +0200 Subject: [PATCH 286/543] [alpine] add sysctl on bootup --- alpine-install-on-disk.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index a0b7cd3..3a48c40 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -83,6 +83,7 @@ run_root rc-update add udev-trigger run_root rc-update add sshd run_root rc-update add networking run_root rc-update add hostname +run_root rc-update add sysctl run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow sudo tee "$rootfs_tmpdir/etc/network/interfaces" < Date: Fri, 25 Jun 2021 09:54:11 +0200 Subject: [PATCH 287/543] [alpine]: updated to v3.14 --- opennebula-images/alpine-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh index e6120d3..87ff9b3 100755 --- a/opennebula-images/alpine-build-opennebula-image.sh +++ b/opennebula-images/alpine-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=v3.13 +RELEASE=v3.14 ARCH=x86_64 IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G From ace073e89a2cc773ea1e34caf46c0c6a42aedfb7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Jul 2021 19:33:04 +0200 Subject: [PATCH 288/543] Add script to install any Linux to any computer --- install-any-linux.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 install-any-linux.sh diff --git a/install-any-linux.sh b/install-any-linux.sh new file mode 100644 index 0000000..edfb948 --- /dev/null +++ b/install-any-linux.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# 2021-07-09 +# Objective: install any Linux automatically to a disk +# Made by ungleich +# Made for bare metal +# Requirements: +# The OS image needs to be in tar format and needs to contain grub + + +if [ $# -ne 2 ]; then + echo "$0 os-image.tar [usb|disk]" + echo "os-image.tar contains the OS" + echo "usb mode: create a bootable usb stick including this script to auto install to disk" + echo "disk mode: actually install os-image.tar to the first disk" + exit 1 +fi + +os=$1; shift +mode=$1; shift + +# isohybrid From 715dd03ab552c2f12c9afa4e28559b1adbb3c841 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Jul 2021 19:42:57 +0200 Subject: [PATCH 289/543] ++ideas for install any linux --- install-any-linux.sh | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/install-any-linux.sh b/install-any-linux.sh index edfb948..eac28b9 100644 --- a/install-any-linux.sh +++ b/install-any-linux.sh @@ -6,16 +6,51 @@ # Requirements: # The OS image needs to be in tar format and needs to contain grub +set -e -if [ $# -ne 2 ]; then - echo "$0 os-image.tar [usb|disk]" +if [ $# -lt 2 ]; then + echo "$0 os-image.tar [usb|disk] [target-usb-disk]" echo "os-image.tar contains the OS" echo "usb mode: create a bootable usb stick including this script to auto install to disk" echo "disk mode: actually install os-image.tar to the first disk" + echo "" + echo "In usb mode, specify the usb disk to install to" exit 1 fi -os=$1; shift +IMAGE=$1; shift mode=$1; shift # isohybrid + +setup_usb() { + rootfs_tmpdir=$(mktemp -d) + + sudo sfdisk "$DISK" < Date: Sun, 11 Jul 2021 14:05:50 +0200 Subject: [PATCH 290/543] alpine: automatically load modules --- alpine-install-on-disk.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 3a48c40..4d88eeb 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -84,6 +84,7 @@ run_root rc-update add sshd run_root rc-update add networking run_root rc-update add hostname run_root rc-update add sysctl +run_root rc-update add modules run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow sudo tee "$rootfs_tmpdir/etc/network/interfaces" < Date: Mon, 19 Jul 2021 21:52:51 +0200 Subject: [PATCH 291/543] viwib: upgrade to 19.07.7 --- openwrt/viwib-1-firmware-upgrade.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 245f6ec..d864323 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -8,6 +8,7 @@ if [ $# -ne 1 ]; then exit 1 fi +set -e set -x viwib_ip=$1; shift From 438dce61c67e1dea46049e614deea654f7776481 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Jul 2021 21:52:59 +0200 Subject: [PATCH 292/543] ++debug --- openwrt/viwib-4-cleanup.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/openwrt/viwib-4-cleanup.sh b/openwrt/viwib-4-cleanup.sh index 5ed6c95..8359c61 100755 --- a/openwrt/viwib-4-cleanup.sh +++ b/openwrt/viwib-4-cleanup.sh @@ -2,8 +2,6 @@ # Nico Schottelius # 2020-06-14 -set -e -set -x if [ $# -ne 2 ]; then echo "$0 viwib-ip-address viwib-id" @@ -12,6 +10,9 @@ if [ $# -ne 2 ]; then exit 1 fi +set -e +set -x + viwib_ip=$1; shift id=$1; shift From fc2aa93d5e49d648c7b58773aaa9082ed2cae021 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 19 Jul 2021 21:53:26 +0200 Subject: [PATCH 293/543] [openwrt] update mjpeg streamer --- .../openwrt-add-camera-with-mjpg-streamer.sh | 50 ++++--------------- 1 file changed, 11 insertions(+), 39 deletions(-) diff --git a/openwrt/openwrt-add-camera-with-mjpg-streamer.sh b/openwrt/openwrt-add-camera-with-mjpg-streamer.sh index 25d98a8..3235c00 100755 --- a/openwrt/openwrt-add-camera-with-mjpg-streamer.sh +++ b/openwrt/openwrt-add-camera-with-mjpg-streamer.sh @@ -27,47 +27,19 @@ uci set mjpg-streamer.core.enabled=1 uci set mjpg-streamer.core.resolution="1920x1080" uci delete mjpg-streamer.core.username uci delete mjpg-streamer.core.password + +if ! uci show firewall | grep "name='Allow-Camera'"; then +uci add firewall rule +uci set firewall.@rule[-1].name='Allow-HTTP' +uci set firewall.@rule[-1].src='wan' +uci set firewall.@rule[-1].proto='tcp' +uci set firewall.@rule[-1].dest_port='8080' +uci set firewall.@rule[-1].target='ACCEPT' +fi + uci commit + /etc/init.d/mjpg-streamer restart EOF - -exit 0 - - -# This is old code / when we used motion - -cat < /etc/motion.conf < Date: Mon, 19 Jul 2021 21:53:55 +0200 Subject: [PATCH 294/543] [openwrt] set -e does not work for updates --- openwrt/viwib-1-firmware-upgrade.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index d864323..245f6ec 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -8,7 +8,6 @@ if [ $# -ne 1 ]; then exit 1 fi -set -e set -x viwib_ip=$1; shift From 4c3147c1b2363150ab6fc74ac48d9aae3902d71e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Jul 2021 10:48:25 +0200 Subject: [PATCH 295/543] Add script to configure k8s vms --- k8s/configure-k8s-vms.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 k8s/configure-k8s-vms.sh diff --git a/k8s/configure-k8s-vms.sh b/k8s/configure-k8s-vms.sh new file mode 100755 index 0000000..e83c513 --- /dev/null +++ b/k8s/configure-k8s-vms.sh @@ -0,0 +1,21 @@ +#!/bin/sh +# Nico Schottelius +# 2021-07-25 + +if [ $# -lt 1 ]; then + echo "$0 cdist-workdir list-of-ipv6-addresses-of-vms" + exit 1 +fi + +workdir=$1; shift + +names="" + +for vm in "$@"; do + # get name and remove trailing dot + name=$(dig +short -x $vm | sed 's/\.$//') + names="$names $name" +done + +cd "${workdir}" +cdist config -vv -j6 -p30 ${names} From 95f162c81135d4aaaa8870483b420ef6a83297e5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Jul 2021 11:05:18 +0200 Subject: [PATCH 296/543] Add script to show disks and their rotational status --- check-disks-rotational.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 check-disks-rotational.sh diff --git a/check-disks-rotational.sh b/check-disks-rotational.sh new file mode 100755 index 0000000..8483145 --- /dev/null +++ b/check-disks-rotational.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +if [ $# -lt 1 ]; then + echo $0: server [server...] + exit 1 +fi + +while [ $# -ge 1 ]; do + server=$1; shift + + ssh root@$server " + cd /sys/block/ + for dev in sd*; do + size=\$(fdisk -l | grep ^Disk | grep \$dev | awk '/bytes/ { print \$3 \" \" \$4 }') + printf \"${server} \${dev} \${size} rotational: \" + cat \$dev/queue/rotational + done + " +done From efb7a5f9fab9fca96fc2342b07f98ce982903bed Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 28 Jul 2021 16:32:43 +0200 Subject: [PATCH 297/543] Update alpine version, fix partition bug --- alpine-install-on-disk.sh | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 3a48c40..b4d520a 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -13,8 +13,8 @@ set -x DISK=$1; shift SSH_KEYS=$1; shift -MAJOR_VERSION=3.13 -MINOR_VERSION=5 +MAJOR_VERSION=3.14 +MINOR_VERSION=0 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz RESOLVCONF=/etc/resolv.conf @@ -24,6 +24,19 @@ rootfs_tmpdir=$(mktemp -d) rootfs_url="http://dl-cdn.alpinelinux.org/alpine/v$MAJOR_VERSION/releases/x86_64/$IMAGE" +case $DISK in + /dev/sd*) + partition=${DISK}1 + ;; + /dev/mmcblk*|/dev/nvme*) + partition=${DISK}p1 + ;; + *) + echo "Unsupported disk - edit this script" >&2 + exit 1 + ;; +esac + run_root () { sudo chroot $rootfs_tmpdir /usr/bin/env \ PATH=/sbin:/bin:/usr/sbin:/usr/bin \ @@ -43,9 +56,9 @@ label: dos EOF # For creation, if an existing filesystem is on the partitions -sudo mkfs.ext4 -F ${DISK}1 +sudo mkfs.ext4 -F ${partition} -sudo mount ${DISK}1 $rootfs_tmpdir +sudo mount ${partition} $rootfs_tmpdir # keep right permissions, use sudo sudo tar xf $IMAGE -C $rootfs_tmpdir From 826906bb4237c8ead9afe60cc54eb2c454cdc63e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 29 Jul 2021 16:15:42 +0200 Subject: [PATCH 298/543] [opennebula] initial openSUSE image definition --- .../opensuse-build-opennebula-image.sh | 172 ++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100755 opennebula-images/opensuse-build-opennebula-image.sh diff --git a/opennebula-images/opensuse-build-opennebula-image.sh b/opennebula-images/opensuse-build-opennebula-image.sh new file mode 100755 index 0000000..aeaf474 --- /dev/null +++ b/opennebula-images/opensuse-build-opennebula-image.sh @@ -0,0 +1,172 @@ +#!/bin/sh + +# This script generates openSUSE images for OpenNebula. +# +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=leap +RELEASE_VERSION=15.2 +IMAGE_PATH=opensuse-${RELEASE}${RELEASE_VERSION}-$(date -I).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd1 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.12.0.2/one-context-5.12.0.2-1.suse.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ "$(lsb_release --short --id)" != "openSUSE" ]; then + echo "WARNING: this script has been designed to run on an openSUSE system." >&2 + echo "WARNING: Not running openSUSE. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +repo_addr=http://download.opensuse.org +case "$RELEASE" in + leap) + distribution_slice=$RELEASE/$RELEASE_VERSION + oss_repo_url="${repo_addr}/distribution/${distribution_slice}/repo/oss" + ;; + tumbleweed) + distribution_slice=$RELEASE + oss_repo_url="${repo_addr}/${distribution_slice}/repo/oss" + ;; + *) + echo "Unkown openSUSE release: $RELEASE." >&2 + exit 1 + ;; +esac +oss_update_repo_url="${repo_addr}/update/${distribution_slice}/oss" + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root zypper -n --no-gpg-checks install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root zypper -n install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install haveged due to lack of entropy in ONE environment. +run_root zypper -n install haveged +run_root systemctl enable haveged.service + +# Install kernel and bootloader. +run_root zypper -n install kernel-default grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <>/mnt/etc/fstab < Date: Thu, 29 Jul 2021 17:01:58 +0200 Subject: [PATCH 299/543] [opennebula] update openSUSE image to Leap 15.3 --- opennebula-images/opensuse-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/opensuse-build-opennebula-image.sh b/opennebula-images/opensuse-build-opennebula-image.sh index aeaf474..baccbe1 100755 --- a/opennebula-images/opensuse-build-opennebula-image.sh +++ b/opennebula-images/opensuse-build-opennebula-image.sh @@ -10,7 +10,7 @@ set -x # XXX: Handle command-line arguments? RELEASE=leap -RELEASE_VERSION=15.2 +RELEASE_VERSION=15.3 IMAGE_PATH=opensuse-${RELEASE}${RELEASE_VERSION}-$(date -I).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd1 From 2e60bbe9d04a630018be4e10b70179398bd30a77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 29 Jul 2021 17:55:27 +0200 Subject: [PATCH 300/543] [opennebula] use base instead of minimal_base pattern for openSUSE image --- opennebula-images/opensuse-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/opensuse-build-opennebula-image.sh b/opennebula-images/opensuse-build-opennebula-image.sh index baccbe1..d90774a 100755 --- a/opennebula-images/opensuse-build-opennebula-image.sh +++ b/opennebula-images/opensuse-build-opennebula-image.sh @@ -91,7 +91,7 @@ mount "${NBD_DEVICE}p1" /mnt/boot zypper --root /mnt addrepo $oss_repo_url opensuse-$RELEASE-oss zypper --root /mnt addrepo $oss_update_repo_url opensuse-$RELEASE-oss-update zypper --root /mnt --gpg-auto-import-keys refresh -zypper --root /mnt --non-interactive install -t pattern minimal_base +zypper --root /mnt --non-interactive install -t pattern base mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts From 996503200879da110832f57dde7b4a5b8d211cec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 Aug 2021 17:23:26 +0200 Subject: [PATCH 301/543] [viirb] modify to go through in mostly one step --- openwrt/viirb-1-firmware-upgrade.sh | 6 ++--- openwrt/viirb-2-configure.sh | 37 ++++++++++------------------- openwrt/viirb-4-cleanup.sh | 3 +++ 3 files changed, 18 insertions(+), 28 deletions(-) diff --git a/openwrt/viirb-1-firmware-upgrade.sh b/openwrt/viirb-1-firmware-upgrade.sh index 786e6da..b65e331 100755 --- a/openwrt/viirb-1-firmware-upgrade.sh +++ b/openwrt/viirb-1-firmware-upgrade.sh @@ -3,8 +3,8 @@ # See https://ungleich.ch/u/products/viirb-ipv6-box/ if [ $# -ne 1 ]; then - echo "$0 interface [address]" - echo " address: connect to this address, ignore the interface" + echo "$0 address" + echo " address: connect to this address" exit 1 fi @@ -13,7 +13,7 @@ set -x viirb_ip=$1; shift # openwrt -version=19.07.6 +version=19.07.7 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin # don't care about other/old known_host entries diff --git a/openwrt/viirb-2-configure.sh b/openwrt/viirb-2-configure.sh index 093d547..3524045 100755 --- a/openwrt/viirb-2-configure.sh +++ b/openwrt/viirb-2-configure.sh @@ -120,36 +120,24 @@ uci set dhcp.wifi.dynamicdhcp='0' # Ensure it is not disabled uci delete wireless.radio0.disabled -# This is temporary - keeping it until the config process is through -# Probably not needet - we can connect to the final IPv6 address! -# This code commented out == The address vanishes due to above reconfiguration -# uci set network.lanv4temp=interface -# uci set network.lanv4temp.proto='static' -# uci set network.lanv4temp.ifname='br-lan' -# uci set network.lanv4temp.ipaddr='192.168.61.1' -# uci set network.lanv4temp.netmask='255.255.255.0' - uci commit -# Need to reboot to restore /etc/resolv.conf -reboot +# This should not be needed, as we connect directly to the dynamic IPv4 +# /etc/init.d/network restart -#/etc/init.d/network restart -EOF +# # ensure viirb is back +# # ensure viirb network is up and running - bridge takes a bit of time +# # ensure viirb gives out ipv6 address +# # Found: 30s is not enough for full reconfiguration +# echo "Public VPN key: ${id} ${public_key}" +# sleep 60 -# ensure viirb is back -# ensure viirb network is up and running - bridge takes a bit of time -# ensure viirb gives out ipv6 address -# Found: 30s is not enough for full reconfiguration -echo "Public VPN key: ${id} ${public_key}" -sleep 60 +# # change to ipv6 +# viirb_ip=${my_lan_ip} -# change to ipv6 -viirb_ip=${my_lan_ip} +# ping -c5 ${viirb_ip} -ping -c5 ${viirb_ip} - -cat < ${viirb_hostname}.public_key diff --git a/openwrt/viirb-4-cleanup.sh b/openwrt/viirb-4-cleanup.sh index a9a116a..77916b8 100755 --- a/openwrt/viirb-4-cleanup.sh +++ b/openwrt/viirb-4-cleanup.sh @@ -36,6 +36,9 @@ rm -f /etc/dropbear/authorized_keys # Setup root password printf "${root_password}\n${root_password}\n" | passwd + +# Ensure everything's good +reboot EOF echo "Submit to user the root password = ${root_password}" From fa45565eb08219703f36982cc8bedf4758d61e89 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 13 Aug 2021 10:43:13 +0200 Subject: [PATCH 302/543] Update megacli-status to python3 --- megaclisas-status | 78 +++++++++++++++++++++++------------------------ 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/megaclisas-status b/megaclisas-status index 3495502..1518baf 100755 --- a/megaclisas-status +++ b/megaclisas-status @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 # $Id: megaclisas-status,v 1.68 2016/10/21 14:38:56 root Exp root $ # # Written by Adam Cecile @@ -40,7 +40,7 @@ Outputs = {} # Startup def print_usage(): - print 'Usage: megaraid-status [--nagios|--debug|--notemp]' + print('Usage: megaraid-status [--nagios|--debug|--notemp]') # We need root access to query if __name__ == '__main__': @@ -49,7 +49,7 @@ if __name__ == '__main__': except AttributeError: root_or_admin = ctypes.windll.shell32.IsUserAnAdmin() !=0 if not root_or_admin: - print '# This script requires Administrator privileges' + print('# This script requires Administrator privileges') sys.exit(5) # Check command line arguments to enable nagios or not @@ -102,19 +102,19 @@ for megabin in "MegaCli64","MegaCli","megacli", "MegaCli.exe": if (megaclipath != None): dbgprint ('Will use MegaCLI from here: '+str(megaclipath)) break - + # Check binary exists (and +x), if not print an error message if (megaclipath != None): if os.path.exists(megaclipath) and os.access(megaclipath, os.X_OK): pass else: if nagiosmode: - print 'UNKNOWN - Cannot find '+megaclipath + print('UNKNOWN - Cannot find '+megaclipath) else: - print 'Cannot find ' + megaclipath + 'in your PATH. Please install it.' + print('Cannot find ' + megaclipath + 'in your PATH. Please install it.') sys.exit(3) else: - print 'Cannot find "MegaCli64","MegaCli" or "megacli" or "MegaCli.exe" in your PATH. Please install it.' + print('Cannot find "MegaCli64","MegaCli" or "megacli" or "MegaCli.exe" in your PATH. Please install it.') sys.exit(3) @@ -130,7 +130,7 @@ def returnWdthFromArrayCol(glarray,idx): # Get command output def getOutput(cmd): lines = [] - if ( Outputs.has_key(cmd) ): + if cmd in Outputs: dbgprint ("Got Cached value: "+str(cmd)) lines = Outputs[cmd] else: @@ -141,7 +141,7 @@ def getOutput(cmd): lines.append(line.strip()) Outputs[cmd] = lines return lines - + def returnControllerNumber(output): for line in output: if re.match(r'^Controller Count.*$',line.strip()): @@ -274,7 +274,7 @@ def returnHBAInfo(table,output,controllerid): cmd = '%s -AdpBbuCmd -GetBbuStatus -a%d -NoLog' % (megaclipath, controllerid) output = getOutput(cmd) controllerbbu = returnBBUStatus(output) - + if controllermodel != 'Unknown': table.append([ 'c'+str(controllerid), controllermodel, controllerram, str(controllertemp), str(controllerbbu), str('FW: '+controllerrev) ]) @@ -469,9 +469,9 @@ def returnDiskInfo(output,controllerid): dbgprint('Disk Info: '+str(arrayid)+' '+str(diskid)+' '+str(oldenclid)) if subfstate == 'Rebuild': cmd = '%s pdrbld -showprog -physdrv\[%s:%s\] -a%d -NoLog' % (megaclipath, enclid, slotid, controllerid) - output = getOutput(cmd) - percent = returnRebuildProgress(output) - fstate = str('Rebuilding (%d%%)' % (percent)) + output = getOutput(cmd) + percent = returnRebuildProgress(output) + fstate = str('Rebuilding (%d%%)' % (percent)) if (( NestedLDTable[controllerid][int(arrayindex)] == True) and (spanid != False)): sarrayid = str(arrayid)+"s"+spanid @@ -566,7 +566,7 @@ bad = False if printcontroller: if controllernumber: if not nagiosmode: - print '-- Controller information --' + print('-- Controller information --') i = 0 controllerid = 0 @@ -585,25 +585,25 @@ if printcontroller: # Header if ( i == 0 ): if not nagiosmode: - print hbafmt % ("-- ID","H/W Model","RAM","Temp","BBU", "Firmware") + print(hbafmt % ("-- ID","H/W Model","RAM","Temp","BBU", "Firmware")) if not nagiosmode: - print hbafmt % ( + print(hbafmt % ( hba[0], hba[1], hba[2], hba[3], hba[4], - hba[5]) + hba[5])) i += 1 if not nagiosmode: - print '' + print('') else: - print "No MegaRAID or PERC adapter detected on your system!" + print("No MegaRAID or PERC adapter detected on your system!") exit(1) if printarray: if not nagiosmode: - print '-- Array information --' + print('-- Array information --') controllerid = 0 pcipath = '' @@ -682,9 +682,9 @@ if printarray: # Header if ( i == 0 ): if not nagiosmode: - print ldfmt % ("-- ID", "Type", "Size", "Strpsz", "Flags", "DskCache", "Status", "OS Path", "CacheCade", "InProgress" ) + print(ldfmt % ("-- ID", "Type", "Size", "Strpsz", "Flags", "DskCache", "Status", "OS Path", "CacheCade", "InProgress" )) if not nagiosmode: - print ldfmt % ( + print(ldfmt % ( arrayinfo[0], arrayinfo[1], arrayinfo[2], @@ -694,7 +694,7 @@ if printarray: arrayinfo[6], arrayinfo[7], arrayinfo[8], - arrayinfo[9]) + arrayinfo[9])) dbgprint("Array state : "+arrayinfo[6]) if arrayinfo[6] not in [ 'Optimal', 'N/A' ]: bad = True @@ -705,7 +705,7 @@ if printarray: i += 1 controllerid += 1 if not nagiosmode: - print '' + print('') controllerid = 0 while controllerid < controllernumber: @@ -716,7 +716,7 @@ while controllerid < controllernumber: if totaldrivenumber: if not nagiosmode: - print '-- Disk information --' + print('-- Disk information --') i = 0 dlen = 0 ; mlen = 0 ; flen = 0 @@ -767,11 +767,11 @@ if totaldrivenumber: # Header if ( i == 0 ): if not nagiosmode: - print drvfmt % ( - "-- ID", "Type", "Drive Model", "Size", "Status", "Speed", "Temp", "Slot ID", "LSI Device ID") + print(drvfmt % ( + "-- ID", "Type", "Drive Model", "Size", "Status", "Speed", "Temp", "Slot ID", "LSI Device ID")) # Drive information if not nagiosmode: - print drvfmt % ( + print(drvfmt % ( str('c'+str(controllerid)+'u'+array[0]+'p'+array[1]), # c0p0 array[2], # HDD/SDD array[3], # Model Information (Variable len) @@ -780,11 +780,11 @@ if totaldrivenumber: array[6], # Speed array[7], # Temp str('['+array[8]+':'+array[9]+']'), # Slot ID - array[10]) # LSI ID + array[10])) # LSI ID i = i + 1 controllerid += 1 if not nagiosmode: - print '' + print('') controllerid = 0 totalconfdrivenumber = 0 @@ -811,7 +811,7 @@ dbgprint('Total Unconfigured Drives : ' + str(totalunconfdrivenumber)) if totalunconfdrivenumber: if not nagiosmode: - print '-- Unconfigured Disk information --' + print('-- Unconfigured Disk information --') controllerid = 0 while controllerid < controllernumber: @@ -844,11 +844,11 @@ if totalunconfdrivenumber: # Header if ( i == 0 ): if not nagiosmode: - print drvfmt % ( - "-- ID", "Type", "Drive Model", "Size", "Status", "Speed", "Temp", "Slot ID", "LSI Device ID") + print(drvfmt % ( + "-- ID", "Type", "Drive Model", "Size", "Status", "Speed", "Temp", "Slot ID", "LSI Device ID")) # Drive information if not nagiosmode: - print drvfmt % ( + print(drvfmt % ( str('c'+str(controllerid)+'uXpY'), # cXpY array[0], # HDD/SDD array[1], # Model Information (Variable len) @@ -857,19 +857,19 @@ if totalunconfdrivenumber: array[4], # Speed array[5], # Temp str('['+array[6]+':'+array[7]+']'), # Slot ID - array[8]) # LSI ID + array[8])) # LSI ID i = i + 1 controllerid += 1 if not nagiosmode: - print '' + print('') if nagiosmode: if bad: - print 'RAID ERROR - Arrays: OK:'+str(nagiosgoodarray)+' Bad:'+str(nagiosbadarray)+' - Disks: OK:'+str(nagiosgooddisk)+' Bad:'+str(nagiosbaddisk) + print('RAID ERROR - Arrays: OK:'+str(nagiosgoodarray)+' Bad:'+str(nagiosbadarray)+' - Disks: OK:'+str(nagiosgooddisk)+' Bad:'+str(nagiosbaddisk)) sys.exit(2) else: - print 'RAID OK - Arrays: OK:'+str(nagiosgoodarray)+' Bad:'+str(nagiosbadarray)+' - Disks: OK:'+str(nagiosgooddisk)+' Bad:'+str(nagiosbaddisk) + print('RAID OK - Arrays: OK:'+str(nagiosgoodarray)+' Bad:'+str(nagiosbadarray)+' - Disks: OK:'+str(nagiosgooddisk)+' Bad:'+str(nagiosbaddisk)) else: if bad: - print '\nThere is at least one disk/array in a NOT OPTIMAL state.' + print('\nThere is at least one disk/array in a NOT OPTIMAL state.') sys.exit(1) From 8e0a6c2e560a3a63b5b3fd6afe7451f2d31e8779 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 13 Aug 2021 12:21:30 +0200 Subject: [PATCH 303/543] [viirb] automatically add into password store for encrypted communication --- openwrt/viirb-4-cleanup.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openwrt/viirb-4-cleanup.sh b/openwrt/viirb-4-cleanup.sh index 77916b8..7d342c0 100755 --- a/openwrt/viirb-4-cleanup.sh +++ b/openwrt/viirb-4-cleanup.sh @@ -41,4 +41,7 @@ printf "${root_password}\n${root_password}\n" | passwd reboot EOF +printf "${root_password}\n${root_password}\n" | pass insert viirb/viirb${id} +pass git push + echo "Submit to user the root password = ${root_password}" From 0e399af0d8e3d23b16c617882ffaae51e905dcc9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 13 Aug 2021 12:22:15 +0200 Subject: [PATCH 304/543] [viirb] remove local password storing --- openwrt/viirb-4-cleanup.sh | 4 ---- 1 file changed, 4 deletions(-) diff --git a/openwrt/viirb-4-cleanup.sh b/openwrt/viirb-4-cleanup.sh index 7d342c0..44e2f38 100755 --- a/openwrt/viirb-4-cleanup.sh +++ b/openwrt/viirb-4-cleanup.sh @@ -19,10 +19,6 @@ viirb_hostname=viirb${id} root_password=$(pwgen -1 32) -# Save for sending to user -# FIXME: future make this more easy / better to transfer -echo $root_password > ${viirb_hostname}.rootpw - cat < Date: Fri, 13 Aug 2021 12:26:13 +0200 Subject: [PATCH 305/543] cleanup --- openwrt/vigir15.public_key | 1 - openwrt/vigir5.public_key | 1 - openwrt/viwib1.public_key | 1 - openwrt/viwib10.public_key | 1 - openwrt/viwib2.public_key | 1 - openwrt/viwib21.public_key | 1 - openwrt/viwib4.public_key | 1 - openwrt/viwib4.rootpw | 1 - 8 files changed, 8 deletions(-) delete mode 100644 openwrt/vigir15.public_key delete mode 100644 openwrt/vigir5.public_key delete mode 100644 openwrt/viwib1.public_key delete mode 100644 openwrt/viwib10.public_key delete mode 100644 openwrt/viwib2.public_key delete mode 100644 openwrt/viwib21.public_key delete mode 100644 openwrt/viwib4.public_key delete mode 100644 openwrt/viwib4.rootpw diff --git a/openwrt/vigir15.public_key b/openwrt/vigir15.public_key deleted file mode 100644 index 3e138c0..0000000 --- a/openwrt/vigir15.public_key +++ /dev/null @@ -1 +0,0 @@ -/QvWXp3v5n6cVdQ78sBpNczY8jI4nzE6uUlrav65N00= diff --git a/openwrt/vigir5.public_key b/openwrt/vigir5.public_key deleted file mode 100644 index ad03e4e..0000000 --- a/openwrt/vigir5.public_key +++ /dev/null @@ -1 +0,0 @@ -020sjVDopRMMLSIgbyviDhSZ2FACa7CQ531DJNDVBRY= diff --git a/openwrt/viwib1.public_key b/openwrt/viwib1.public_key deleted file mode 100644 index 58596f5..0000000 --- a/openwrt/viwib1.public_key +++ /dev/null @@ -1 +0,0 @@ -bKobh51z1mg2p3b2zPyj+gu8vTq5RtBpxvKo5+XSCic= diff --git a/openwrt/viwib10.public_key b/openwrt/viwib10.public_key deleted file mode 100644 index 19c544b..0000000 --- a/openwrt/viwib10.public_key +++ /dev/null @@ -1 +0,0 @@ -1GdDCQM3wxRzSLSSXy7OiXavD8a+XAK4UbYIa8och2g= diff --git a/openwrt/viwib2.public_key b/openwrt/viwib2.public_key deleted file mode 100644 index b24c4b5..0000000 --- a/openwrt/viwib2.public_key +++ /dev/null @@ -1 +0,0 @@ -ifKKyAHG0T7mT7D0QmMk+t8btvaTpQIpgYpD9kfXhko= diff --git a/openwrt/viwib21.public_key b/openwrt/viwib21.public_key deleted file mode 100644 index 7539b33..0000000 --- a/openwrt/viwib21.public_key +++ /dev/null @@ -1 +0,0 @@ -iKTwnViboBnM1/7eBUPhNWpqYPWKbAs07d4/Z7Qhggg= diff --git a/openwrt/viwib4.public_key b/openwrt/viwib4.public_key deleted file mode 100644 index 27e3640..0000000 --- a/openwrt/viwib4.public_key +++ /dev/null @@ -1 +0,0 @@ -r/MGL7pNWaF8qL/g8v0e/1wlQqgCbdeF15pMwEFvMW0= diff --git a/openwrt/viwib4.rootpw b/openwrt/viwib4.rootpw deleted file mode 100644 index 9a8673d..0000000 --- a/openwrt/viwib4.rootpw +++ /dev/null @@ -1 +0,0 @@ -ta9oc1eecahP1iutho3eixeel8eich2u From f691230af5e412ed99a6a709cdf369a63d18ad3e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 13 Aug 2021 12:26:50 +0200 Subject: [PATCH 306/543] +ignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 1de0e42..bb932ed 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ opennebula-vm-etcd/config-and-secrets.conf ipxe/ openwrt-*-*.bin +alpine-minirootfs-*.tar.gz +opennebula-images/*.qcow2 From 13283db2c0c22cb95e35f336dc0ca3847d7d08d1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 18 Aug 2021 11:42:56 +0200 Subject: [PATCH 307/543] viwib: update to 19.07.8 and automatic password store usage --- openwrt/viwib-1-firmware-upgrade.sh | 2 +- openwrt/viwib-4-cleanup.sh | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 245f6ec..3bbcf45 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=19.07.7 +version=19.07.8 filename=openwrt-${version}-ramips-mt76x8-gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries diff --git a/openwrt/viwib-4-cleanup.sh b/openwrt/viwib-4-cleanup.sh index 8359c61..fa1dcbb 100755 --- a/openwrt/viwib-4-cleanup.sh +++ b/openwrt/viwib-4-cleanup.sh @@ -20,10 +20,6 @@ viwib_hostname=viwib${id} root_password=$(pwgen -1 32) -# Save for sending to user -# FIXME: future make this more easy / better to transfer -echo $root_password > ${viwib_hostname}.rootpw - cat < Date: Wed, 18 Aug 2021 12:19:08 +0200 Subject: [PATCH 308/543] [viwib] do not write down public key anymore --- openwrt/viwib-2-configure.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 71db5ba..72c6e7d 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -188,7 +188,6 @@ reboot EOF echo "Wireguard public key and id: ${id} ${public_key}" -echo ${public_key} > ${viwib_hostname}.public_key # change to ipv6 viwib_ip=${my_lan_ip} From b17039d74e8fbc32e9f62b86277bce1d4d2b297d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 19 Aug 2021 12:55:23 +0200 Subject: [PATCH 309/543] Begin script for disk installation of debian/devuan --- debian-devuan-install-on-disk.sh | 144 +++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100755 debian-devuan-install-on-disk.sh diff --git a/debian-devuan-install-on-disk.sh b/debian-devuan-install-on-disk.sh new file mode 100755 index 0000000..15b8979 --- /dev/null +++ b/debian-devuan-install-on-disk.sh @@ -0,0 +1,144 @@ +#!/bin/sh +# Nico Schottelius, 2019-12-09 + +set -e +set -x + +if [ $# -ne 2 ]; then + echo $0 suite disk + exit 1 +fi + +suite=$1; shift +disk=$1; shift + +keyurl=https://key.wf + + +case $disk in + /dev/sd*) + partition=${disk}1 + ;; + /dev/mmcblk*|/dev/nvme*) + partition=${disk}p1 + ;; + *) + echo "Unsupported disk - edit this script" >&2 + exit 1 + ;; +esac + +chroot_dir=$(mktemp -d) +date=$(date +%F) + +basename=${suite}-${date} + +################################################################################ +# Disk preparation + +# Clean the first 2M - getting rid of old things +# in the gap and also the paritition table +dd if=/dev/zero of=${disk} bs=1M count=2 + +# Partition disk with 1 Linux partition +sudo sfdisk "$DISK" < ${chroot_dir}/etc/apt/sources.list + +chroot ${chroot_dir} apt update +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 ifenslave vlan + +echo "unconfigured-host" > ${chroot_dir}/etc/hostname + +cp ${chroot_dir}/boot/vmlinuz-* ${kernel} + +echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${chroot_dir}/etc/cron.d/ipv6addr + +mkdir -p ${chroot_dir}/root/.ssh + +for key in sami dominique jinguk nico; do + curl -s ${keyurl}/${key} >> ${chroot_dir}/root/.ssh/authorized_keys +done + +# Fix possible permission issue from above +chown -R root:root ${chroot_dir}/root/ + +################################################################################ +# networking + +# echo bonding + +cat > ${chroot_dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +# I would like to have a generic block like this below +# But as long as interface +auto bond0 +iface bond0 inet manual + bond-miimon 500 + bond-mode 4 + post-up /sbin/ip link set \$IFACE mtu 9000 + bond-slaves none + +auto eth0 +iface eth0 inet manual + bond-master bond0 + post-up /sbin/ip link set \$IFACE mtu 9000 + +auto eth1 +iface eth1 inet manual + bond-master bond0 + post-up /sbin/ip link set \$IFACE mtu 9000 + +# server network +auto bond0.11 +iface bond0.11 inet6 auto + post-up /sbin/ip link set \$IFACE mtu 9000 + vlan-raw-device bond0 + +EOF + +# # find the boot interfaces at boot: HP servers still have ifnames=1 +# cat > ${chroot_dir}/etc/rc.local <> /etc/network/interfaces << eof +# auto \$dev +# iface \$dev inet6 auto +# eof + +# ifup "\${dev}" + +# exit 0 +# EOF + +# chmod a+rx "${chroot_dir}/etc/rc.local" + +# ensure there is /init in the initramfs -> otherwise there is a kernel panic +# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init +# in our case, they are just the same +ln -fs /sbin/init ${chroot_dir}/init + +# Finally building the initramfs +( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) + +# Fix paranoid permissions +chmod a+rx ${abs_outdir} +chmod a+r ${kernel} ${initramfs} + + +exit 0 From 93832b5a90ba06e8c0e2b7d5a31f65d9e7bac0a3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 19 Aug 2021 15:32:17 +0200 Subject: [PATCH 310/543] devuan install: +loop support --- debian-devuan-install-on-disk.sh | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/debian-devuan-install-on-disk.sh b/debian-devuan-install-on-disk.sh index 15b8979..2eec1f1 100755 --- a/debian-devuan-install-on-disk.sh +++ b/debian-devuan-install-on-disk.sh @@ -19,7 +19,7 @@ case $disk in /dev/sd*) partition=${disk}1 ;; - /dev/mmcblk*|/dev/nvme*) + /dev/mmcblk*|/dev/nvme*|/dev/loop*) partition=${disk}p1 ;; *) @@ -31,8 +31,6 @@ esac chroot_dir=$(mktemp -d) date=$(date +%F) -basename=${suite}-${date} - ################################################################################ # Disk preparation From 4787bb200cf7e7e6a10032ea556974d144fc3fb7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 19 Aug 2021 15:44:59 +0200 Subject: [PATCH 311/543] Update install script for including keyfile --- debian-devuan-install-on-disk.sh | 74 +++++++++++--------------------- 1 file changed, 25 insertions(+), 49 deletions(-) diff --git a/debian-devuan-install-on-disk.sh b/debian-devuan-install-on-disk.sh index 2eec1f1..106b4f8 100755 --- a/debian-devuan-install-on-disk.sh +++ b/debian-devuan-install-on-disk.sh @@ -4,17 +4,18 @@ set -e set -x -if [ $# -ne 2 ]; then - echo $0 suite disk +if [ $# -ne 3 ]; then + echo $0 suite keyfile disk + echo suite: beowulf or similar + echo keyfile: file containing the ssh keys + echo disk: the block device exit 1 fi suite=$1; shift +keyfile=$1; shift disk=$1; shift -keyurl=https://key.wf - - case $disk in /dev/sd*) partition=${disk}1 @@ -39,15 +40,17 @@ date=$(date +%F) dd if=/dev/zero of=${disk} bs=1M count=2 # Partition disk with 1 Linux partition -sudo sfdisk "$DISK" < ${chroot_dir}/etc/apt/sources.list chroot ${chroot_dir} apt update -chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 ifenslave vlan +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 ifenslave vlan grub echo "unconfigured-host" > ${chroot_dir}/etc/hostname -cp ${chroot_dir}/boot/vmlinuz-* ${kernel} - echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${chroot_dir}/etc/cron.d/ipv6addr mkdir -p ${chroot_dir}/root/.ssh -for key in sami dominique jinguk nico; do - curl -s ${keyurl}/${key} >> ${chroot_dir}/root/.ssh/authorized_keys -done +cat ${keyfile} > ${chroot_dir}/root/.ssh/authorized_keys # Fix possible permission issue from above -chown -R root:root ${chroot_dir}/root/ +chmod -R og-rwx ${chroot_dir}/root/ ################################################################################ # networking @@ -101,42 +100,19 @@ iface eth1 inet manual bond-master bond0 post-up /sbin/ip link set \$IFACE mtu 9000 -# server network -auto bond0.11 -iface bond0.11 inet6 auto - post-up /sbin/ip link set \$IFACE mtu 9000 - vlan-raw-device bond0 - EOF -# # find the boot interfaces at boot: HP servers still have ifnames=1 -# cat > ${chroot_dir}/etc/rc.local <> /etc/network/interfaces << eof -# auto \$dev -# iface \$dev inet6 auto -# eof +chroot ${chroot_dir} grub-install ${disk} -# ifup "\${dev}" +for dir in dev sys proc; do + umount ${chroot_dir}/${dir} +done -# exit 0 -# EOF +umount ${chroot_dir} +sync -# chmod a+rx "${chroot_dir}/etc/rc.local" - -# ensure there is /init in the initramfs -> otherwise there is a kernel panic -# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init -# in our case, they are just the same -ln -fs /sbin/init ${chroot_dir}/init - -# Finally building the initramfs -( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) - -# Fix paranoid permissions -chmod a+rx ${abs_outdir} -chmod a+r ${kernel} ${initramfs} - - -exit 0 +rmdir ${chroot_dir} From 43e2cf3ee2fd165d14769ffdaa94b916a675bad9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 19 Aug 2021 15:50:23 +0200 Subject: [PATCH 312/543] Use grub-pc, which is grub2 --- debian-devuan-install-on-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-devuan-install-on-disk.sh b/debian-devuan-install-on-disk.sh index 106b4f8..0e5a2b7 100755 --- a/debian-devuan-install-on-disk.sh +++ b/debian-devuan-install-on-disk.sh @@ -59,7 +59,7 @@ debootstrap "${suite}" "${chroot_dir}" echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${chroot_dir}/etc/apt/sources.list chroot ${chroot_dir} apt update -chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 ifenslave vlan grub +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 ifenslave vlan grub-pc echo "unconfigured-host" > ${chroot_dir}/etc/hostname From ef38b13e553416c9e49a559a9ec0f85deb7c4722 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 19 Aug 2021 15:51:32 +0200 Subject: [PATCH 313/543] openwrt: add script to support block devices --- openwrt/openwrt-add-usb-stick.sh | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100755 openwrt/openwrt-add-usb-stick.sh diff --git a/openwrt/openwrt-add-usb-stick.sh b/openwrt/openwrt-add-usb-stick.sh new file mode 100755 index 0000000..3637ca0 --- /dev/null +++ b/openwrt/openwrt-add-usb-stick.sh @@ -0,0 +1,28 @@ +#!/bin/sh +# Nico Schottelius +# 2021-01-07 +# Based on https://openwrt.org/docs/guide-user/storage/usb-drives-quickstart + +if [ $# -ne 1 ]; then + echo "$0 ip-address nat64-prefix" + echo " ip-address: where to find the OpenWRT device" + exit 1 +fi + +my_ip=$1; shift + +cat < Date: Sun, 29 Aug 2021 14:58:55 +0200 Subject: [PATCH 314/543] wait for partprobe to settle --- debian-devuan-install-on-disk.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/debian-devuan-install-on-disk.sh b/debian-devuan-install-on-disk.sh index 0e5a2b7..5d7c5a1 100755 --- a/debian-devuan-install-on-disk.sh +++ b/debian-devuan-install-on-disk.sh @@ -46,6 +46,7 @@ label: dos EOF partprobe "${disk}" +sleep 3 # For creation, if an existing filesystem is on the partitions mkfs.ext4 -F ${partition} From a4ac6cf02a69b3fe6b39f6837aaf48694c9820f8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 5 Sep 2021 08:59:17 +0200 Subject: [PATCH 315/543] [ceph] Update activate script to support LVM --- ceph/ceph-osd-activate-all | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ceph/ceph-osd-activate-all b/ceph/ceph-osd-activate-all index 95ef7bd..2765e93 100755 --- a/ceph/ceph-osd-activate-all +++ b/ceph/ceph-osd-activate-all @@ -28,4 +28,10 @@ for dev in $(fdisk -l | awk '$6 ~/Ceph/ { print $1 }'); do fi done +# LVM based +for osdid in $(lvs -o lv_tags | grep ceph.block_device| sed -e 's/.*ceph.osd_id=//' -e 's/,.*//'); do + ceph-volume lvm activate --no-systemd $osdid + /opt/ungleich-tools/monit-ceph-create-start "osd.$osdid" +done + rmdir "$tmpdir" From 422fcf7237889145ba696204ba45b6f60383ea6b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 5 Sep 2021 11:24:40 +0200 Subject: [PATCH 316/543] monit: overwrite to ensure no appending is done --- monit-ceph-create-start | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monit-ceph-create-start b/monit-ceph-create-start index a11dbb7..9b4ac23 100755 --- a/monit-ceph-create-start +++ b/monit-ceph-create-start @@ -22,7 +22,7 @@ id=$(echo $to_monitor | awk -F . '{ print $2 }') case "$daemon" in osd) depends="${depends}, ${to_monitor}-whoami" - cat >> "$conf" < "$conf" < Date: Sun, 5 Sep 2021 11:48:00 +0200 Subject: [PATCH 317/543] Use ceph-volume activate --all --- ceph/ceph-osd-activate-all | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ceph/ceph-osd-activate-all b/ceph/ceph-osd-activate-all index 2765e93..0ed7a0f 100755 --- a/ceph/ceph-osd-activate-all +++ b/ceph/ceph-osd-activate-all @@ -28,10 +28,18 @@ for dev in $(fdisk -l | awk '$6 ~/Ceph/ { print $1 }'); do fi done -# LVM based -for osdid in $(lvs -o lv_tags | grep ceph.block_device| sed -e 's/.*ceph.osd_id=//' -e 's/,.*//'); do - ceph-volume lvm activate --no-systemd $osdid - /opt/ungleich-tools/monit-ceph-create-start "osd.$osdid" -done +# LVM based / manual does not work atm: +# + grep ceph.block_device +# + sed -e s/.*ceph.osd_id=// -e s/,.*// +# + ceph-volume lvm activate --no-systemd 112 +# --> UnboundLocalError: local variable 'tags' referenced before assignment + +#for osdid in $(lvs -o lv_tags | grep ceph.block_device| sed -e 's/.*ceph.osd_id=//' -e 's/,.*//'); do +# ceph-volume lvm activate --no-systemd $osdid +# /opt/ungleich-tools/monit-ceph-create-start "osd.$osdid" +#done + +# Using this with a fake /bin/systemctl +ceph-volume lvm activate --all rmdir "$tmpdir" From 4c8051b5a23f8af288e9f58f21c46a568f50253d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 5 Sep 2021 11:54:33 +0200 Subject: [PATCH 318/543] ++generate grub.cfg --- debian-devuan-install-on-disk.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian-devuan-install-on-disk.sh b/debian-devuan-install-on-disk.sh index 5d7c5a1..9474f97 100755 --- a/debian-devuan-install-on-disk.sh +++ b/debian-devuan-install-on-disk.sh @@ -73,6 +73,9 @@ cat ${keyfile} > ${chroot_dir}/root/.ssh/authorized_keys # Fix possible permission issue from above chmod -R og-rwx ${chroot_dir}/root/ +# Ensure boot loader has a configuration +chroot ${chroot_dir} grub-mkconfig -o /boot/grub/grub.cfg + ################################################################################ # networking From 41fd5a8812ee1eab3f232c6f863fe57f8704666f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 5 Sep 2021 12:41:04 +0200 Subject: [PATCH 319/543] grub config needs to be generated later! --- debian-devuan-install-on-disk.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/debian-devuan-install-on-disk.sh b/debian-devuan-install-on-disk.sh index 9474f97..a816120 100755 --- a/debian-devuan-install-on-disk.sh +++ b/debian-devuan-install-on-disk.sh @@ -73,9 +73,6 @@ cat ${keyfile} > ${chroot_dir}/root/.ssh/authorized_keys # Fix possible permission issue from above chmod -R og-rwx ${chroot_dir}/root/ -# Ensure boot loader has a configuration -chroot ${chroot_dir} grub-mkconfig -o /boot/grub/grub.cfg - ################################################################################ # networking @@ -111,6 +108,8 @@ for dir in dev sys proc; do done chroot ${chroot_dir} grub-install ${disk} +# Ensure boot loader has a configuration +chroot ${chroot_dir} grub-mkconfig -o /boot/grub/grub.cfg for dir in dev sys proc; do umount ${chroot_dir}/${dir} From 9aaf1eb733121254b96bf468bc7ea22e50771267 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 9 Sep 2021 09:16:14 +0200 Subject: [PATCH 320/543] ++vgchange for lvm based osds --- ceph/ceph-osd-activate-all | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ceph/ceph-osd-activate-all b/ceph/ceph-osd-activate-all index 0ed7a0f..8f696de 100755 --- a/ceph/ceph-osd-activate-all +++ b/ceph/ceph-osd-activate-all @@ -39,6 +39,8 @@ done # /opt/ungleich-tools/monit-ceph-create-start "osd.$osdid" #done +# Activate all volumes in the OS +vgchange -ay # Using this with a fake /bin/systemctl ceph-volume lvm activate --all From 2dcfce10353942478fadda401320e7ba6715f49f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Sep 2021 07:42:39 +0200 Subject: [PATCH 321/543] ceph: on activation, skip already mounted OSDs --- ceph/ceph-osd-activate-all | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ceph/ceph-osd-activate-all b/ceph/ceph-osd-activate-all index 8f696de..4a69fa0 100755 --- a/ceph/ceph-osd-activate-all +++ b/ceph/ceph-osd-activate-all @@ -9,6 +9,10 @@ tmpdir=$(mktemp -d) # XFS based partition scheme for dev in $(fdisk -l | awk '$6 ~/Ceph/ { print $1 }'); do + if mount | grep ^$dev ; then + echo Skipping $dev, already mounted + continue + fi mount "$dev" "$tmpdir" id=$(cat "${tmpdir}/whoami") From 5b4ee78da8ec1a8e7ad2bd86da69a912e3a22145 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Sep 2021 07:43:30 +0200 Subject: [PATCH 322/543] ceph/lvm: also create monit entry --- ceph/ceph-osd-activate-all | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ceph/ceph-osd-activate-all b/ceph/ceph-osd-activate-all index 4a69fa0..4f5ee02 100755 --- a/ceph/ceph-osd-activate-all +++ b/ceph/ceph-osd-activate-all @@ -38,14 +38,14 @@ done # + ceph-volume lvm activate --no-systemd 112 # --> UnboundLocalError: local variable 'tags' referenced before assignment -#for osdid in $(lvs -o lv_tags | grep ceph.block_device| sed -e 's/.*ceph.osd_id=//' -e 's/,.*//'); do -# ceph-volume lvm activate --no-systemd $osdid -# /opt/ungleich-tools/monit-ceph-create-start "osd.$osdid" -#done - # Activate all volumes in the OS vgchange -ay # Using this with a fake /bin/systemctl ceph-volume lvm activate --all +for osdid in $(lvs -o lv_tags | grep ceph.block_device| sed -e 's/.*ceph.osd_id=//' -e 's/,.*//'); do + #ceph-volume lvm activate --no-systemd $osdid + /opt/ungleich-tools/monit-ceph-create-start "osd.$osdid" +done + rmdir "$tmpdir" From cf2f84de792dd344c6304b9e46c0cd50b03e4a50 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Sep 2021 16:05:04 +0200 Subject: [PATCH 323/543] [vigir] update openwrt to 21.02.0 --- openwrt/vigir-1-firmware-upgrade.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index 4cc4d19..3345852 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,9 +12,8 @@ set -x vigir_ip=$1; shift # openwrt -version=19.07.6 -filename=openwrt-${version}-ramips-mt7621-zbt-wg3526-16M-squashfs-sysupgrade.bin - +version=21.02.0 +filename=openwrt-${version}-ramips-mt7621-zbtlink_zbt-wg3526-16m-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} From 11415886bbd6261e014c7d73ed56f6569f280c1d Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Tue, 28 Sep 2021 10:36:52 +0200 Subject: [PATCH 324/543] [image]Update arch-build-opennebula-image.sh for Task#9418 --- .../arch-build-opennebula-image.sh | 33 +++++++++++-------- 1 file changed, 20 insertions(+), 13 deletions(-) diff --git a/opennebula-images/arch-build-opennebula-image.sh b/opennebula-images/arch-build-opennebula-image.sh index f809de5..6415f7a 100755 --- a/opennebula-images/arch-build-opennebula-image.sh +++ b/opennebula-images/arch-build-opennebula-image.sh @@ -1,6 +1,6 @@ #!/bin/sh -# This script generates Debian images for OpenNebula. +# This script generates arch linux images for OpenNebula. # # Test image locally (without network) with: # qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 @@ -13,7 +13,7 @@ IMAGE_PATH=arch-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 -ONE_CONTEXT_VERSION=5.12.0.2 +ONE_CONTEXT_VERSION=6.0.0 ONE_CONTEXT_SOURCE_ARCHIVE="https://github.com/OpenNebula/addon-context-linux/archive/v$ONE_CONTEXT_VERSION.tar.gz" cleanup() { @@ -108,37 +108,44 @@ Server = http://mirror.puzzle.ch/archlinux/\$repo/os/\$arch Server = https://mirror.puzzle.ch/archlinux/\$repo/os/\$arch Server = https://mirror.ungleich.ch/mirror/packages/archlinux/\$repo/os/\$arch EOF -run_root pacman --sync --refresh --upgrade +run_root pacman -Syu +#run_root pacman --sync --refresh --upgrade # Guest networking is to be handled by the one-context package. # See https://github.com/OpenNebula/addon-context-linux for details. -run_root pacman --sync curl tar +run_root pacman -Sy curl tar rsync --noconfirm run_root curl -L "$ONE_CONTEXT_SOURCE_ARCHIVE" -o one-context.tar.gz run_root tar xf one-context.tar.gz -run_root cp -rT addon-context-linux-${ONE_CONTEXT_VERSION}/src/ / +run_root rsync -avrh addon-context-linux-${ONE_CONTEXT_VERSION}/src/* / +#run_root cp -rT addon-context-linux-${ONE_CONTEXT_VERSION}/src/ / run_root rm -r addon-context-linux-${ONE_CONTEXT_VERSION} -run_root systemctl enable one-context.service +run_root cp -r /usr/lib/systemd/system/one-context.service##arch.one /usr/lib/systemd/system/one-context.service +#run_root systemctl enable one-context.service # Initalize base services. run_root systemd-machine-id-setup run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service - +#run_root pacman -Q linux # Install kernel and generate initramfs. -run_root pacman --sync linux mkinitcpio -run_rot mkinitcpio -P +run_root pacman -Sy linux +run_root rsync -avrh /usr/lib/modules/ /lib/modules +#run_root pacman -Sy linux +run_root pacman -Sy mkinitcpio +#run_root pacman -S linux +run_root mkinitcpio -P # Install and configure bootloader. -run_root pacman --sync grub +run_root pacman -Sy grub --noconfirm run_root grub-install --target=i386-pc "${NBD_DEVICE}" run_root grub-mkconfig -o /boot/grub/grub.cfg # Install en configure SSH daemon. -run_root pacman --sync openssh-server +run_root pacman -Sy openssh --noconfirm # Install haveged due to lack of entropy in ONE environment. -run_root pacman --sync haveged +run_root pacman -Sy haveged run_root systemctl enable haveged.service # Generate fstab file. @@ -155,4 +162,4 @@ run_root touch /etc/machine-id rm -f /var/lib/systemd/random-seed # Make sure everything is written to disk before exiting. -sync +sync \ No newline at end of file From eb421d4dcdff8100b83f92f295bbcafd0a1ef90b Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Tue, 28 Sep 2021 17:50:08 +0200 Subject: [PATCH 325/543] [image]Update arch-build-opennebula-image.sh for Task#9418 --- .../arch-build-opennebula-image.sh | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/opennebula-images/arch-build-opennebula-image.sh b/opennebula-images/arch-build-opennebula-image.sh index 6415f7a..3376cf6 100755 --- a/opennebula-images/arch-build-opennebula-image.sh +++ b/opennebula-images/arch-build-opennebula-image.sh @@ -1,6 +1,6 @@ #!/bin/sh -# This script generates arch linux images for OpenNebula. +# This script generates Debian images for OpenNebula. # # Test image locally (without network) with: # qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 @@ -120,7 +120,7 @@ run_root rsync -avrh addon-context-linux-${ONE_CONTEXT_VERSION}/src/* / #run_root cp -rT addon-context-linux-${ONE_CONTEXT_VERSION}/src/ / run_root rm -r addon-context-linux-${ONE_CONTEXT_VERSION} run_root cp -r /usr/lib/systemd/system/one-context.service##arch.one /usr/lib/systemd/system/one-context.service -#run_root systemctl enable one-context.service +run_root systemctl enable one-context.service # Initalize base services. run_root systemd-machine-id-setup @@ -129,24 +129,30 @@ run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service #run_root pacman -Q linux # Install kernel and generate initramfs. -run_root pacman -Sy linux +run_root pacman -Sy linux --noconfirm run_root rsync -avrh /usr/lib/modules/ /lib/modules #run_root pacman -Sy linux -run_root pacman -Sy mkinitcpio +run_root pacman -Sy mkinitcpio --noconfirm #run_root pacman -S linux run_root mkinitcpio -P # Install and configure bootloader. run_root pacman -Sy grub --noconfirm run_root grub-install --target=i386-pc "${NBD_DEVICE}" +#run_root sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0"/' /etc/default/grub run_root grub-mkconfig -o /boot/grub/grub.cfg - # Install en configure SSH daemon. -run_root pacman -Sy openssh --noconfirm +run_root pacman -Sy openssh netctl --noconfirm # Install haveged due to lack of entropy in ONE environment. -run_root pacman -Sy haveged +run_root pacman -Sy haveged --noconfirm run_root systemctl enable haveged.service +run_root systemctl enable sshd +run_root systemctl enable systemd-networkd +run_root systemctl enable systemd-resolved +run_root systemctl enable systemd-timesyncd + + # Generate fstab file. boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") From a7ae4001edf9f1c58314196a9785c63bc2e7b93c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 18 Oct 2021 13:53:38 +0200 Subject: [PATCH 326/543] ceph: add script to delete/clean disk --- ceph/ceph-delete-disk | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 ceph/ceph-delete-disk diff --git a/ceph/ceph-delete-disk b/ceph/ceph-delete-disk new file mode 100644 index 0000000..9cb1f05 --- /dev/null +++ b/ceph/ceph-delete-disk @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Inspired from https://rook.io/docs/rook/v1.7/ceph-teardown.html + +if [ $# -ne 1 ]; then + echo $0 disk + echo f.i. $0 /dev/sdx + exit 1 +fi + +DISK="$1"; shift + +# Zap the disk to a fresh, usable state (zap-all is important, b/c MBR has to be clean) + +# You will have to run this step for all disks. +sgdisk --zap-all $DISK + +# Clean hdds with dd +dd if=/dev/zero of="$DISK" bs=1M count=100 oflag=direct,dsync + +# Clean disks such as ssd with blkdiscard instead of dd +blkdiscard $DISK + +# These steps only have to be run once on each node +# If rook sets up osds using ceph-volume, teardown leaves some devices mapped that lock the disks. +ls /dev/mapper/ceph-* | xargs -I% -- dmsetup remove % + +# ceph-volume setup can leave ceph- directories in /dev and /dev/mapper (unnecessary clutter) +rm -rf /dev/ceph-* +rm -rf /dev/mapper/ceph--* + +# Inform the OS of partition table changes +partprobe $DISK From 728fb0b9f77851ca5558207c7d0a49e2825b7381 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 18 Oct 2021 14:29:05 +0200 Subject: [PATCH 327/543] ceph-disk-delete: remove unsupported flags on alpine --- ceph/ceph-delete-disk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceph/ceph-delete-disk b/ceph/ceph-delete-disk index 9cb1f05..12bc282 100644 --- a/ceph/ceph-delete-disk +++ b/ceph/ceph-delete-disk @@ -15,7 +15,7 @@ DISK="$1"; shift sgdisk --zap-all $DISK # Clean hdds with dd -dd if=/dev/zero of="$DISK" bs=1M count=100 oflag=direct,dsync +dd if=/dev/zero of="$DISK" bs=1M count=100 # Clean disks such as ssd with blkdiscard instead of dd blkdiscard $DISK From 886ddc1f97728db78e0664ddc531a38bffef2ff4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 18 Oct 2021 14:29:38 +0200 Subject: [PATCH 328/543] ceph-delete-disk: make executable --- ceph/ceph-delete-disk | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ceph/ceph-delete-disk diff --git a/ceph/ceph-delete-disk b/ceph/ceph-delete-disk old mode 100644 new mode 100755 From 877c5e9b13a443d3dd5bdd4adf068a9cc6dae081 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 1 Nov 2021 11:10:44 +0100 Subject: [PATCH 329/543] update jool ipv4 island --- openwrt/openwrt-add-jool-ipv4island.sh | 18 ++++++++++++------ openwrt/viwib-2-configure.sh | 2 +- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/openwrt/openwrt-add-jool-ipv4island.sh b/openwrt/openwrt-add-jool-ipv4island.sh index 3b1c7d6..3349385 100755 --- a/openwrt/openwrt-add-jool-ipv4island.sh +++ b/openwrt/openwrt-add-jool-ipv4island.sh @@ -1,15 +1,19 @@ #!/bin/sh -if [ $# -ne 2 ]; then +if [ $# -ne 1 ]; then echo "$0 ip-address nat64-prefix" echo " ip-address: where to find the OpenWRT device" - echo " nat64-prefix: which network to use for INCOMING NAT64 (often 2a0a:e5c1:XXX:b00::/96" + echo " nat64-prefix: how to reach the ipv4 hosts" + echo "" + echo "Example:" + echo "$0 2a0a:e5c1:301::42 2a0a:e5c1:301:b00::/96" + echo "" + echo "This maps 2a0a:e5c1:301:b00::/96 to the IPv4 addresses on the device 2a0a:e5c1:301::42" exit 1 fi my_ip=$1; shift nat64_prefix=$1; shift -v4addr=$1; shift cat < /etc/rc.local << EO2 +sysctl -w net.ipv6.conf.all.accept_ra=2 modprobe jool jool -6 ${nat64_prefix} -EO2 -sh /etc/rc.local +# OpenWRT has 192.168.1. assigned on LAN by default +# Probably not even necessary +# jool -4 -a 192.168.1.1 + EOF diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 72c6e7d..b38f41c 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -119,7 +119,7 @@ uci delete wireless.radio0.disabled uci commit -ping6 -c5 ungleich.ch || exit 1 +ping -c5 ungleich.ch || exit 1 # update the sources opkg update From 06142299382a02bf10029cab346045c439b8a7e3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 1 Nov 2021 11:12:27 +0100 Subject: [PATCH 330/543] ipv4island: fix argc check --- openwrt/openwrt-add-jool-ipv4island.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-add-jool-ipv4island.sh b/openwrt/openwrt-add-jool-ipv4island.sh index 3349385..ca24bb4 100755 --- a/openwrt/openwrt-add-jool-ipv4island.sh +++ b/openwrt/openwrt-add-jool-ipv4island.sh @@ -1,6 +1,6 @@ #!/bin/sh -if [ $# -ne 1 ]; then +if [ $# -ne 2 ]; then echo "$0 ip-address nat64-prefix" echo " ip-address: where to find the OpenWRT device" echo " nat64-prefix: how to reach the ipv4 hosts" From 757998a05508f3c4274a115dc124345c5831d681 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 7 Nov 2021 16:25:02 +0100 Subject: [PATCH 331/543] openwrt: update for openwrt 21.02. --- openwrt/openwrt-add-jool-ipv4island.sh | 8 +++++++- openwrt/vigir-2-configure.sh | 3 +-- openwrt/viirb-2-configure.sh | 3 +-- openwrt/viwib-2-configure.sh | 3 +-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/openwrt/openwrt-add-jool-ipv4island.sh b/openwrt/openwrt-add-jool-ipv4island.sh index ca24bb4..01f4556 100755 --- a/openwrt/openwrt-add-jool-ipv4island.sh +++ b/openwrt/openwrt-add-jool-ipv4island.sh @@ -21,15 +21,21 @@ set -x opkg update opkg install kmod-jool jool-tools + +cat < /etc/rc.local + sysctl -w net.ipv6.conf.all.accept_ra=2 modprobe jool -jool -6 ${nat64_prefix} +jool instance add default --netfilter -6 ${nat64_prefix} + +EO2 # OpenWRT has 192.168.1. assigned on LAN by default # Probably not even necessary # jool -4 -a 192.168.1.1 +sh /etc/rc.local EOF diff --git a/openwrt/vigir-2-configure.sh b/openwrt/vigir-2-configure.sh index e071265..74369d5 100755 --- a/openwrt/vigir-2-configure.sh +++ b/openwrt/vigir-2-configure.sh @@ -117,8 +117,7 @@ ping6 -c5 ungleich.ch || exit 1 opkg update # install wireguard + gui -opkg install wireguard -opkg install luci-app-wireguard +opkg install luci-app-wireguard luci-proto-wireguard # VPN / Wireguard uci set network.wg0=interface diff --git a/openwrt/viirb-2-configure.sh b/openwrt/viirb-2-configure.sh index 3524045..8f0f7e9 100755 --- a/openwrt/viirb-2-configure.sh +++ b/openwrt/viirb-2-configure.sh @@ -144,8 +144,7 @@ ping -c5 ungleich.ch || exit 1 opkg update # install wireguard + gui -opkg install wireguard -opkg install luci-app-wireguard +opkg install luci-app-wireguard luci-proto-wireguard # VPN / Wireguard uci set network.wg0=interface diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index b38f41c..6b49059 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -125,8 +125,7 @@ ping -c5 ungleich.ch || exit 1 opkg update # install wireguard + gui -opkg install wireguard -opkg install luci-app-wireguard +opkg install luci-app-wireguard luci-proto-wireguard # VPN / Wireguard uci set network.wg0=interface From 2d4fe869967eb49ee237d52c940a1c7d6c8089f7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 16 Nov 2021 14:17:08 +0100 Subject: [PATCH 332/543] viwib: crorect firmware link --- openwrt/viwib-1-firmware-upgrade.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 3bbcf45..57ddc70 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -4,7 +4,7 @@ if [ $# -ne 1 ]; then echo "$0 address" - echo " address: connect to this address, ignore the interface" + echo " address: connect to this address (default: 192.168.8.1)" exit 1 fi @@ -13,8 +13,8 @@ set -x viwib_ip=$1; shift # openwrt -version=19.07.8 -filename=openwrt-${version}-ramips-mt76x8-gl-mt300n-v2-squashfs-sysupgrade.bin +version=21.02.1 +filename=openwrt-${version}-ramips-mt76x8-glinet_gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${viwib_ip} From ee7a504bba93c1a5aef3d484e0051dc960b20aee Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 16 Nov 2021 14:17:22 +0100 Subject: [PATCH 333/543] vigir: store password in password store --- openwrt/vigir-4-cleanup.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/openwrt/vigir-4-cleanup.sh b/openwrt/vigir-4-cleanup.sh index 6e16ea4..57c80f3 100755 --- a/openwrt/vigir-4-cleanup.sh +++ b/openwrt/vigir-4-cleanup.sh @@ -19,10 +19,6 @@ vigir_hostname=vigir${id} root_password=$(pwgen -1 32) -# Save for sending to user -# FIXME: future make this more easy / better to transfer -echo $root_password > ${vigir_hostname}.rootpw - cat < Date: Tue, 16 Nov 2021 14:29:46 +0100 Subject: [PATCH 334/543] openwrt: rename for better distinguishing --- ...{openwrt-add-usb-stick.sh => openwrt-add-usb-storage-stick.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openwrt/{openwrt-add-usb-stick.sh => openwrt-add-usb-storage-stick.sh} (100%) diff --git a/openwrt/openwrt-add-usb-stick.sh b/openwrt/openwrt-add-usb-storage-stick.sh similarity index 100% rename from openwrt/openwrt-add-usb-stick.sh rename to openwrt/openwrt-add-usb-storage-stick.sh From 920833fa9b8ff7dc7070ed0979f9b21b247753d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 18 Nov 2021 16:31:57 +0100 Subject: [PATCH 335/543] ONE / Debian: update to bulleyes --- opennebula-images/debian-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index d18e349..5c6c4f7 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=buster # 10.X +RELEASE=bulleyes # 11.X ARCH=amd64 IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G From 9fec5ad1eddb9664eac26df174e9f17a19767574 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 20 Nov 2021 11:55:14 +0100 Subject: [PATCH 336/543] k8s: add script to do something on all nodes of the cluster --- k8s/pssh-all-nodes.sh | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100755 k8s/pssh-all-nodes.sh diff --git a/k8s/pssh-all-nodes.sh b/k8s/pssh-all-nodes.sh new file mode 100755 index 0000000..424106c --- /dev/null +++ b/k8s/pssh-all-nodes.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +#set -x + +tmp=$(mktemp) +kubectl get node -o name | sed 's,node/,,' > "$tmp" +#cat "$tmp" +pssh -h "$tmp" -l root -i "$@" +rm -f "$tmp" From e7b56339f12510abbf03f1667c357cb78f1b520c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 21 Nov 2021 11:19:19 +0100 Subject: [PATCH 337/543] ++comment --- k8s/pssh-all-nodes.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/pssh-all-nodes.sh b/k8s/pssh-all-nodes.sh index 424106c..5cca477 100755 --- a/k8s/pssh-all-nodes.sh +++ b/k8s/pssh-all-nodes.sh @@ -1,4 +1,6 @@ #!/bin/sh +# Execute commands on all hosts of the currently selected kubernetes cluster +# Do export KUBECONFIG=~/your-admin.conf before using this script #set -x From 295a36106f8f829eb01263dce967b7f03841f236 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Dec 2021 00:02:45 +0100 Subject: [PATCH 338/543] add domain at the beginning/k8s pssh --- k8s/pssh-all-nodes.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/k8s/pssh-all-nodes.sh b/k8s/pssh-all-nodes.sh index 5cca477..a4e0298 100755 --- a/k8s/pssh-all-nodes.sh +++ b/k8s/pssh-all-nodes.sh @@ -4,8 +4,10 @@ #set -x +domain=$1; shift + tmp=$(mktemp) -kubectl get node -o name | sed 's,node/,,' > "$tmp" +kubectl get node -o name | sed -e 's,node/,,' -e "s,\$,.$domain,"> "$tmp" #cat "$tmp" pssh -h "$tmp" -l root -i "$@" rm -f "$tmp" From dc81a3fbc11938feeef5c13a85644d38b3b2086f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Dec 2021 16:01:23 +0100 Subject: [PATCH 339/543] ++coupon code --- generate-coupon.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 generate-coupon.py diff --git a/generate-coupon.py b/generate-coupon.py new file mode 100644 index 0000000..8f59a8d --- /dev/null +++ b/generate-coupon.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 + +import random +import ipaddress + +net = ipaddress.IPv6Network("2a0a:e5c0:11:2::/64") +offset = random.randint(0, 2**64) +coupon = net[offset] +print(coupon) From 5b3892bde410b7db7d167f895b0ddf0635a9cb27 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Dec 2021 16:02:07 +0100 Subject: [PATCH 340/543] ++coupon mode --- generate-coupon.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 generate-coupon.py diff --git a/generate-coupon.py b/generate-coupon.py old mode 100644 new mode 100755 From 2cab333fd61528246c044ed3741a93a7d40399c0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Dec 2021 16:02:19 +0100 Subject: [PATCH 341/543] add k8s helper --- k8s/nodes-with-suffix.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 k8s/nodes-with-suffix.sh diff --git a/k8s/nodes-with-suffix.sh b/k8s/nodes-with-suffix.sh new file mode 100755 index 0000000..60b6fb9 --- /dev/null +++ b/k8s/nodes-with-suffix.sh @@ -0,0 +1,8 @@ +#!/bin/sh +# Execute commands on all hosts of the currently selected kubernetes cluster +# Do export KUBECONFIG=~/your-admin.conf before using this script +# Can be used to pass into cdist + +domain=$1; shift + +echo $(kubectl get node -o name | sed -e 's,node/,,' -e "s,\$,.$domain,") From d0db1ef9ec53c74c5522f097549cca0e11451d16 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 Dec 2021 16:15:44 +0100 Subject: [PATCH 342/543] coupon++ --- generate-coupon.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/generate-coupon.py b/generate-coupon.py index 8f59a8d..c3d5938 100755 --- a/generate-coupon.py +++ b/generate-coupon.py @@ -2,8 +2,15 @@ import random import ipaddress +import datetime net = ipaddress.IPv6Network("2a0a:e5c0:11:2::/64") offset = random.randint(0, 2**64) coupon = net[offset] -print(coupon) + +today = datetime.datetime.now() +today_in_2y = today + datetime.timedelta(days=365*2) + +print(f"Coupon ID : {coupon}") +print(f"Today : {today}") +print(f"Today in 2 years: {today_in_2y}") From efdfb94c725952ebc5656aab0546db46f8e1f5d2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Dec 2021 22:53:18 +0100 Subject: [PATCH 343/543] ++monit script --- monit-start-all-from-commandline | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 monit-start-all-from-commandline diff --git a/monit-start-all-from-commandline b/monit-start-all-from-commandline new file mode 100644 index 0000000..af3a9be --- /dev/null +++ b/monit-start-all-from-commandline @@ -0,0 +1,8 @@ +#!/bin/sh + +for c in "$@"; do + echo $c + start=$(grep "start program" $c | sed -e 's/.*start program = "//' -e 's/".*//') + echo $start + ( $start & ) +done From 8d4803b888bdcf59febdf1b1f101056ecf03e72a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 31 Dec 2021 16:03:13 +0100 Subject: [PATCH 344/543] ++ tool to generate wireguard tunnels --- wireguard/.gitignore | 1 + wireguard/gen-tunnels.sh | 67 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 wireguard/.gitignore create mode 100755 wireguard/gen-tunnels.sh diff --git a/wireguard/.gitignore b/wireguard/.gitignore new file mode 100644 index 0000000..fee9217 --- /dev/null +++ b/wireguard/.gitignore @@ -0,0 +1 @@ +*.conf diff --git a/wireguard/gen-tunnels.sh b/wireguard/gen-tunnels.sh new file mode 100755 index 0000000..987bfae --- /dev/null +++ b/wireguard/gen-tunnels.sh @@ -0,0 +1,67 @@ +#!/bin/sh +# 2021-12-30 +# Nico Schottelius + +if [ $# -ne 7 ] ; then + echo $0 "v6|v4" vpngw vpnpubkey prefix mask start end + echo "f.i. $0 v4 vpn-....ungleich.ch:51820 6BRnQ.. 192.0.0. 32 22 43" + echo "f.i. $0 v6 vpn-....ungleich.ch:51820 6BRnQ.. 2a0a:e5c0: 48 22 333" + exit 1 +fi + +v4v6=$1; shift +vpngw=$1; shift +vpnpub=$1; shift +prefix=$1; shift +mask=$1; shift +start=$1; shift +end=$1; shift + +case "$v4v6" in + v6) + sep=":" + allowed_ips="::/0" + ;; + v4) + sep="." + allowed_ips="0.0.0.0/0" + ;; + + *) + echo "Unsupported, use v6 or v4" >&2 + exit 1 + ;; +esac + +: > gw.conf + +for ip in $(seq $start $end); do + privkey=$(wg genkey) + pubkey=$(echo $privkey | wg pubkey) + + addr=$prefix${sep}${ip}/${mask} + addr_nomask=$prefix${sep}${ip} + file="vpn-${addr_nomask}.conf" + echo "Writing ${file} and updating gw.conf" + + cat < $file +[Interface] +PrivateKey = $privkey +ListenPort = 51820 +Address = ${addr} + +[Peer] +PublicKey = 6BRnQ+dmeFzVCH9RbM1pbJ7u3y3qrl+zUzzYCmC88kE= +Endpoint = vpn-18515529.ungleich.ch:51820 +AllowedIPs = $allowed_ips + +EOF + cat <> gw.conf + +[Peer] +PublicKey = ${pubkey} +AllowedIPs = ${addr} +EOF + + +done From 307761fcc7f001dd73be3abb519c14f8310cbb2c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 31 Dec 2021 20:22:37 +0100 Subject: [PATCH 345/543] [wireguard] add variables for endpoints --- wireguard/gen-tunnels.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/wireguard/gen-tunnels.sh b/wireguard/gen-tunnels.sh index 987bfae..0251c07 100755 --- a/wireguard/gen-tunnels.sh +++ b/wireguard/gen-tunnels.sh @@ -41,19 +41,19 @@ for ip in $(seq $start $end); do addr=$prefix${sep}${ip}/${mask} addr_nomask=$prefix${sep}${ip} - file="vpn-${addr_nomask}.conf" + file="${addr_nomask}.conf" echo "Writing ${file} and updating gw.conf" cat < $file [Interface] -PrivateKey = $privkey +PrivateKey = ${privkey} ListenPort = 51820 Address = ${addr} [Peer] -PublicKey = 6BRnQ+dmeFzVCH9RbM1pbJ7u3y3qrl+zUzzYCmC88kE= -Endpoint = vpn-18515529.ungleich.ch:51820 -AllowedIPs = $allowed_ips +PublicKey = ${vpnpub} +Endpoint = ${vpngw} +AllowedIPs = ${allowed_ips} EOF cat <> gw.conf From 24521e82ddc26d76400c41c1e82398a4997775c4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 20 Jan 2022 20:14:03 +0100 Subject: [PATCH 346/543] Add script for firmware update on gl inet microuter n300 --- openwrt/.gitignore | 2 ++ openwrt/microuter-n300-firmware-upgrade.sh | 39 ++++++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 openwrt/.gitignore create mode 100755 openwrt/microuter-n300-firmware-upgrade.sh diff --git a/openwrt/.gitignore b/openwrt/.gitignore new file mode 100644 index 0000000..55a0f9b --- /dev/null +++ b/openwrt/.gitignore @@ -0,0 +1,2 @@ +*.rootpw +*.public_key diff --git a/openwrt/microuter-n300-firmware-upgrade.sh b/openwrt/microuter-n300-firmware-upgrade.sh new file mode 100755 index 0000000..ed1e21b --- /dev/null +++ b/openwrt/microuter-n300-firmware-upgrade.sh @@ -0,0 +1,39 @@ +#!/bin/sh +# 2022-01-20, Nico Schottelius +# See https://ungleich.ch/u/products/device-ipv6-box/ + +if [ $# -ne 1 ]; then + echo "$0 address" + echo " address: connect to this address" + exit 1 +fi + +set -x + +device_ip=$1; shift + +# openwrt +version=21.02.0 +filename=openwrt-${version}-ramips-mt76x8-glinet_microuter-n300-squashfs-sysupgrade.bin + +# don't care about other/old known_host entries +ssh-keygen -R ${device_ip} + +while ! ping -c1 ${device_ip}; do + echo "Cannot ping $device_ip yet - waiting" + sleep 1 +done + +cat ~/.ssh/id_rsa.pub | ssh root@${device_ip} "cat > /etc/dropbear/authorized_keys" + +# Don't re-download if we already have it +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} + +if echo $device_ip | grep -q :; then + scp_ip="[$device_ip]" +else + scp_ip="$device_ip" +fi + +scp ${filename} root@${scp_ip}:/tmp +ssh root@${device_ip} "sysupgrade -n /tmp/*.bin" From 7e45b4fba8583ddac7d2e98c2c11d2630aead5b3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 21 Jan 2022 15:55:33 +0100 Subject: [PATCH 347/543] Update viwib setup script to retain ipv4 and default settings --- openwrt/openwrt-add-usb-lte.sh | 2 +- openwrt/viwib-2-configure.sh | 52 +++++++++++++++------------------- 2 files changed, 24 insertions(+), 30 deletions(-) diff --git a/openwrt/openwrt-add-usb-lte.sh b/openwrt/openwrt-add-usb-lte.sh index d666fe7..808700b 100755 --- a/openwrt/openwrt-add-usb-lte.sh +++ b/openwrt/openwrt-add-usb-lte.sh @@ -5,7 +5,7 @@ if [ $# -ne 2 ]; then echo "$0 ip-address interface" echo " ip-address: where to find the OpenWRT device" echo " interface: which interface is the LTE device" - echo " Usually eth1 on VIIRB, eth3 on PIB" + echo " Usually eth1 on VIIRB, VIWIB eth3 on PIB" exit 1 fi diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 6b49059..47a93ef 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -55,28 +55,37 @@ fi cat < Date: Fri, 21 Jan 2022 16:58:06 +0100 Subject: [PATCH 348/543] [microuter-n300] bump openwrt version --- openwrt/microuter-n300-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/microuter-n300-firmware-upgrade.sh b/openwrt/microuter-n300-firmware-upgrade.sh index ed1e21b..6b317bb 100755 --- a/openwrt/microuter-n300-firmware-upgrade.sh +++ b/openwrt/microuter-n300-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x device_ip=$1; shift # openwrt -version=21.02.0 +version=21.02.1 filename=openwrt-${version}-ramips-mt76x8-glinet_microuter-n300-squashfs-sysupgrade.bin # don't care about other/old known_host entries From e1785e168b1803b403e48d1202ea0a39091c73b9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 25 Jan 2022 14:00:23 +0100 Subject: [PATCH 349/543] [openwrt] phase in new script for vpn generalisation --- openwrt/vpn-add.sh | 222 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 222 insertions(+) create mode 100755 openwrt/vpn-add.sh diff --git a/openwrt/vpn-add.sh b/openwrt/vpn-add.sh new file mode 100755 index 0000000..e1a79c1 --- /dev/null +++ b/openwrt/vpn-add.sh @@ -0,0 +1,222 @@ +#!/bin/sh +# 2020-06-13, 2022-01-20, Nico Schottelius +# Add a VPN to either of our devices + +if [ $# -lt 2 ]; then + echo "$0 device-ip vpn-server" + echo " device-ip-address: where to find the device" + echo " device-name: device we configure" + echo " [wireguard-private-key]: wg key to reuse" + exit 1 +fi + +# Get & set vpnserver network & co. via rest? + +device_ip=$1; shift +device_name=$1; shift + +# Whitelisting of enabled networks +case $device_name in + *-09??|*-0a??) + + device_id=$(echo $device_name | sed 's/.*-\(....\)$/\1/') + vpnserver_id=$(echo $device_id | sed 's/\(..\)..$/\1/') + prefix_base=2a0a:e5c1:${device_id} + vpn_endpoint_host=vpn-2a0ae5c1${vpnserver_id}.ungleich.ch + + case $vpnserver_id in + 09) + vpn_endpoint_pubkey="vnDJHqkAdMs8QkiIQizGGcPlaQfAwVBUvTBrYKfDZmE=" + ;; + a0) + vpn_endpoint_pubkey="Hxb5lV5r90r3hT9/JsMM8zIzrzYfXHCtt1jFtPgDlCQ=" + ;; + esac + ;; + + *) + echo "$device_type currently unsupported" + exit 1 + ;; +esac + +echo Setting up $device_name connecting to $vpn_endpoint_host for $prefix_base + +set -x + +hex_id=$device_id +device_hostname=${device_name} + +my_prefix=${prefix_base} +my_network=${my_prefix}::/48 + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +# wireguard +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi +public_key=$(echo $private_key | wg pubkey) + +echo "Trying to reach ${device_ip} ..." +ping -c3 ${device_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach ${device_ip}, aborting" + exit 1 +fi + +cat < Date: Tue, 25 Jan 2022 14:08:27 +0100 Subject: [PATCH 350/543] [vpn] add new endpoints --- openwrt/vpn-add.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/openwrt/vpn-add.sh b/openwrt/vpn-add.sh index e1a79c1..f13b08e 100755 --- a/openwrt/vpn-add.sh +++ b/openwrt/vpn-add.sh @@ -17,7 +17,7 @@ device_name=$1; shift # Whitelisting of enabled networks case $device_name in - *-09??|*-0a??) + *-03??|*-09??|*-0a??) device_id=$(echo $device_name | sed 's/.*-\(....\)$/\1/') vpnserver_id=$(echo $device_id | sed 's/\(..\)..$/\1/') @@ -25,10 +25,16 @@ case $device_name in vpn_endpoint_host=vpn-2a0ae5c1${vpnserver_id}.ungleich.ch case $vpnserver_id in - 09) + 03) # cdist, viirb, linthal + vpn_endpoint_pubkey="ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE=" + ;; + 05) # cdist, vigir, linthal + vpn_endpoint_pubkey="oaFiIVV1NjvDcfdtwJqR4F3k2XIC07npNgj0YjIEem4=" + ;; + 09) # k8s, viwib2 made, linthal vpn_endpoint_pubkey="vnDJHqkAdMs8QkiIQizGGcPlaQfAwVBUvTBrYKfDZmE=" ;; - a0) + a0) # k8s, viwib2 made, diesbach vpn_endpoint_pubkey="Hxb5lV5r90r3hT9/JsMM8zIzrzYfXHCtt1jFtPgDlCQ=" ;; esac From d77ce797fdc742b8f4e2d8b63bb6bc68f4a808e8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 26 Jan 2022 17:27:11 +0100 Subject: [PATCH 351/543] [openwrt] add support for teltonika --- openwrt/teltonika-firmware-upgrade.sh | 29 +++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 openwrt/teltonika-firmware-upgrade.sh diff --git a/openwrt/teltonika-firmware-upgrade.sh b/openwrt/teltonika-firmware-upgrade.sh new file mode 100644 index 0000000..2d8b3e7 --- /dev/null +++ b/openwrt/teltonika-firmware-upgrade.sh @@ -0,0 +1,29 @@ +#!/bin/sh +# Nico Schottelius, 2022-01-26 + +if [ $# -ne 1 ]; then + echo "$0 address" + echo " address: connect to this address (often: 192.168.1.1)" + exit 1 +fi + +set -x + +filename=RUT2_R_00.07.01.2_WEBUI.bin +device_ip=$1; shift + +while ! ping -c1 ${device_ip}; do + echo "Cannot ping $device_ip yet - waiting" + sleep 1 +done + +if echo $device_ip | grep -q :; then + scp_ip="[$device_ip]" +else + scp_ip="$device_ip" +fi + +wget -c https://wiki.teltonika-networks.com/wikibase/images/8/85/${filename} + +scp ${filename} root@${scp_ip}:/tmp +ssh root@${device_ip} "sysupgrade -n /tmp/*.bin" From 0e44b7587c60ead1151a8e6c2ae39328463875bb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 26 Jan 2022 17:27:27 +0100 Subject: [PATCH 352/543] [openwrt] vpn-add: support 06 prefix --- openwrt/vpn-add.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openwrt/vpn-add.sh b/openwrt/vpn-add.sh index f13b08e..12f4e94 100755 --- a/openwrt/vpn-add.sh +++ b/openwrt/vpn-add.sh @@ -31,6 +31,9 @@ case $device_name in 05) # cdist, vigir, linthal vpn_endpoint_pubkey="oaFiIVV1NjvDcfdtwJqR4F3k2XIC07npNgj0YjIEem4=" ;; + 06) # cdist, viwib, linthal + vpn_endpoint_pubkey="ygZQW3OSiMJl/RpKyaJVE0GSt6bjEDnoxdMJsNiloRE=" + ;; 09) # k8s, viwib2 made, linthal vpn_endpoint_pubkey="vnDJHqkAdMs8QkiIQizGGcPlaQfAwVBUvTBrYKfDZmE=" ;; From 0b2ef8216d47a02aacb734e9ce25398469b8be64 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 26 Jan 2022 17:27:47 +0100 Subject: [PATCH 353/543] [openwrt] begin to add default firewall --- openwrt/openwrt-default-firewall.sh | 49 +++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 openwrt/openwrt-default-firewall.sh diff --git a/openwrt/openwrt-default-firewall.sh b/openwrt/openwrt-default-firewall.sh new file mode 100644 index 0000000..fe6badf --- /dev/null +++ b/openwrt/openwrt-default-firewall.sh @@ -0,0 +1,49 @@ +#!/bin/sh +# 2022-01-21, Nico Schottelius + +# Firewall configuration +if ! uci show firewall | grep "name='Allow-SSH'"; then +uci add firewall rule +uci set firewall.@rule[-1].name='Allow-SSH' +uci set firewall.@rule[-1].src='wan' +uci set firewall.@rule[-1].dest='lan' +uci set firewall.@rule[-1].proto='tcp' +uci set firewall.@rule[-1].dest_port='22' +uci set firewall.@rule[-1].target='ACCEPT' +fi + +if ! uci show firewall | grep "name='Allow-HTTPS'"; then +uci add firewall rule +uci set firewall.@rule[-1].name='Allow-HTTPS' +uci set firewall.@rule[-1].src='wan' +uci set firewall.@rule[-1].dest='lan' +uci set firewall.@rule[-1].proto='tcp' +uci set firewall.@rule[-1].dest_port='443' +uci set firewall.@rule[-1].target='ACCEPT' +fi + +if ! uci show firewall | grep "name='Allow-HTTP'"; then +uci add firewall rule +uci set firewall.@rule[-1].name='Allow-HTTP' +uci set firewall.@rule[-1].src='wan' +uci set firewall.@rule[-1].dest='lan' +uci set firewall.@rule[-1].proto='tcp' +uci set firewall.@rule[-1].dest_port='80' +uci set firewall.@rule[-1].target='ACCEPT' +fi + +if ! uci show firewall | grep "name='Allow-Remote-SSH-Access'"; then +uci add firewall rule +uci set firewall.@rule[-1].name='Allow-Remote-SSH-Access' +uci set firewall.@rule[-1].src='wan' +uci set firewall.@rule[-1].proto='tcp' +uci set firewall.@rule[-1].dest_port='22' +uci set firewall.@rule[-1].enabled='0' +uci set firewall.@rule[-1].target='ACCEPT' +fi + + +# Add interfaces to the right network zone +uci set firewall.@zone[1].network='wan wan6 wg0' + +uci commit From c7dd06160932e1f2bbae32e66d197392354666d5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 26 Jan 2022 17:33:18 +0100 Subject: [PATCH 354/543] [openwrt] +teltonika details --- openwrt/teltonika-firmware-upgrade.sh | 2 + openwrt/vpn-add.sh | 72 +++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/openwrt/teltonika-firmware-upgrade.sh b/openwrt/teltonika-firmware-upgrade.sh index 2d8b3e7..e20de15 100644 --- a/openwrt/teltonika-firmware-upgrade.sh +++ b/openwrt/teltonika-firmware-upgrade.sh @@ -1,5 +1,7 @@ #!/bin/sh # Nico Schottelius, 2022-01-26 +# https://wiki.teltonika-networks.com/view/Firmware_upgrade_via_command_line +# https://wiki.teltonika-networks.com/view/RUT240_Firmware_Downloads if [ $# -ne 1 ]; then echo "$0 address" diff --git a/openwrt/vpn-add.sh b/openwrt/vpn-add.sh index 12f4e94..759f59d 100755 --- a/openwrt/vpn-add.sh +++ b/openwrt/vpn-add.sh @@ -229,3 +229,75 @@ uci delete wireless.radio0.disabled uci set network.lan.proto='dhcp' uci delete network.lan.ipaddr uci delete network.lan.netmask + + +# Teltonika syntax: network + +config interface 'wg_wg0' + option proto 'wireguard' + option private_key '...=' + option public_key 'ZZYoBV8b2LhqCrcQ8wFv6e6mu41w9i1g5kh4LiKefFI=' + list addresses '185.155.30.3/32' + option listen_port '51820' + option disabled '0' + +config wireguard_wg_wg0 + option description 'ungleich-ipv4' + option public_key '5ach7pUQ57aa402LHz1MYh7lyBZS0GvBEw2PC6dMHW4=' + list allowed_ips '0.0.0.0/0' + option route_allowed_ips '1' + option endpoint_host 'vpn-18515530.ungleich.ch' + option persistent_keepalive '25' + +config interface 'wg_ipv6' + option proto 'wireguard' + option private_key '...=' + option public_key 'N2buXMy2IJDH+Au/e0ripdiWYlpTQVdWeCeGHpXyjB0=' + list addresses '2a0a:e5c1:19e::42/48' + option listen_port '51821' + option disabled '0' + +config wireguard_wg_ipv6 + option public_key 'hi60lGP+xEUQ+kVnqA7PlJAO1SVqTS1W36g0LhFP0xQ=' + list allowed_ips '::/0' + option route_allowed_ips '1' + option endpoint_host 'vpn-2a0ae5c1.ungleich.ch' + option endpoint_port '51820' + option persistent_keepalive '25' + option description 'ungleich-ipv6' + + # Teltonika firewall + +config zone + option name 'wireguard' + option input 'ACCEPT' + option output 'ACCEPT' + option forward 'REJECT' + option masq '1' + option device 'wg_+' + +config rule + option name 'Allow-WireGuard' + option src 'wan' + option proto 'udp' + option target 'ACCEPT' + option enabled '1' + option dest_port '51820 51821' + +config rule + option dest_port '51820' + option src 'wan' + option name 'Allow-wireguard_wg_wg0-traffic' + option target 'ACCEPT' + option vpn_type 'wireguard' + option proto 'udp' + option family 'ipv4' + +config rule + option dest_port '51821' + option src 'wan' + option name 'Allow-wireguard_wg_ipv6-traffic' + option target 'ACCEPT' + option vpn_type 'wireguard' + option proto 'udp' + option family 'ipv4' From 9c0beb18d34b490018e360acf191f91c4b663aa1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 11 Feb 2022 22:24:43 +0100 Subject: [PATCH 355/543] alpine: add uefi support --- alpine-install-on-disk.sh | 44 +++++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 45d8eb9..ba6c733 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -1,9 +1,10 @@ #!/bin/sh -if [ $# -ne 2 ]; then - echo "$0 disk ssh-keyfile" +if [ $# -ne 3 ]; then + echo "$0 disk ssh-keyfile [efi|bios]" echo " disk: which disk to install to" echo " ssh-keyfile: ssh keys to add into the image" + echo " use efi or bios partitioning" exit 1 fi @@ -12,8 +13,9 @@ set -x DISK=$1; shift SSH_KEYS=$1; shift +BOOT_VIA=$1; shift -MAJOR_VERSION=3.14 +MAJOR_VERSION=3.15 MINOR_VERSION=0 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz @@ -26,10 +28,12 @@ rootfs_url="http://dl-cdn.alpinelinux.org/alpine/v$MAJOR_VERSION/releases/x86_64 case $DISK in /dev/sd*) - partition=${DISK}1 + partition1=${DISK}1 + partition2=${DISK}2 ;; /dev/mmcblk*|/dev/nvme*) - partition=${DISK}p1 + partition1=${DISK}p1 + partition2=${DISK}p2 ;; *) echo "Unsupported disk - edit this script" >&2 @@ -49,16 +53,34 @@ wget -c "$rootfs_url" -O "$IMAGE" # in the gap and also the paritition table dd if=/dev/zero of=${DISK} bs=1M count=2 -# Partition disk with 1 Linux partition -sudo sfdisk "$DISK" <&2 + exit + ;; +esac -# For creation, if an existing filesystem is on the partitions -sudo mkfs.ext4 -F ${partition} - -sudo mount ${partition} $rootfs_tmpdir # keep right permissions, use sudo sudo tar xf $IMAGE -C $rootfs_tmpdir From 86535b43bc16ad03bbb901a1964bfa199f0611d2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 14 Feb 2022 16:12:44 +0100 Subject: [PATCH 356/543] ++notes --- alpine-install-on-disk.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index ba6c733..43e0e05 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -156,3 +156,6 @@ rmdir ${rootfs_tmpdir} echo "${DISK} has been setup with Alpine Linux" exit 0 + +apk add grub-efi +grub-install --no-nvram --efi-directory /boot/ From ebe45dc3dfb80b41ea57767f87060268ecdba2af Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 22 Feb 2022 19:26:55 +0100 Subject: [PATCH 357/543] alpine: enhance uefi support --- alpine-install-on-disk.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 43e0e05..a135c6b 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -138,10 +138,22 @@ EOF # Setup bootloader -run_root apk add grub-bios +run_root apk add grub-bios grub-efi echo 'GRUB_CMDLINE_LINUX_DEFAULT="quiet rootfstype=ext4"' >> ${rootfs_tmpdir}/etc/default/grub run_root grub-mkconfig -o /boot/grub/grub.cfg -run_root grub-install --target=i386-pc ${DISK} + +case "$BOOT_VIA" in + bios) + run_root grub-install --target=i386-pc ${DISK} + ;; + efi) + run_root grub-install --efi-directory=/boot --no-nvram + run_root mkdir /boot/EFI/boot + run_root cp /boot/EFI/alpine/grubx64.efi /boot/EFI/boot/bootx64.efi + run_root cp /boot/grub/grub.cfg /boot/EFI/boot/ + ;; +esac + # Cleanup run_root rm -f /etc/resolv.conf From 45cccd0471ec8679353678a4e418fdde8828c2f6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 24 Feb 2022 16:58:21 +0100 Subject: [PATCH 358/543] ++alpine/uefi --- alpine-install-on-disk.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index a135c6b..ecbd32e 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -151,6 +151,7 @@ case "$BOOT_VIA" in run_root mkdir /boot/EFI/boot run_root cp /boot/EFI/alpine/grubx64.efi /boot/EFI/boot/bootx64.efi run_root cp /boot/grub/grub.cfg /boot/EFI/boot/ + sudo umount ${rootfs_tmpdir}/boot ;; esac From a12e54aa315566de431d1d8dd2e3b488024ed055 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 2 Mar 2022 12:32:10 +0100 Subject: [PATCH 359/543] Update openwrt ipv4 script to support 2nd vpn server --- openwrt/openwrt-add-ipv4-vpn.sh | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/openwrt/openwrt-add-ipv4-vpn.sh b/openwrt/openwrt-add-ipv4-vpn.sh index 9a824e3..32e56e8 100755 --- a/openwrt/openwrt-add-ipv4-vpn.sh +++ b/openwrt/openwrt-add-ipv4-vpn.sh @@ -13,9 +13,6 @@ fi my_ip=$1; shift my_wireguard_ip=$1; shift -interface=ungleichipv4 -vpn_endpoint_host=vpn-18515529.ungleich.ch - if [ $# -eq 1 ]; then private_key=$1; shift else @@ -23,9 +20,21 @@ else fi public_key=$(echo $private_key | wg pubkey) -vpn_endpoint_host=vpn-18515529.ungleich.ch -vpn_endpoint_pubkey=6BRnQ+dmeFzVCH9RbM1pbJ7u3y3qrl+zUzzYCmC88kE= +case $my_wireguard_ip in + 185.155.29.*) + vpn_endpoint_pubkey=6BRnQ+dmeFzVCH9RbM1pbJ7u3y3qrl+zUzzYCmC88kE= + ;; + 185.155.30.*) + vpn_endpoint_pubkey=5ach7pUQ57aa402LHz1MYh7lyBZS0GvBEw2PC6dMHW4= + ;; + *) + echo "Unknown VPN host for IP $my_wireguard_ip" >&2 + exit 1 + ;; +esac +interface="ip$(echo $my_wireguard_ip | awk -F. '{ print $1 $2 $3 $4 }')" +vpn_endpoint_host=vpn-$(echo $my_wireguard_ip | awk -F. '{ print $1 $2 $3 }').ungleich.ch cat < Date: Wed, 2 Mar 2022 12:57:12 +0100 Subject: [PATCH 360/543] Note bug in our setup --- openwrt/openwrt-add-ipv4-vpn.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openwrt/openwrt-add-ipv4-vpn.sh b/openwrt/openwrt-add-ipv4-vpn.sh index 32e56e8..c97050f 100755 --- a/openwrt/openwrt-add-ipv4-vpn.sh +++ b/openwrt/openwrt-add-ipv4-vpn.sh @@ -1,6 +1,8 @@ #!/bin/sh # 2021-04-27 +echo BUG: cannot use more than one due to port assignment +echo All WG IPv4 have same source port if [ $# -lt 2 ]; then echo "$0 host ipv4-address interface [private-key]" From 888a1bbd14dd4ad0dda3c46946a451689ce43690 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 13 Mar 2022 17:35:19 +0100 Subject: [PATCH 361/543] [alpine installer] add fs type on mount --- alpine-install-on-disk.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index ecbd32e..4964bb5 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -61,7 +61,7 @@ label: dos EOF # For creation, if an existing filesystem is on the partitions sudo mkfs.ext4 -F ${partition1} - sudo mount ${partition1} $rootfs_tmpdir + sudo mount -t ext4 ${partition1} $rootfs_tmpdir ;; efi) sudo sfdisk "$DISK" <&2 From 9f3f014a84151a0d0c48c566a1301e65da7ab08a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 13 Mar 2022 17:35:31 +0100 Subject: [PATCH 362/543] Add script to copy alpine linux installer and execute it --- copy-tools-and-install.sh | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100755 copy-tools-and-install.sh diff --git a/copy-tools-and-install.sh b/copy-tools-and-install.sh new file mode 100755 index 0000000..1434a55 --- /dev/null +++ b/copy-tools-and-install.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +if [ $# -ne 4 ]; then + echo "$0 host [args for alpine-install-on-disk.sh]" + exit 1 +fi + +host=$1; shift + +disk=$1; shift +key=$1; shift +mode=$1; shift + +cat < /etc/resolv.conf +apk add git sfdisk sudo util-linux-misc wget +cd /opt +if [ ! -d /opt/ungleich-tools ]; then git clone https://code.ungleich.ch/ungleich-public/ungleich-tools.git; fi +/opt/ungleich-tools/alpine-install-on-disk.sh $disk $key $mode + +EOF From ccdc89acee23296abbfe75e734f3c4a10dc7b634 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 13 Mar 2022 17:35:51 +0100 Subject: [PATCH 363/543] openwrt: clarify cli usage --- openwrt/openwrt-add-usb-lte.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-add-usb-lte.sh b/openwrt/openwrt-add-usb-lte.sh index 808700b..130fdcf 100755 --- a/openwrt/openwrt-add-usb-lte.sh +++ b/openwrt/openwrt-add-usb-lte.sh @@ -5,7 +5,7 @@ if [ $# -ne 2 ]; then echo "$0 ip-address interface" echo " ip-address: where to find the OpenWRT device" echo " interface: which interface is the LTE device" - echo " Usually eth1 on VIIRB, VIWIB eth3 on PIB" + echo " Usually eth1 on VIIRB and VIWIB; eth3 on PIB" exit 1 fi From dc1f42ab0271b932bbdde79bca8f08ed6d06778a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 1 Apr 2022 15:26:19 +0200 Subject: [PATCH 364/543] Run apk update before adding git and co. --- copy-tools-and-install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/copy-tools-and-install.sh b/copy-tools-and-install.sh index 1434a55..470b38c 100755 --- a/copy-tools-and-install.sh +++ b/copy-tools-and-install.sh @@ -13,7 +13,8 @@ mode=$1; shift cat < /etc/resolv.conf -apk add git sfdisk sudo util-linux-misc wget +apk update +apk add git sfdisk sudo wget cd /opt if [ ! -d /opt/ungleich-tools ]; then git clone https://code.ungleich.ch/ungleich-public/ungleich-tools.git; fi /opt/ungleich-tools/alpine-install-on-disk.sh $disk $key $mode From 048697a8122bbd3a87f484e502b5dea07af70ec1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 1 Feb 2022 15:37:23 +0100 Subject: [PATCH 365/543] ++ ceph emergency script --- ceph/start-all-monit-osds.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 ceph/start-all-monit-osds.sh diff --git a/ceph/start-all-monit-osds.sh b/ceph/start-all-monit-osds.sh new file mode 100644 index 0000000..e8b44ba --- /dev/null +++ b/ceph/start-all-monit-osds.sh @@ -0,0 +1,4 @@ +#!/bin/sh +# 2022-02-01 + +for osd in /etc/monit/conf.d/osd.*; do cmd=$(cat $osd | grep "start pro" | sed -e 's/.* = "//' -e 's/".*//'); echo $cmd; ( $cmd & ); done From 10eac966bc198acd03e9d8b7b249079234b47d51 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 17 Apr 2022 19:20:32 +0200 Subject: [PATCH 366/543] [viwib+vigir] update openwrt to 21.02.1 --- openwrt/vigir-1-firmware-upgrade.sh | 2 +- openwrt/viwib-1-firmware-upgrade.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index 3345852..a93bb4b 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,7 +12,7 @@ set -x vigir_ip=$1; shift # openwrt -version=21.02.0 +version=21.02.2 filename=openwrt-${version}-ramips-mt7621-zbtlink_zbt-wg3526-16m-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 57ddc70..a486d0b 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=21.02.1 +version=21.02.2 filename=openwrt-${version}-ramips-mt76x8-glinet_gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries From 03afb6161bbb8c3ba3edd4b9fd2a10f3af7de45b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 17 Apr 2022 19:20:53 +0200 Subject: [PATCH 367/543] Add a ceph start script --- ceph/start-all-monit-osds.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ceph/start-all-monit-osds.sh b/ceph/start-all-monit-osds.sh index e8b44ba..d1108fd 100644 --- a/ceph/start-all-monit-osds.sh +++ b/ceph/start-all-monit-osds.sh @@ -1,4 +1,12 @@ #!/bin/sh # 2022-02-01 +/etc/init.d/lvm2 start + + +/opt/ungleich-tools/ceph/ceph-osd-activate-all + +ceph-volume lvm activate --all + + for osd in /etc/monit/conf.d/osd.*; do cmd=$(cat $osd | grep "start pro" | sed -e 's/.* = "//' -e 's/".*//'); echo $cmd; ( $cmd & ); done From 2828b4e79d38794d19fc60da4d418596a9bc61ab Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 May 2022 14:02:27 +0200 Subject: [PATCH 368/543] FreeBSD: bump to 13.0 --- opennebula-images/freebsd-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 535d2a6..a03d5d3 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -7,7 +7,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=12.1-RELEASE +RELEASE=13.0-RELEASE ARCH=amd64 IMAGE_PATH=freebsd-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G From f31189a08db3c971376301e2d51b0a4f3f58dadd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 May 2022 10:17:56 +0200 Subject: [PATCH 369/543] [viwib] allow ssh in for people to remotely access their router --- openwrt/viwib-2-configure.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/openwrt/viwib-2-configure.sh b/openwrt/viwib-2-configure.sh index 47a93ef..1f1711c 100755 --- a/openwrt/viwib-2-configure.sh +++ b/openwrt/viwib-2-configure.sh @@ -174,6 +174,15 @@ uci set firewall.@rule[-1].dest_port='80' uci set firewall.@rule[-1].target='ACCEPT' fi +if ! uci show firewall | grep "name='Allow-SSH-in'"; then +uci add firewall rule +uci set firewall.@rule[-1].name='Allow-SSH-in' +uci set firewall.@rule[-1].src='wan' +uci set firewall.@rule[-1].proto='tcp' +uci set firewall.@rule[-1].dest_port='22' +uci set firewall.@rule[-1].target='ACCEPT' +fi + # Add interfaces to the right network zone uci set firewall.@zone[1].network='wan wan6 wg0' From 0d7367df9ad9d00c0eb3e51e02d75bb42a9ebbe8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 May 2022 14:04:50 +0200 Subject: [PATCH 370/543] freebsd: also bump the ports release --- opennebula-images/freebsd-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index a03d5d3..c76f9fc 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -13,7 +13,7 @@ IMAGE_PATH=freebsd-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" -PORTS_BASE="https://download.freebsd.org/ftp/snapshots/$ARCH/12.1-STABLE" +PORTS_BASE="https://download.freebsd.org/ftp/snapshots/$ARCH/13.0-STABLE" ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.12.0/one-context-5.12.0_1.txz" From 1724933e877fa07a780bdc9a384f04c7538b846d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 May 2022 10:23:16 +0200 Subject: [PATCH 371/543] openwrt/ipv4: output peer section at the end --- openwrt/openwrt-add-ipv4-vpn.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/openwrt/openwrt-add-ipv4-vpn.sh b/openwrt/openwrt-add-ipv4-vpn.sh index c97050f..0b1f795 100755 --- a/openwrt/openwrt-add-ipv4-vpn.sh +++ b/openwrt/openwrt-add-ipv4-vpn.sh @@ -24,10 +24,10 @@ public_key=$(echo $private_key | wg pubkey) case $my_wireguard_ip in 185.155.29.*) - vpn_endpoint_pubkey=6BRnQ+dmeFzVCH9RbM1pbJ7u3y3qrl+zUzzYCmC88kE= + vpn_endpoint_pubkey="6BRnQ+dmeFzVCH9RbM1pbJ7u3y3qrl+zUzzYCmC88kE=" ;; 185.155.30.*) - vpn_endpoint_pubkey=5ach7pUQ57aa402LHz1MYh7lyBZS0GvBEw2PC6dMHW4= + vpn_endpoint_pubkey="5ach7pUQ57aa402LHz1MYh7lyBZS0GvBEw2PC6dMHW4=" ;; *) echo "Unknown VPN host for IP $my_wireguard_ip" >&2 @@ -77,4 +77,10 @@ uci commit EOF -echo "Host ${my_ip} uses ip ${my_wireguard_ip} with public key ${public_key}" +echo "Host ${my_ip} uses ip ${my_wireguard_ip} with public key ${public_key}:" + +cat < Date: Mon, 2 May 2022 10:30:03 +0200 Subject: [PATCH 372/543] openwrt/ipv4: add whitespace --- openwrt/openwrt-add-ipv4-vpn.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/openwrt/openwrt-add-ipv4-vpn.sh b/openwrt/openwrt-add-ipv4-vpn.sh index 0b1f795..9e6fb52 100755 --- a/openwrt/openwrt-add-ipv4-vpn.sh +++ b/openwrt/openwrt-add-ipv4-vpn.sh @@ -80,6 +80,8 @@ EOF echo "Host ${my_ip} uses ip ${my_wireguard_ip} with public key ${public_key}:" cat < Date: Mon, 9 May 2022 14:53:51 +0000 Subject: [PATCH 373/543] [VNC] update get_info.py for Task#10544 --- vnc_console_connection/get_info.py | 41 +++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/vnc_console_connection/get_info.py b/vnc_console_connection/get_info.py index e98ae72..466417b 100755 --- a/vnc_console_connection/get_info.py +++ b/vnc_console_connection/get_info.py @@ -6,6 +6,7 @@ from xmltodict import parse from config import config from ldap_list import vm_list from db_export import setconn +from db_export import delconn # Constants ALL_VM_STATES = -1 @@ -61,6 +62,22 @@ class VM: } +class tVM: + def __init__(self, tvm: dict): + self.id = vm.get('ID', None) + self.owner = { + 'id': vm.get('UID', None), + 'name': vm.get('UNAME', None), + 'gname': vm.get('GNAME', None) + } + self.name = vm.get('NAME', None) + self.status = vm.get('STATE', None) + if self.status: + self.status = VMState(int(self.status)).name.lower() + + template = vm['TEMPLATE'] + + def main(): with RPCClient(opnserver) as rpc_client: success, response, *_ = rpc_client.one.vmpool.infoextended( @@ -74,15 +91,33 @@ def main(): vm_user = vm['UNAME'] vm_id = vm['ID'] vm_port = vm['TEMPLATE']['GRAPHICS'].get('PORT') - vm_host = vm['HISTORY_RECORDS']['HISTORY']['HOSTNAME'] + vm_host = vm['HISTORY_RECORDS']['HISTORY']['HOSTNAME'] if vm['UNAME'] == temp_uname: - #print(entry.uid, vm_id, vm_port, vm_host) + print(entry.uid, vm_id, vm_port, vm_host) setconn(entry.uid, vm_id, vm_port, vm_host) - + else: print(response) + with RPCClient(opnserver) as rpc_client2: + success, response, *_ = rpc_client2.one.vmpool.infoextended( + session_string , VmFilterFlag.AllResources.value, START_ID, END_ID, VMState.DONE.value + ) + if success: + vms2 = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + for entry in vm_list.entries: + temp_uname = entry.uid + for i, tvm in enumerate(vms2): + vm_user = tvm['UNAME'] + vm_id = tvm['ID'] + if tvm['UNAME'] == temp_uname: + print("terminated VM : ", entry.uid, vm_id) + delconn(entry.uid, vm_id) + + else: + print(response) if __name__ == "__main__": main() + \ No newline at end of file From 16d7e8013360caa51cfe25abcb12d59a593c8978 Mon Sep 17 00:00:00 2001 From: kjg Date: Mon, 9 May 2022 14:55:11 +0000 Subject: [PATCH 374/543] [VNC] update db_export.py for Task#10544 --- vnc_console_connection/db_export.py | 47 ++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/vnc_console_connection/db_export.py b/vnc_console_connection/db_export.py index d283eb4..b7fab12 100755 --- a/vnc_console_connection/db_export.py +++ b/vnc_console_connection/db_export.py @@ -1,5 +1,14 @@ import psycopg2 as pg2 from config import config +import logging + +logger = logging.getLogger() +logger.setLevel(logging.INFO) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') +file_handler = logging.FileHandler('/var/log/desktop.log') +file_handler.setFormatter(formatter) +logger.addHandler(file_handler) + db_name = config['db']['db_name'] db_user = config['db']['db_user'] @@ -10,15 +19,15 @@ db_port = config['db']['db_port'] def setconn(u_id, vm_num, vm_port,vm_host): conn = pg2.connect("host = localhost dbname={} user={} password={} port={}".format(db_name,db_user,db_password,db_port)) conn.autocommit = True - cur = conn.cursor() + cur = conn.cursor() cur.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) row = cur.fetchone() if row == None: - cur.execute("INSERT INTO guacamole_entity (name, type) VALUES ('{}','USER')".format(u_id)) + cur.execute("INSERT INTO guacamole_entity (name, type) VALUES ('{}','USER')".format(u_id)) cur.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) row = cur.fetchone() en_id = row[0] - cur.execute("INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('{}', '\x74657374', now())".format(en_id)) + cur.execute("INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('{}', '\x74657374', now())".format(en_id)) print("create user : " , u_id) else: en_id = row[0] @@ -43,7 +52,9 @@ def setconn(u_id, vm_num, vm_port,vm_host): #clipboard-encoding cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','clipboard-encoding','UTF-8')".format(cn_id)) print("create connection") - else: + log = "create connection : " + cn + logging.info(log) + else: cur.execute("SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '{}' AND parent_id IS NULL".format(cn)) temp_cn_id = cur.fetchone() cn_id = temp_cn_id[0] @@ -52,4 +63,30 @@ def setconn(u_id, vm_num, vm_port,vm_host): #cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='UTF-8' where connection_id='{}' and parameter_name='clipboard-encoding'".format(cn_id)) print("no connection") conn.close() - return None \ No newline at end of file + return None + + +def delconn(u_id, vm_num): + conn2 = pg2.connect("host = localhost dbname={} user={} password={} port={}".format(db_name,db_user,db_password,db_port)) + conn2.autocommit = True + cur2 = conn2.cursor() + cur2.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) + row2 = cur2.fetchone() + if row2 == None: + print("no user : " , u_id) + else: + cn2 = "{}{}".format(u_id,vm_num) + cur2.execute("SELECT connection_id FROM guacamole_connection WHERE connection_name = '{}'".format(cn2)) + row2 = cur2.fetchone() + if row2 != None: + print("cn_id : ", row2[0]) + #delete connection + cur2.execute("SELECT connection_id from guacamole_connection_permission where connection_id = '{}'".format(row2[0])) + row2 = cur2.fetchone() + if row2 != None: + print("delete connection : ",row2[0]) + cur2.execute("delete from guacamole_connection_permission where connection_id = '{}'".format(row2[0])) + log = "delete connection : " + cn2 + logging.info(log) + conn2.close() + \ No newline at end of file From d5d327b93655f4929f06864e3ab87e6040e7af72 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 3 Jun 2022 17:15:11 +0200 Subject: [PATCH 375/543] Update ubuntu build script to build 22.04 --- opennebula-images/ubuntu-build-opennebula-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennebula-images/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh index 01a4c3b..6f1db12 100755 --- a/opennebula-images/ubuntu-build-opennebula-image.sh +++ b/opennebula-images/ubuntu-build-opennebula-image.sh @@ -9,14 +9,14 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=groovy # 20.10 +RELEASE=jammy # 22.04 LTS ARCH=amd64 IMAGE_PATH=ubuntu-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 # TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.4.0/one-context_6.4.0-1.deb" ONE_CONTEXT_DEB_PATH=/root/one-context.deb cleanup() { From 4fbd39ea1e4f3b9f73cadadd60e4427dae40ac83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 13:04:27 +0200 Subject: [PATCH 376/543] Add FreeBSD on ZFS install script --- .../freebsd-zfs-build-opennebula-image.sh | 140 ++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 opennebula-images/freebsd-zfs-build-opennebula-image.sh diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh new file mode 100644 index 0000000..a58edac --- /dev/null +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -0,0 +1,140 @@ +#!/bin/sh + +# This script generates FreeBSD images for OpenNebula, being heavily inspired +# from srht's FreeBSD build image definition. It assumes running on a FreeBSD host. +# ZFS installation as documented by the FreeBSD project +# https://wiki.freebsd.org/RootOnZFS/GPTZFSBoot + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=13.0-RELEASE +ARCH=amd64 +IMAGE_PATH=freebsd-$RELEASE-$(date -I).img.qcow2 +IMAGE_SIZE=10G + +DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" +ZPOOL=zroot + +ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.2.0/one-context-6.2.0_1.txz" + +if [ "$(zpool list -Ho name $ZPOOL)" = $ZPOOL ]; then + echo "The pool $ZPOOL is already imported." >&2 + exit 1 +fi + +cleanup() { + sync || true + umount /mnt/dev || true + zpool export $ZPOOL || true + mdconfig -du md0 || true +} +trap cleanup EXIT + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +# Allocate and partition/format disk image. +# We use "legacy boot", aka BIOS boot +# Preferably, we'd use EFI boot here, check the FreeBSD wiki link in the header +# to see how to make that change, but make the EFI partition larger +disk=$(mktemp) +truncate -s 6G $disk +mdconfig -a -t vnode -f $disk -u md0 +gpart create -s gpt /dev/md0 +gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 +gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 md0 +gpart add -t freebsd-zfs -l zfs0 -b 1M -s 5G md0 +zpool create -O compression=on -o ashift=12 -o altroot=/mnt -m none $ZPOOL md0p2 + +zfs create -o mountpoint=none $ZPOOL/ROOT +zfs create -o mountpoint=/ -o canmount=noauto $ZPOOL/ROOT/default +mount -t zfs $ZPOOL/ROOT/default /mnt +zpool set bootfs=$ZPOOL/ROOT/default $ZPOOL + +zfs create -o mountpoint=/tmp -o exec=on -o setuid=off $ZPOOL/tmp +zfs create -o canmount=off -o mountpoint=/usr $ZPOOL/usr +zfs create $ZPOOL/usr/home +zfs create -o exec=off -o setuid=off $ZPOOL/usr/src +zfs create $ZPOOL/usr/obj +zfs create -o mountpoint=/usr/ports -o setuid=off $ZPOOL/usr/ports +zfs create -o exec=off -o setuid=off $ZPOOL/usr/ports/distfiles +zfs create -o exec=off -o setuid=off $ZPOOL/usr/ports/packages +zfs create -o canmount=off -o mountpoint=/var $ZPOOL/var +zfs create -o exec=off -o setuid=off $ZPOOL/var/audit +zfs create -o exec=off -o setuid=off $ZPOOL/var/crash +zfs create -o exec=off -o setuid=off $ZPOOL/var/log +zfs create -o atime=on -o exec=off -o setuid=off $ZPOOL/var/mail +zfs create -o exec=on -o setuid=off $ZPOOL/var/tmp + +ln -s /usr/home /mnt/home +chmod 1777 /mnt/var/tmp +chmod 1777 /mnt/tmp + +# Mount allocated image. +mkdir -p /mnt/dev +mount -t devfs devfs /mnt/dev + +# Download and extract base system. +dist_files="kernel.txz base.txz" +dist_dir="/usr/freebsd-dist/$ARCH/$RELEASE" + +mkdir -p "$dist_dir" +for f in $dist_files +do + fetch -m -o "$dist_dir/$f" "$DIST_BASE/$f" + tar -C /mnt -xJf "$dist_dir/$f" +done + +# Configure new system. +printf '# Device\tMountpoint\tFStype\tOptions\tDump\tPass#\n' >/mnt/etc/fstab +touch /mnt/firstboot +sysrc -f /mnt/boot/loader.conf zfs_load=YES autoboot_delay=-1 + +sysrc -f /mnt/etc/rc.conf ntpd_enable=YES sshd_enable=YES growfs_enable=YES hostname=freebsd + +cp /etc/resolv.conf /mnt/etc/resolv.conf +tzsetup -s -C /mnt UTC + +cat >>/mnt/etc/ssh/sshd_config </mnt/usr/local/etc/pkg/repos/FreeBSD.conf + +# freebsd-update is only supported for RELEASE +if [ "${release%-RELEASE}" != "$RELEASE" ] +then + env PAGER=true /usr/sbin/freebsd-update \ + -b /mnt \ + --currently-running "$RELEASE" \ + --not-running-from-cron -F \ + fetch install +fi + +env ASSUME_ALWAYS_YES=YES pkg -c /mnt bootstrap -f + +fetch -m -o /mnt/one-context.txz "$ONE_CONTEXT_PKG_URL" +# OpenNebula has dependencies, but these are not included in the package for some reason +# https://github.com/OpenNebula/addon-context-linux/blob/40efc929487b2955e6f32643853a5cdc93c548da/targets.sh#L25 +# It would be useful to see if there is an alternative to OpenNebula without so many dependencies, +# so we can run on FreeBSD base, and avoid breaking OpenNebula when the admin removes a dependency. +env ASSUME_ALWAYS_YES=YES pkg -c /mnt install sudo bash curl base64 ruby open-vm-tools-nox11 gawk virt-what one-context.txz +rm /mnt/one-context.txz + +cleanup +trap : EXIT + +mkdir -p "$ARCH" +qemu-img convert -f raw -O qcow2 "$disk" "$IMAGE_PATH" +rm "$disk" + +# Filesystem will be enlarged by growfs(7) on next startup +qemu-img resize "$IMAGE_PATH" "$IMAGE_SIZE" From 059a0d6bae81f23cded593451140ef8f49f5e115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 14:22:45 +0200 Subject: [PATCH 377/543] Fix overly zealous double-zpool check --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index a58edac..fec98e4 100644 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -19,7 +19,7 @@ ZPOOL=zroot ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.2.0/one-context-6.2.0_1.txz" -if [ "$(zpool list -Ho name $ZPOOL)" = $ZPOOL ]; then +if [ "$(zpool list -Ho name $ZPOOL || true)" = $ZPOOL ]; then echo "The pool $ZPOOL is already imported." >&2 exit 1 fi From eaa5ed671d44eb74928d388cd5791c2d87f7fa1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 15:51:22 +0200 Subject: [PATCH 378/543] Make freebsd-zfs-build-opennebula-image.sh executable --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 opennebula-images/freebsd-zfs-build-opennebula-image.sh diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh old mode 100644 new mode 100755 From c4b91aa64ab5ce368287237e25899b595e2f6585 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 15:58:09 +0200 Subject: [PATCH 379/543] Remove caches before creating the image --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index fec98e4..4328416 100755 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -117,6 +117,7 @@ then --currently-running "$RELEASE" \ --not-running-from-cron -F \ fetch install + rm -rf /mnt/var/db/freebsd-update/* fi env ASSUME_ALWAYS_YES=YES pkg -c /mnt bootstrap -f @@ -127,6 +128,7 @@ fetch -m -o /mnt/one-context.txz "$ONE_CONTEXT_PKG_URL" # It would be useful to see if there is an alternative to OpenNebula without so many dependencies, # so we can run on FreeBSD base, and avoid breaking OpenNebula when the admin removes a dependency. env ASSUME_ALWAYS_YES=YES pkg -c /mnt install sudo bash curl base64 ruby open-vm-tools-nox11 gawk virt-what one-context.txz +env ASSUME_ALWAYS_YES=YES pkg -c /mnt clean --all rm /mnt/one-context.txz cleanup From 9e926a90dce2485990bdf86178491bfb2024f3ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 16:02:54 +0200 Subject: [PATCH 380/543] Remove the trap, THEN cleanup --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index 4328416..4964c16 100755 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -131,8 +131,8 @@ env ASSUME_ALWAYS_YES=YES pkg -c /mnt install sudo bash curl base64 ruby open-vm env ASSUME_ALWAYS_YES=YES pkg -c /mnt clean --all rm /mnt/one-context.txz -cleanup trap : EXIT +cleanup mkdir -p "$ARCH" qemu-img convert -f raw -O qcow2 "$disk" "$IMAGE_PATH" From 2263c0df2e696d69496856c8f5344456855c0a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 17:14:27 +0200 Subject: [PATCH 381/543] Shorter duplicate zpool check --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index 4964c16..e0d687b 100755 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -19,7 +19,7 @@ ZPOOL=zroot ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.2.0/one-context-6.2.0_1.txz" -if [ "$(zpool list -Ho name $ZPOOL || true)" = $ZPOOL ]; then +if zpool list -Ho name $ZPOOL 2>/dev/null; then echo "The pool $ZPOOL is already imported." >&2 exit 1 fi From a389fdd795b5bcaff1bc6ae5317a35ddc016c1db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 17:14:42 +0200 Subject: [PATCH 382/543] Install qemu-tools as part of the script --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index e0d687b..363cee3 100755 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -37,6 +37,8 @@ if [ "$(whoami)" != 'root' ]; then exit 1 fi +env ASSUME_ALWAYS_YES=YES pkg install -y qemu-tools + # Allocate and partition/format disk image. # We use "legacy boot", aka BIOS boot # Preferably, we'd use EFI boot here, check the FreeBSD wiki link in the header From ad5e1d5fa3a3d761256ef3608ebd78bf807b9bcd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 17:14:53 +0200 Subject: [PATCH 383/543] Add -zfs to image name --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index 363cee3..fcb9a33 100755 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -11,7 +11,7 @@ set -x # XXX: Handle command-line arguments? RELEASE=13.0-RELEASE ARCH=amd64 -IMAGE_PATH=freebsd-$RELEASE-$(date -I).img.qcow2 +IMAGE_PATH=freebsd-zfs-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" From 805ad9228b23fa648cd5cda98154b6985bb955f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 1 May 2022 22:19:23 +0200 Subject: [PATCH 384/543] Do not create empty pkg directory --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index fcb9a33..41b845f 100755 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -107,8 +107,8 @@ PasswordAuthentication no PermitEmptyPasswords no EOF -mkdir -p /mnt/usr/local/etc/pkg/repos/ # It doesn't appear to be necessary to use "latest", "quarterly" is new enough +#mkdir -p /mnt/usr/local/etc/pkg/repos/ #sed -es@quarterly@latest@ /mnt/usr/local/etc/pkg/repos/FreeBSD.conf # freebsd-update is only supported for RELEASE From 33cc65ecc266068aa1e8f92d8db25e18c9f612a8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jun 2022 08:37:24 +0200 Subject: [PATCH 385/543] Upgrade to FreeBSD 13 and sync with ZFS script --- .../freebsd-build-opennebula-image.sh | 40 ++++++++----------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index c76f9fc..af5e6a8 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -13,9 +13,8 @@ IMAGE_PATH=freebsd-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" -PORTS_BASE="https://download.freebsd.org/ftp/snapshots/$ARCH/13.0-STABLE" -ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.12.0/one-context-5.12.0_1.txz" +ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.2.0/one-context-6.2.0_1.txz" cleanup() { sync || true @@ -30,6 +29,8 @@ if [ "$(whoami)" != 'root' ]; then exit 1 fi +env ASSUME_ALWAYS_YES=YES pkg install -y qemu-tools + # Allocate and partition/format disk image. disk=$(mktemp) truncate -s 6G $disk @@ -59,14 +60,9 @@ done # Configure new system. echo "/dev/gpt/rootfs / ufs rw,noatime 1 1" >/mnt/etc/fstab touch /mnt/firstboot -echo 'autoboot_delay="-1"' >>/mnt/boot/loader.conf +sysrc -f /mnt/boot/loader.conf autoboot_delay=-1 -cat >>/mnt/etc/rc.conf </mnt/usr/local/etc/pkg/repos/FreeBSD.conf </mnt/usr/local/etc/pkg/repos/FreeBSD.conf # freebsd-update is only supported for RELEASE if [ "${release%-RELEASE}" != "$RELEASE" ] @@ -93,20 +85,22 @@ then --currently-running "$RELEASE" \ --not-running-from-cron -F \ fetch install + rm -rf /mnt/var/db/freebsd-update/* fi env ASSUME_ALWAYS_YES=YES pkg -c /mnt bootstrap -f -env ASSUME_ALWAYS_YES=YES pkg -c /mnt install bash curl -curl -L "$ONE_CONTEXT_PKG_URL" -o /mnt/one-context.txz -env ASSUME_ALWAYS_YES=YES pkg -c /mnt add one-context.txz +fetch -m -o /mnt/one-context.txz "$ONE_CONTEXT_PKG_URL" +# OpenNebula has dependencies, but these are not included in the package for some reason +# https://github.com/OpenNebula/addon-context-linux/blob/40efc929487b2955e6f32643853a5cdc93c548da/targets.sh#L25 +# It would be useful to see if there is an alternative to OpenNebula without so many dependencies, +# so we can run on FreeBSD base, and avoid breaking OpenNebula when the admin removes a dependency. +env ASSUME_ALWAYS_YES=YES pkg -c /mnt install sudo bash curl base64 ruby open-vm-tools-nox11 gawk virt-what one-context.txz +env ASSUME_ALWAYS_YES=YES pkg -c /mnt clean --all rm /mnt/one-context.txz -fetch -m -o "$dist_dir/ports.txz" "$PORTS_BASE/ports.txz" -tar -C /mnt -xJf "$dist_dir/ports.txz" - -cleanup trap : EXIT +cleanup mkdir -p "$ARCH" qemu-img convert -f raw -O qcow2 "$disk" "$IMAGE_PATH" From f24303d021b6af46870be237b58bdb6d01e96ebd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Mon, 2 May 2022 09:27:59 +0200 Subject: [PATCH 386/543] Make the image zstd-19 compressed --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index 41b845f..d3a2847 100755 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -53,7 +53,9 @@ gpart add -t freebsd-zfs -l zfs0 -b 1M -s 5G md0 zpool create -O compression=on -o ashift=12 -o altroot=/mnt -m none $ZPOOL md0p2 zfs create -o mountpoint=none $ZPOOL/ROOT -zfs create -o mountpoint=/ -o canmount=noauto $ZPOOL/ROOT/default +# We set zstd-19 so our image will become smaller, at the cost of a longer build time +# At the end, we remove zstd-19 again, but all files already written will remain zstd-19 compressed +zfs create -o mountpoint=/ -o canmount=noauto -o compression=zstd-19 $ZPOOL/ROOT/default mount -t zfs $ZPOOL/ROOT/default /mnt zpool set bootfs=$ZPOOL/ROOT/default $ZPOOL @@ -133,6 +135,11 @@ env ASSUME_ALWAYS_YES=YES pkg -c /mnt install sudo bash curl base64 ruby open-vm env ASSUME_ALWAYS_YES=YES pkg -c /mnt clean --all rm /mnt/one-context.txz +# Remove zstd-19 again, as it would be too slow for daily use. +# But all files that were already writtne will remain zstd-19 compressed. +# zstd-19 is slow to compress but fast to read. +zfs inherit compression $ZPOOL/ROOT/default + trap : EXIT cleanup From 31431ef8db7be6784c5b60edd5c4031c0e8761d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Mon, 2 May 2022 09:43:56 +0200 Subject: [PATCH 387/543] Remove mountpoints not in an 13.0-RELASE install --- opennebula-images/freebsd-zfs-build-opennebula-image.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh index d3a2847..39992ce 100755 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ b/opennebula-images/freebsd-zfs-build-opennebula-image.sh @@ -63,10 +63,7 @@ zfs create -o mountpoint=/tmp -o exec=on -o setuid=off $ZPOOL/tmp zfs create -o canmount=off -o mountpoint=/usr $ZPOOL/usr zfs create $ZPOOL/usr/home zfs create -o exec=off -o setuid=off $ZPOOL/usr/src -zfs create $ZPOOL/usr/obj zfs create -o mountpoint=/usr/ports -o setuid=off $ZPOOL/usr/ports -zfs create -o exec=off -o setuid=off $ZPOOL/usr/ports/distfiles -zfs create -o exec=off -o setuid=off $ZPOOL/usr/ports/packages zfs create -o canmount=off -o mountpoint=/var $ZPOOL/var zfs create -o exec=off -o setuid=off $ZPOOL/var/audit zfs create -o exec=off -o setuid=off $ZPOOL/var/crash From 7bac0537572b30034ee3c47380db7973ae3f1403 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Tue, 24 May 2022 23:05:09 +0200 Subject: [PATCH 388/543] Update FreeBSD 13.1 and remove dependencies - Upgrade to 13.1-RELEASE - Build ZFS and UFS images in same script - Replace OpenNebula's addon-context-linux script with a new script - New script does not have dependency on bash, Python and Ruby - pkg is no longer preinstalled, but FreeBSD still offers on first invocation --- .../freebsd-build-opennebula-image.sh | 178 +++++++++++++----- .../freebsd-zfs-build-opennebula-image.sh | 148 --------------- 2 files changed, 129 insertions(+), 197 deletions(-) delete mode 100755 opennebula-images/freebsd-zfs-build-opennebula-image.sh diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index af5e6a8..4c4d841 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -2,25 +2,43 @@ # This script generates FreeBSD images for OpenNebula, being heavily inspired # from srht's FreeBSD build image definition. It assumes running on a FreeBSD host. +# ZFS installation as documented by the FreeBSD project +# https://wiki.freebsd.org/RootOnZFS/GPTZFSBoot set -e set -x # XXX: Handle command-line arguments? -RELEASE=13.0-RELEASE +RELEASE=13.1-RELEASE ARCH=amd64 -IMAGE_PATH=freebsd-$RELEASE-$(date -I).img.qcow2 +IMAGE_PATH_ZFS=freebsd-zfs-$RELEASE-$(date -I).img.qcow2 +IMAGE_PATH_UFS=freebsd-ufs-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" +CLSETUP_COMMIT=82d7d16ff14e1893f06f39788bb7cd8604284583 +CLSETUP_URL="https://git.sr.ht/~jornane/clsetup/archive/$CLSETUP_COMMIT.tar.gz" +ZPOOL=zroot -ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.2.0/one-context-6.2.0_1.txz" +ZFSTARGET="$(mktemp -d /var/tmp/zfsbuild.XXXXX)" +UFSTARGET="$(mktemp -d /var/tmp/ufsbuild.XXXXX)" +CLSETUP_WORK="$(mktemp -d /var/tmp/clsetup.XXXXX)" + +if zpool list -Ho name $ZPOOL 2>/dev/null; then + echo "The pool $ZPOOL is already imported." >&2 + exit 1 +fi cleanup() { - sync || true - umount /mnt/dev || true - umount /mnt || true - mdconfig -du md0 || true + sync ||: + umount "$UFSTARGET/dev" ||: + umount "$UFSTARGET/tmp" ||: + umount "$UFSTARGET/var/tmp" ||: + umount "$UFSTARGET" ||: + zpool export $ZPOOL ||: + mdconfig -du md0 ||: + mdconfig -du md1 ||: + rm -rf "$CLSETUP_WORK" } trap cleanup EXIT @@ -29,82 +47,144 @@ if [ "$(whoami)" != 'root' ]; then exit 1 fi -env ASSUME_ALWAYS_YES=YES pkg install -y qemu-tools +if ! command -v rsync >/dev/null +then + env ASSUME_ALWAYS_YES=YES pkg install -y rsync +fi +if ! command -v qemu-img >/dev/null +then + env ASSUME_ALWAYS_YES=YES pkg install -y qemu-tools +fi -# Allocate and partition/format disk image. -disk=$(mktemp) -truncate -s 6G $disk -mdconfig -a -t vnode -f $disk -u md0 -gpart create -s gpt /dev/md0 -gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 -gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md0 -gpart add -t freebsd-ufs -l rootfs -b 1M -s 5G md0 -newfs -U /dev/md0p2 +fetch -qo- "$CLSETUP_URL" | tar -C "$CLSETUP_WORK" --strip-components 1 -xzf- + +ufsdisk=$(mktemp /var/tmp/ufsdisk.XXXXX) +truncate -s 6G $ufsdisk +mdconfig -a -t vnode -f $ufsdisk -u md1 +gpart create -s gpt /dev/md1 +gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md1 +gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md1 +gpart add -t freebsd-ufs -l rootfs -b 1M -s 5G md1 +newfs -U /dev/md1p2 # Mount allocated image. -mount /dev/md0p2 /mnt -mkdir -p /mnt/dev -mount -t devfs devfs /mnt/dev +mount /dev/md1p2 "$UFSTARGET" + +# Allocate and partition/format disk image. +# We use "legacy boot", aka BIOS boot +# Preferably, we'd use EFI boot here, check the FreeBSD wiki link in the header +# to see how to make that change, but make the EFI partition larger +zfsdisk=$(mktemp /var/tmp/zfsdisk.XXXXX) +truncate -s 6G $zfsdisk +mdconfig -a -t vnode -f $zfsdisk -u md0 +gpart create -s gpt /dev/md0 +gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 +gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 md0 +gpart add -t freebsd-zfs -l zfs0 -b 1M -s 5G md0 +zpool create -O compression=on -o ashift=12 -o altroot="$ZFSTARGET" -m none $ZPOOL md0p2 + +zfs create -o mountpoint=none $ZPOOL/ROOT +# We set zstd-19 so our image will become smaller, at the cost of a longer build time. +# At the end of the process, we disable zstd-19 again using zfs inherit compression, +# but all files already written will remain zstd-19 compressed +zfs create -o mountpoint=/ -o canmount=noauto $ZPOOL/ROOT/default +mount -t zfs $ZPOOL/ROOT/default "$ZFSTARGET" +zpool set bootfs=$ZPOOL/ROOT/default $ZPOOL + +zfs create -o mountpoint=/tmp -o exec=on -o setuid=off $ZPOOL/tmp +zfs create -o canmount=off -o mountpoint=/usr $ZPOOL/usr +zfs create $ZPOOL/usr/home +zfs create -o exec=off -o setuid=off $ZPOOL/usr/src +zfs create -o mountpoint=/usr/ports -o setuid=off $ZPOOL/usr/ports +zfs create -o canmount=off -o mountpoint=/var $ZPOOL/var +zfs create -o exec=off -o setuid=off $ZPOOL/var/audit +zfs create -o exec=off -o setuid=off $ZPOOL/var/crash +zfs create -o exec=off -o setuid=off $ZPOOL/var/log +zfs create -o atime=on -o exec=off -o setuid=off $ZPOOL/var/mail +zfs create -o exec=on -o setuid=off $ZPOOL/var/tmp + +ln -s /usr/home "$ZFSTARGET/home" +chmod 1777 "$ZFSTARGET/var/tmp" +chmod 1777 "$ZFSTARGET/tmp" + +# Mount dev in chroot +mkdir -p "$UFSTARGET/dev" +mount -t devfs devfs "$UFSTARGET/dev" # Download and extract base system. dist_files="kernel.txz base.txz" dist_dir="/usr/freebsd-dist/$ARCH/$RELEASE" -mkdir -p "$dist_dir" +mkdir -p "$dist_dir" "$UFSTARGET" for f in $dist_files do fetch -m -o "$dist_dir/$f" "$DIST_BASE/$f" - tar -C /mnt -xJf "$dist_dir/$f" + tar -C "$UFSTARGET" -xJf "$dist_dir/$f" done +# Avoid writing temporary files while building +mount_nullfs /tmp "$UFSTARGET/tmp" +mount_nullfs /var/tmp "$UFSTARGET/var/tmp" + +# Install the first-boot script that configures the network and ssh key +make -C "$CLSETUP_WORK/" PREFIX="$UFSTARGET/usr/local" install + # Configure new system. -echo "/dev/gpt/rootfs / ufs rw,noatime 1 1" >/mnt/etc/fstab -touch /mnt/firstboot -sysrc -f /mnt/boot/loader.conf autoboot_delay=-1 +printf '# Device\tMountpoint\tFStype\tOptions\t\tDump\tPass#\n' >"$UFSTARGET/etc/fstab" +touch "$UFSTARGET/firstboot" +sysrc -f "$UFSTARGET/boot/loader.conf" \ + zfs_load=YES \ + autoboot_delay=-1 \ -sysrc -f /mnt/etc/rc.conf ntpd_enable=YES sshd_enable=YES growfs_enable=YES hostname=freebsd +sysrc -f "$UFSTARGET/etc/rc.conf" \ + ntpd_enable=YES \ + sshd_enable=YES \ + growfs_enable=YES \ + hostname=freebsd \ + firstboot_clsetup_enable=YES \ -cp /etc/resolv.conf /mnt/etc/resolv.conf -tzsetup -s -C /mnt UTC +# The resolv.conf file is written by firstboot_clsetup +#cp /etc/resolv.conf "$UFSTARGET/etc/resolv.conf" -cat >>/mnt/etc/ssh/sshd_config <>"$UFSTARGET/etc/ssh/sshd_config" </mnt/usr/local/etc/pkg/repos/FreeBSD.conf - # freebsd-update is only supported for RELEASE -if [ "${release%-RELEASE}" != "$RELEASE" ] +if printf %s "$RELEASE" | grep -q '.-RELEASE$' then env PAGER=true /usr/sbin/freebsd-update \ - -b /mnt \ + -b "$UFSTARGET" \ --currently-running "$RELEASE" \ --not-running-from-cron -F \ fetch install - rm -rf /mnt/var/db/freebsd-update/* fi +rm -rf "$UFSTARGET/var/db/freebsd-update/"* ||: -env ASSUME_ALWAYS_YES=YES pkg -c /mnt bootstrap -f - -fetch -m -o /mnt/one-context.txz "$ONE_CONTEXT_PKG_URL" -# OpenNebula has dependencies, but these are not included in the package for some reason -# https://github.com/OpenNebula/addon-context-linux/blob/40efc929487b2955e6f32643853a5cdc93c548da/targets.sh#L25 -# It would be useful to see if there is an alternative to OpenNebula without so many dependencies, -# so we can run on FreeBSD base, and avoid breaking OpenNebula when the admin removes a dependency. -env ASSUME_ALWAYS_YES=YES pkg -c /mnt install sudo bash curl base64 ruby open-vm-tools-nox11 gawk virt-what one-context.txz -env ASSUME_ALWAYS_YES=YES pkg -c /mnt clean --all -rm /mnt/one-context.txz +# Set zstd-19 compression, copy all data to the pool, and then set compression to default again +# This will make the base image smaller, at the cost of taking longer to generate, as zstd-19 is slow to write +# Therefore, afterwards we restore compression to default, so written files stay zstd-19, which is fast to read, +# but files written by the user afterwards will be written with the default compression algorihtm. +zfs set compression=zstd-19 $ZPOOL/ROOT/default +umount "$UFSTARGET/dev" "$UFSTARGET/tmp" "$UFSTARGET/var/tmp" +rsync -aH --fileflags --inplace "$UFSTARGET/." "$ZFSTARGET" +sysrc -f "$UFSTARGET/boot/loader.conf" -x zfs_load +printf '%s\t%s\t\t%s\t%s\t%s\t%s\n' /dev/gpt/rootfs / ufs rw,noatime 1 1 >>"$UFSTARGET/etc/fstab" +sync ||: +zfs inherit compression $ZPOOL/ROOT/default trap : EXIT cleanup mkdir -p "$ARCH" -qemu-img convert -f raw -O qcow2 "$disk" "$IMAGE_PATH" -rm "$disk" +qemu-img convert -f raw -O qcow2 "$zfsdisk" "$ARCH/$IMAGE_PATH_ZFS" +qemu-img convert -f raw -O qcow2 "$ufsdisk" "$ARCH/$IMAGE_PATH_UFS" +rm "$zfsdisk" "$ufsdisk" # Filesystem will be enlarged by growfs(7) on next startup -qemu-img resize "$IMAGE_PATH" "$IMAGE_SIZE" +qemu-img resize "$ARCH/$IMAGE_PATH_ZFS" "$IMAGE_SIZE" +qemu-img resize "$ARCH/$IMAGE_PATH_UFS" "$IMAGE_SIZE" diff --git a/opennebula-images/freebsd-zfs-build-opennebula-image.sh b/opennebula-images/freebsd-zfs-build-opennebula-image.sh deleted file mode 100755 index 39992ce..0000000 --- a/opennebula-images/freebsd-zfs-build-opennebula-image.sh +++ /dev/null @@ -1,148 +0,0 @@ -#!/bin/sh - -# This script generates FreeBSD images for OpenNebula, being heavily inspired -# from srht's FreeBSD build image definition. It assumes running on a FreeBSD host. -# ZFS installation as documented by the FreeBSD project -# https://wiki.freebsd.org/RootOnZFS/GPTZFSBoot - -set -e -set -x - -# XXX: Handle command-line arguments? -RELEASE=13.0-RELEASE -ARCH=amd64 -IMAGE_PATH=freebsd-zfs-$RELEASE-$(date -I).img.qcow2 -IMAGE_SIZE=10G - -DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" -ZPOOL=zroot - -ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.2.0/one-context-6.2.0_1.txz" - -if zpool list -Ho name $ZPOOL 2>/dev/null; then - echo "The pool $ZPOOL is already imported." >&2 - exit 1 -fi - -cleanup() { - sync || true - umount /mnt/dev || true - zpool export $ZPOOL || true - mdconfig -du md0 || true -} -trap cleanup EXIT - -if [ "$(whoami)" != 'root' ]; then - echo "This script must be run as root." >&2 - exit 1 -fi - -env ASSUME_ALWAYS_YES=YES pkg install -y qemu-tools - -# Allocate and partition/format disk image. -# We use "legacy boot", aka BIOS boot -# Preferably, we'd use EFI boot here, check the FreeBSD wiki link in the header -# to see how to make that change, but make the EFI partition larger -disk=$(mktemp) -truncate -s 6G $disk -mdconfig -a -t vnode -f $disk -u md0 -gpart create -s gpt /dev/md0 -gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 -gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 md0 -gpart add -t freebsd-zfs -l zfs0 -b 1M -s 5G md0 -zpool create -O compression=on -o ashift=12 -o altroot=/mnt -m none $ZPOOL md0p2 - -zfs create -o mountpoint=none $ZPOOL/ROOT -# We set zstd-19 so our image will become smaller, at the cost of a longer build time -# At the end, we remove zstd-19 again, but all files already written will remain zstd-19 compressed -zfs create -o mountpoint=/ -o canmount=noauto -o compression=zstd-19 $ZPOOL/ROOT/default -mount -t zfs $ZPOOL/ROOT/default /mnt -zpool set bootfs=$ZPOOL/ROOT/default $ZPOOL - -zfs create -o mountpoint=/tmp -o exec=on -o setuid=off $ZPOOL/tmp -zfs create -o canmount=off -o mountpoint=/usr $ZPOOL/usr -zfs create $ZPOOL/usr/home -zfs create -o exec=off -o setuid=off $ZPOOL/usr/src -zfs create -o mountpoint=/usr/ports -o setuid=off $ZPOOL/usr/ports -zfs create -o canmount=off -o mountpoint=/var $ZPOOL/var -zfs create -o exec=off -o setuid=off $ZPOOL/var/audit -zfs create -o exec=off -o setuid=off $ZPOOL/var/crash -zfs create -o exec=off -o setuid=off $ZPOOL/var/log -zfs create -o atime=on -o exec=off -o setuid=off $ZPOOL/var/mail -zfs create -o exec=on -o setuid=off $ZPOOL/var/tmp - -ln -s /usr/home /mnt/home -chmod 1777 /mnt/var/tmp -chmod 1777 /mnt/tmp - -# Mount allocated image. -mkdir -p /mnt/dev -mount -t devfs devfs /mnt/dev - -# Download and extract base system. -dist_files="kernel.txz base.txz" -dist_dir="/usr/freebsd-dist/$ARCH/$RELEASE" - -mkdir -p "$dist_dir" -for f in $dist_files -do - fetch -m -o "$dist_dir/$f" "$DIST_BASE/$f" - tar -C /mnt -xJf "$dist_dir/$f" -done - -# Configure new system. -printf '# Device\tMountpoint\tFStype\tOptions\tDump\tPass#\n' >/mnt/etc/fstab -touch /mnt/firstboot -sysrc -f /mnt/boot/loader.conf zfs_load=YES autoboot_delay=-1 - -sysrc -f /mnt/etc/rc.conf ntpd_enable=YES sshd_enable=YES growfs_enable=YES hostname=freebsd - -cp /etc/resolv.conf /mnt/etc/resolv.conf -tzsetup -s -C /mnt UTC - -cat >>/mnt/etc/ssh/sshd_config </mnt/usr/local/etc/pkg/repos/FreeBSD.conf - -# freebsd-update is only supported for RELEASE -if [ "${release%-RELEASE}" != "$RELEASE" ] -then - env PAGER=true /usr/sbin/freebsd-update \ - -b /mnt \ - --currently-running "$RELEASE" \ - --not-running-from-cron -F \ - fetch install - rm -rf /mnt/var/db/freebsd-update/* -fi - -env ASSUME_ALWAYS_YES=YES pkg -c /mnt bootstrap -f - -fetch -m -o /mnt/one-context.txz "$ONE_CONTEXT_PKG_URL" -# OpenNebula has dependencies, but these are not included in the package for some reason -# https://github.com/OpenNebula/addon-context-linux/blob/40efc929487b2955e6f32643853a5cdc93c548da/targets.sh#L25 -# It would be useful to see if there is an alternative to OpenNebula without so many dependencies, -# so we can run on FreeBSD base, and avoid breaking OpenNebula when the admin removes a dependency. -env ASSUME_ALWAYS_YES=YES pkg -c /mnt install sudo bash curl base64 ruby open-vm-tools-nox11 gawk virt-what one-context.txz -env ASSUME_ALWAYS_YES=YES pkg -c /mnt clean --all -rm /mnt/one-context.txz - -# Remove zstd-19 again, as it would be too slow for daily use. -# But all files that were already writtne will remain zstd-19 compressed. -# zstd-19 is slow to compress but fast to read. -zfs inherit compression $ZPOOL/ROOT/default - -trap : EXIT -cleanup - -mkdir -p "$ARCH" -qemu-img convert -f raw -O qcow2 "$disk" "$IMAGE_PATH" -rm "$disk" - -# Filesystem will be enlarged by growfs(7) on next startup -qemu-img resize "$IMAGE_PATH" "$IMAGE_SIZE" From 2a0b713a78a6d9beaed18e694cda2996518b2657 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Wed, 25 May 2022 10:59:20 +0200 Subject: [PATCH 389/543] Use temporary pool name while installing This will allow running the script when the OS is already running off a pool named "zroot". --- .../freebsd-build-opennebula-image.sh | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 4c4d841..5899a6a 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -19,13 +19,14 @@ DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" CLSETUP_COMMIT=82d7d16ff14e1893f06f39788bb7cd8604284583 CLSETUP_URL="https://git.sr.ht/~jornane/clsetup/archive/$CLSETUP_COMMIT.tar.gz" ZPOOL=zroot +ZPOOL_TMP="zinstalling" ZFSTARGET="$(mktemp -d /var/tmp/zfsbuild.XXXXX)" UFSTARGET="$(mktemp -d /var/tmp/ufsbuild.XXXXX)" CLSETUP_WORK="$(mktemp -d /var/tmp/clsetup.XXXXX)" -if zpool list -Ho name $ZPOOL 2>/dev/null; then - echo "The pool $ZPOOL is already imported." >&2 +if zpool list -Ho name $ZPOOL_TMP 2>/dev/null; then + echo "The pool $ZPOOL_TMP is already imported." >&2 exit 1 fi @@ -35,7 +36,7 @@ cleanup() { umount "$UFSTARGET/tmp" ||: umount "$UFSTARGET/var/tmp" ||: umount "$UFSTARGET" ||: - zpool export $ZPOOL ||: + zpool export $ZPOOL_TMP ||: mdconfig -du md0 ||: mdconfig -du md1 ||: rm -rf "$CLSETUP_WORK" @@ -81,27 +82,27 @@ gpart create -s gpt /dev/md0 gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 md0 gpart add -t freebsd-zfs -l zfs0 -b 1M -s 5G md0 -zpool create -O compression=on -o ashift=12 -o altroot="$ZFSTARGET" -m none $ZPOOL md0p2 +zpool create -O compression=on -o ashift=12 -o altroot="$ZFSTARGET" -m none -t $ZPOOL_TMP $ZPOOL md0p2 -zfs create -o mountpoint=none $ZPOOL/ROOT +zfs create -o mountpoint=none $ZPOOL_TMP/ROOT # We set zstd-19 so our image will become smaller, at the cost of a longer build time. # At the end of the process, we disable zstd-19 again using zfs inherit compression, # but all files already written will remain zstd-19 compressed -zfs create -o mountpoint=/ -o canmount=noauto $ZPOOL/ROOT/default -mount -t zfs $ZPOOL/ROOT/default "$ZFSTARGET" -zpool set bootfs=$ZPOOL/ROOT/default $ZPOOL +zfs create -o mountpoint=/ -o canmount=noauto $ZPOOL_TMP/ROOT/default +mount -t zfs $ZPOOL_TMP/ROOT/default "$ZFSTARGET" +zpool set bootfs=$ZPOOL_TMP/ROOT/default $ZPOOL_TMP -zfs create -o mountpoint=/tmp -o exec=on -o setuid=off $ZPOOL/tmp -zfs create -o canmount=off -o mountpoint=/usr $ZPOOL/usr -zfs create $ZPOOL/usr/home -zfs create -o exec=off -o setuid=off $ZPOOL/usr/src -zfs create -o mountpoint=/usr/ports -o setuid=off $ZPOOL/usr/ports -zfs create -o canmount=off -o mountpoint=/var $ZPOOL/var -zfs create -o exec=off -o setuid=off $ZPOOL/var/audit -zfs create -o exec=off -o setuid=off $ZPOOL/var/crash -zfs create -o exec=off -o setuid=off $ZPOOL/var/log -zfs create -o atime=on -o exec=off -o setuid=off $ZPOOL/var/mail -zfs create -o exec=on -o setuid=off $ZPOOL/var/tmp +zfs create -o mountpoint=/tmp -o exec=on -o setuid=off $ZPOOL_TMP/tmp +zfs create -o canmount=off -o mountpoint=/usr $ZPOOL_TMP/usr +zfs create $ZPOOL_TMP/usr/home +zfs create -o exec=off -o setuid=off $ZPOOL_TMP/usr/src +zfs create -o mountpoint=/usr/ports -o setuid=off $ZPOOL_TMP/usr/ports +zfs create -o canmount=off -o mountpoint=/var $ZPOOL_TMP/var +zfs create -o exec=off -o setuid=off $ZPOOL_TMP/var/audit +zfs create -o exec=off -o setuid=off $ZPOOL_TMP/var/crash +zfs create -o exec=off -o setuid=off $ZPOOL_TMP/var/log +zfs create -o atime=on -o exec=off -o setuid=off $ZPOOL_TMP/var/mail +zfs create -o exec=on -o setuid=off $ZPOOL_TMP/var/tmp ln -s /usr/home "$ZFSTARGET/home" chmod 1777 "$ZFSTARGET/var/tmp" @@ -169,13 +170,13 @@ rm -rf "$UFSTARGET/var/db/freebsd-update/"* ||: # This will make the base image smaller, at the cost of taking longer to generate, as zstd-19 is slow to write # Therefore, afterwards we restore compression to default, so written files stay zstd-19, which is fast to read, # but files written by the user afterwards will be written with the default compression algorihtm. -zfs set compression=zstd-19 $ZPOOL/ROOT/default +zfs set compression=zstd-19 $ZPOOL_TMP/ROOT/default umount "$UFSTARGET/dev" "$UFSTARGET/tmp" "$UFSTARGET/var/tmp" rsync -aH --fileflags --inplace "$UFSTARGET/." "$ZFSTARGET" sysrc -f "$UFSTARGET/boot/loader.conf" -x zfs_load printf '%s\t%s\t\t%s\t%s\t%s\t%s\n' /dev/gpt/rootfs / ufs rw,noatime 1 1 >>"$UFSTARGET/etc/fstab" sync ||: -zfs inherit compression $ZPOOL/ROOT/default +zfs inherit compression $ZPOOL_TMP/ROOT/default trap : EXIT cleanup From 2b715244381a2958859434f689ed82ea9e2c30d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Wed, 25 May 2022 11:04:46 +0200 Subject: [PATCH 390/543] Bump clsetup and rename to cloudsetup --- .../freebsd-build-opennebula-image.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 5899a6a..3d0add4 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -16,14 +16,14 @@ IMAGE_PATH_UFS=freebsd-ufs-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" -CLSETUP_COMMIT=82d7d16ff14e1893f06f39788bb7cd8604284583 -CLSETUP_URL="https://git.sr.ht/~jornane/clsetup/archive/$CLSETUP_COMMIT.tar.gz" +CLOUDSETUP_COMMIT=4ac15b8647d5525048c5faa5fd4b28491905d000 +CLOUDSETUP_URL="https://git.sr.ht/~jornane/cloudsetup/archive/$CLOUDSETUP_COMMIT.tar.gz" ZPOOL=zroot ZPOOL_TMP="zinstalling" ZFSTARGET="$(mktemp -d /var/tmp/zfsbuild.XXXXX)" UFSTARGET="$(mktemp -d /var/tmp/ufsbuild.XXXXX)" -CLSETUP_WORK="$(mktemp -d /var/tmp/clsetup.XXXXX)" +CLOUDSETUP_WORK="$(mktemp -d /var/tmp/cloudsetup.XXXXX)" if zpool list -Ho name $ZPOOL_TMP 2>/dev/null; then echo "The pool $ZPOOL_TMP is already imported." >&2 @@ -39,7 +39,7 @@ cleanup() { zpool export $ZPOOL_TMP ||: mdconfig -du md0 ||: mdconfig -du md1 ||: - rm -rf "$CLSETUP_WORK" + rm -rf "$CLOUDSETUP_WORK" } trap cleanup EXIT @@ -57,7 +57,7 @@ then env ASSUME_ALWAYS_YES=YES pkg install -y qemu-tools fi -fetch -qo- "$CLSETUP_URL" | tar -C "$CLSETUP_WORK" --strip-components 1 -xzf- +fetch -qo- "$CLOUDSETUP_URL" | tar -C "$CLOUDSETUP_WORK" --strip-components 1 -xzf- ufsdisk=$(mktemp /var/tmp/ufsdisk.XXXXX) truncate -s 6G $ufsdisk @@ -128,7 +128,7 @@ mount_nullfs /tmp "$UFSTARGET/tmp" mount_nullfs /var/tmp "$UFSTARGET/var/tmp" # Install the first-boot script that configures the network and ssh key -make -C "$CLSETUP_WORK/" PREFIX="$UFSTARGET/usr/local" install +make -C "$CLOUDSETUP_WORK/" PREFIX="$UFSTARGET/usr/local" install # Configure new system. printf '# Device\tMountpoint\tFStype\tOptions\t\tDump\tPass#\n' >"$UFSTARGET/etc/fstab" @@ -142,9 +142,9 @@ sysrc -f "$UFSTARGET/etc/rc.conf" \ sshd_enable=YES \ growfs_enable=YES \ hostname=freebsd \ - firstboot_clsetup_enable=YES \ + firstboot_cloudsetup_enable=YES \ -# The resolv.conf file is written by firstboot_clsetup +# The resolv.conf file is written by firstboot_cloudsetup #cp /etc/resolv.conf "$UFSTARGET/etc/resolv.conf" tzsetup -s -C "$UFSTARGET" UTC From 02e273faf442b133da3cdced273351cf3070b90a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Wed, 25 May 2022 12:04:36 +0200 Subject: [PATCH 391/543] Remove target directories on cleanup --- opennebula-images/freebsd-build-opennebula-image.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 3d0add4..cb3dc82 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -39,7 +39,9 @@ cleanup() { zpool export $ZPOOL_TMP ||: mdconfig -du md0 ||: mdconfig -du md1 ||: - rm -rf "$CLOUDSETUP_WORK" + rm -rf "$CLOUDSETUP_WORK" ||: + rmdir "$ZFSTARGET" ||: + rmdir "$UFSTARGET" ||: } trap cleanup EXIT From 39ff63706406214439271b97125751db6294f105 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Wed, 25 May 2022 12:09:19 +0200 Subject: [PATCH 392/543] Add quotes around variables --- .../freebsd-build-opennebula-image.sh | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index cb3dc82..a29e5e9 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -11,8 +11,8 @@ set -x # XXX: Handle command-line arguments? RELEASE=13.1-RELEASE ARCH=amd64 -IMAGE_PATH_ZFS=freebsd-zfs-$RELEASE-$(date -I).img.qcow2 -IMAGE_PATH_UFS=freebsd-ufs-$RELEASE-$(date -I).img.qcow2 +IMAGE_PATH_ZFS="freebsd-zfs-$RELEASE-$(date -I).img.qcow2" +IMAGE_PATH_UFS="freebsd-ufs-$RELEASE-$(date -I).img.qcow2" IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" @@ -25,7 +25,7 @@ ZFSTARGET="$(mktemp -d /var/tmp/zfsbuild.XXXXX)" UFSTARGET="$(mktemp -d /var/tmp/ufsbuild.XXXXX)" CLOUDSETUP_WORK="$(mktemp -d /var/tmp/cloudsetup.XXXXX)" -if zpool list -Ho name $ZPOOL_TMP 2>/dev/null; then +if zpool list -Ho name "$ZPOOL_TMP" 2>/dev/null; then echo "The pool $ZPOOL_TMP is already imported." >&2 exit 1 fi @@ -36,7 +36,7 @@ cleanup() { umount "$UFSTARGET/tmp" ||: umount "$UFSTARGET/var/tmp" ||: umount "$UFSTARGET" ||: - zpool export $ZPOOL_TMP ||: + zpool export "$ZPOOL_TMP" ||: mdconfig -du md0 ||: mdconfig -du md1 ||: rm -rf "$CLOUDSETUP_WORK" ||: @@ -61,9 +61,9 @@ fi fetch -qo- "$CLOUDSETUP_URL" | tar -C "$CLOUDSETUP_WORK" --strip-components 1 -xzf- -ufsdisk=$(mktemp /var/tmp/ufsdisk.XXXXX) -truncate -s 6G $ufsdisk -mdconfig -a -t vnode -f $ufsdisk -u md1 +ufsdisk="$(mktemp /var/tmp/ufsdisk.XXXXX)" +truncate -s 6G "$ufsdisk" +mdconfig -a -t vnode -f "$ufsdisk" -u md1 gpart create -s gpt /dev/md1 gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md1 gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md1 @@ -77,34 +77,34 @@ mount /dev/md1p2 "$UFSTARGET" # We use "legacy boot", aka BIOS boot # Preferably, we'd use EFI boot here, check the FreeBSD wiki link in the header # to see how to make that change, but make the EFI partition larger -zfsdisk=$(mktemp /var/tmp/zfsdisk.XXXXX) -truncate -s 6G $zfsdisk -mdconfig -a -t vnode -f $zfsdisk -u md0 +zfsdisk="$(mktemp /var/tmp/zfsdisk.XXXXX)" +truncate -s 6G "$zfsdisk" +mdconfig -a -t vnode -f "$zfsdisk" -u md0 gpart create -s gpt /dev/md0 gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 md0 gpart add -t freebsd-zfs -l zfs0 -b 1M -s 5G md0 -zpool create -O compression=on -o ashift=12 -o altroot="$ZFSTARGET" -m none -t $ZPOOL_TMP $ZPOOL md0p2 +zpool create -O compression=on -o ashift=12 -o "altroot=$ZFSTARGET" -m none -t "$ZPOOL_TMP" "$ZPOOL" md0p2 -zfs create -o mountpoint=none $ZPOOL_TMP/ROOT +zfs create -o mountpoint=none "$ZPOOL_TMP/ROOT" # We set zstd-19 so our image will become smaller, at the cost of a longer build time. # At the end of the process, we disable zstd-19 again using zfs inherit compression, # but all files already written will remain zstd-19 compressed -zfs create -o mountpoint=/ -o canmount=noauto $ZPOOL_TMP/ROOT/default -mount -t zfs $ZPOOL_TMP/ROOT/default "$ZFSTARGET" -zpool set bootfs=$ZPOOL_TMP/ROOT/default $ZPOOL_TMP +zfs create -o mountpoint=/ -o canmount=noauto "$ZPOOL_TMP/ROOT/default" +mount -t zfs "$ZPOOL_TMP/ROOT/default" "$ZFSTARGET" +zpool set "bootfs=$ZPOOL_TMP/ROOT/default" "$ZPOOL_TMP" -zfs create -o mountpoint=/tmp -o exec=on -o setuid=off $ZPOOL_TMP/tmp -zfs create -o canmount=off -o mountpoint=/usr $ZPOOL_TMP/usr -zfs create $ZPOOL_TMP/usr/home -zfs create -o exec=off -o setuid=off $ZPOOL_TMP/usr/src -zfs create -o mountpoint=/usr/ports -o setuid=off $ZPOOL_TMP/usr/ports -zfs create -o canmount=off -o mountpoint=/var $ZPOOL_TMP/var -zfs create -o exec=off -o setuid=off $ZPOOL_TMP/var/audit -zfs create -o exec=off -o setuid=off $ZPOOL_TMP/var/crash -zfs create -o exec=off -o setuid=off $ZPOOL_TMP/var/log -zfs create -o atime=on -o exec=off -o setuid=off $ZPOOL_TMP/var/mail -zfs create -o exec=on -o setuid=off $ZPOOL_TMP/var/tmp +zfs create -o mountpoint=/tmp -o exec=on -o setuid=off "$ZPOOL_TMP/tmp" +zfs create -o canmount=off -o mountpoint=/usr "$ZPOOL_TMP/usr" +zfs create "$ZPOOL_TMP/usr/home" +zfs create -o exec=off -o setuid=off "$ZPOOL_TMP/usr/src" +zfs create -o mountpoint=/usr/ports -o setuid=off "$ZPOOL_TMP/usr/ports" +zfs create -o canmount=off -o mountpoint=/var "$ZPOOL_TMP/var" +zfs create -o exec=off -o setuid=off "$ZPOOL_TMP/var/audit" +zfs create -o exec=off -o setuid=off "$ZPOOL_TMP/var/crash" +zfs create -o exec=off -o setuid=off "$ZPOOL_TMP/var/log" +zfs create -o atime=on -o exec=off -o setuid=off "$ZPOOL_TMP/var/mail" +zfs create -o exec=on -o setuid=off "$ZPOOL_TMP/var/tmp" ln -s /usr/home "$ZFSTARGET/home" chmod 1777 "$ZFSTARGET/var/tmp" @@ -172,13 +172,13 @@ rm -rf "$UFSTARGET/var/db/freebsd-update/"* ||: # This will make the base image smaller, at the cost of taking longer to generate, as zstd-19 is slow to write # Therefore, afterwards we restore compression to default, so written files stay zstd-19, which is fast to read, # but files written by the user afterwards will be written with the default compression algorihtm. -zfs set compression=zstd-19 $ZPOOL_TMP/ROOT/default +zfs set compression=zstd-19 "$ZPOOL_TMP/ROOT/default" umount "$UFSTARGET/dev" "$UFSTARGET/tmp" "$UFSTARGET/var/tmp" rsync -aH --fileflags --inplace "$UFSTARGET/." "$ZFSTARGET" sysrc -f "$UFSTARGET/boot/loader.conf" -x zfs_load printf '%s\t%s\t\t%s\t%s\t%s\t%s\n' /dev/gpt/rootfs / ufs rw,noatime 1 1 >>"$UFSTARGET/etc/fstab" sync ||: -zfs inherit compression $ZPOOL_TMP/ROOT/default +zfs inherit compression "$ZPOOL_TMP/ROOT/default" trap : EXIT cleanup From 906754e8413457df72df7935bc934a8d684bd204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Fri, 27 May 2022 22:04:42 +0200 Subject: [PATCH 393/543] Prepare for later EFI boot --- .../freebsd-build-opennebula-image.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index a29e5e9..3081154 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -65,7 +65,8 @@ ufsdisk="$(mktemp /var/tmp/ufsdisk.XXXXX)" truncate -s 6G "$ufsdisk" mdconfig -a -t vnode -f "$ufsdisk" -u md1 gpart create -s gpt /dev/md1 -gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md1 +#gpart add -t efi -l efiboot0 -s 260M md1 +gpart add -t freebsd-boot -l gptboot0 -b 40 -s 512K md1 gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md1 gpart add -t freebsd-ufs -l rootfs -b 1M -s 5G md1 newfs -U /dev/md1p2 @@ -81,7 +82,8 @@ zfsdisk="$(mktemp /var/tmp/zfsdisk.XXXXX)" truncate -s 6G "$zfsdisk" mdconfig -a -t vnode -f "$zfsdisk" -u md0 gpart create -s gpt /dev/md0 -gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 +#gpart add -t efi -l efiboot0 -s 260M md1 +gpart add -t freebsd-boot -l gptboot0 -b 40 -s 512K md0 gpart bootcode -b /boot/pmbr -p /boot/gptzfsboot -i 1 md0 gpart add -t freebsd-zfs -l zfs0 -b 1M -s 5G md0 zpool create -O compression=on -o ashift=12 -o "altroot=$ZFSTARGET" -m none -t "$ZPOOL_TMP" "$ZPOOL" md0p2 @@ -133,7 +135,6 @@ mount_nullfs /var/tmp "$UFSTARGET/var/tmp" make -C "$CLOUDSETUP_WORK/" PREFIX="$UFSTARGET/usr/local" install # Configure new system. -printf '# Device\tMountpoint\tFStype\tOptions\t\tDump\tPass#\n' >"$UFSTARGET/etc/fstab" touch "$UFSTARGET/firstboot" sysrc -f "$UFSTARGET/boot/loader.conf" \ zfs_load=YES \ @@ -175,8 +176,13 @@ rm -rf "$UFSTARGET/var/db/freebsd-update/"* ||: zfs set compression=zstd-19 "$ZPOOL_TMP/ROOT/default" umount "$UFSTARGET/dev" "$UFSTARGET/tmp" "$UFSTARGET/var/tmp" rsync -aH --fileflags --inplace "$UFSTARGET/." "$ZFSTARGET" + sysrc -f "$UFSTARGET/boot/loader.conf" -x zfs_load -printf '%s\t%s\t\t%s\t%s\t%s\t%s\n' /dev/gpt/rootfs / ufs rw,noatime 1 1 >>"$UFSTARGET/etc/fstab" +printf '# Device\tMountpoint\tFStype\tOptions\t\tDump\tPass#\n' \ + >"$ZFSTARGET/etc/fstab" +printf '# Device\tMountpoint\tFStype\tOptions\t\tDump\tPass#\n%s\t%s\t\t%s\t%s\t%s\t%s\n' \ + /dev/gpt/rootfs / ufs rw,noatime 1 1 \ + >"$UFSTARGET/etc/fstab" sync ||: zfs inherit compression "$ZPOOL_TMP/ROOT/default" From c9353f173622002fe9e21e28c11b97daa1d79ca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Fri, 27 May 2022 22:13:42 +0200 Subject: [PATCH 394/543] Install firstboot script through ports --- .../freebsd-build-opennebula-image.sh | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 3081154..d801698 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -16,14 +16,11 @@ IMAGE_PATH_UFS="freebsd-ufs-$RELEASE-$(date -I).img.qcow2" IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" -CLOUDSETUP_COMMIT=4ac15b8647d5525048c5faa5fd4b28491905d000 -CLOUDSETUP_URL="https://git.sr.ht/~jornane/cloudsetup/archive/$CLOUDSETUP_COMMIT.tar.gz" ZPOOL=zroot ZPOOL_TMP="zinstalling" ZFSTARGET="$(mktemp -d /var/tmp/zfsbuild.XXXXX)" UFSTARGET="$(mktemp -d /var/tmp/ufsbuild.XXXXX)" -CLOUDSETUP_WORK="$(mktemp -d /var/tmp/cloudsetup.XXXXX)" if zpool list -Ho name "$ZPOOL_TMP" 2>/dev/null; then echo "The pool $ZPOOL_TMP is already imported." >&2 @@ -59,7 +56,18 @@ then env ASSUME_ALWAYS_YES=YES pkg install -y qemu-tools fi -fetch -qo- "$CLOUDSETUP_URL" | tar -C "$CLOUDSETUP_WORK" --strip-components 1 -xzf- +portsnap fetch +if [ -f /usr/ports/README ] +then + portsnap update || portsnap extract +else + portsnap extract +fi +make -C /usr/ports/sysutils/firstboot-cloudsetup clean package +CLOUDSETUP_VERSION="$(fgrep VERSION /usr/ports/sysutils/firstboot-cloudsetup/Makefile | cut -f2- | tr -d \\t)" +CLOUDSETUP_PKG="/usr/ports/sysutils/firstboot-cloudsetup/work/pkg/firstboot-cloudsetup-${CLOUDSETUP_VERSION}.pkg" +tar -tzf "$CLOUDSETUP_PKG" >/dev/null # check that it's a valid tar, or we crash due to set -e +# tar -t lists the contents of a tar file, but does not extract ufsdisk="$(mktemp /var/tmp/ufsdisk.XXXXX)" truncate -s 6G "$ufsdisk" @@ -132,7 +140,11 @@ mount_nullfs /tmp "$UFSTARGET/tmp" mount_nullfs /var/tmp "$UFSTARGET/var/tmp" # Install the first-boot script that configures the network and ssh key -make -C "$CLOUDSETUP_WORK/" PREFIX="$UFSTARGET/usr/local" install +# We must use --rootdir and not --chroot, because the file is read from within the chroot +# --automatic means that the package is considered to be installed "automatically", +# aka as a dependency of something, so pkg autoremove will remove it. +# We do not run pkg autoremove ourselves, that's up to the administrator. +pkg --rootdir "$UFSTARGET" add --automatic "$CLOUDSETUP_PKG" # Configure new system. touch "$UFSTARGET/firstboot" From d3369d321afa101279270c6aa5acd84b7046c335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 29 May 2022 16:29:01 +0200 Subject: [PATCH 395/543] Rename gptboot0 to gptboot on single-disk UFS --- opennebula-images/freebsd-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index d801698..8051921 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -74,7 +74,7 @@ truncate -s 6G "$ufsdisk" mdconfig -a -t vnode -f "$ufsdisk" -u md1 gpart create -s gpt /dev/md1 #gpart add -t efi -l efiboot0 -s 260M md1 -gpart add -t freebsd-boot -l gptboot0 -b 40 -s 512K md1 +gpart add -t freebsd-boot -l gptboot -b 40 -s 512K md1 gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md1 gpart add -t freebsd-ufs -l rootfs -b 1M -s 5G md1 newfs -U /dev/md1p2 From a4daf87a3497d4d85a02eb473afe3ec51e1fd8da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 29 May 2022 16:29:43 +0200 Subject: [PATCH 396/543] Make cloudsetup version overrideable This is useful for applying hotfixes before the updated port lands in FreeBSD ports. --- opennebula-images/freebsd-build-opennebula-image.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 8051921..5161cbd 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -15,6 +15,10 @@ IMAGE_PATH_ZFS="freebsd-zfs-$RELEASE-$(date -I).img.qcow2" IMAGE_PATH_UFS="freebsd-ufs-$RELEASE-$(date -I).img.qcow2" IMAGE_SIZE=10G +# Comment out to simply use latest version +# Hash checking is disabled when specifying this +CLOUDSETUP_VERSION=1.1 + DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" ZPOOL=zroot ZPOOL_TMP="zinstalling" @@ -63,6 +67,12 @@ then else portsnap extract fi + +if [ -n "$CLOUDSETUP_VERSION" ] +then + sed -i .bak -e '/^PORTVERSION=/ s/[0-9]*\.[0-9]*/'"$CLOUDSETUP_VERSION/" /usr/ports/sysutils/firstboot-cloudsetup/Makefile + make -C /usr/ports/sysutils/firstboot-cloudsetup makesum +fi make -C /usr/ports/sysutils/firstboot-cloudsetup clean package CLOUDSETUP_VERSION="$(fgrep VERSION /usr/ports/sysutils/firstboot-cloudsetup/Makefile | cut -f2- | tr -d \\t)" CLOUDSETUP_PKG="/usr/ports/sysutils/firstboot-cloudsetup/work/pkg/firstboot-cloudsetup-${CLOUDSETUP_VERSION}.pkg" From d35d04801ad288941a310d58f5e80628006a98cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 29 May 2022 17:17:42 +0200 Subject: [PATCH 397/543] Run freebsd-update at first boot instead of build Since the image probably isn't going to be rebuild that often, it's better to update at first boot, even though that will take longer. --- .../freebsd-build-opennebula-image.sh | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 5161cbd..d94bb81 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -79,6 +79,11 @@ CLOUDSETUP_PKG="/usr/ports/sysutils/firstboot-cloudsetup/work/pkg/firstboot-clou tar -tzf "$CLOUDSETUP_PKG" >/dev/null # check that it's a valid tar, or we crash due to set -e # tar -t lists the contents of a tar file, but does not extract +make -C /usr/ports/sysutils/firstboot-freebsd-update clean package +FBUPDATE_VERSION="$(fgrep VERSION /usr/ports/sysutils/firstboot-freebsd-update/Makefile | cut -f2- | tr -d \\t)" +FBUPDATE_PKG="/usr/ports/sysutils/firstboot-freebsd-update/work/pkg/firstboot-freebsd-update-${FBUPDATE_VERSION}.pkg" +tar -tzf "$FBUPDATE_PKG" >/dev/null # check that it's a valid tar, or we crash due to set -e + ufsdisk="$(mktemp /var/tmp/ufsdisk.XXXXX)" truncate -s 6G "$ufsdisk" mdconfig -a -t vnode -f "$ufsdisk" -u md1 @@ -154,7 +159,7 @@ mount_nullfs /var/tmp "$UFSTARGET/var/tmp" # --automatic means that the package is considered to be installed "automatically", # aka as a dependency of something, so pkg autoremove will remove it. # We do not run pkg autoremove ourselves, that's up to the administrator. -pkg --rootdir "$UFSTARGET" add --automatic "$CLOUDSETUP_PKG" +pkg --rootdir "$UFSTARGET" add --automatic "$CLOUDSETUP_PKG" "$FBUPDATE_PKG" # Configure new system. touch "$UFSTARGET/firstboot" @@ -168,6 +173,7 @@ sysrc -f "$UFSTARGET/etc/rc.conf" \ growfs_enable=YES \ hostname=freebsd \ firstboot_cloudsetup_enable=YES \ + firstboot_freebsd_update_enable=YES \ # The resolv.conf file is written by firstboot_cloudsetup #cp /etc/resolv.conf "$UFSTARGET/etc/resolv.conf" @@ -180,17 +186,6 @@ PasswordAuthentication no PermitEmptyPasswords no EOF -# freebsd-update is only supported for RELEASE -if printf %s "$RELEASE" | grep -q '.-RELEASE$' -then - env PAGER=true /usr/sbin/freebsd-update \ - -b "$UFSTARGET" \ - --currently-running "$RELEASE" \ - --not-running-from-cron -F \ - fetch install -fi -rm -rf "$UFSTARGET/var/db/freebsd-update/"* ||: - # Set zstd-19 compression, copy all data to the pool, and then set compression to default again # This will make the base image smaller, at the cost of taking longer to generate, as zstd-19 is slow to write # Therefore, afterwards we restore compression to default, so written files stay zstd-19, which is fast to read, From 03aee8ad688a6f6526ae8a3a2ec7c273ec5f1e6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sun, 29 May 2022 17:18:35 +0200 Subject: [PATCH 398/543] Mount dev after tarbombing --- opennebula-images/freebsd-build-opennebula-image.sh | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index d94bb81..31d9e8c 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -135,22 +135,19 @@ ln -s /usr/home "$ZFSTARGET/home" chmod 1777 "$ZFSTARGET/var/tmp" chmod 1777 "$ZFSTARGET/tmp" -# Mount dev in chroot -mkdir -p "$UFSTARGET/dev" -mount -t devfs devfs "$UFSTARGET/dev" - # Download and extract base system. dist_files="kernel.txz base.txz" dist_dir="/usr/freebsd-dist/$ARCH/$RELEASE" -mkdir -p "$dist_dir" "$UFSTARGET" +mkdir -p "$dist_dir" for f in $dist_files do fetch -m -o "$dist_dir/$f" "$DIST_BASE/$f" tar -C "$UFSTARGET" -xJf "$dist_dir/$f" done -# Avoid writing temporary files while building +# Mount dev and tmp in chroot +mount -t devfs devfs "$UFSTARGET/dev" mount_nullfs /tmp "$UFSTARGET/tmp" mount_nullfs /var/tmp "$UFSTARGET/var/tmp" From b14fa3db427ba382a612121d72e032644e1dffed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Mon, 30 May 2022 16:19:37 +0200 Subject: [PATCH 399/543] Hotfix the pkg-plist file for the cloudsetup port --- opennebula-images/freebsd-build-opennebula-image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 31d9e8c..87f4215 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -71,6 +71,7 @@ fi if [ -n "$CLOUDSETUP_VERSION" ] then sed -i .bak -e '/^PORTVERSION=/ s/[0-9]*\.[0-9]*/'"$CLOUDSETUP_VERSION/" /usr/ports/sysutils/firstboot-cloudsetup/Makefile + echo '%%DATADIR%%/userconf-getent.sh' >> /usr/ports/sysutils/firstboot-cloudsetup/pkg-plist make -C /usr/ports/sysutils/firstboot-cloudsetup makesum fi make -C /usr/ports/sysutils/firstboot-cloudsetup clean package From beb967c1963b274d96dba76145ad31936bca00d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Mon, 30 May 2022 20:39:10 +0200 Subject: [PATCH 400/543] Update cloudsetup to 1.2 --- opennebula-images/freebsd-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 87f4215..5ff345c 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -17,7 +17,7 @@ IMAGE_SIZE=10G # Comment out to simply use latest version # Hash checking is disabled when specifying this -CLOUDSETUP_VERSION=1.1 +CLOUDSETUP_VERSION=1.2 DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" ZPOOL=zroot From 84daa8eca4ba32d6a7b854f2902779efe756c911 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Thu, 2 Jun 2022 20:55:19 +0200 Subject: [PATCH 401/543] Set PermitRootLogin without-password This replaces the PermitRootLogin yes + PasswordAuthentication no construction. --- .../freebsd-build-opennebula-image.sh | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 5ff345c..a63bba2 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -178,11 +178,19 @@ sysrc -f "$UFSTARGET/etc/rc.conf" \ tzsetup -s -C "$UFSTARGET" UTC -cat >>"$UFSTARGET/etc/ssh/sshd_config" <>"$UFSTARGET/etc/ssh/sshd_config" < Date: Wed, 8 Jun 2022 18:43:24 +0200 Subject: [PATCH 402/543] Vanity: use quotes as in rc.conf --- .../freebsd-build-opennebula-image.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index a63bba2..cca57a7 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -162,16 +162,16 @@ pkg --rootdir "$UFSTARGET" add --automatic "$CLOUDSETUP_PKG" "$FBUPDATE_PKG" # Configure new system. touch "$UFSTARGET/firstboot" sysrc -f "$UFSTARGET/boot/loader.conf" \ - zfs_load=YES \ - autoboot_delay=-1 \ + zfs_load="YES" \ + autoboot_delay="-1" \ sysrc -f "$UFSTARGET/etc/rc.conf" \ - ntpd_enable=YES \ - sshd_enable=YES \ - growfs_enable=YES \ - hostname=freebsd \ - firstboot_cloudsetup_enable=YES \ - firstboot_freebsd_update_enable=YES \ + ntpd_enable="YES" \ + sshd_enable="YES" \ + growfs_enable="YES" \ + hostname="freebsd" \ + firstboot_cloudsetup_enable="YES" \ + firstboot_freebsd_update_enable="YES" \ # The resolv.conf file is written by firstboot_cloudsetup #cp /etc/resolv.conf "$UFSTARGET/etc/resolv.conf" From 9266f02268a1a91d5cd1b3154f7ac8b5e8354cfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sat, 11 Jun 2022 22:00:10 +0200 Subject: [PATCH 403/543] cloudsetup 1.2 is now in ports, no need to override anymore --- opennebula-images/freebsd-build-opennebula-image.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index cca57a7..b77e68c 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -17,7 +17,7 @@ IMAGE_SIZE=10G # Comment out to simply use latest version # Hash checking is disabled when specifying this -CLOUDSETUP_VERSION=1.2 +#CLOUDSETUP_VERSION=1.2 DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" ZPOOL=zroot @@ -71,7 +71,6 @@ fi if [ -n "$CLOUDSETUP_VERSION" ] then sed -i .bak -e '/^PORTVERSION=/ s/[0-9]*\.[0-9]*/'"$CLOUDSETUP_VERSION/" /usr/ports/sysutils/firstboot-cloudsetup/Makefile - echo '%%DATADIR%%/userconf-getent.sh' >> /usr/ports/sysutils/firstboot-cloudsetup/pkg-plist make -C /usr/ports/sysutils/firstboot-cloudsetup makesum fi make -C /usr/ports/sysutils/firstboot-cloudsetup clean package From a1cad581b9ab194fc4216301f1d42d7b61d0cbe5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B8rn=20=C3=85ne=20de=20Jong?= Date: Sat, 11 Jun 2022 22:00:22 +0200 Subject: [PATCH 404/543] Add zfs_enable to rc.conf --- opennebula-images/freebsd-build-opennebula-image.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index b77e68c..bf3f06c 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -165,6 +165,7 @@ sysrc -f "$UFSTARGET/boot/loader.conf" \ autoboot_delay="-1" \ sysrc -f "$UFSTARGET/etc/rc.conf" \ + zfs_enable="YES" \ ntpd_enable="YES" \ sshd_enable="YES" \ growfs_enable="YES" \ @@ -200,6 +201,7 @@ umount "$UFSTARGET/dev" "$UFSTARGET/tmp" "$UFSTARGET/var/tmp" rsync -aH --fileflags --inplace "$UFSTARGET/." "$ZFSTARGET" sysrc -f "$UFSTARGET/boot/loader.conf" -x zfs_load +sysrc -f "$UFSTARGET/etc/rc.conf" -x zfs_enable printf '# Device\tMountpoint\tFStype\tOptions\t\tDump\tPass#\n' \ >"$ZFSTARGET/etc/fstab" printf '# Device\tMountpoint\tFStype\tOptions\t\tDump\tPass#\n%s\t%s\t\t%s\t%s\t%s\t%s\n' \ From 2e09e7ef5967d4378510767a2d4ee19c7eaa8c23 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 8 Jul 2022 16:07:21 +0200 Subject: [PATCH 405/543] [openwrt/mjpeg] allow camera/fix name --- openwrt/openwrt-add-camera-with-mjpg-streamer.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-add-camera-with-mjpg-streamer.sh b/openwrt/openwrt-add-camera-with-mjpg-streamer.sh index 3235c00..4ac82e2 100755 --- a/openwrt/openwrt-add-camera-with-mjpg-streamer.sh +++ b/openwrt/openwrt-add-camera-with-mjpg-streamer.sh @@ -30,7 +30,7 @@ uci delete mjpg-streamer.core.password if ! uci show firewall | grep "name='Allow-Camera'"; then uci add firewall rule -uci set firewall.@rule[-1].name='Allow-HTTP' +uci set firewall.@rule[-1].name='Allow-Camera' uci set firewall.@rule[-1].src='wan' uci set firewall.@rule[-1].proto='tcp' uci set firewall.@rule[-1].dest_port='8080' From 7d486e1a26e9c61608ff38901ec7d4fd96840612 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 8 Jul 2022 16:13:24 +0200 Subject: [PATCH 406/543] [mjpeg streamer] need to add additional packages in openwrt 21.x --- openwrt/openwrt-add-camera-with-mjpg-streamer.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openwrt/openwrt-add-camera-with-mjpg-streamer.sh b/openwrt/openwrt-add-camera-with-mjpg-streamer.sh index 4ac82e2..0d3ddca 100755 --- a/openwrt/openwrt-add-camera-with-mjpg-streamer.sh +++ b/openwrt/openwrt-add-camera-with-mjpg-streamer.sh @@ -21,7 +21,8 @@ cat < Date: Mon, 11 Jul 2022 11:14:47 +0200 Subject: [PATCH 407/543] Update viirb firmware upgrader --- openwrt/viirb-1-firmware-upgrade.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openwrt/viirb-1-firmware-upgrade.sh b/openwrt/viirb-1-firmware-upgrade.sh index b65e331..88310ac 100755 --- a/openwrt/viirb-1-firmware-upgrade.sh +++ b/openwrt/viirb-1-firmware-upgrade.sh @@ -13,8 +13,8 @@ set -x viirb_ip=$1; shift # openwrt -version=19.07.7 -filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin +version=21.02.3 +filename=openwrt-${version}-ramips-mt76x8-vocore_vocore2-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${viirb_ip} From 9672f5eb7c105ebc628a30004080f6ee7d6d179e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 20 Jul 2022 12:36:37 +0200 Subject: [PATCH 408/543] [alpine/opennebula] begin upgrade to 3.16 - unfinished --- .../alpine-build-opennebula-image.sh | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/opennebula-images/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh index 87ff9b3..1866cc0 100755 --- a/opennebula-images/alpine-build-opennebula-image.sh +++ b/opennebula-images/alpine-build-opennebula-image.sh @@ -9,14 +9,14 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=v3.14 +RELEASE=v3.16 ARCH=x86_64 IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 APK_MIRROR=http://dl-2.alpinelinux.org/alpine/ # Mind the trailing / -ONE_CONTEXT_APK_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-r1.apk" +ONE_CONTEXT_APK_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.4.0/one-context-6.4.0-r1.apk" ONE_CONTEXT_APK_PATH=/root/one-context.apk cleanup() { @@ -82,7 +82,7 @@ apk add -U -X $APK_MIRROR$RELEASE/main/ \ --arch="$ARCH" \ --root=/mnt \ --initdb \ - alpine-base alpine-conf openssh sudo tzdata gnupg haveged bash eudev + alpine-base alpine-conf openssh tzdata gnupg haveged bash eudev mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts @@ -154,18 +154,8 @@ UUID=$root_uuid / ext4 rw,relatime,data=ordered 0 1 EOF # Install kernel and bootloader. -run_root apk add linux-virt syslinux - -dd if=/usr/share/syslinux/mbr.bin of="$NBD_DEVICE" bs=1 count=440 -extlinux -i /mnt/boot - -cat >/mnt/boot/extlinux.conf < "/mnt$ONE_CONTEXT_APK_PATH" From 44f201c96d37b5a5ce5162ec0129a3da8b68632c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 5 Aug 2022 23:29:24 +0200 Subject: [PATCH 409/543] Alpine disk installer: fix mount points for efi --- alpine-install-on-disk.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 4964bb5..f1867e9 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -108,7 +108,8 @@ sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf # nb2:~# blkid| grep ^${DISK}1 | awk '{ print $2 }' # UUID="fecf4182-f6dd-4d2c-9af7-8f36444ee25c" eval $(blkid | grep ^${DISK}1 | awk '{ print $2 }') -echo "UUID=$UUID / ext4 defaults 0 1" >> ${rootfs_tmpdir}/etc/fstab +UUID_1=$UUID + run_root apk update run_root apk add linux-lts openrc udev openssh e2fsprogs @@ -145,8 +146,15 @@ run_root grub-mkconfig -o /boot/grub/grub.cfg case "$BOOT_VIA" in bios) run_root grub-install --target=i386-pc ${DISK} + echo "UUID=$UUID_1 / ext4 defaults 0 1" >> ${rootfs_tmpdir}/etc/fstab ;; efi) + eval $(blkid | grep ^${DISK}2 | awk '{ print $2 }') + UUID_2=$UUID + + echo "UUID=$UUID_2 / ext4 defaults 0 1" >> ${rootfs_tmpdir}/etc/fstab + echo "UUID=$UUID_1 / vfat defaults 0 2" >> ${rootfs_tmpdir}/etc/fstab + run_root grub-install --efi-directory=/boot --no-nvram run_root mkdir /boot/EFI/boot run_root cp /boot/EFI/alpine/grubx64.efi /boot/EFI/boot/bootx64.efi From 09ceca72c94e3a1f1da28f6e9f65424f4c4ebf5a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 5 Sep 2022 19:48:03 +0200 Subject: [PATCH 410/543] operwrt: add script for viwib2 firmware upgrade --- openwrt/viwib2-1-firmware-upgrade.sh | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 openwrt/viwib2-1-firmware-upgrade.sh diff --git a/openwrt/viwib2-1-firmware-upgrade.sh b/openwrt/viwib2-1-firmware-upgrade.sh new file mode 100755 index 0000000..05efa71 --- /dev/null +++ b/openwrt/viwib2-1-firmware-upgrade.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 1 ]; then + echo "$0 address" + echo " address: connect to this address (default: 192.168.8.1)" + exit 1 +fi + +set -x + +viwib_ip=$1; shift + +# openwrt +version=21.02.3 +filename=openwrt-${version}-ath79-nand-glinet_gl-ar300m-nor-squashfs-sysupgrade.bin +url=https://downloads.openwrt.org/releases/${version}/targets/ath79/nand + +# don't care about other/old known_host entries +ssh-keygen -R ${viwib_ip} + +while ! ping -c1 ${viwib_ip}; do + echo "Cannot ping $viwib_ip yet - waiting" + sleep 1 +done + +cat ~/.ssh/id_rsa.pub | ssh root@${viwib_ip} "cat > /etc/dropbear/authorized_keys" + + +# Don't re-download if we already have it +wget -c ${url}/${filename} + +if echo $viwib_ip | grep -q :; then + scp_ip="[$viwib_ip]" +else + scp_ip="$viwib_ip" +fi + +scp ${filename} root@${scp_ip}:/tmp +ssh root@${viwib_ip} "sysupgrade -n /tmp/*.bin" + +# It still pings for some time - wait for the reboot to happen +echo "Waiting for viwib to really disappear" +sleep 15 + +wait=0 +found="" + +while [ $wait -lt 180 ]; do + ping -c1 ${viwib_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + break + fi + + sleep 1 + wait=$((wait+1)) + done + +if [ ! "$found" ]; then + echo "Did not find updated viwib - debug / restart it" + exit 1 +fi + +echo "viwib successfully updated to ${version}" From 9f4b5a172ee71d0accf30bfc5fcc5e217e0b181d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 29 Sep 2022 15:56:39 +0200 Subject: [PATCH 411/543] add ripe.sh --- ripe.sh | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100755 ripe.sh diff --git a/ripe.sh b/ripe.sh new file mode 100755 index 0000000..dfda77a --- /dev/null +++ b/ripe.sh @@ -0,0 +1,81 @@ +#!/bin/sh +# 2022-03-18, ungleich (foss at ungleich.ch) +# Copying: GPL3+ + +if [ $# -lt 2 ] ; then + echo "$0 network asn [nodryrun]" + echo "ASN needs to be in ASXXXXXX format" + echo "By default uses dry-run and does not create objects" + exit 1 +fi + +set -x + +network=$1; shift +asn=$1; shift + +if [ $# -ge 1 ]; then + dryrun="" +else + dryrun="&dry-run" +fi + +form=$(mktemp) + +case ${network} in + *:*) + obj_type=route6 + ;; + *.*) + obj_type=route + ;; + *) + echo "No idea what to do with $network" + exit 1 + ;; +esac + +cat > $form < Date: Thu, 29 Sep 2022 15:57:27 +0200 Subject: [PATCH 412/543] ++abort ripe if password is not set --- ripe.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/ripe.sh b/ripe.sh index dfda77a..038789c 100755 --- a/ripe.sh +++ b/ripe.sh @@ -6,6 +6,7 @@ if [ $# -lt 2 ] ; then echo "$0 network asn [nodryrun]" echo "ASN needs to be in ASXXXXXX format" echo "By default uses dry-run and does not create objects" + echo "Requires environment variable RIPE_API_PASSWORD to be set" exit 1 fi @@ -20,6 +21,11 @@ else dryrun="&dry-run" fi +if [ -z "$RIPE_API_PASSWORD" ]; then + echo "You need to set RIPE_API_PASSWORD" >&2 + exit 1 +fi + form=$(mktemp) case ${network} in From 4415a0ae0678a930de74a36afc642db498548e8c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Oct 2022 20:35:09 +0200 Subject: [PATCH 413/543] ++debug --- alpine-install-on-disk.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index f1867e9..4fcee56 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -163,6 +163,8 @@ case "$BOOT_VIA" in ;; esac +# Debug +run_root cat /etc/fstab # Cleanup run_root rm -f /etc/resolv.conf From 1b6eb1cc9325a3b31e2844e232a00b774df33c44 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 9 Oct 2022 12:00:02 +0200 Subject: [PATCH 414/543] alpine-installer: put /boot on the right mountpoint --- alpine-install-on-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 4fcee56..294651b 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -153,7 +153,7 @@ case "$BOOT_VIA" in UUID_2=$UUID echo "UUID=$UUID_2 / ext4 defaults 0 1" >> ${rootfs_tmpdir}/etc/fstab - echo "UUID=$UUID_1 / vfat defaults 0 2" >> ${rootfs_tmpdir}/etc/fstab + echo "UUID=$UUID_1 /boot vfat defaults 0 2" >> ${rootfs_tmpdir}/etc/fstab run_root grub-install --efi-directory=/boot --no-nvram run_root mkdir /boot/EFI/boot From 27e127abdc15798586797695ce558703c916cdd1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 11 Nov 2022 11:05:52 +0100 Subject: [PATCH 415/543] ++admin script --- admin/pricing_mastodon.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 admin/pricing_mastodon.py diff --git a/admin/pricing_mastodon.py b/admin/pricing_mastodon.py new file mode 100644 index 0000000..3c690eb --- /dev/null +++ b/admin/pricing_mastodon.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 + +maintenance=35 +networking=10 +cpu=3 +ram=4 +hdd=(2/100) +ssd=(3.5/10) + +package = {} + # maint, net, cpu, ram, hdd, ssd +package['starter'] = [ 1, 1, 4, 6, 200, 20 ] +package['community'] = [ 1, 1, 8, 12, 1000, 100 ] +package['pro'] = [ 1, 1, 16, 24, 5000, 500 ] + +for k,v in package.items(): + price=v[0] * maintenance + price+=v[1] * networking + price+=v[2] * cpu + price+=v[3] * ram + price+=v[4] * hdd + price+=v[5] * ssd + + print(f"Price for {k} with {v[2]} CPUs, {v[3]} GB RAM, {v[4]} GB HDD, {v[5]} GB SSD = {price}") From e39e8a66c203269a22eb46d344dc839f7a01593b Mon Sep 17 00:00:00 2001 From: kjg Date: Tue, 15 Nov 2022 10:05:22 +0000 Subject: [PATCH 416/543] [tools] update devuan-build-opennebula-image.sh for chimaera --- opennebula-images/devuan-build-opennebula-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennebula-images/devuan-build-opennebula-image.sh b/opennebula-images/devuan-build-opennebula-image.sh index ab9b598..5e8bd37 100755 --- a/opennebula-images/devuan-build-opennebula-image.sh +++ b/opennebula-images/devuan-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=beowulf +RELEASE=chimaera ARCH=amd64 IMAGE_PATH=devuan-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G @@ -18,7 +18,7 @@ HOSTNAME=devuan MIRROR=http://pkgmaster.devuan.org/merged # TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.4.0/one-context_6.4.0-1.deb" ONE_CONTEXT_DEB_PATH=/root/one-context.deb cleanup() { From 824680e123ef0b04b5a3809430a66ef648c541c6 Mon Sep 17 00:00:00 2001 From: kjg Date: Tue, 15 Nov 2022 13:57:49 +0000 Subject: [PATCH 417/543] [tools] update fedora-build-opennebula-image.sh for Fedora 36 --- opennebula-images/fedora-build-opennebula-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennebula-images/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh index d7e4ddc..787a42e 100755 --- a/opennebula-images/fedora-build-opennebula-image.sh +++ b/opennebula-images/fedora-build-opennebula-image.sh @@ -18,14 +18,14 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=33 +RELEASE=36 ARCH=x86_64 IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd1 # TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.4.0/one-context-6.4.0-1.el8.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm cleanup() { From 9bb54276aae6f86abdaaff8be57850220775cff1 Mon Sep 17 00:00:00 2001 From: kjg Date: Wed, 16 Nov 2022 15:45:35 +0000 Subject: [PATCH 418/543] [tools] update fedora-build-opennebula-image.sh for Fedora 36 --- opennebula-images/fedora-build-opennebula-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennebula-images/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh index 787a42e..c23049d 100755 --- a/opennebula-images/fedora-build-opennebula-image.sh +++ b/opennebula-images/fedora-build-opennebula-image.sh @@ -25,7 +25,7 @@ IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd1 # TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.4.0/one-context-6.4.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm cleanup() { @@ -90,7 +90,7 @@ dnf -y \ --enablerepo=fedora \ --enablerepo=updates install \ --setopt=install_weak_deps=False \ - basesystem systemd systemd-udev passwd dnf fedora-release + basesystem systemd systemd-udev passwd dnf fedora-release glibc-langpack-en.x86_64 mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts From 26cefb3e9a3682fff1d4574935724c807178ae65 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 7 Dec 2022 10:46:44 +0100 Subject: [PATCH 419/543] [ceph] add script for only removing the osd without disk --- ceph/ceph-osd-remove-from-cluster-only.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 ceph/ceph-osd-remove-from-cluster-only.sh diff --git a/ceph/ceph-osd-remove-from-cluster-only.sh b/ceph/ceph-osd-remove-from-cluster-only.sh new file mode 100755 index 0000000..3bb7bc0 --- /dev/null +++ b/ceph/ceph-osd-remove-from-cluster-only.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# 2022-12-07, 10:46 +# Nico Schottelius + +set -x +set -e + +if [ $# -ne 1 ]; then + echo "$0 osd.id" + echo "i.e. $0 17" + exit 1 +fi + +osd_id=$1; shift +osd_name=osd.${osd_id} + +ceph osd crush remove $osd_name +ceph osd rm $osd_name +ceph auth del $osd_name From 0c27d8ae21f5455904036d60a9f2878fadb8b0ec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 4 Feb 2023 13:24:10 +0100 Subject: [PATCH 420/543] [viwib] upgrade to 22.03.3 --- openwrt/viwib-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index a486d0b..c489062 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=21.02.2 +version=22.03.3 filename=openwrt-${version}-ramips-mt76x8-glinet_gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries From 18ea33de66d2d30c1df0fcf833b978197f8f4c37 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 4 Feb 2023 13:29:06 +0100 Subject: [PATCH 421/543] [viwib] use scp -O (legacy) for openwrt --- openwrt/viwib-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index c489062..dbc73a7 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -35,7 +35,7 @@ else scp_ip="$viwib_ip" fi -scp ${filename} root@${scp_ip}:/tmp +scp -O ${filename} root@${scp_ip}:/tmp ssh root@${viwib_ip} "sysupgrade -n /tmp/*.bin" # It still pings for some time - wait for the reboot to happen From 20b64796606c3fec50aa8a4ffa776f28b612ea29 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 4 Feb 2023 13:31:36 +0100 Subject: [PATCH 422/543] [viwib] switch to viwib2 by default --- ...-upgrade.sh => legacy-viwib1-1-firmware-upgrade.sh} | 10 ++++------ openwrt/viwib-1-firmware-upgrade.sh | 6 ++++-- 2 files changed, 8 insertions(+), 8 deletions(-) rename openwrt/{viwib2-1-firmware-upgrade.sh => legacy-viwib1-1-firmware-upgrade.sh} (83%) diff --git a/openwrt/viwib2-1-firmware-upgrade.sh b/openwrt/legacy-viwib1-1-firmware-upgrade.sh similarity index 83% rename from openwrt/viwib2-1-firmware-upgrade.sh rename to openwrt/legacy-viwib1-1-firmware-upgrade.sh index 05efa71..dbc73a7 100755 --- a/openwrt/viwib2-1-firmware-upgrade.sh +++ b/openwrt/legacy-viwib1-1-firmware-upgrade.sh @@ -13,9 +13,8 @@ set -x viwib_ip=$1; shift # openwrt -version=21.02.3 -filename=openwrt-${version}-ath79-nand-glinet_gl-ar300m-nor-squashfs-sysupgrade.bin -url=https://downloads.openwrt.org/releases/${version}/targets/ath79/nand +version=22.03.3 +filename=openwrt-${version}-ramips-mt76x8-glinet_gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${viwib_ip} @@ -27,9 +26,8 @@ done cat ~/.ssh/id_rsa.pub | ssh root@${viwib_ip} "cat > /etc/dropbear/authorized_keys" - # Don't re-download if we already have it -wget -c ${url}/${filename} +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} if echo $viwib_ip | grep -q :; then scp_ip="[$viwib_ip]" @@ -37,7 +35,7 @@ else scp_ip="$viwib_ip" fi -scp ${filename} root@${scp_ip}:/tmp +scp -O ${filename} root@${scp_ip}:/tmp ssh root@${viwib_ip} "sysupgrade -n /tmp/*.bin" # It still pings for some time - wait for the reboot to happen diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index dbc73a7..0d67ab3 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -14,7 +14,8 @@ viwib_ip=$1; shift # openwrt version=22.03.3 -filename=openwrt-${version}-ramips-mt76x8-glinet_gl-mt300n-v2-squashfs-sysupgrade.bin +filename=openwrt-${version}-ath79-nand-glinet_gl-ar300m-nor-squashfs-sysupgrade.bin +url=https://downloads.openwrt.org/releases/${version}/targets/ath79/nand # don't care about other/old known_host entries ssh-keygen -R ${viwib_ip} @@ -26,8 +27,9 @@ done cat ~/.ssh/id_rsa.pub | ssh root@${viwib_ip} "cat > /etc/dropbear/authorized_keys" + # Don't re-download if we already have it -wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} +wget -c ${url}/${filename} if echo $viwib_ip | grep -q :; then scp_ip="[$viwib_ip]" From 331efdaf160733c7b9f8c33286083e354bd1bfbb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 4 Feb 2023 13:34:09 +0100 Subject: [PATCH 423/543] [viwib] wait 30s for device to disappear --- openwrt/viwib-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-firmware-upgrade.sh index 0d67ab3..45e550e 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-firmware-upgrade.sh @@ -42,7 +42,7 @@ ssh root@${viwib_ip} "sysupgrade -n /tmp/*.bin" # It still pings for some time - wait for the reboot to happen echo "Waiting for viwib to really disappear" -sleep 15 +sleep 30 wait=0 found="" From ca42fdc2dc83c8873b93640b6b38439a1754bb75 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Feb 2023 11:09:46 +0100 Subject: [PATCH 424/543] [vigir] update firmware --- openwrt/vigir-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index a93bb4b..d77a368 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,7 +12,7 @@ set -x vigir_ip=$1; shift # openwrt -version=21.02.2 +version=22.03.3 filename=openwrt-${version}-ramips-mt7621-zbtlink_zbt-wg3526-16m-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} From 2108400c40565a99b06e6c6f2cecfa3d61d8aea4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Feb 2023 11:11:04 +0100 Subject: [PATCH 425/543] [vigir] use scp -O --- openwrt/vigir-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index d77a368..f967fbd 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -33,7 +33,7 @@ else scp_ip="$vigir_ip" fi -scp ${filename} root@${scp_ip}:/tmp +scp -O ${filename} root@${scp_ip}:/tmp ssh root@${vigir_ip} "sysupgrade -n /tmp/*.bin" # It still pings for some time - wait for the reboot to happen From f76db6ea6b5191c704777bfe37d2761aceb99200 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 17 Apr 2023 13:32:30 +0200 Subject: [PATCH 426/543] [ceph] prevent from removing monitor on itself --- ceph/ceph-mon-stop-delete | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ceph/ceph-mon-stop-delete b/ceph/ceph-mon-stop-delete index 1246a96..c40e58b 100755 --- a/ceph/ceph-mon-stop-delete +++ b/ceph/ceph-mon-stop-delete @@ -1,11 +1,12 @@ #!/bin/sh -if [ $# -ne 0 ]; then - echo "$0 (no arguments" +if [ $# -ne 1 ]; then + echo "$0 " + echo "f.i. $0 serverX" exit 1 fi -mon=mon.$(hostname) +mon=mon.$1 # Starting with monit, if available if [ -e /etc/monit ]; then From dc641911d4510904b78e81bbbb70fd75873ee1e5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 20 Apr 2023 13:51:56 +0200 Subject: [PATCH 427/543] [wireguard] correct masks --- openwrt/openwrt-add-wireguard.sh | 2 +- wireguard/gen-tunnels.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/openwrt/openwrt-add-wireguard.sh b/openwrt/openwrt-add-wireguard.sh index d44d9e9..b8b6325 100755 --- a/openwrt/openwrt-add-wireguard.sh +++ b/openwrt/openwrt-add-wireguard.sh @@ -10,7 +10,7 @@ if [ $# -lt 4 ]; then echo " ipv6-network: which network to use for us (/48 expected)" echo " private-key: specify wireguard key optionally" exit 1 -fi +qnfi my_ip=$1; shift vpn_endpoint_host=$1; shift diff --git a/wireguard/gen-tunnels.sh b/wireguard/gen-tunnels.sh index 0251c07..787869a 100755 --- a/wireguard/gen-tunnels.sh +++ b/wireguard/gen-tunnels.sh @@ -21,10 +21,12 @@ case "$v4v6" in v6) sep=":" allowed_ips="::/0" + gw_mask="/128" ;; v4) sep="." allowed_ips="0.0.0.0/0" + gw_mask="/32" ;; *) @@ -47,7 +49,6 @@ for ip in $(seq $start $end); do cat < $file [Interface] PrivateKey = ${privkey} -ListenPort = 51820 Address = ${addr} [Peer] @@ -60,7 +61,7 @@ EOF [Peer] PublicKey = ${pubkey} -AllowedIPs = ${addr} +AllowedIPs = ${addr_nomask}${gw_mask} EOF From 7f712a2c9853f66aa74808fd1317b10b70f56225 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 20 Apr 2023 13:52:07 +0200 Subject: [PATCH 428/543] [k8s] add script to connect to ceph pod --- k8s/k8s-ceph-tools.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 k8s/k8s-ceph-tools.sh diff --git a/k8s/k8s-ceph-tools.sh b/k8s/k8s-ceph-tools.sh new file mode 100644 index 0000000..cf2d363 --- /dev/null +++ b/k8s/k8s-ceph-tools.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# Connect to the pod running ceph-tools + +POD=$(kubectl -n rook-ceph get pods \ + -l app=rook-ceph-tools --output=jsonpath={.items..metadata.name}) + +kubectl -n rook-ceph exec -ti $POD -- bash From 276f2eeaa84b744076e021ccfbdcf63b99506c05 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 20 Apr 2023 13:53:25 +0200 Subject: [PATCH 429/543] k8s-ceph-tools: fix permissions --- k8s/k8s-ceph-tools.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 k8s/k8s-ceph-tools.sh diff --git a/k8s/k8s-ceph-tools.sh b/k8s/k8s-ceph-tools.sh old mode 100644 new mode 100755 From 1495d7c2354c9c57c4302b27056cc98d143b59b4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 22 Apr 2023 09:27:54 +0200 Subject: [PATCH 430/543] [jool] begin to document --- openwrt/openwrt-add-jool-ipv4island.sh | 2 ++ openwrt/openwrt-add-jool.sh | 3 +-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/openwrt/openwrt-add-jool-ipv4island.sh b/openwrt/openwrt-add-jool-ipv4island.sh index 01f4556..c3a86fd 100755 --- a/openwrt/openwrt-add-jool-ipv4island.sh +++ b/openwrt/openwrt-add-jool-ipv4island.sh @@ -1,5 +1,7 @@ #!/bin/sh +# The purpose of this setup is to ... + if [ $# -ne 2 ]; then echo "$0 ip-address nat64-prefix" echo " ip-address: where to find the OpenWRT device" diff --git a/openwrt/openwrt-add-jool.sh b/openwrt/openwrt-add-jool.sh index c73838c..8dab557 100755 --- a/openwrt/openwrt-add-jool.sh +++ b/openwrt/openwrt-add-jool.sh @@ -8,8 +8,7 @@ if [ $# -ne 1 ]; then exit 1 fi -my_ip=$1; shift -nat64_prefix=$1; shift +my_ip=$1; shiftnat64_prefix=$1; shift cat < Date: Sat, 22 Apr 2023 10:16:43 +0200 Subject: [PATCH 431/543] k8s-router.py: add initial script for accessing routers --- k8s/k8s-router.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100755 k8s/k8s-router.py diff --git a/k8s/k8s-router.py b/k8s/k8s-router.py new file mode 100755 index 0000000..65fc132 --- /dev/null +++ b/k8s/k8s-router.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# Connect to the router pod +# On Alpine: nb3:~# apk add py3-kubernetes + + +import sys +import os +from kubernetes import client, config + +# ~/k8s contains the config files +K8SCONFIGDIR=os.path.join(os.environ['HOME'], "k8s") + +routermap = { + "p5-r1": "server137", + "p5-r2": "server138", + "p6-r1": "server139", + "p6-r2": "server140", + "p10-r1": "server122", + "p10-r2": "server123", + "p15-r1": "server120", + "p15-r2": "server121", +} + + +if not len(sys.argv) == 2: + print(f"{sys.argv[0]} ") + sys.exit(1) + +router=sys.argv[1] + + +if not router in routermap: + print(f"Router {router} not known") + sys.exit(1) + + +k8sconfig = os.path.join(K8SCONFIGDIR, f"{routermap[router]}.conf") + +if not os.path.exists(k8sconfig): + print(f"You need to have {k8sconfig} for accessing {router}") + sys.exit(1) + +config.load_kube_config(config_file=k8sconfig) + +#app.kubernetes.io/instance=bird-router-server120 + +#POD=$(kubectl -n rook-ceph get pods \ +# -l app=rook-ceph-tools --output=jsonpath={.items..metadata.name}) +# +#kubectl -n rook-ceph exec -ti $POD -- bash From 71182ce5507e8f338ed8f78d528220517e3dab3c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 22 Apr 2023 16:02:15 +0200 Subject: [PATCH 432/543] [k8s] update router script to connect to bird --- k8s/k8s-router.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/k8s/k8s-router.py b/k8s/k8s-router.py index 65fc132..b01187c 100755 --- a/k8s/k8s-router.py +++ b/k8s/k8s-router.py @@ -5,6 +5,7 @@ import sys import os +import subprocess from kubernetes import client, config # ~/k8s contains the config files @@ -36,13 +37,27 @@ if not router in routermap: k8sconfig = os.path.join(K8SCONFIGDIR, f"{routermap[router]}.conf") +print(f"Using KUBECONFIG={k8sconfig} for accessing {router} ...") + if not os.path.exists(k8sconfig): print(f"You need to have {k8sconfig} for accessing {router}") sys.exit(1) config.load_kube_config(config_file=k8sconfig) +v1 = client.CoreV1Api() -#app.kubernetes.io/instance=bird-router-server120 +pods = v1.list_pod_for_all_namespaces(watch=False, + label_selector="app.kubernetes.io/component=bird") + +print("Number of pods: " + str(len(pods.items))) +for i in pods.items: + pod=i.metadata.name + print(f"Pod: {pod}") + +os.environ["KUBECONFIG"] = k8sconfig + +cmd = f"kubectl exec -ti {pod} -c bird -- sh" +p = subprocess.run(cmd, shell=True) #POD=$(kubectl -n rook-ceph get pods \ # -l app=rook-ceph-tools --output=jsonpath={.items..metadata.name}) From 0e43bcca1eb037c496eff9553b3e04dd04f06a71 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 5 May 2023 11:12:36 +0200 Subject: [PATCH 433/543] [alpine installer] switch to alpine 3.17.3 --- alpine-install-on-disk.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 294651b..4ad15f5 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -15,8 +15,8 @@ DISK=$1; shift SSH_KEYS=$1; shift BOOT_VIA=$1; shift -MAJOR_VERSION=3.15 -MINOR_VERSION=0 +MAJOR_VERSION=3.17 +MINOR_VERSION=3 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz RESOLVCONF=/etc/resolv.conf From 1b6a599bcf717ad22c323426164e964ffc1d8d8c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 30 May 2023 13:22:05 +0200 Subject: [PATCH 434/543] [alpine-install-on-disk] switch to doas --- alpine-install-on-disk.sh | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 4ad15f5..164aed5 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -55,25 +55,25 @@ dd if=/dev/zero of=${DISK} bs=1M count=2 case "$BOOT_VIA" in bios) - sudo sfdisk "$DISK" <&2 @@ -82,24 +82,24 @@ EOF esac -# keep right permissions, use sudo -sudo tar xf $IMAGE -C $rootfs_tmpdir +# keep right permissions, use doas +doas tar xf $IMAGE -C $rootfs_tmpdir # These are required by grub-install # And also for generating grub config that contains rootfstype for dir in dev proc sys; do - sudo mount --bind /${dir} ${rootfs_tmpdir}/${dir} + doas mount --bind /${dir} ${rootfs_tmpdir}/${dir} done # Add SSH keys run_root mkdir -p root/.ssh -sudo cp $SSH_KEYS $rootfs_tmpdir/root/.ssh/authorized_keys +doas cp $SSH_KEYS $rootfs_tmpdir/root/.ssh/authorized_keys run_root chown root:root /root/.ssh/authorized_keys run_root chmod 0600 /root/.ssh/authorized_keys run_root chmod 0700 /root/.ssh # Import local resolv.conf. -sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf +doas cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf # Generate fstab which is later included in the initramfs @@ -123,7 +123,7 @@ run_root rc-update add sysctl run_root rc-update add modules run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow -sudo tee "$rootfs_tmpdir/etc/network/interfaces" < Date: Tue, 30 May 2023 13:26:54 +0200 Subject: [PATCH 435/543] alpine-install-on-disk: use hdparm to force reread of table --- alpine-install-on-disk.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 164aed5..baae22c 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -59,6 +59,7 @@ case "$BOOT_VIA" in label: dos ,,L EOF + doas hdparm -z $DISK # For creation, if an existing filesystem is on the partitions doas mkfs.ext4 -F ${partition1} doas mount -t ext4 ${partition1} $rootfs_tmpdir @@ -69,6 +70,7 @@ label: gpt ,500MiB,U ,,L EOF + doas hdparm -z $DISK doas mkfs.vfat ${partition1} doas mkfs.ext4 -F ${partition2} doas mount -t ext4 ${partition2} "$rootfs_tmpdir" From f2df5e8c48f3711094f6b07875dc5cc1d2dbe423 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Jun 2023 08:55:49 +0200 Subject: [PATCH 436/543] [k8s] add command to find and run commands in the ceph-toolbox --- k8s/rook-toolbox.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 k8s/rook-toolbox.sh diff --git a/k8s/rook-toolbox.sh b/k8s/rook-toolbox.sh new file mode 100755 index 0000000..81554ae --- /dev/null +++ b/k8s/rook-toolbox.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Nico Schottelius, 2023-06-09 +# Enter the ceph toolbox + +if [ -z "$@" ]; then + # set $1 to bash + set -- bash +fi + +kubectl exec -n rook-ceph -ti $(kubectl -n rook-ceph get pods -l app=rook-ceph-tools -o jsonpath='{.items[*].metadata.name}') -- "$@" From 20ed1abc37c17f521f414b8f95586c886dcaca77 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Jun 2023 13:58:21 +0200 Subject: [PATCH 437/543] [ceph] add tool to locate the block device --- ceph/find-osd-device.sh | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100755 ceph/find-osd-device.sh diff --git a/ceph/find-osd-device.sh b/ceph/find-osd-device.sh new file mode 100755 index 0000000..9572e6c --- /dev/null +++ b/ceph/find-osd-device.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Locate which block device corresponds to the OSD +# Nico Schottelius, 2023-06-10 + + +if [ $# -ne 1 ]; then + echo $0 osdnum + echo f.i. $0 99 + exit 1 +fi + +osdid=$1; shift + +osd_path=/var/lib/ceph/osd/ceph-${osdid} + +mountpath=$(mount | grep "on ${osd_path} ") + +if [ -z ${mountpath} ]; then + echo "Nothing mounted on ${osd_path}, are you on the right host?" + exit 1 +fi + +blockdev=$(readlink -f ${mountpath}/block) + +# Is directly referring to sdX? print and exit +if echo $blockdev | grep -q ^/dev/sd; then + echo $blockdev + exit 0 +fi + +# try the non-recursive variant, resulting in finding pv/vg +blockdev=$(readlink ${mountpath}/block) +lvm_vg=$(echo $blockdev | awk -F/ '{ print $3 }') + +pv_name=$(pvdisplay | grep -B1 $lvm_vg | awk '/PV Name/ { print $3 }') + +if [ "$pv_name" ]; then + echo $pv_name + exit 0 +fi + +echo "Cannot determine block device for osd.${osdid}" >&2 +exit 1 From 32442484f9a3cd9d8cbc436b5088a4833e5d989e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Jun 2023 13:58:48 +0200 Subject: [PATCH 438/543] [k8s] continue on finding router pod --- k8s/k8s-router.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/k8s/k8s-router.py b/k8s/k8s-router.py index b01187c..bd427c8 100755 --- a/k8s/k8s-router.py +++ b/k8s/k8s-router.py @@ -49,17 +49,16 @@ v1 = client.CoreV1Api() pods = v1.list_pod_for_all_namespaces(watch=False, label_selector="app.kubernetes.io/component=bird") -print("Number of pods: " + str(len(pods.items))) -for i in pods.items: - pod=i.metadata.name - print(f"Pod: {pod}") +num_pods = len(pods.items) +print("Number of pods: " + str(num_pods)) +if not num_pods == 1: + print(f"There should be exactly 1 matching pod - there are {num_pods} pods") + sys.exit(1) + +pod=pods.items[0].metadata.name +print(f"Pod: {pod}") os.environ["KUBECONFIG"] = k8sconfig cmd = f"kubectl exec -ti {pod} -c bird -- sh" p = subprocess.run(cmd, shell=True) - -#POD=$(kubectl -n rook-ceph get pods \ -# -l app=rook-ceph-tools --output=jsonpath={.items..metadata.name}) -# -#kubectl -n rook-ceph exec -ti $POD -- bash From 59031104a83bb7bfc25fa303e5a5ea3cc187d2da Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Jun 2023 13:59:59 +0200 Subject: [PATCH 439/543] [find-osd-device] Fix quotes --- ceph/find-osd-device.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ceph/find-osd-device.sh b/ceph/find-osd-device.sh index 9572e6c..52f8c7e 100755 --- a/ceph/find-osd-device.sh +++ b/ceph/find-osd-device.sh @@ -15,7 +15,7 @@ osd_path=/var/lib/ceph/osd/ceph-${osdid} mountpath=$(mount | grep "on ${osd_path} ") -if [ -z ${mountpath} ]; then +if [ -z "${mountpath}" ]; then echo "Nothing mounted on ${osd_path}, are you on the right host?" exit 1 fi From f3187d110a93c3596a4ae217bb4cf50f7dd0d03d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 10 Jun 2023 14:07:14 +0200 Subject: [PATCH 440/543] [find-osd-device] fix initial script issues --- ceph/find-osd-device.sh | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/ceph/find-osd-device.sh b/ceph/find-osd-device.sh index 52f8c7e..75806d9 100755 --- a/ceph/find-osd-device.sh +++ b/ceph/find-osd-device.sh @@ -9,18 +9,16 @@ if [ $# -ne 1 ]; then exit 1 fi -osdid=$1; shift +osd_id=$1; shift -osd_path=/var/lib/ceph/osd/ceph-${osdid} +osd_path=/var/lib/ceph/osd/ceph-${osd_id} -mountpath=$(mount | grep "on ${osd_path} ") - -if [ -z "${mountpath}" ]; then +if ! mount | grep -q " on ${osd_path} "; then echo "Nothing mounted on ${osd_path}, are you on the right host?" exit 1 fi -blockdev=$(readlink -f ${mountpath}/block) +blockdev=$(readlink -f ${osd_path}/block) # Is directly referring to sdX? print and exit if echo $blockdev | grep -q ^/dev/sd; then @@ -29,7 +27,7 @@ if echo $blockdev | grep -q ^/dev/sd; then fi # try the non-recursive variant, resulting in finding pv/vg -blockdev=$(readlink ${mountpath}/block) +blockdev=$(readlink ${osd_path}/block) lvm_vg=$(echo $blockdev | awk -F/ '{ print $3 }') pv_name=$(pvdisplay | grep -B1 $lvm_vg | awk '/PV Name/ { print $3 }') From 40c37ae68161cda5b0a10b818cb24aa75e0951aa Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Jun 2023 20:57:20 +0200 Subject: [PATCH 441/543] [k8s] add script to access router pod --- k8s/bird-router.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 k8s/bird-router.sh diff --git a/k8s/bird-router.sh b/k8s/bird-router.sh new file mode 100755 index 0000000..113ddea --- /dev/null +++ b/k8s/bird-router.sh @@ -0,0 +1,10 @@ +#!/bin/sh +# Nico Schottelius, 2023-06-09 +# Enter the ceph toolbox + +if [ -z "$@" ]; then + set -- -c bird -- birdc +fi +set -x + +kubectl exec -ti $(kubectl get pods -l app.kubernetes.io/component=bird -o jsonpath='{.items[*].metadata.name}') "$@" From 21f95dad132451aef1cdf3223dd0c7978e63a3f5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 4 Jul 2023 13:20:21 +0200 Subject: [PATCH 442/543] [openwrt/viwib] update firmware --- ...-firmware-upgrade.sh => viwib-1-black-firmware-upgrade.sh} | 4 ++-- ...firmware-upgrade.sh => viwib-1-yellow-firmware-upgrade.sh} | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) rename openwrt/{viwib-1-firmware-upgrade.sh => viwib-1-black-firmware-upgrade.sh} (95%) rename openwrt/{legacy-viwib1-1-firmware-upgrade.sh => viwib-1-yellow-firmware-upgrade.sh} (95%) diff --git a/openwrt/viwib-1-firmware-upgrade.sh b/openwrt/viwib-1-black-firmware-upgrade.sh similarity index 95% rename from openwrt/viwib-1-firmware-upgrade.sh rename to openwrt/viwib-1-black-firmware-upgrade.sh index 45e550e..4f3afac 100755 --- a/openwrt/viwib-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-black-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=22.03.3 +version=22.03.5 filename=openwrt-${version}-ath79-nand-glinet_gl-ar300m-nor-squashfs-sysupgrade.bin url=https://downloads.openwrt.org/releases/${version}/targets/ath79/nand @@ -38,7 +38,7 @@ else fi scp -O ${filename} root@${scp_ip}:/tmp -ssh root@${viwib_ip} "sysupgrade -n /tmp/*.bin" +ssh root@${viwib_ip} "sysupgrade -n /tmp/${filename}" # It still pings for some time - wait for the reboot to happen echo "Waiting for viwib to really disappear" diff --git a/openwrt/legacy-viwib1-1-firmware-upgrade.sh b/openwrt/viwib-1-yellow-firmware-upgrade.sh similarity index 95% rename from openwrt/legacy-viwib1-1-firmware-upgrade.sh rename to openwrt/viwib-1-yellow-firmware-upgrade.sh index dbc73a7..761625e 100755 --- a/openwrt/legacy-viwib1-1-firmware-upgrade.sh +++ b/openwrt/viwib-1-yellow-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=22.03.3 +version=22.03.5 filename=openwrt-${version}-ramips-mt76x8-glinet_gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries @@ -36,7 +36,7 @@ else fi scp -O ${filename} root@${scp_ip}:/tmp -ssh root@${viwib_ip} "sysupgrade -n /tmp/*.bin" +ssh root@${viwib_ip} "sysupgrade -n /tmp/${filename}" # It still pings for some time - wait for the reboot to happen echo "Waiting for viwib to really disappear" From c6d03c77356150f449d8d7970d8a8b3295fa3d8b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 18 Jul 2023 14:17:12 +0200 Subject: [PATCH 443/543] [alpin installer] switch back to sudo --- alpine-install-on-disk.sh | 42 +++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index baae22c..8b5a632 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -55,27 +55,27 @@ dd if=/dev/zero of=${DISK} bs=1M count=2 case "$BOOT_VIA" in bios) - doas sfdisk "$DISK" <&2 @@ -84,24 +84,24 @@ EOF esac -# keep right permissions, use doas -doas tar xf $IMAGE -C $rootfs_tmpdir +# keep right permissions, use sudo +sudo tar xf $IMAGE -C $rootfs_tmpdir # These are required by grub-install # And also for generating grub config that contains rootfstype for dir in dev proc sys; do - doas mount --bind /${dir} ${rootfs_tmpdir}/${dir} + sudo mount --bind /${dir} ${rootfs_tmpdir}/${dir} done # Add SSH keys run_root mkdir -p root/.ssh -doas cp $SSH_KEYS $rootfs_tmpdir/root/.ssh/authorized_keys +sudo cp $SSH_KEYS $rootfs_tmpdir/root/.ssh/authorized_keys run_root chown root:root /root/.ssh/authorized_keys run_root chmod 0600 /root/.ssh/authorized_keys run_root chmod 0700 /root/.ssh # Import local resolv.conf. -doas cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf +sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf # Generate fstab which is later included in the initramfs @@ -125,7 +125,7 @@ run_root rc-update add sysctl run_root rc-update add modules run_root sed -i 's/root:!::0:::::/root:*::0:::::/' /etc/shadow -doas tee "$rootfs_tmpdir/etc/network/interfaces" < Date: Sun, 23 Jul 2023 09:32:56 +0200 Subject: [PATCH 444/543] [mystrom] add script for power cycling --- mystrom/mystrom-10soff-and-then-on.sh | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 mystrom/mystrom-10soff-and-then-on.sh diff --git a/mystrom/mystrom-10soff-and-then-on.sh b/mystrom/mystrom-10soff-and-then-on.sh new file mode 100755 index 0000000..1231c6e --- /dev/null +++ b/mystrom/mystrom-10soff-and-then-on.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# 2020-12-07, Nico Schottelius + +if [ $# -lt 1 ]; then + echo "$0 hostname [hostname...]" + echo " hostname: which mystrom to connect to" + exit 1 +fi + +while [ $# -ge 1 ]; do + hostname=$1; shift + + http --json GET "http://${hostname}/report" + curl -s --location \ + --request GET \ + "http://${hostname}/relay?state=0" + http --json GET "http://${hostname}/report" + echo "Waiting..." + sleep 10 + curl -s --location \ + --request GET \ + "http://${hostname}/relay?state=1" + http --json GET "http://${hostname}/report" + +done From 19a5834124e80ee66f82c5cd9cd995675bd67f75 Mon Sep 17 00:00:00 2001 From: kjg Date: Tue, 8 Aug 2023 11:48:27 +0000 Subject: [PATCH 445/543] [ungleich-tools] update debian-build-opennebula-image.sh for Task#11911 --- opennebula-images/debian-build-opennebula-image.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 5c6c4f7..dd7d7bf 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=bulleyes # 11.X +RELEASE=bookworm # 12.X ARCH=amd64 IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G @@ -17,7 +17,7 @@ NBD_DEVICE=/dev/nbd0 HOSTNAME=debian # TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.6.1/one-context_6.6.1-1.deb" ONE_CONTEXT_DEB_PATH=/root/one-context.deb cleanup() { @@ -124,7 +124,7 @@ run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" run_root rm "$ONE_CONTEXT_DEB_PATH" # Manually install legacy network scripts used by one-context. -run_root apt-get -y install ifupdown +run_root apt-get -y install ifupdown systemd-timesyncd.service # Initalize base services. run_root systemd-machine-id-setup From 0084b0897ec0ba472bd7b1ccb1f91d7a44832386 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 11 Sep 2023 18:14:54 +0200 Subject: [PATCH 446/543] alpine linux installer: bump to 3.18.3 --- alpine-install-on-disk.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 8b5a632..056a145 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -5,6 +5,7 @@ if [ $# -ne 3 ]; then echo " disk: which disk to install to" echo " ssh-keyfile: ssh keys to add into the image" echo " use efi or bios partitioning" + echo "post install in OS: apk add hdparm sudo wget efibootmgr git" exit 1 fi @@ -15,7 +16,7 @@ DISK=$1; shift SSH_KEYS=$1; shift BOOT_VIA=$1; shift -MAJOR_VERSION=3.17 +MAJOR_VERSION=3.18 MINOR_VERSION=3 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz From 8d00a1afcf7c6ec0fb57be9f86eccd22b55a2f35 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 11 Sep 2023 18:27:57 +0200 Subject: [PATCH 447/543] alpine: add support for nvram writing --- alpine-install-on-disk.sh | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 056a145..7b8570b 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -1,7 +1,7 @@ #!/bin/sh if [ $# -ne 3 ]; then - echo "$0 disk ssh-keyfile [efi|bios]" + echo "$0 disk ssh-keyfile [efi|bios|efinvram]" echo " disk: which disk to install to" echo " ssh-keyfile: ssh keys to add into the image" echo " use efi or bios partitioning" @@ -65,7 +65,7 @@ EOF sudo mkfs.ext4 -F ${partition1} sudo mount -t ext4 ${partition1} $rootfs_tmpdir ;; - efi) + efi*) sudo sfdisk "$DISK" <> ${rootfs_tmpdir}/etc/fstab ;; - efi) + efi*) eval $(blkid | grep ^${DISK}2 | awk '{ print $2 }') UUID_2=$UUID @@ -166,6 +166,15 @@ case "$BOOT_VIA" in ;; esac +# Actually do add us to the bios +if [ $BOOT_VIA = "efinvram" ]; then + dir=/sys/firmware/efi/efivars/ + sudo mount --bind ${dir} ${rootfs_tmpdir}${dir} + run_root apk add efibootmgr + run_root grub-install --efi-directory=/boot +fi + + # Debug run_root cat /etc/fstab @@ -182,6 +191,3 @@ rmdir ${rootfs_tmpdir} echo "${DISK} has been setup with Alpine Linux" exit 0 - -apk add grub-efi -grub-install --no-nvram --efi-directory /boot/ From 342ed596341db76d1a7a8f2dd1da3dcd181411bd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 16 Nov 2023 12:30:40 +0100 Subject: [PATCH 448/543] update alpine linux installer to 3.18.4 --- alpine-install-on-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 7b8570b..3d0a12b 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -17,7 +17,7 @@ SSH_KEYS=$1; shift BOOT_VIA=$1; shift MAJOR_VERSION=3.18 -MINOR_VERSION=3 +MINOR_VERSION=4 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz RESOLVCONF=/etc/resolv.conf From 466a9c7dd1d71e6f5054a1e8e69c6239ebce1226 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 6 Dec 2023 18:58:09 +0100 Subject: [PATCH 449/543] alpine-install-on-disk: add python3 --- alpine-install-on-disk.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 3d0a12b..ff0516e 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -117,6 +117,9 @@ UUID_1=$UUID run_root apk update run_root apk add linux-lts openrc udev openssh e2fsprogs +# For ansible +run_root apk add python3 + run_root rc-update add udev run_root rc-update add udev-trigger run_root rc-update add sshd From 5d5e94d05fffaae0ca4128ee234c0a3f08752ec5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 6 Jan 2024 15:23:46 +0100 Subject: [PATCH 450/543] alpine: update to installer to 3.19.0 --- alpine-install-on-disk.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index ff0516e..493b2c4 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -5,7 +5,7 @@ if [ $# -ne 3 ]; then echo " disk: which disk to install to" echo " ssh-keyfile: ssh keys to add into the image" echo " use efi or bios partitioning" - echo "post install in OS: apk add hdparm sudo wget efibootmgr git" + echo "pre install in OS: apk add hdparm sudo wget efibootmgr git sfdisk" exit 1 fi @@ -16,8 +16,9 @@ DISK=$1; shift SSH_KEYS=$1; shift BOOT_VIA=$1; shift -MAJOR_VERSION=3.18 -MINOR_VERSION=4 +MAJOR_VERSION=3.19 +MINOR_VERSION=0 + IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz RESOLVCONF=/etc/resolv.conf From 6910bd8931280a8b7800fbf8aebbf41771812cf9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 6 Jan 2024 16:58:36 +0100 Subject: [PATCH 451/543] alpine: fix efinvram installation --- alpine-install-on-disk.sh | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 493b2c4..b1487a3 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -162,22 +162,26 @@ case "$BOOT_VIA" in echo "UUID=$UUID_2 / ext4 defaults 0 1" >> ${rootfs_tmpdir}/etc/fstab echo "UUID=$UUID_1 /boot vfat defaults 0 2" >> ${rootfs_tmpdir}/etc/fstab - run_root grub-install --efi-directory=/boot --no-nvram - run_root mkdir /boot/EFI/boot - run_root cp /boot/EFI/alpine/grubx64.efi /boot/EFI/boot/bootx64.efi - run_root cp /boot/grub/grub.cfg /boot/EFI/boot/ + + # Actually do add us to the bios + if [ $BOOT_VIA = "efinvram" ]; then + dir=/sys/firmware/efi/efivars/ + sudo mount --bind ${dir} ${rootfs_tmpdir}${dir} + run_root apk add efibootmgr + run_root grub-install --efi-directory=/boot + else + run_root grub-install --efi-directory=/boot --no-nvram + + # FIX for some machines (?) + run_root mkdir /boot/EFI/boot + run_root cp /boot/EFI/alpine/grubx64.efi /boot/EFI/boot/bootx64.efi + run_root cp /boot/grub/grub.cfg /boot/EFI/boot/ + fi + sudo umount ${rootfs_tmpdir}/boot ;; esac -# Actually do add us to the bios -if [ $BOOT_VIA = "efinvram" ]; then - dir=/sys/firmware/efi/efivars/ - sudo mount --bind ${dir} ${rootfs_tmpdir}${dir} - run_root apk add efibootmgr - run_root grub-install --efi-directory=/boot -fi - # Debug run_root cat /etc/fstab From 1bcd1b90b4e4ce37b28a2d70cc44fbab9dcb484d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 6 Jan 2024 17:11:37 +0100 Subject: [PATCH 452/543] alpine-installer: umount efivars dir in case of writing to nvram --- alpine-install-on-disk.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index b1487a3..fadcf29 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -169,6 +169,7 @@ case "$BOOT_VIA" in sudo mount --bind ${dir} ${rootfs_tmpdir}${dir} run_root apk add efibootmgr run_root grub-install --efi-directory=/boot + sudo umount ${rootfs_tmpdir}${dir} else run_root grub-install --efi-directory=/boot --no-nvram From 8ecb1efd1600c7de70b32db3996d99e3f9d49b41 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 6 Jan 2024 17:25:46 +0100 Subject: [PATCH 453/543] Phase in initial openstack script --- openstack/setup-openstack-client.sh | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 openstack/setup-openstack-client.sh diff --git a/openstack/setup-openstack-client.sh b/openstack/setup-openstack-client.sh new file mode 100644 index 0000000..a1f5f3b --- /dev/null +++ b/openstack/setup-openstack-client.sh @@ -0,0 +1,26 @@ +#!/bin/sh + +mkdir -p ~/osh +python3 -m venv ~/osh/venv +. ~/osh/venv/bin/activate +UPPER_CONSTRAINTS_FILE=https://releases.openstack.org/constraints/upper/${OPENSTACK_RELEASE:-xena}} pip install cmd2 python-openstackclient python-heatclient --ignore-installed + +# Create /etc/openstack for me +sudo -H mkdir -p /etc/openstack +sudo -H chown -R $(id -un): /etc/openstack + +# create config - probably need to fix auth_url +tee /etc/openstack/clouds.yaml << EOF + clouds: + openstack_helm: + region_name: RegionOne + identity_api_version: 3 + cacert: /etc/openstack-helm/certs/ca/ca.pem + auth: + username: 'admin' + password: 'password' + project_name: 'admin' + project_domain_name: 'default' + user_domain_name: 'default' + auth_url: 'https://keystone.openstack.svc.cluster.local/v3' +EOF From ece4c00dd057183eccd9d52e5db488a88a4d6286 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 6 Jan 2024 19:29:52 +0100 Subject: [PATCH 454/543] [openstack] add initial scripts --- openstack/04-ceph-adapter-rook.sh | 7 +++++++ openstack/activate-openstack.sh | 7 +++++++ openstack/init-openstack-env.sh | 15 +++++++++++++++ openstack/setup-openstack-client.sh | 0 openstack/values/ceph-adapter-rook-ceph.yaml | 8 ++++++++ 5 files changed, 37 insertions(+) create mode 100755 openstack/04-ceph-adapter-rook.sh create mode 100755 openstack/activate-openstack.sh create mode 100755 openstack/init-openstack-env.sh mode change 100644 => 100755 openstack/setup-openstack-client.sh create mode 100644 openstack/values/ceph-adapter-rook-ceph.yaml diff --git a/openstack/04-ceph-adapter-rook.sh b/openstack/04-ceph-adapter-rook.sh new file mode 100755 index 0000000..9f53925 --- /dev/null +++ b/openstack/04-ceph-adapter-rook.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +cd ~/osh/openstack-helm-infra +helm upgrade --install ceph-adapter-rook \ + ./ceph-adapter-rook/ \ + --namespace=rook-ceph \ + --values=$HOME/vcs/ungleich-tools/openstack/values/ceph-adapter-rook-ceph.yaml diff --git a/openstack/activate-openstack.sh b/openstack/activate-openstack.sh new file mode 100755 index 0000000..ffa2928 --- /dev/null +++ b/openstack/activate-openstack.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +echo "This script should be sourced for setting env variables, like venv activate" + +export OPENSTACK_RELEASE=2023.2 +export CONTAINER_DISTRO_NAME=ubuntu +export CONTAINER_DISTRO_VERSION=jammy diff --git a/openstack/init-openstack-env.sh b/openstack/init-openstack-env.sh new file mode 100755 index 0000000..ab45bc9 --- /dev/null +++ b/openstack/init-openstack-env.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +mkdir -p ~/osh +cd ~/osh +if [ ! -d openstack-helm ]; then + git clone https://opendev.org/openstack/openstack-helm.git +else + cd openstack-helm && git pull +fi + +if [ ! -d openstack-helm-infa ]; then + git clone https://opendev.org/openstack/openstack-helm-infra.git +else + cd openstack-helm-infra && git pull +fi diff --git a/openstack/setup-openstack-client.sh b/openstack/setup-openstack-client.sh old mode 100644 new mode 100755 diff --git a/openstack/values/ceph-adapter-rook-ceph.yaml b/openstack/values/ceph-adapter-rook-ceph.yaml new file mode 100644 index 0000000..bfd01a8 --- /dev/null +++ b/openstack/values/ceph-adapter-rook-ceph.yaml @@ -0,0 +1,8 @@ +manifests: + configmap_bin: true + configmap_templates: true + configmap_etc: false + job_storage_admin_keys: true + job_namespace_client_key: false + job_namespace_client_ceph_config: false + service_mon_discovery: true From 39d16e6dedff68f79e106c401cf6116593858247 Mon Sep 17 00:00:00 2001 From: kjg Date: Wed, 24 Jan 2024 04:12:47 +0000 Subject: [PATCH 455/543] [ungleich-tools] update alpine linux script for Task#11912 --- .../alpine-build-opennebula-image.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/opennebula-images/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh index 1866cc0..aa148ca 100755 --- a/opennebula-images/alpine-build-opennebula-image.sh +++ b/opennebula-images/alpine-build-opennebula-image.sh @@ -16,7 +16,7 @@ IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 APK_MIRROR=http://dl-2.alpinelinux.org/alpine/ # Mind the trailing / -ONE_CONTEXT_APK_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.4.0/one-context-6.4.0-r1.apk" +ONE_CONTEXT_APK_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.6.1/one-context-6.6.1-r1.apk" ONE_CONTEXT_APK_PATH=/root/one-context.apk cleanup() { @@ -145,6 +145,9 @@ done # Set hostname. run_root setup-hostname -n alpine +#debug +blkid + # Generate fstab file. boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") @@ -153,9 +156,22 @@ UUID=$boot_uuid /boot ext4 rw,relatime,data=ordered 0 2 UUID=$root_uuid / ext4 rw,relatime,data=ordered 0 1 EOF +#debug +echo $boot_uuid +echo $root_uuid +cat /mnt/etc/fstab +run_root cat /etc/fstab + # Install kernel and bootloader. run_root apk add linux-virt grub grub-bios +echo 'GRUB_CMDLINE_LINUX_DEFAULT="quiet rootfstype=ext4"' >> /mnt/etc/default/grub +cat /mnt/etc/default/grub run_root grub-install --target=i386-pc $NBD_DEVICE +run_root grub-mkconfig -o /boot/grub/grub.cfg + +#debug +run_root cat /etc/default/grub +run_root cat /etc/fstab # Install one-context APK and hope things works as expected. curl -L "$ONE_CONTEXT_APK_URL" > "/mnt$ONE_CONTEXT_APK_PATH" From 409214218f0be2912aa7108b4b65ecccd2d43528 Mon Sep 17 00:00:00 2001 From: kjg Date: Wed, 24 Jan 2024 12:21:48 +0000 Subject: [PATCH 456/543] [ungleich-tools] update alpine-build-opennebula-image.sh for Task#11912 --- opennebula-images/alpine-build-opennebula-image.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/opennebula-images/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh index aa148ca..5f2361d 100755 --- a/opennebula-images/alpine-build-opennebula-image.sh +++ b/opennebula-images/alpine-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=v3.16 +RELEASE=v3.19 ARCH=x86_64 IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G @@ -130,7 +130,7 @@ for i in devfs dmesg hwdrivers mdev; do run_root rc-update add $i sysinit done -for i in bootmisc hostname hwclock modules sysctl syslog acpid networking urandom haveged; do +for i in bootmisc hostname hwclock modules sysctl syslog acpid networking haveged; do run_root rc-update add $i boot done From 7640f1ec44110baace5578ed6b2242aa2c45885b Mon Sep 17 00:00:00 2001 From: kjg Date: Wed, 31 Jan 2024 02:06:51 +0000 Subject: [PATCH 457/543] [ungleich-tools] almalinux-build-opennebula-image.sh for Task#11914 --- opennebula-images/almalinux-build-opennebula-image.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/opennebula-images/almalinux-build-opennebula-image.sh b/opennebula-images/almalinux-build-opennebula-image.sh index 079e5d7..fcc558f 100755 --- a/opennebula-images/almalinux-build-opennebula-image.sh +++ b/opennebula-images/almalinux-build-opennebula-image.sh @@ -12,14 +12,16 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=8.3 +RELEASE=9.3 ARCH=x86_64 IMAGE_PATH=almalinux-$RELEASE-$(date --iso-8601).img IMAGE_SIZE=10G LOOPBACK_DEVICE=/dev/loop0 +# since v9.3, there is an issue is the compilation of RHEL 9 and it's derivatives to use the x86-64-v2 instruction set. +# refer to Task#12351, change cpu type # TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.6.1/one-context-6.6.1-1.el8.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm cleanup() { @@ -111,7 +113,8 @@ EOF # See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. # network-scripts.x86_64 : Legacy scripts for manipulating of network devices -run_root dnf -y install network-scripts +#run_root dnf -y install network-scripts +run_root dnf -y install NetworkManager # Install (magic?) one-context RPM and hope things works as expected. curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" From 0f9bfed4074a03bfcb52f63c065d7f0ee80f4d62 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 31 Jan 2024 12:05:00 +0100 Subject: [PATCH 458/543] [openwrt] bump version for vigir --- openwrt/vigir-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index f967fbd..939e437 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,7 +12,7 @@ set -x vigir_ip=$1; shift # openwrt -version=22.03.3 +version=23.05.2 filename=openwrt-${version}-ramips-mt7621-zbtlink_zbt-wg3526-16m-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} From 23e953cd663941fc1bf0e600c9678623b041b61a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 31 Jan 2024 12:05:16 +0100 Subject: [PATCH 459/543] add notes, add sleep for alpine installer --- alpine-install-on-disk.sh | 1 + openwrt/openwrt-add-ipv4-vpn.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index fadcf29..c86a826 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -61,6 +61,7 @@ case "$BOOT_VIA" in label: dos ,,L EOF + sleep 10 sudo hdparm -z $DISK # For creation, if an existing filesystem is on the partitions sudo mkfs.ext4 -F ${partition1} diff --git a/openwrt/openwrt-add-ipv4-vpn.sh b/openwrt/openwrt-add-ipv4-vpn.sh index 9e6fb52..e81c98d 100755 --- a/openwrt/openwrt-add-ipv4-vpn.sh +++ b/openwrt/openwrt-add-ipv4-vpn.sh @@ -9,6 +9,8 @@ if [ $# -lt 2 ]; then echo " host: where to find the OpenWRT device" echo " ipv4-address: which ipv4 address to use" echo " private-key: Use this wireguard key instead of generating one" + echo "" + echo "f.i.: $0 2a0a:e5c1:6XX:cafe::42 185.155.a.b" exit 1 fi From 09b4825918dd6b04f11748ab584c9e1e184fff2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 31 Jan 2024 12:06:55 +0100 Subject: [PATCH 460/543] [vigir] update help text --- openwrt/vigir-1-firmware-upgrade.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index 939e437..c3be5d9 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -3,8 +3,8 @@ # See https://ungleich.ch/u/products/viirb-ipv6-box/ if [ $# -ne 1 ]; then - echo "$0 interface [address]" - echo " address: connect to this address, ignore the interface" + echo "$0 address" + echo " address: connect to this address" exit 1 fi From b5990f5e6a2441d81e9d43f9f6c794b21b8861ec Mon Sep 17 00:00:00 2001 From: kjg Date: Wed, 31 Jan 2024 12:44:01 +0000 Subject: [PATCH 461/543] [ungleich-tools] update freebsd-build-opennebula-image.sh for Task#11913 --- opennebula-images/freebsd-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index bf3f06c..ce62438 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=13.1-RELEASE +RELEASE=13.2-RELEASE ARCH=amd64 IMAGE_PATH_ZFS="freebsd-zfs-$RELEASE-$(date -I).img.qcow2" IMAGE_PATH_UFS="freebsd-ufs-$RELEASE-$(date -I).img.qcow2" From c080415b9d8a3e28c216e8afa89e59bde2aa7936 Mon Sep 17 00:00:00 2001 From: kjg Date: Thu, 1 Feb 2024 05:01:25 +0000 Subject: [PATCH 462/543] [ungleich-tools] update freebsd-build-opennebula-image.sh for Task#11913 --- opennebula-images/freebsd-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index ce62438..24b0d1a 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=13.2-RELEASE +RELEASE=14.0-RELEASE ARCH=amd64 IMAGE_PATH_ZFS="freebsd-zfs-$RELEASE-$(date -I).img.qcow2" IMAGE_PATH_UFS="freebsd-ufs-$RELEASE-$(date -I).img.qcow2" From 41918e1162b2551beaf054adea96807caf404b8a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 28 Mar 2024 11:28:52 +0100 Subject: [PATCH 463/543] [ONE images] refresh ubuntu and devuan images --- opennebula-images/devuan-build-opennebula-image.sh | 7 +++---- opennebula-images/ubuntu-build-opennebula-image.sh | 14 +++++--------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/opennebula-images/devuan-build-opennebula-image.sh b/opennebula-images/devuan-build-opennebula-image.sh index 5e8bd37..922aea9 100755 --- a/opennebula-images/devuan-build-opennebula-image.sh +++ b/opennebula-images/devuan-build-opennebula-image.sh @@ -9,16 +9,15 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=chimaera +RELEASE=daedalus ARCH=amd64 IMAGE_PATH=devuan-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G -NBD_DEVICE=/dev/nbd0 +NBD_DEVICE=/dev/nbd4 HOSTNAME=devuan MIRROR=http://pkgmaster.devuan.org/merged -# TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.4.0/one-context_6.4.0-1.deb" +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.6.1/one-context_6.6.1-1.deb" ONE_CONTEXT_DEB_PATH=/root/one-context.deb cleanup() { diff --git a/opennebula-images/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh index 6f1db12..223ce1b 100755 --- a/opennebula-images/ubuntu-build-opennebula-image.sh +++ b/opennebula-images/ubuntu-build-opennebula-image.sh @@ -9,14 +9,13 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=jammy # 22.04 LTS +RELEASE=mantic # 23.10 ARCH=amd64 IMAGE_PATH=ubuntu-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G -NBD_DEVICE=/dev/nbd0 +NBD_DEVICE=/dev/nbd2 -# TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v6.4.0/one-context_6.4.0-1.deb" +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/one-apps/releases/download/v6.8.1/one-context_6.8.1-1.deb" ONE_CONTEXT_DEB_PATH=/root/one-context.deb cleanup() { @@ -70,7 +69,6 @@ mkfs.ext4 "${NBD_DEVICE}p1" mkfs.ext4 "${NBD_DEVICE}p2" # Mount partitions, install base OS. - mount "${NBD_DEVICE}p2" /mnt mkdir /mnt/boot mount "${NBD_DEVICE}p1" /mnt/boot @@ -89,9 +87,6 @@ mount --bind /sys /mnt/sys # Guest networking is to be handled by the one-context package. # See https://github.com/OpenNebula/addon-context-linux for details. -# Required to resolve package mirror in chroot. -cp /etc/resolv.conf /mnt/etc/resolv.conf - # Initialize /etc/hosts. cat > /mnt/etc/hosts << EOF 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 @@ -123,7 +118,7 @@ run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service # Install kernel and bootloader. Do not autoconfigure grub. -run_root echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections +run_root "echo 'grub-pc grub-pc/install_devices_empty boolean true' | debconf-set-selections" run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-base linux-image-generic grub-pc # Configure grub. @@ -145,6 +140,7 @@ EOF run_root rm -f /etc/machine-id run_root touch /etc/machine-id rm -f /var/lib/systemd/random-seed +run_root hostnamectl set-hostname ubuntu # Remove temporary files and reclaim freed disk space. run_root apt-get clean From 7ec8bb3a2bad94ee52b87f8e8f509e9d9b10eef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 28 Mar 2024 11:37:06 +0100 Subject: [PATCH 464/543] [ONE images] refresh fedora image, add rocky image build script --- .../fedora-build-opennebula-image.sh | 22 +-- .../rocky-build-opennebula-image.sh | 177 ++++++++++++++++++ 2 files changed, 186 insertions(+), 13 deletions(-) create mode 100755 opennebula-images/rocky-build-opennebula-image.sh diff --git a/opennebula-images/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh index c23049d..76a649b 100755 --- a/opennebula-images/fedora-build-opennebula-image.sh +++ b/opennebula-images/fedora-build-opennebula-image.sh @@ -18,14 +18,14 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=36 +RELEASE=39 ARCH=x86_64 IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G -NBD_DEVICE=/dev/nbd1 +NBD_DEVICE=/dev/nbd0 # TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/one-apps/releases/download/v6.8.1/one-context-6.8.1-1.el9.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm cleanup() { @@ -90,7 +90,7 @@ dnf -y \ --enablerepo=fedora \ --enablerepo=updates install \ --setopt=install_weak_deps=False \ - basesystem systemd systemd-udev passwd dnf fedora-release glibc-langpack-en.x86_64 + basesystem systemd systemd-udev passwd dnf fedora-release glibc-langpack-en mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts @@ -100,12 +100,7 @@ mount --bind /run /mnt/run mount --bind /sys /mnt/sys # Guest networking is to be handled by the one-context package. -# See https://github.com/OpenNebula/addon-context-linux for details. -# Note: as of writing, one-context does not support NetworkManager or -# systemd-networkd. - -# Required to resolve package mirror in chroot. -cp /etc/resolv.conf /mnt/etc/resolv.conf +# See https://github.com/OpenNebula/one-apps for details. # Initialize /etc/hosts. cat > /mnt/etc/hosts << EOF @@ -114,9 +109,9 @@ cat > /mnt/etc/hosts << EOF EOF -# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. -# network-scripts.x86_64 : Legacy scripts for manipulating of network devices -run_root dnf -y install network-scripts +# Install and enable NetworkManager. +run_root dnf -y install NetworkManager +run_root systemctl enable NetworkManager # Install (magic?) one-context RPM and hope things works as expected. curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" @@ -168,6 +163,7 @@ EOF run_root rm -f /etc/machine-id run_root touch /etc/machine-id rm -f /var/lib/systemd/random-seed +run_root hostnamectl set-hostname fedora # Remove temporary files and reclaim freed disk space. # Note: build logs could be removed as well. diff --git a/opennebula-images/rocky-build-opennebula-image.sh b/opennebula-images/rocky-build-opennebula-image.sh new file mode 100755 index 0000000..67f29ef --- /dev/null +++ b/opennebula-images/rocky-build-opennebula-image.sh @@ -0,0 +1,177 @@ +#!/bin/sh + +# This script generates Rocky Linux images for OpenNebula. + +# Depends on the following packages (as of Fedora 31): +# qemu-img util-linux coreutils dnf curl e2fsprogs + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=9.3 +ARCH=x86_64 +IMAGE_PATH=rocky-$RELEASE-$(date +%+F).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/one-apps/releases/download/v6.8.1/one-context-6.8.1-1.el9.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/fedora-release' ]; then + echo "WARNING: this script has been designed to run on a Fedora system." >&2 + echo "WARNING: Not running Fedora. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Required to resolve package mirror in chroot. +cp /etc/resolv.conf /mnt/etc/resolv.conf + +# Re-run dnf/install hooks that couldn't be executed in initial call. +# Install a few extra dependencies. +run_root dnf -y --releasever $RELEASE install rocky-release epel-release + +# Set locale. +run_root localectl set-locale LANG=en_GB.UTF-8 + +# Install and enable NetworkManager. +# Guest networking is to be handled by the one-context package. +# See https://github.com/OpenNebula/one-apps for details. +run_root dnf install -y NetworkManager +run_root systemctl enable NetworkManager + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +#run_root systemctl enable systemd-timesyncd.service + +# Install haveged due to lack of entropy in ONE environment. +run_root dnf -y install haveged +run_root systemctl enable haveged.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <>/mnt/etc/fstab < Date: Thu, 28 Mar 2024 11:55:26 +0100 Subject: [PATCH 465/543] [ONE images] add RPM repositories for Rocky Linux --- .../{ => centos}/centos-7-minus.repo | 0 .../{ => centos}/centos-extras.repo | 0 .../rpm-repositories/rocky/rocky-extras.repo | 65 +++++++++++++ .../rpm-repositories/rocky/rocky.repo | 93 +++++++++++++++++++ 4 files changed, 158 insertions(+) rename opennebula-images/rpm-repositories/{ => centos}/centos-7-minus.repo (100%) rename opennebula-images/rpm-repositories/{ => centos}/centos-extras.repo (100%) create mode 100644 opennebula-images/rpm-repositories/rocky/rocky-extras.repo create mode 100644 opennebula-images/rpm-repositories/rocky/rocky.repo diff --git a/opennebula-images/rpm-repositories/centos-7-minus.repo b/opennebula-images/rpm-repositories/centos/centos-7-minus.repo similarity index 100% rename from opennebula-images/rpm-repositories/centos-7-minus.repo rename to opennebula-images/rpm-repositories/centos/centos-7-minus.repo diff --git a/opennebula-images/rpm-repositories/centos-extras.repo b/opennebula-images/rpm-repositories/centos/centos-extras.repo similarity index 100% rename from opennebula-images/rpm-repositories/centos-extras.repo rename to opennebula-images/rpm-repositories/centos/centos-extras.repo diff --git a/opennebula-images/rpm-repositories/rocky/rocky-extras.repo b/opennebula-images/rpm-repositories/rocky/rocky-extras.repo new file mode 100644 index 0000000..8402708 --- /dev/null +++ b/opennebula-images/rpm-repositories/rocky/rocky-extras.repo @@ -0,0 +1,65 @@ +# rocky-extras.repo +# +# The mirrorlist system uses the connecting IP address of the client and the +# update status of each mirror to pick current mirrors that are geographically +# close to the client. You should use this for Rocky updates unless you are +# manually picking other mirrors. +# +# If the mirrorlist does not work for you, you can try the commented out +# baseurl line instead. + +[extras] +name=Rocky Linux $releasever - Extras +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=extras-$releasever +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/extras/$basearch/os/ +gpgcheck=1 +enabled=1 +countme=1 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[extras-debug] +name=Rocky Linux $releasever - Extras Debug +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=extras-$releasever-debug +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/extras/$basearch/debug/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[extras-source] +name=Rocky Linux $releasever - Extras Source +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=extras-$releasever-source +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/extras/source/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[plus] +name=Rocky Linux $releasever - Plus +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=plus-$releasever +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/plus/$basearch/os/ +gpgcheck=1 +enabled=0 +countme=1 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[plus-debug] +name=Rocky Linux $releasever - Plus - Debug +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=plus-$releasever-debug +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/plus/$basearch/debug/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[plus-source] +name=Rocky Linux $releasever - Plus - Source +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=source&repo=plus-$releasever-source +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/plus/source/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 diff --git a/opennebula-images/rpm-repositories/rocky/rocky.repo b/opennebula-images/rpm-repositories/rocky/rocky.repo new file mode 100644 index 0000000..99da7a8 --- /dev/null +++ b/opennebula-images/rpm-repositories/rocky/rocky.repo @@ -0,0 +1,93 @@ +# rocky.repo +# +# The mirrorlist system uses the connecting IP address of the client and the +# update status of each mirror to pick current mirrors that are geographically +# close to the client. You should use this for Rocky updates unless you are +# manually picking other mirrors. +# +# If the mirrorlist does not work for you, you can try the commented out +# baseurl line instead. + +[baseos] +name=Rocky Linux $releasever - BaseOS +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=BaseOS-$releasever +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/BaseOS/$basearch/os/ +gpgcheck=1 +enabled=1 +countme=1 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[baseos-debug] +name=Rocky Linux $releasever - BaseOS - Debug +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=BaseOS-$releasever-debug +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/BaseOS/$basearch/debug/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[baseos-source] +name=Rocky Linux $releasever - BaseOS - Source +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=source&repo=BaseOS-$releasever-source +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/BaseOS/source/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[appstream] +name=Rocky Linux $releasever - AppStream +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=AppStream-$releasever +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/AppStream/$basearch/os/ +gpgcheck=1 +enabled=1 +countme=1 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[appstream-debug] +name=Rocky Linux $releasever - AppStream - Debug +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=AppStream-$releasever-debug +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/AppStream/$basearch/debug/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[appstream-source] +name=Rocky Linux $releasever - AppStream - Source +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=source&repo=AppStream-$releasever-source +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/AppStream/source/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[crb] +name=Rocky Linux $releasever - CRB +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=CRB-$releasever +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/CRB/$basearch/os/ +gpgcheck=1 +enabled=1 +countme=1 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[crb-debug] +name=Rocky Linux $releasever - CRB - Debug +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=$basearch&repo=CRB-$releasever-debug +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/CRB/$basearch/debug/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 + +[crb-source] +name=Rocky Linux $releasever - CRB - Source +mirrorlist=https://mirrors.rockylinux.org/mirrorlist?arch=source&repo=CRB-$releasever-source +#baseurl=http://dl.rockylinux.org/$contentdir/$releasever/CRB/source/tree/ +gpgcheck=1 +enabled=0 +metadata_expire=6h +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-Rocky-9 From 7f735f515e11083fc37b3153aa85141c16fcab88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 28 Mar 2024 11:57:10 +0100 Subject: [PATCH 466/543] [ONE images] initial OpenBSD image build script --- .../openbsd-build-opennebula-image.sh | 143 ++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100755 opennebula-images/openbsd-build-opennebula-image.sh diff --git a/opennebula-images/openbsd-build-opennebula-image.sh b/opennebula-images/openbsd-build-opennebula-image.sh new file mode 100755 index 0000000..aa67061 --- /dev/null +++ b/opennebula-images/openbsd-build-opennebula-image.sh @@ -0,0 +1,143 @@ +#!/bin/sh + +# This script generates OpenBSD images for OpenNebula, being inspired from +# srht's OpenBSD build image definition. It assumes running on an OpenBSD host. + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=7.4 +ARCH=amd64 +IMAGE_PATH="$(pwd)/openbsd-$RELEASE-$(date +"%Y-%m-%d").img" +IMAGE_SIZE=10G +VIRTUAL_DEVICE=vnd0 + +# Setup working directory. +workdir="$(mktemp -d)" +cd "${workdir:?}" + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + vnconfig -u "$VIRTUAL_DEVICE" + rm -r "${workdir:?}" +} + +# Create base image. +vmctl create -s "$IMAGE_SIZE" "$IMAGE_PATH" +vnconfig "$VIRTUAL_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Format disk, mount /mnt. +fdisk -iy "${VIRTUAL_DEVICE}" +cat > "${workdir}/partitions" < /mnt/etc/fstab <>/mnt/etc/ssh/sshd_config < /mnt/etc/myname +echo "nameserver 2606:4700:4700::1111" >> /mnt/etc/resolv.conf +echo "nameserver 1.1.1.1" > /mnt/etc/resolv.conf +echo "127.0.0.1 localhost.localdomain localhost" > /mnt/etc/hosts +echo "::1 localhost.localdomain localhost" >> /mnt/etc/hosts +echo "$openbsd_installurl" > /mnt/etc/installurl +ln -sf /usr/share/zoneinfo/UTC /mnt/etc/localtime + + +cat >>/mnt/etc/rc.conf.local </mnt/var/db/kernel.SHA256) +rm -rf /mnt/usr/share/relink/kernel +mkdir -m 700 /mnt/usr/share/relink/kernel +tar -C /mnt/usr/share/relink/kernel -xzf /mnt/usr/share/relink/kernel.tgz GENERIC.MP +rm -f /mnt/usr/share/relink/kernel.tgz +echo "Relinking kernel" +chroot /mnt /bin/ksh -e -c "cd /usr/share/relink/kernel/GENERIC.MP; make newbsd; make newinstall" > /dev/null + +# Update and install utilities. +chroot /mnt /usr/sbin/pkg_add $pkg_add_params -u +chroot /mnt /usr/sbin/pkg_add $pkg_add_params bash cloud-agent + +# Remove useless kernel object files. This saves about 300MB of space in the final image +rm -rf /mnt/usr/share/relink/kernel/GENERIC.MP/ + +# Disable boot wait. Saves 5 seconds +echo "boot" > /mnt/etc/boot.conf + +# Dump root filesystem in OS image. +makefs "/dev/${VIRTUAL_DEVICE}a" /mnt +growfs -y "/dev/${VIRTUAL_DEVICE}a" +fsck -y "/dev/${VIRTUAL_DEVICE}a" +sync + +# Setup bootloader. +mount /dev/vnd0a /mnt +installboot -vr /mnt ${VIRTUAL_DEVICE:?} +umount /mnt + +# Convert raw image to qcow. +vmctl create -i "$IMAGE_PATH" "$IMAGE_PATH.qcow2" From 2659ec10c97271ab5cd8e2823f1b99db9fb1721b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 2 Apr 2024 13:40:21 +0200 Subject: [PATCH 467/543] [ONE images] refresh archlinux image --- .../arch-build-opennebula-image.sh | 98 +++++++++---------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/opennebula-images/arch-build-opennebula-image.sh b/opennebula-images/arch-build-opennebula-image.sh index 3376cf6..619bda1 100755 --- a/opennebula-images/arch-build-opennebula-image.sh +++ b/opennebula-images/arch-build-opennebula-image.sh @@ -13,8 +13,8 @@ IMAGE_PATH=arch-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 -ONE_CONTEXT_VERSION=6.0.0 -ONE_CONTEXT_SOURCE_ARCHIVE="https://github.com/OpenNebula/addon-context-linux/archive/v$ONE_CONTEXT_VERSION.tar.gz" +ONE_CONTEXT_VERSION=6.8.1 +ONE_CONTEXT_SOURCE_ARCHIVE="https://github.com/OpenNebula/one-apps/archive/refs/tags/v${ONE_CONTEXT_VERSION:?}.tar.gz" cleanup() { # The order here is important. @@ -96,76 +96,74 @@ EOF cat > /mnt/etc/pacman.d/mirrorlist << EOF ## ## Arch Linux repository mirrorlist -## Generated on 2020-05-12 +## Generated on 2024-03-07 ## ## Switzerland -Server = http://pkg.adfinis-sygroup.ch/archlinux/\$repo/os/\$arch -Server = https://pkg.adfinis-sygroup.ch/archlinux/\$repo/os/\$arch +Server = http://pkg.adfinis.com/archlinux/\$repo/os/\$arch +Server = https://pkg.adfinis.com/archlinux/\$repo/os/\$arch Server = http://mirror.init7.net/archlinux/\$repo/os/\$arch Server = https://mirror.init7.net/archlinux/\$repo/os/\$arch +Server = http://mirror.metanet.ch/archlinux/\$repo/os/\$arch +Server = https://mirror.metanet.ch/archlinux/\$repo/os/\$arch Server = http://mirror.puzzle.ch/archlinux/\$repo/os/\$arch Server = https://mirror.puzzle.ch/archlinux/\$repo/os/\$arch Server = https://mirror.ungleich.ch/mirror/packages/archlinux/\$repo/os/\$arch EOF -run_root pacman -Syu -#run_root pacman --sync --refresh --upgrade - -# Guest networking is to be handled by the one-context package. -# See https://github.com/OpenNebula/addon-context-linux for details. -run_root pacman -Sy curl tar rsync --noconfirm -run_root curl -L "$ONE_CONTEXT_SOURCE_ARCHIVE" -o one-context.tar.gz -run_root tar xf one-context.tar.gz -run_root rsync -avrh addon-context-linux-${ONE_CONTEXT_VERSION}/src/* / -#run_root cp -rT addon-context-linux-${ONE_CONTEXT_VERSION}/src/ / -run_root rm -r addon-context-linux-${ONE_CONTEXT_VERSION} -run_root cp -r /usr/lib/systemd/system/one-context.service##arch.one /usr/lib/systemd/system/one-context.service -run_root systemctl enable one-context.service +run_root pacman -Syu --noconfirm # Initalize base services. run_root systemd-machine-id-setup -run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime -run_root systemctl enable systemd-timesyncd.service -#run_root pacman -Q linux -# Install kernel and generate initramfs. -run_root pacman -Sy linux --noconfirm -run_root rsync -avrh /usr/lib/modules/ /lib/modules -#run_root pacman -Sy linux -run_root pacman -Sy mkinitcpio --noconfirm -#run_root pacman -S linux -run_root mkinitcpio -P - -# Install and configure bootloader. -run_root pacman -Sy grub --noconfirm -run_root grub-install --target=i386-pc "${NBD_DEVICE}" -#run_root sed -i 's/^GRUB_CMDLINE_LINUX=.*$/GRUB_CMDLINE_LINUX="net.ifnames=0 biosdevname=0"/' /etc/default/grub -run_root grub-mkconfig -o /boot/grub/grub.cfg -# Install en configure SSH daemon. -run_root pacman -Sy openssh netctl --noconfirm - -# Install haveged due to lack of entropy in ONE environment. -run_root pacman -Sy haveged --noconfirm -run_root systemctl enable haveged.service -run_root systemctl enable sshd -run_root systemctl enable systemd-networkd -run_root systemctl enable systemd-resolved -run_root systemctl enable systemd-timesyncd - - - # Generate fstab file. boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") cat >>/mnt/etc/fstab < /mnt/etc/hostname # Make sure everything is written to disk before exiting. -sync \ No newline at end of file +sync From 1ee1e661555aca4d163329a6c7945a2747849098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 2 Apr 2024 13:40:41 +0200 Subject: [PATCH 468/543] [ONE images] do not use hostnamectl in chroot (ubuntu build script) --- opennebula-images/ubuntu-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh index 223ce1b..b9f64de 100755 --- a/opennebula-images/ubuntu-build-opennebula-image.sh +++ b/opennebula-images/ubuntu-build-opennebula-image.sh @@ -140,7 +140,7 @@ EOF run_root rm -f /etc/machine-id run_root touch /etc/machine-id rm -f /var/lib/systemd/random-seed -run_root hostnamectl set-hostname ubuntu +echo "ubuntu" > /mnt/etc/hostname # Remove temporary files and reclaim freed disk space. run_root apt-get clean From 390313eac8819b1bfcb4cc5fc21c2d6f7042b80e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 11 Apr 2024 11:53:47 +0200 Subject: [PATCH 469/543] [ONE images] properly initialize/configure OpenBSD networking --- opennebula-images/openbsd-build-opennebula-image.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/opennebula-images/openbsd-build-opennebula-image.sh b/opennebula-images/openbsd-build-opennebula-image.sh index aa67061..266448a 100755 --- a/opennebula-images/openbsd-build-opennebula-image.sh +++ b/opennebula-images/openbsd-build-opennebula-image.sh @@ -121,6 +121,7 @@ chroot /mnt /bin/ksh -e -c "cd /usr/share/relink/kernel/GENERIC.MP; make newbsd; # Update and install utilities. chroot /mnt /usr/sbin/pkg_add $pkg_add_params -u chroot /mnt /usr/sbin/pkg_add $pkg_add_params bash cloud-agent +echo '!/usr/local/libexec/cloud-agent "\$if"' > /mnt/etc/hostname.vio0 # Remove useless kernel object files. This saves about 300MB of space in the final image rm -rf /mnt/usr/share/relink/kernel/GENERIC.MP/ From f6566612b98bf4c4f61f5ba981f2bf68b8b875d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 16 Apr 2024 09:56:12 +0200 Subject: [PATCH 470/543] [ONE images] fedora/rocky, disable grub's os-prober, set hostname --- opennebula-images/fedora-build-opennebula-image.sh | 3 ++- opennebula-images/rocky-build-opennebula-image.sh | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/opennebula-images/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh index 76a649b..20d584d 100755 --- a/opennebula-images/fedora-build-opennebula-image.sh +++ b/opennebula-images/fedora-build-opennebula-image.sh @@ -144,6 +144,7 @@ kernel_version=$(ls /mnt/boot | grep "vmlinuz.*.$ARCH" | cut -d- -f2-) run_root dracut --force --kver $kernel_version # Configure grub2. +echo "GRUB_DISABLE_OS_PROBER=true" >> /mnt/etc/default/grub run_root grub2-install --target=i386-pc "${NBD_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg @@ -163,7 +164,7 @@ EOF run_root rm -f /etc/machine-id run_root touch /etc/machine-id rm -f /var/lib/systemd/random-seed -run_root hostnamectl set-hostname fedora +echo "fedora" > /mnt/etc/hostname # Remove temporary files and reclaim freed disk space. # Note: build logs could be removed as well. diff --git a/opennebula-images/rocky-build-opennebula-image.sh b/opennebula-images/rocky-build-opennebula-image.sh index 67f29ef..3b17c1e 100755 --- a/opennebula-images/rocky-build-opennebula-image.sh +++ b/opennebula-images/rocky-build-opennebula-image.sh @@ -148,6 +148,7 @@ kernel_version=$(ls /mnt/boot | grep "vmlinuz.*.$ARCH" | cut -d- -f2-) run_root dracut --force --kver $kernel_version # Configure grub2. +echo "GRUB_DISABLE_OS_PROBER=true" >> /mnt/etc/default/grub run_root grub2-install --target=i386-pc "${NBD_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg @@ -167,7 +168,7 @@ EOF run_root rm -f /etc/machine-id run_root touch /etc/machine-id rm -f /var/lib/systemd/random-seed -run_root hostnamectl set-hostname rocky +echo "rocky" > /mnt/etc/hostname # Remove temporary files and reclaim freed disk space. # Note: build logs could be removed as well. From 0cb14991764d06b3db913475cb22effc8aa877d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 30 Apr 2024 08:50:19 +0200 Subject: [PATCH 471/543] [ONE images] bump Fedora (-> 40), Ubuntu (-> 24.04) and OpenBSD (-> 7.5) images --- opennebula-images/fedora-build-opennebula-image.sh | 2 +- opennebula-images/openbsd-build-opennebula-image.sh | 2 +- opennebula-images/ubuntu-build-opennebula-image.sh | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/opennebula-images/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh index 20d584d..a8d6071 100755 --- a/opennebula-images/fedora-build-opennebula-image.sh +++ b/opennebula-images/fedora-build-opennebula-image.sh @@ -18,7 +18,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=39 +RELEASE=40 ARCH=x86_64 IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G diff --git a/opennebula-images/openbsd-build-opennebula-image.sh b/opennebula-images/openbsd-build-opennebula-image.sh index 266448a..72de0b3 100755 --- a/opennebula-images/openbsd-build-opennebula-image.sh +++ b/opennebula-images/openbsd-build-opennebula-image.sh @@ -7,7 +7,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=7.4 +RELEASE=7.5 ARCH=amd64 IMAGE_PATH="$(pwd)/openbsd-$RELEASE-$(date +"%Y-%m-%d").img" IMAGE_SIZE=10G diff --git a/opennebula-images/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh index b9f64de..db1b37b 100755 --- a/opennebula-images/ubuntu-build-opennebula-image.sh +++ b/opennebula-images/ubuntu-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=mantic # 23.10 +RELEASE=noble # 24.04 LTS ARCH=amd64 IMAGE_PATH=ubuntu-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G @@ -122,6 +122,7 @@ run_root "echo 'grub-pc grub-pc/install_devices_empty boolean true' | debconf-se run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-base linux-image-generic grub-pc # Configure grub. +echo "GRUB_DISABLE_OS_PROBER=true" >> /mnt/etc/default/grub run_root grub-install --target=i386-pc "${NBD_DEVICE}" run_root grub-mkconfig -o /boot/grub/grub.cfg From 492e3e5ab389a27f8411631a643779704e31f858 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 9 May 2024 13:50:42 +0200 Subject: [PATCH 472/543] [openwrt/viwib black] update to openwrt 23.05.3 --- openwrt/viwib-1-black-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viwib-1-black-firmware-upgrade.sh b/openwrt/viwib-1-black-firmware-upgrade.sh index 4f3afac..9a1479d 100755 --- a/openwrt/viwib-1-black-firmware-upgrade.sh +++ b/openwrt/viwib-1-black-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=22.03.5 +version=23.05.3 filename=openwrt-${version}-ath79-nand-glinet_gl-ar300m-nor-squashfs-sysupgrade.bin url=https://downloads.openwrt.org/releases/${version}/targets/ath79/nand From ff171bb9e33edbe7e2b9790989b5e7356242cffa Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 May 2024 13:13:06 +0200 Subject: [PATCH 473/543] [vigir] bump to 23.05.3 --- openwrt/vigir-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index c3be5d9..fcad3e7 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,7 +12,7 @@ set -x vigir_ip=$1; shift # openwrt -version=23.05.2 +version=23.05.3 filename=openwrt-${version}-ramips-mt7621-zbtlink_zbt-wg3526-16m-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} From 44b338e6b4484df55e01ea5c79a18e52e240d6f2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 14 May 2024 13:22:59 +0200 Subject: [PATCH 474/543] [vigir-2] use ping, not forced ping6 --- openwrt/vigir-2-configure.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/vigir-2-configure.sh b/openwrt/vigir-2-configure.sh index 74369d5..9c08251 100755 --- a/openwrt/vigir-2-configure.sh +++ b/openwrt/vigir-2-configure.sh @@ -111,7 +111,7 @@ uci delete wireless.radio1.disabled uci commit -ping6 -c5 ungleich.ch || exit 1 +ping -c5 ungleich.ch || exit 1 # update the sources opkg update From d2d393862d55600d670462f537a029764aa79011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 28 May 2024 11:11:05 +0200 Subject: [PATCH 475/543] [ONE images] refresh alpine image to v3.20 --- opennebula-images/alpine-build-opennebula-image.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/opennebula-images/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh index 5f2361d..645d7a8 100755 --- a/opennebula-images/alpine-build-opennebula-image.sh +++ b/opennebula-images/alpine-build-opennebula-image.sh @@ -8,8 +8,7 @@ set -e set -x -# XXX: Handle command-line arguments? -RELEASE=v3.19 +RELEASE=v3.20 ARCH=x86_64 IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G From 2342d9beb1b94e948132e069e76d7ce14bae7cf9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 1 Jul 2024 15:24:06 +0200 Subject: [PATCH 476/543] alpine linux: bump to 3.20.1 --- alpine-install-on-disk.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index c86a826..5112959 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -16,8 +16,8 @@ DISK=$1; shift SSH_KEYS=$1; shift BOOT_VIA=$1; shift -MAJOR_VERSION=3.19 -MINOR_VERSION=0 +MAJOR_VERSION=3.20 +MINOR_VERSION=1 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz From f97a13a87758a208f0307fdb5e071d58277e7868 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 3 Jul 2024 11:57:26 +0200 Subject: [PATCH 477/543] ceph-start-all-monit: add execute permissions --- ceph/start-all-monit-osds.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ceph/start-all-monit-osds.sh diff --git a/ceph/start-all-monit-osds.sh b/ceph/start-all-monit-osds.sh old mode 100644 new mode 100755 From be852526b8c8f1628bf0b0c3ea548ef8da1e0843 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 22 Jul 2024 12:06:04 +0200 Subject: [PATCH 478/543] [openwrt] start nat64 configuration script --- openwrt/openwrt-nat64-bootstrap.sh | 33 ++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100755 openwrt/openwrt-nat64-bootstrap.sh diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh new file mode 100755 index 0000000..9e9c0d3 --- /dev/null +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -0,0 +1,33 @@ +#!/bin/sh +# Nico Schottelius, 2024-07-22 +# This script assumes a clean/newly setup openwrt device +# +# Assumption: +# WAN = IPv4, dhcp provided externally +# LAN = IPv6, "clients" that want to reach IPv4 Internet +# Consequences +# - do not provide IPv4 or IPv6 dhcp/ra on any interface + +if [ $# -ne 3 ] ; then + echo $0 address hostname nat64prefix + exit 1 +fi + +address=$1; shift +hostname=$1; shift +nat64=$1; shift + +cat < Date: Mon, 22 Jul 2024 12:18:48 +0200 Subject: [PATCH 479/543] [openwrt/nat64] uci commit --- openwrt/openwrt-nat64-bootstrap.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 9e9c0d3..6d1030c 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -20,6 +20,7 @@ nat64=$1; shift cat < Date: Tue, 23 Jul 2024 10:27:58 +0200 Subject: [PATCH 480/543] openwrt/nat64: begin to introduce iBGP syntax --- openwrt/openwrt-nat64-bootstrap.sh | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 6d1030c..167d718 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -8,19 +8,24 @@ # Consequences # - do not provide IPv4 or IPv6 dhcp/ra on any interface -if [ $# -ne 3 ] ; then - echo $0 address hostname nat64prefix +if [ $# -lt 4 ] ; then + echo $0 "address hostname nat64prefix asn iBGPpeer1 [iBGPpeer2...]" exit 1 fi address=$1; shift hostname=$1; shift nat64=$1; shift +asn=$1; shift + +# Now $@ only contains iBGP peers cat < Date: Tue, 23 Jul 2024 13:38:49 +0200 Subject: [PATCH 481/543] [openwrt/nat64] add bgp config generator --- openwrt/openwrt-nat64-bootstrap.sh | 33 +++++++++++++++++++++++++++--- 1 file changed, 30 insertions(+), 3 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 167d718..aab739a 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -9,14 +9,17 @@ # - do not provide IPv4 or IPv6 dhcp/ra on any interface if [ $# -lt 4 ] ; then - echo $0 "address hostname nat64prefix asn iBGPpeer1 [iBGPpeer2...]" + echo $0 "address hostname nat64prefix nat64route asn routerid iBGPpeer1 [iBGPpeer2...]" exit 1 fi address=$1; shift hostname=$1; shift -nat64=$1; shift +nat64prefix=$1; shift +nat64route=$1; shift asn=$1; shift +routerid=$1; shift +ibgp_peers="$@" # Now $@ only contains iBGP peers @@ -36,8 +39,32 @@ uci set system.@system[0].hostname="${hostname}" # Do something wireless (?) # disable? -echo "Setting up ibgp for $@ .." +echo "Setting up bird ..." + +cat > /etc/bird.conf <> /etc/bird.conf < Date: Tue, 23 Jul 2024 13:41:54 +0200 Subject: [PATCH 482/543] openwrt/nat64: fix syntax error --- openwrt/openwrt-nat64-bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index aab739a..8dc1121 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -63,6 +63,7 @@ protocol bgp { }; } BBB +done uci commit /etc/init.d/bird restart From 0431b89a54a5d46b40839990e7ae8376ecccb1ee Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 23 Jul 2024 13:49:44 +0200 Subject: [PATCH 483/543] [openwrt/nat64] escape local var, add birdc --- openwrt/openwrt-nat64-bootstrap.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 8dc1121..02f18d5 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -28,7 +28,7 @@ set -x opkg update # add jool + bird2 -opkg install jool-tools-netfilter bird2 +opkg install jool-tools-netfilter bird2 bird2c # Do not announce ULA - we have GUA uci delete network.globals.ula_prefix @@ -55,7 +55,7 @@ for ibgp_peer in ${ibgp_peers}; do cat >> /etc/bird.conf < Date: Tue, 23 Jul 2024 13:51:05 +0200 Subject: [PATCH 484/543] [openwrt/nat64] add todos --- openwrt/openwrt-nat64-bootstrap.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 02f18d5..e29736e 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -65,6 +65,12 @@ protocol bgp { BBB done +# TODO: configure jool +# TODO: start jool +# TODO: ensure jool is started at boot +# TODO: ensure bird is started at boot + + uci commit /etc/init.d/bird restart From 9fd12ac43bdd8d457df0830791848a47a6159304 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 24 Jul 2024 11:05:28 +0200 Subject: [PATCH 485/543] openwrt/nat64: add jool support --- openwrt/openwrt-nat64-bootstrap.sh | 39 +++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 3 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index e29736e..e09d5c2 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -9,7 +9,7 @@ # - do not provide IPv4 or IPv6 dhcp/ra on any interface if [ $# -lt 4 ] ; then - echo $0 "address hostname nat64prefix nat64route asn routerid iBGPpeer1 [iBGPpeer2...]" + echo $0 "address hostname nat64prefix nat64route ipv4out asn routerid iBGPpeer1 [iBGPpeer2...]" exit 1 fi @@ -17,6 +17,7 @@ address=$1; shift hostname=$1; shift nat64prefix=$1; shift nat64route=$1; shift +ipv4out=$1; shift asn=$1; shift routerid=$1; shift ibgp_peers="$@" @@ -66,11 +67,43 @@ BBB done # TODO: configure jool -# TODO: start jool +cat >> /etc/jool/jool-nat64.conf.json < Date: Wed, 24 Jul 2024 11:53:18 +0200 Subject: [PATCH 486/543] [openwrt] bump to 23.05.4 --- openwrt/vigir-1-firmware-upgrade.sh | 2 +- openwrt/viirb-1-firmware-upgrade.sh | 2 +- openwrt/viwib-1-black-firmware-upgrade.sh | 2 +- openwrt/viwib-1-yellow-firmware-upgrade.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index fcad3e7..2b5579e 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,7 +12,7 @@ set -x vigir_ip=$1; shift # openwrt -version=23.05.3 +version=23.05.4 filename=openwrt-${version}-ramips-mt7621-zbtlink_zbt-wg3526-16m-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} diff --git a/openwrt/viirb-1-firmware-upgrade.sh b/openwrt/viirb-1-firmware-upgrade.sh index 88310ac..07a309a 100755 --- a/openwrt/viirb-1-firmware-upgrade.sh +++ b/openwrt/viirb-1-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viirb_ip=$1; shift # openwrt -version=21.02.3 +version=23.05.4 filename=openwrt-${version}-ramips-mt76x8-vocore_vocore2-squashfs-sysupgrade.bin # don't care about other/old known_host entries diff --git a/openwrt/viwib-1-black-firmware-upgrade.sh b/openwrt/viwib-1-black-firmware-upgrade.sh index 9a1479d..306c435 100755 --- a/openwrt/viwib-1-black-firmware-upgrade.sh +++ b/openwrt/viwib-1-black-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=23.05.3 +version=23.05.4 filename=openwrt-${version}-ath79-nand-glinet_gl-ar300m-nor-squashfs-sysupgrade.bin url=https://downloads.openwrt.org/releases/${version}/targets/ath79/nand diff --git a/openwrt/viwib-1-yellow-firmware-upgrade.sh b/openwrt/viwib-1-yellow-firmware-upgrade.sh index 761625e..e1b3cdf 100755 --- a/openwrt/viwib-1-yellow-firmware-upgrade.sh +++ b/openwrt/viwib-1-yellow-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=22.03.5 +version=23.05.4 filename=openwrt-${version}-ramips-mt76x8-glinet_gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries From a8bc794e4768dca86c109a470757106da1bc4a01 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 24 Jul 2024 11:53:27 +0200 Subject: [PATCH 487/543] openwrt/nat64: bump argument requirement --- openwrt/openwrt-nat64-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index e09d5c2..ed24a81 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -8,7 +8,7 @@ # Consequences # - do not provide IPv4 or IPv6 dhcp/ra on any interface -if [ $# -lt 4 ] ; then +if [ $# -lt 7 ] ; then echo $0 "address hostname nat64prefix nat64route ipv4out asn routerid iBGPpeer1 [iBGPpeer2...]" exit 1 fi From c168fdc1700dc7bd527be66b9336fd1a1ca591a1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 25 Jul 2024 11:53:50 +0200 Subject: [PATCH 488/543] openwrt/nat64: add stuff --- openwrt/openwrt-nat64-bootstrap.sh | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index ed24a81..3afee24 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -1,3 +1,4 @@ + #!/bin/sh # Nico Schottelius, 2024-07-22 # This script assumes a clean/newly setup openwrt device @@ -9,7 +10,7 @@ # - do not provide IPv4 or IPv6 dhcp/ra on any interface if [ $# -lt 7 ] ; then - echo $0 "address hostname nat64prefix nat64route ipv4out asn routerid iBGPpeer1 [iBGPpeer2...]" + echo $0 "address hostname nat64prefix nat64route ipv4address ipv4gw asn routerid iBGPpeer1 [iBGPpeer2...]" exit 1 fi @@ -17,7 +18,8 @@ address=$1; shift hostname=$1; shift nat64prefix=$1; shift nat64route=$1; shift -ipv4out=$1; shift +ipv4address=$1; shift +ipv4gw=$1; shift asn=$1; shift routerid=$1; shift ibgp_peers="$@" @@ -34,9 +36,20 @@ opkg install jool-tools-netfilter bird2 bird2c # Do not announce ULA - we have GUA uci delete network.globals.ula_prefix +# Remove IPv6 assign, we are using static IPv6 +uci delete network.lan.ip6assign + +# Disable firewalling effectively to allow traffic any direction +uci set firewall.@defaults[0].input=ACCEPT +uci set firewall.@defaults[0].forward=ACCEPT + # Set hostname uci set system.@system[0].hostname="${hostname}" +# Set IPv4 address on LAN +uci set network.lan.ipaddr="${ipv4address}" +uci set network.lan.gateway="${ipv4gw}" + # Do something wireless (?) # disable? @@ -83,15 +96,15 @@ cat >> /etc/jool/jool-nat64.conf.json < Date: Fri, 26 Jul 2024 09:29:40 +0200 Subject: [PATCH 489/543] stuff --- openwrt/openwrt-nat64-bootstrap.sh | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 3afee24..6c5ed42 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -46,9 +46,18 @@ uci set firewall.@defaults[0].forward=ACCEPT # Set hostname uci set system.@system[0].hostname="${hostname}" -# Set IPv4 address on LAN -uci set network.lan.ipaddr="${ipv4address}" -uci set network.lan.gateway="${ipv4gw}" +# Set IPv4 address on WAN for NAT64 upstream +uci set network.wan.ipaddr="${ipv4address}" +uci set network.wan.netmask="255.255.255.0" # hardcoded, usually correct, fix this script if needed +uci set network.wan.gateway="${ipv4gw}" +uci set network.wan.proto="static" + +# Make LAN IPv6 dynamic +uci delete network.lan.ipaddr +uci delete network.lan.netmask +uci delete network.lan.gateway +uci set network.wan.proto="dhcpv6" + # Do something wireless (?) # disable? From 847f9c1dd12eac2e766fcef19312d156f9254dfc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 26 Jul 2024 09:46:57 +0200 Subject: [PATCH 490/543] openwrt/nat64: change lan/wan proto --- openwrt/openwrt-nat64-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 6c5ed42..9419e5e 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -56,7 +56,7 @@ uci set network.wan.proto="static" uci delete network.lan.ipaddr uci delete network.lan.netmask uci delete network.lan.gateway -uci set network.wan.proto="dhcpv6" +uci set network.lan.proto="dhcpv6" # Do something wireless (?) From b8829f96acc6951e1d7ecf438cafdc5ed645c53d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 26 Jul 2024 09:58:27 +0200 Subject: [PATCH 491/543] openwrt/nat64: disable dhcp on lan --- openwrt/openwrt-nat64-bootstrap.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 9419e5e..d25e2b6 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -58,6 +58,11 @@ uci delete network.lan.netmask uci delete network.lan.gateway uci set network.lan.proto="dhcpv6" +# Disable DHCP on LAN +uci delete dhcp.lan.dhcpv4 +uci delete dhcp.lan.dhcpv6 +uci delete dhcp.lan.ra +uci set dhcp.lan.ignore=1 # Do something wireless (?) # disable? @@ -130,3 +135,5 @@ uci commit /etc/init.d/bird restart EOF + +echo "Restart router to restart firewall, network, dhcp" From 310de6a4ed1694ab970b064c36a862e35a20fd04 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 26 Jul 2024 10:02:47 +0200 Subject: [PATCH 492/543] openwrt/nat64: enable jool --- openwrt/openwrt-nat64-bootstrap.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index d25e2b6..1a79401 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -63,9 +63,9 @@ uci delete dhcp.lan.dhcpv4 uci delete dhcp.lan.dhcpv6 uci delete dhcp.lan.ra uci set dhcp.lan.ignore=1 +uci commit -# Do something wireless (?) -# disable? +# TODO: Do something wireless (?), maybe disable? echo "Setting up bird ..." @@ -94,7 +94,7 @@ BBB done # TODO: configure jool -cat >> /etc/jool/jool-nat64.conf.json < /etc/jool/jool-nat64.conf.json <> /etc/jool/jool-nat64.conf.json < Date: Fri, 26 Jul 2024 10:07:35 +0200 Subject: [PATCH 493/543] [openwrt/nat64] export route, not prefix via bgp --- openwrt/openwrt-nat64-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 1a79401..fbcc9b0 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -75,7 +75,7 @@ router id ${routerid}; protocol static static6 { ipv6; - route ${nat64prefix} unreachable; + route ${nat64route} unreachable; } BBB From a84ffc49e981f06f874db8ec2908a7ce80805db7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 26 Jul 2024 10:43:17 +0200 Subject: [PATCH 494/543] [openwrt/nat64] only use prefix, nt --- openwrt/openwrt-nat64-bootstrap.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index fbcc9b0..e86ea9f 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -9,15 +9,14 @@ # Consequences # - do not provide IPv4 or IPv6 dhcp/ra on any interface -if [ $# -lt 7 ] ; then - echo $0 "address hostname nat64prefix nat64route ipv4address ipv4gw asn routerid iBGPpeer1 [iBGPpeer2...]" +if [ $# -lt 8 ] ; then + echo $0 "address hostname nat64prefix ipv4address ipv4gw asn routerid iBGPpeer1 [iBGPpeer2...]" exit 1 fi address=$1; shift hostname=$1; shift nat64prefix=$1; shift -nat64route=$1; shift ipv4address=$1; shift ipv4gw=$1; shift asn=$1; shift @@ -30,8 +29,8 @@ cat < /etc/bird.conf < Date: Fri, 26 Jul 2024 11:04:35 +0200 Subject: [PATCH 495/543] [openwrt/nat64] add babel->kernel support Needed for local ibgp routes --- openwrt/openwrt-nat64-bootstrap.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index e86ea9f..d9748fd 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -10,7 +10,7 @@ # - do not provide IPv4 or IPv6 dhcp/ra on any interface if [ $# -lt 8 ] ; then - echo $0 "address hostname nat64prefix ipv4address ipv4gw asn routerid iBGPpeer1 [iBGPpeer2...]" + echo $0 "address hostname nat64prefix ipv4address ipv4gw asn routerid babelpw iBGPpeer1 [iBGPpeer2...]" exit 1 fi @@ -21,6 +21,7 @@ ipv4address=$1; shift ipv4gw=$1; shift asn=$1; shift routerid=$1; shift +babelpw=$1; shift ibgp_peers="$@" # Now $@ only contains iBGP peers @@ -72,11 +73,22 @@ cat > /etc/bird.conf < Date: Fri, 26 Jul 2024 11:14:37 +0200 Subject: [PATCH 496/543] [openwrt/nat64] add atop --- openwrt/openwrt-nat64-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index d9748fd..95c957e 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -31,7 +31,7 @@ set -x opkg update # add jool + bird2 + tcpdump -opkg install jool-tools-netfilter bird2 bird2c tcpdump tmux +opkg install jool-tools-netfilter bird2 bird2c tcpdump tmux atop # Do not announce ULA - we have GUA uci delete network.globals.ula_prefix From b46a9082dc33a31ece1f83afc9efde5dd0509aac Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 31 Jul 2024 16:21:12 +0200 Subject: [PATCH 497/543] [openwrt-nat64] fix bird syntax error --- openwrt/openwrt-nat64-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 95c957e..94b0692 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -87,7 +87,7 @@ protocol babel { ipv6 { export where (source = RTS_DEVICE) || (source = RTS_BABEL); }; } protocol kernel kernel_v6 { - ipv6 { export export where source ~ [ RTS_BABEL ]; }; + ipv6 { export where source ~ [ RTS_BABEL ]; }; } BBB From 0c8776d5d26f6e4c93908586698d9e68207de289 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 31 Jul 2024 16:23:09 +0200 Subject: [PATCH 498/543] [openwrt-nat64] add nload --- openwrt/openwrt-nat64-bootstrap.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 94b0692..e89de63 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -31,7 +31,7 @@ set -x opkg update # add jool + bird2 + tcpdump -opkg install jool-tools-netfilter bird2 bird2c tcpdump tmux atop +opkg install jool-tools-netfilter bird2 bird2c tcpdump tmux atop nload # Do not announce ULA - we have GUA uci delete network.globals.ula_prefix From 4c1c08ca4a77f16718507fc414e791dec6728500 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 2 Aug 2024 10:05:18 +0200 Subject: [PATCH 499/543] [openwrt/nat64] extend port range --- openwrt/openwrt-nat64-bootstrap.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index e89de63..0fe691a 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -123,15 +123,15 @@ cat > /etc/jool/jool-nat64.conf.json < Date: Sun, 4 Aug 2024 12:05:09 +0200 Subject: [PATCH 500/543] [openwrt/nat64] disable source routing to fix routing issue --- openwrt/openwrt-nat64-bootstrap.sh | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 0fe691a..beeca6f 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -24,7 +24,6 @@ routerid=$1; shift babelpw=$1; shift ibgp_peers="$@" -# Now $@ only contains iBGP peers cat < /etc/jool/jool-nat64.conf.json < Date: Mon, 5 Aug 2024 13:05:23 +0200 Subject: [PATCH 501/543] ceph: update script for startup --- ceph/start-all-monit-osds.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/ceph/start-all-monit-osds.sh b/ceph/start-all-monit-osds.sh index d1108fd..ba4a430 100755 --- a/ceph/start-all-monit-osds.sh +++ b/ceph/start-all-monit-osds.sh @@ -1,5 +1,6 @@ #!/bin/sh # 2022-02-01 +# Update 2024-08-05: include mgr and osds /etc/init.d/lvm2 start @@ -9,4 +10,12 @@ ceph-volume lvm activate --all -for osd in /etc/monit/conf.d/osd.*; do cmd=$(cat $osd | grep "start pro" | sed -e 's/.* = "//' -e 's/".*//'); echo $cmd; ( $cmd & ); done +for osd in + /etc/monit/conf.d/osd.* \ + /etc/monit/conf.d/mgr.* \ + /etc/monit/conf.d/mon.* ; do + + cmd=$(cat $osd | grep "start pro" | sed -e 's/.* = "//' -e 's/".*//') + echo $cmd + ( $cmd & ) +done From 4b740b4b3be3a85e75115021b1c9b8aa4e98e253 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 5 Aug 2024 13:07:08 +0200 Subject: [PATCH 502/543] rename startup script for ceph components Signed-off-by: Nico Schottelius --- ...l-monit-osds.sh => ceph-start-FAST-monit-defined-processes.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename ceph/{start-all-monit-osds.sh => ceph-start-FAST-monit-defined-processes.sh} (100%) diff --git a/ceph/start-all-monit-osds.sh b/ceph/ceph-start-FAST-monit-defined-processes.sh similarity index 100% rename from ceph/start-all-monit-osds.sh rename to ceph/ceph-start-FAST-monit-defined-processes.sh From 8582e73afc7d2d9acc5723530754ea9f60ed7f54 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 5 Aug 2024 13:18:49 +0200 Subject: [PATCH 503/543] ceph: fast starter: start mon, mgr first, fix syntax error --- ceph/ceph-start-FAST-monit-defined-processes.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ceph/ceph-start-FAST-monit-defined-processes.sh b/ceph/ceph-start-FAST-monit-defined-processes.sh index ba4a430..10ee906 100755 --- a/ceph/ceph-start-FAST-monit-defined-processes.sh +++ b/ceph/ceph-start-FAST-monit-defined-processes.sh @@ -10,10 +10,10 @@ ceph-volume lvm activate --all -for osd in - /etc/monit/conf.d/osd.* \ +for osd in \ + /etc/monit/conf.d/mon.* \ /etc/monit/conf.d/mgr.* \ - /etc/monit/conf.d/mon.* ; do + /etc/monit/conf.d/osd.* ; do cmd=$(cat $osd | grep "start pro" | sed -e 's/.* = "//' -e 's/".*//') echo $cmd From 1d378c74adfadf4e2544d8f62608954a1023cbbd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Aug 2024 12:27:26 +0200 Subject: [PATCH 504/543] --whitespace Signed-off-by: Nico Schottelius --- openwrt/openwrt-nat64-bootstrap.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index beeca6f..25e7598 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -1,4 +1,3 @@ - #!/bin/sh # Nico Schottelius, 2024-07-22 # This script assumes a clean/newly setup openwrt device From 877fe7e377df7b5866b1cad1903a70f5dec771a6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Aug 2024 13:02:47 +0200 Subject: [PATCH 505/543] openwrt/nat64/bgp: enable bfd --- openwrt/openwrt-nat64-bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 25e7598..0045054 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -110,6 +110,7 @@ cat >> /etc/bird.conf < Date: Thu, 8 Aug 2024 13:03:53 +0200 Subject: [PATCH 506/543] openwrt/nat64/bgp: enable direct --- openwrt/openwrt-nat64-bootstrap.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index 0045054..bc7249b 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -110,6 +110,7 @@ cat >> /etc/bird.conf < Date: Sat, 10 Aug 2024 14:31:40 +0200 Subject: [PATCH 507/543] openwrt/nat64: disable bfd, add gateway recursive --- openwrt/openwrt-nat64-bootstrap.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openwrt/openwrt-nat64-bootstrap.sh b/openwrt/openwrt-nat64-bootstrap.sh index bc7249b..d45c034 100755 --- a/openwrt/openwrt-nat64-bootstrap.sh +++ b/openwrt/openwrt-nat64-bootstrap.sh @@ -111,11 +111,12 @@ protocol bgp { local as ${asn}; neighbor \${ibgp_peer} as ${asn}; direct; - bfd on; + # bfd on; ipv6 { import none; export where source ~ [ RTS_STATIC ]; + gateway recursive; }; } BBB From 98f168c2d25c3a94656c7cf3f936fb6fda129b72 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 13 Aug 2024 16:43:13 +0200 Subject: [PATCH 508/543] [openwrt] start viirb/monitor --- openwrt/openwrt-viirb-monitor.sh | 77 ++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100755 openwrt/openwrt-viirb-monitor.sh diff --git a/openwrt/openwrt-viirb-monitor.sh b/openwrt/openwrt-viirb-monitor.sh new file mode 100755 index 0000000..3134149 --- /dev/null +++ b/openwrt/openwrt-viirb-monitor.sh @@ -0,0 +1,77 @@ +#!/bin/sh +# Nico Schottelius, 2024-07-22 +# This script assumes a clean/newly setup openwrt device +# +# Assumption: +# VIIRB being used to... +# - monitor camera +# - monitor UPS via USB + +if [ $# -lt 8 ] ; then + echo $0 "address hostname wifi-ssid wifi-psk" +fi + +address=$1; shift +hostname=$1; shift +wifi_ssid=$1; shift +wifi_psk=$1; shift + +cat < Date: Wed, 14 Aug 2024 09:46:06 +0200 Subject: [PATCH 509/543] openwrt/viirb monitor: update bootstrapping --- openwrt/openwrt-viirb-monitor.sh | 47 ++++++++++++++++++++------------ 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/openwrt/openwrt-viirb-monitor.sh b/openwrt/openwrt-viirb-monitor.sh index 3134149..0c16cd1 100755 --- a/openwrt/openwrt-viirb-monitor.sh +++ b/openwrt/openwrt-viirb-monitor.sh @@ -8,14 +8,18 @@ # - monitor UPS via USB if [ $# -lt 8 ] ; then - echo $0 "address hostname wifi-ssid wifi-psk" + echo $0 "address hostname passwd wifi-ssid wifi-psk" fi address=$1; shift hostname=$1; shift +root_password=$1; shift wifi_ssid=$1; shift wifi_psk=$1; shift +# Allow passwordless to avoid nightmare +cat ~/.ssh/id_rsa.pub | ssh root@${address} "cat > /etc/dropbear/authorized_keys" + cat < Date: Wed, 14 Aug 2024 10:35:21 +0200 Subject: [PATCH 510/543] openwrt/viirb monitor: add to lan list --- openwrt/openwrt-viirb-monitor.sh | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/openwrt/openwrt-viirb-monitor.sh b/openwrt/openwrt-viirb-monitor.sh index 0c16cd1..f3c63d8 100755 --- a/openwrt/openwrt-viirb-monitor.sh +++ b/openwrt/openwrt-viirb-monitor.sh @@ -22,7 +22,7 @@ cat ~/.ssh/id_rsa.pub | ssh root@${address} "cat > /etc/dropbear/authorized_keys cat < Date: Thu, 15 Aug 2024 13:39:44 +0200 Subject: [PATCH 511/543] Add script to generate dns entries from interface addresses --- dns-generate-for-device.sh | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 dns-generate-for-device.sh diff --git a/dns-generate-for-device.sh b/dns-generate-for-device.sh new file mode 100644 index 0000000..85c8f61 --- /dev/null +++ b/dns-generate-for-device.sh @@ -0,0 +1,18 @@ +#!/bin/sh +# Nico Schottelius, 2024-08-14 +# +# Retrieve an IPv6 interface address and generate a generic +# DNS zone file + +if [ #$ -ne 3 ]; then + echo "$0 address interface hostname" + exit 1 +fi + +address=$1; shift +interface=$1; shift +hostname=$1; shift + +interface_addr=$(ssh "root@${address}" "ip -o a sh dev ${interface}" | awk '/inet6/ { print $4 }' | grep -v ^fe80| sed 's,/.*,,') + +echo "${hostname} AAAA ${interface_addr}" From 33f8f69be31229a35d9abe70ab4d0dcd1201f19e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 15 Aug 2024 13:41:14 +0200 Subject: [PATCH 512/543] dns/generate: fix permissions, args --- dns-generate-for-device.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 dns-generate-for-device.sh diff --git a/dns-generate-for-device.sh b/dns-generate-for-device.sh old mode 100644 new mode 100755 index 85c8f61..a96d46f --- a/dns-generate-for-device.sh +++ b/dns-generate-for-device.sh @@ -4,7 +4,7 @@ # Retrieve an IPv6 interface address and generate a generic # DNS zone file -if [ #$ -ne 3 ]; then +if [ $# -ne 3 ]; then echo "$0 address interface hostname" exit 1 fi From e0f6cfe931c7e410ca38c6bb3f4f4059724c9304 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 17 Aug 2024 17:46:45 +0200 Subject: [PATCH 513/543] openwrt: add script for creating ups entries --- openwrt/openwrt-configure-nut-ups.sh | 30 ++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 openwrt/openwrt-configure-nut-ups.sh diff --git a/openwrt/openwrt-configure-nut-ups.sh b/openwrt/openwrt-configure-nut-ups.sh new file mode 100644 index 0000000..d87e478 --- /dev/null +++ b/openwrt/openwrt-configure-nut-ups.sh @@ -0,0 +1,30 @@ +#!/bin/sh +# Nico Schottelius, 2024-08-17 +# create configuration for nut based on available usb ups + + +if [ $# -lt 8 ] ; then + echo $0 "address" + exit 1 +fi + +address=$1; shift + +cat </dev/null | grep -e "iProduct.* UPS" -A1 | awk '/iSerial/ { print \$3 }'); do + +uci set nut_server.ups\${i}=driver +uci set nut_server.ups\${i}.port=auto +uci set nut_server.ups\${i}.driver=usbhid-ups +uci set nut_server.ups\${i}.serial=\$ups + +i=\$((i+1)) + +done + +uci commit From c0908940091f9b1003b2b49bf4d6a0b26e6769f7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 17 Aug 2024 18:12:56 +0200 Subject: [PATCH 514/543] ups: include Innova... --- openwrt/openwrt-configure-nut-ups.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) mode change 100644 => 100755 openwrt/openwrt-configure-nut-ups.sh diff --git a/openwrt/openwrt-configure-nut-ups.sh b/openwrt/openwrt-configure-nut-ups.sh old mode 100644 new mode 100755 index d87e478..1c793dc --- a/openwrt/openwrt-configure-nut-ups.sh +++ b/openwrt/openwrt-configure-nut-ups.sh @@ -3,7 +3,7 @@ # create configuration for nut based on available usb ups -if [ $# -lt 8 ] ; then +if [ $# -lt 1 ] ; then echo $0 "address" exit 1 fi @@ -12,11 +12,9 @@ address=$1; shift cat </dev/null | grep -e "iProduct.* UPS" -A1 | awk '/iSerial/ { print \$3 }'); do +for ups in \$(lsusb -vv 2>/dev/null | grep -e "iProduct.* UPS" -e "iProduct.*Innova Unity" -A1 | awk '/iSerial/ { print \$3 }'); do uci set nut_server.ups\${i}=driver uci set nut_server.ups\${i}.port=auto From db567a9fbd5a5e7c9da33c440e8501a37aa8ae42 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 17 Aug 2024 18:14:50 +0200 Subject: [PATCH 515/543] ups: add support for LIHVX2K0 --- openwrt/openwrt-configure-nut-ups.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-configure-nut-ups.sh b/openwrt/openwrt-configure-nut-ups.sh index 1c793dc..af9f6b1 100755 --- a/openwrt/openwrt-configure-nut-ups.sh +++ b/openwrt/openwrt-configure-nut-ups.sh @@ -14,7 +14,7 @@ cat </dev/null | grep -e "iProduct.* UPS" -e "iProduct.*Innova Unity" -A1 | awk '/iSerial/ { print \$3 }'); do +for ups in \$(lsusb -vv 2>/dev/null | grep -e "iProduct.* UPS" -e "iProduct.*Innova Unity" -e "iProduct.*LIHVX2K0" -A1 | awk '/iSerial/ { print \$3 }'); do uci set nut_server.ups\${i}=driver uci set nut_server.ups\${i}.port=auto From 8fd5745ce67460133a6365e07f4c9bfdd35c55f1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 17 Aug 2024 20:15:39 +0200 Subject: [PATCH 516/543] openwrt: add file to turn router into wifi client --- openwrt/openwrt-viwib-camera.sh | 80 +++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 openwrt/openwrt-viwib-camera.sh diff --git a/openwrt/openwrt-viwib-camera.sh b/openwrt/openwrt-viwib-camera.sh new file mode 100755 index 0000000..8411a61 --- /dev/null +++ b/openwrt/openwrt-viwib-camera.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# Nico Schottelius, 2024-07-22 +# This script assumes a clean/newly setup openwrt device +# +# Assumption: +# VIWIB being used to... +# - monitor camera + +if [ $# -lt 8 ] ; then + echo $0 "address hostname passwd wifi-ssid wifi-psk" +fi + +address=$1; shift +hostname=$1; shift +root_password=$1; shift +wifi_ssid=$1; shift +wifi_psk=$1; shift + +# Allow passwordless to avoid nightmare +cat ~/.ssh/id_rsa.pub | ssh root@${address} "cat > /etc/dropbear/authorized_keys" + +cat < Date: Sat, 17 Aug 2024 20:16:54 +0200 Subject: [PATCH 517/543] openwrt: rename to better suit actual usage Signed-off-by: Nico Schottelius --- openwrt/{openwrt-viwib-camera.sh => openwrt-make-wifi-client.sh} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename openwrt/{openwrt-viwib-camera.sh => openwrt-make-wifi-client.sh} (100%) diff --git a/openwrt/openwrt-viwib-camera.sh b/openwrt/openwrt-make-wifi-client.sh similarity index 100% rename from openwrt/openwrt-viwib-camera.sh rename to openwrt/openwrt-make-wifi-client.sh From 2ecf88132498d388ce4a8334b02a8ecadc562b7e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 18 Aug 2024 14:05:50 +0200 Subject: [PATCH 518/543] alpine-linux: bump to 3.20.2 --- alpine-install-on-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 5112959..4dbace7 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -17,7 +17,7 @@ SSH_KEYS=$1; shift BOOT_VIA=$1; shift MAJOR_VERSION=3.20 -MINOR_VERSION=1 +MINOR_VERSION=2 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz From 6694face6a971f6c963ca044945ebaac3a369c8a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 19:30:46 +0100 Subject: [PATCH 519/543] Add initial code for managing encrypted messages on matrix --- ...messages-from-encrypted-room-via-backup.py | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100755 matrix/matrix-get-messages-from-encrypted-room-via-backup.py diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py new file mode 100755 index 0000000..f69d2e7 --- /dev/null +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +# ungleich glarus ag, 2025-01-23 + +import base58 +import base64 +import json +import sys +import argparse +import requests + +import pprint + +class UngleichMatrixClient: + def __init__(self, args): + self.server = args.server_url + self.room_id = args.room_id + self.username = args.login_username + self.password = args.login_password + + self.access_token = False + + self.matrix_url = {} + self.matrix_url['login'] = f"{args.server_url}/_matrix/client/v3/login" + + + def ensure_logged_in(self): + if not self.access_token: + self.login_response = self.login_to_server() + self.access_token = self.login_response.json()['access_token'] + + print(self.access_token) + + def login_to_server(self): + login_data = { + 'identifier': { + "type": "m.id.user", + "user": f"{self.username}" + }, + 'type': "m.login.password", + 'device_id': "ungleich-matrix-client", + 'initial_device_display_name' : "ungleich-matrix-client", + 'password': f"{self.password}" + } + r = requests.post(self.matrix_url['login'], json=login_data) + + if not r.status_code == 200: + raise Exception("Login Failed") + return r + + def get_messages(self): + self.ensure_logged_in() + + +# Login, get token + # matrix_server=https://ungleich.matrix.ungleich.cloud + # matrix_key_url=https://${matrix_server}/_matrix/client/v3/room_keys/keys + + # matrix_user="\@nico:ungleich.ch" + # matrix_password=$(pass ldap/nico) + # matrix_room_id='!pkPjzozYKUNEbcZKKA:unchat.cat' + + # matrix_access_token=$(echo $matrix_login | jq .access_token | sed 's/"//g') + + + +# Get room_keys + +# Get messages from room + +# Decrypt each message: +# Retrieve the session key + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument("--server-url", required=True, help="Matrix Server URL, i.e. https://your-server ") + parser.add_argument("--room-id", required=True, help="ID of the room to get messages from, i.e. !...:your-matrix-domain ") + parser.add_argument("--login-username", required=True, help="Username for logging into the server, i.e. @you:your-matrix-domain ") + parser.add_argument("--login-password", required=True, help="Password for logging into the server, i.e. your-very-safe-password!! ") + + args = parser.parse_args() + client = UngleichMatrixClient(args) + client.get_messages() From 65852b4006795cb5d9e33c2617d1206ae83aef25 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 19:31:02 +0100 Subject: [PATCH 520/543] alpine-installer: update to 3.21 --- alpine-install-on-disk.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 4dbace7..4b8efb7 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -16,8 +16,8 @@ DISK=$1; shift SSH_KEYS=$1; shift BOOT_VIA=$1; shift -MAJOR_VERSION=3.20 -MINOR_VERSION=2 +MAJOR_VERSION=3.21 +MINOR_VERSION=0 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz From ca3dee2c0e6afb8f4d22f9b7df90c826a630222a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 20:12:39 +0100 Subject: [PATCH 521/543] matrix-client: add basic message getting --- ...messages-from-encrypted-room-via-backup.py | 79 ++++++++++++++----- 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index f69d2e7..ff71b1e 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -16,20 +16,18 @@ class UngleichMatrixClient: self.room_id = args.room_id self.username = args.login_username self.password = args.login_password + self.security_key = args.security_key self.access_token = False + self.room_keys = False + self.room_messages = [] self.matrix_url = {} self.matrix_url['login'] = f"{args.server_url}/_matrix/client/v3/login" + self.matrix_url['room_keys'] = f"{args.server_url}/_matrix/client/v3/room_keys/keys?version=1" + self.matrix_url['room_messages'] = f"{args.server_url}/_matrix/client/v3/rooms/{self.room_id}/messages" - def ensure_logged_in(self): - if not self.access_token: - self.login_response = self.login_to_server() - self.access_token = self.login_response.json()['access_token'] - - print(self.access_token) - def login_to_server(self): login_data = { 'identifier': { @@ -47,36 +45,77 @@ class UngleichMatrixClient: raise Exception("Login Failed") return r + def _ensure_logged_in(self): + if not self.access_token: + self.login_response = self.login_to_server() + self.access_token = self.login_response.json()['access_token'] + + def get_room_keys(self): + self._ensure_logged_in() + + params = { + 'version': "1", + 'access_token': self.access_token + } + + if not self.room_keys: + r = requests.get(self.matrix_url['room_keys'], + params=params) + self.room_keys = r.json() + + def get_room_messages(self): + """ + Get messages from a room, requires to use pagination! + Continue until no end property is in the reply anyomer + + """ + self._ensure_logged_in() + + params = { + 'access_token': self.access_token + } + + more_messages = True + next_batch = "" + while more_messages: + if next_batch: + params['from'] = next_batch + + r = requests.get(self.matrix_url['room_messages'], + params=params) + self.room_messages.append(r.json()) + + if 'end' in r.json(): + next_batch = r.json()['end'] + else: + more_messages = False + def get_messages(self): - self.ensure_logged_in() - - -# Login, get token - # matrix_server=https://ungleich.matrix.ungleich.cloud - # matrix_key_url=https://${matrix_server}/_matrix/client/v3/room_keys/keys - - # matrix_user="\@nico:ungleich.ch" - # matrix_password=$(pass ldap/nico) - # matrix_room_id='!pkPjzozYKUNEbcZKKA:unchat.cat' - - # matrix_access_token=$(echo $matrix_login | jq .access_token | sed 's/"//g') + # self.get_room_keys() + # print(self.room_keys) + self.get_room_messages() + for message in self.room_messages: + print(message) # Get room_keys + # http 'https://ungleich.matrix.ungleich.cloud/_matrix/client/v3/room_keys/keys' version==1 access_token=="$matrix_access_token" + # jq '.rooms."!pkP......."' < key-backup.json > room-specific-keys.json + # Get messages from room # Decrypt each message: # Retrieve the session key - if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument("--server-url", required=True, help="Matrix Server URL, i.e. https://your-server ") parser.add_argument("--room-id", required=True, help="ID of the room to get messages from, i.e. !...:your-matrix-domain ") parser.add_argument("--login-username", required=True, help="Username for logging into the server, i.e. @you:your-matrix-domain ") parser.add_argument("--login-password", required=True, help="Password for logging into the server, i.e. your-very-safe-password!! ") + parser.add_argument("--security-key", required=True, help="Your security backup key, i.e. ABCf defg aaaa - ensure to quote as one argument! ") args = parser.parse_args() client = UngleichMatrixClient(args) From 2e2b8da121d26acb716a89f0b96909ea0204098f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 20:35:14 +0100 Subject: [PATCH 522/543] [matrix-client] begin matching messages on session keys --- ...messages-from-encrypted-room-via-backup.py | 86 ++++++++++++++++--- 1 file changed, 72 insertions(+), 14 deletions(-) diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index ff71b1e..889970e 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -4,11 +4,23 @@ import base58 import base64 import json -import sys import argparse import requests -import pprint +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, padding +from cryptography.hazmat.primitives.asymmetric import ec +from cryptography.hazmat.primitives.kdf.hkdf import HKDF +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.ciphers.algorithms import AES +from secrets import token_bytes +from cryptography.hazmat.primitives.hmac import HMAC +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey,X25519PublicKey + +from cryptography.hazmat.primitives.serialization import load_pem_private_key + +from olm import Account,InboundGroupSession + class UngleichMatrixClient: def __init__(self, args): @@ -16,7 +28,7 @@ class UngleichMatrixClient: self.room_id = args.room_id self.username = args.login_username self.password = args.login_password - self.security_key = args.security_key + self.security_key_unparsed = args.security_key self.access_token = False self.room_keys = False @@ -83,29 +95,75 @@ class UngleichMatrixClient: r = requests.get(self.matrix_url['room_messages'], params=params) - self.room_messages.append(r.json()) + + for message in r.json()['chunk']: + self.room_messages.append(message) if 'end' in r.json(): next_batch = r.json()['end'] else: more_messages = False + def parse_security_key(self): + security_key = self.security_key_unparsed.replace(" ", "") + security_key_binary = base58.b58decode(security_key) + + self.security_key = security_key_binary + + # without useless bytes and without parity + self.real_security_key = security_key_binary[2:-1] + + + def check_security_key_parity(self): + parity_byte = self.security_key[-1] + calculated_parity=0 + for key_byte in self.security_key[:-1]: + calculated_parity ^= key_byte + + print(f"Parity byte = {parity_byte} calculated parity = {calculated_parity}") + + if parity_byte != calculated_parity: + raise Exception("Security key is broken") + + def setup_security_key_pair(self): + self.security_private_key = X25519PrivateKey.from_private_bytes(self.real_security_key) + print(f"Private key = {self.security_private_key}") + self.security_public_key = self.security_private_key.public_key() + print(f"Public key = {self.security_public_key}") + + + def decrypt_message(self, ciphertext, session_id): + room_key = self.room_keys['rooms'][self.room_id]['sessions'] + print(f"Messages key data: {room_key}") + + def decrypt_room_messages(self): + """ + Decrypt messages that are of type 'm.room.encrypted' + + {'type': 'm.room.encrypted', 'room_id': '!fDjvLemgiriPvvWEeG:ungleich.ch', 'sender': '@nico:ungleich.ch', 'content': {'algorithm': 'm.megolm.v1.aes-sha2', 'ciphertext': 'AwgBEqABNL8ztRQA67gXxkpbeiSp3zkJTkPXUwjQh0VnnFh6+Tff/dWjfF2rYu9q7MhG7BQgtaAoBoFNot8bPan23Y8Niip714ntI7t89F1t79TkUOcn5H0STydqGOOoZqnDf/l63ggWfD8EbudFSxoO7sJLL9iGO2+9HYWTMdTFAhcHg5c/k3aG+fQrXkbv+5afZXH3CxKnWxe4ukkoGMaDAo7jm3l2killUJ/J6NynCiJ/XinFWIdbRXSIUx3cwnFS/KWvdVmhu2iXYFtIvV65UE/JFhDjZ+rCH7lZ9DBD5jKjsVPQJqtFule0CQ', 'device_id': 'SSAUACUQKJ', 'sender_key': 'pEDLuq1RlDI2bxO6/lx9OQZt0NYma+gs6jg3QVYl4Vk', 'session_id': 'nkx3WnUpLL7hblZ9LNBkx0RPrKp3weX2o/aAgp7hx0c'}, 'origin_server_ts': 1738264304685, 'unsigned': {'membership': 'join', 'age': 126031}, 'event_id': '$k9dYdD6b5eG_AZaZtO6imeHU8HGBpiZt3dqM8C3T8-8', 'user_id': '@nico:ungleich.ch', 'age': 126031} + """ + + for message in self.room_messages: + if message['type'] == 'm.room.encrypted': + sender = message['sender'] + ciphertext = message['content']['ciphertext'] + session_id = message['content']['session_id'] + + plaintext = self.decrypt_message(ciphertext, session_id) + + def get_messages(self): - # self.get_room_keys() - # print(self.room_keys) + self.parse_security_key() + self.check_security_key_parity() + self.setup_security_key_pair() + self.get_room_messages() for message in self.room_messages: print(message) + self.get_room_keys() + self.decrypt_room_messages() - -# Get room_keys - # http 'https://ungleich.matrix.ungleich.cloud/_matrix/client/v3/room_keys/keys' version==1 access_token=="$matrix_access_token" - # jq '.rooms."!pkP......."' < key-backup.json > room-specific-keys.json - - -# Get messages from room - # Decrypt each message: # Retrieve the session key From ce204af6c244a4fb2135c0cbced28611649ec823 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 20:39:21 +0100 Subject: [PATCH 523/543] add stuff --- matrix/matrix-get-messages-from-encrypted-room-via-backup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index 889970e..af48846 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -136,6 +136,10 @@ class UngleichMatrixClient: room_key = self.room_keys['rooms'][self.room_id]['sessions'] print(f"Messages key data: {room_key}") + session_key_encrypted = room_key['session_data']['ciphertext'] + ephemeral_key = room_key['session_data']['ephemeral'] + session_mac = room_key['session_data']['mac'] + def decrypt_room_messages(self): """ Decrypt messages that are of type 'm.room.encrypted' From d8b14f594d751516d7411e3f1fc30cbcf779a3af Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 20:52:26 +0100 Subject: [PATCH 524/543] [matrix-client] start decrypting the session keys --- ...messages-from-encrypted-room-via-backup.py | 65 ++++++++++++++++++- 1 file changed, 62 insertions(+), 3 deletions(-) diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index af48846..879513f 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -132,13 +132,72 @@ class UngleichMatrixClient: print(f"Public key = {self.security_public_key}") + def decrypt_session_key(self, encrypted_session_key, ephemeral_key, session_mac): + + # Construct the public ephemeral key + # use + b'==') to expand padding https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding + ephemeral_key_bytes = base64.b64decode(ephemeral_key + '==') + ephemeral_public_key = X25519PublicKey.from_public_bytes(ephemeral_key_bytes) + shared_key = private_key.exchange(ephemeral_public_key) + + # when we have shared secret, use HDKF to get the AES part + # "Using the shared secret, + # generate 80 bytes + # by performing an HKDF + # using SHA-256 as the hash, + # with a salt of 32 bytes of 0, + # and with the empty string as the info. + + # The first 32 bytes are used as the AES key, + # the next 32 bytes are used as the MAC key, + # and the last 16 bytes are used as the AES initialization vector." + derived_key = HKDF( + algorithm=hashes.SHA256(), + length=80, + salt=bytes(32), + info=b'', + ).derive(shared_key) + + print(f"Derived key = %s, len=%s" % (derived_key, len(derived_key) )) + + aes_key = derived_key[:32] + mac_key = derived_key[32:64] + aes_iv = derived_key[64:] + + print("AES key = {0} / len = {1}".format(aes_key, len(aes_key))) + print("Mac key = {0} / len = {1}".format(mac_key, len(mac_key))) + print("AES IV = {0} / len = {1}".format(aes_iv, len(aes_iv))) + + # Pass an empty string through HMAC-SHA-256 using the MAC key generated above. The first 8 bytes of the resulting MAC are base64-encoded, and become the mac property of the session_data. + + mac = HMAC(mac_key, hashes.SHA256()) + mac.update(b'') + + # only use first 8 bytes + signature = mac.finalize()[:8] + print(f"Calculated signature over empty string = {signature}") + + session_signature = base64.b64decode(session_mac + '==') + print(f"Session signature = {session_signature}") + + if signature == session_signature: + print("Signature seems to be correct") + else: + print("Signature likely incorrect") + raise Exception("Session key signature broken") + + def decrypt_message(self, ciphertext, session_id): room_key = self.room_keys['rooms'][self.room_id]['sessions'] print(f"Messages key data: {room_key}") - session_key_encrypted = room_key['session_data']['ciphertext'] - ephemeral_key = room_key['session_data']['ephemeral'] - session_mac = room_key['session_data']['mac'] + encrypted_session_key = room_key[session_id]['session_data']['ciphertext'] + ephemeral_key = room_key[session_id]['session_data']['ephemeral'] + session_mac = room_key[session_id]['session_data']['mac'] + + session_key = self.decrypt_session_key(encrypted_session_key, + ephemeral_key, + session_mac) def decrypt_room_messages(self): """ From fa7451c791ba4458f2d4a9cfae5e2588c2228ff7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 21:07:31 +0100 Subject: [PATCH 525/543] phase in message decryption --- ...messages-from-encrypted-room-via-backup.py | 30 +++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index 879513f..7f0558b 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -138,7 +138,7 @@ class UngleichMatrixClient: # use + b'==') to expand padding https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding ephemeral_key_bytes = base64.b64decode(ephemeral_key + '==') ephemeral_public_key = X25519PublicKey.from_public_bytes(ephemeral_key_bytes) - shared_key = private_key.exchange(ephemeral_public_key) + shared_key = self.security_private_key.exchange(ephemeral_public_key) # when we have shared secret, use HDKF to get the AES part # "Using the shared secret, @@ -186,6 +186,27 @@ class UngleichMatrixClient: print("Signature likely incorrect") raise Exception("Session key signature broken") + cipher = Cipher(algorithms.AES(aes_key), modes.CBC(aes_iv)) + decryptor = cipher.decryptor() + + # use + b'==') to expand padding https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding + encrypted_session_key_bytes = base64.b64decode(encrypted_session_key + '==') + session_key_bytes = decryptor.update(encrypted_session_key_bytes) + decryptor.finalize() + + # Remove PKCS7 padding - block size 128 was guessed / tested to be correct + unpadder = padding.PKCS7(128).unpadder() + data = unpadder.update(session_key_bytes) + data += unpadder.finalize() + + session_key_json_string = data.decode("utf8") + + print(f"Unencrypted session key JSON: {session_key_json_string}") + session_key_json = json.loads(session_key_json_string) + session_key_base64 = session_key_json['session_key'] + + print("session key = {session_key_base64}, {length}".format(session_key_base64=session_key_base64, length=len(session_key_base64))) + + return session_key def decrypt_message(self, ciphertext, session_id): room_key = self.room_keys['rooms'][self.room_id]['sessions'] @@ -195,10 +216,15 @@ class UngleichMatrixClient: ephemeral_key = room_key[session_id]['session_data']['ephemeral'] session_mac = room_key[session_id]['session_data']['mac'] - session_key = self.decrypt_session_key(encrypted_session_key, + session_key_base64 = self.decrypt_session_key(encrypted_session_key, ephemeral_key, session_mac) + inbound_group = InboundGroupSession.import_session(session_key_base64) + plaintext = inbound_group.decrypt(message_ciphertext_base64) + + print(f"Encrypted {ciphertext} = {plaintext}") + def decrypt_room_messages(self): """ Decrypt messages that are of type 'm.room.encrypted' From 47337c87919025d93120501de01bad7f3c0f1571 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 21:10:05 +0100 Subject: [PATCH 526/543] finalise decryption of messages --- .../matrix-get-messages-from-encrypted-room-via-backup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index 7f0558b..53bf85e 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -206,7 +206,7 @@ class UngleichMatrixClient: print("session key = {session_key_base64}, {length}".format(session_key_base64=session_key_base64, length=len(session_key_base64))) - return session_key + return session_key_base64 def decrypt_message(self, ciphertext, session_id): room_key = self.room_keys['rooms'][self.room_id]['sessions'] @@ -221,9 +221,9 @@ class UngleichMatrixClient: session_mac) inbound_group = InboundGroupSession.import_session(session_key_base64) - plaintext = inbound_group.decrypt(message_ciphertext_base64) + plaintext = inbound_group.decrypt(ciphertext) - print(f"Encrypted {ciphertext} = {plaintext}") + print(f"Encrypted message {ciphertext} = {plaintext}") def decrypt_room_messages(self): """ From 19ecb2a705cb5eb3108c1cb63e15e05ed385b15f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jan 2025 21:30:46 +0100 Subject: [PATCH 527/543] ++hints for matrix client --- matrix/matrix-get-messages-from-encrypted-room-via-backup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index 53bf85e..b9ca36a 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -71,6 +71,7 @@ class UngleichMatrixClient: } if not self.room_keys: + print("Getting room keys ... this can take a while ...") r = requests.get(self.matrix_url['room_keys'], params=params) self.room_keys = r.json() From d42a50b3c6d58111777c13f5773e07baf88fe75b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 31 Jan 2025 15:25:10 +0100 Subject: [PATCH 528/543] matrix++ Signed-off-by: Nico Schottelius --- matrix/matrix-client.py | 16 +++++++ matrix/matrix-client.sh | 45 +++++++++++++++++++ ...messages-from-encrypted-room-via-backup.py | 10 +++++ 3 files changed, 71 insertions(+) create mode 100755 matrix/matrix-client.py create mode 100755 matrix/matrix-client.sh diff --git a/matrix/matrix-client.py b/matrix/matrix-client.py new file mode 100755 index 0000000..f4d0097 --- /dev/null +++ b/matrix/matrix-client.py @@ -0,0 +1,16 @@ +#!/usr/bin/env python3 + +import sys +import urllib +from mautrix.client import ClientAPI + +matrix_server="ungleich.matrix.ungleich.cloud" +matrix_user="@nico:ungleich.ch" +matrix_password=sys.argv[1] + +client = ClientAPI(matrixuser, + base_url=f"https://{matrix_server}" + token="syt_123_456") + + +await print(client.whoami()) diff --git a/matrix/matrix-client.sh b/matrix/matrix-client.sh new file mode 100755 index 0000000..af5f3ae --- /dev/null +++ b/matrix/matrix-client.sh @@ -0,0 +1,45 @@ +#!/bin/sh + +matrix_server=$1; shift +# matrix_user=$1; shift +# matrix_user=$(echo $matrix_user | sed 's/^@/\@/') +# matrix_password=$1; shift + + +# Login +# http -j -p "HBhb" \ +# https://${matrix_server}/_matrix/client/v3/login \ +# identifier:="{ \"type\":\"m.id.user\", \"user\":\"$matrix_user\" }" \ +# device_id=nicoshell initial_device_display_name="matrixclientsh" \ +# type=m.login.password \ +# password=$matrix_password + +# { +# "access_token": "..", +# "device_id": "nicoshell", +# "home_server": "ungleich.ch", +# "user_id": "@nico:ungleich.ch", +# "well_known": { +# "m.homeserver": { +# "base_url": "https://ungleich.matrix.ungleich.cloud/" +# } +# } +# } + +key_url=https://${matrix_server}/_matrix/client/v3/room_keys/keys +token=$1; shift +http -j -p "HBhb" $key_url \ + identifier:="{ \"type\":\"m.id.user\", \"user\":\"$matrix_user\" }" \ + device_id=nicoshell initial_device_display_name="matrixclientsh" \ + type=m.login.password \ + password=$matrix_password + + +# get device keys + master_keys of Nico +http https://ungleich.matrix.ungleich.cloud/_matrix/client/v3/keys/query access_token=="$token" device_keys:='{ "@nico:ungleich.ch": [] }' + +# Get messages +room_id="!..:..." +http 'https://ungleich.matrix.ungleich.cloud/_matrix/client/v3/rooms/!pkPjzozYKUNEbcZKKA:unchat.cat/messages' access_token=="$token" + +# Get room keys diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index b9ca36a..f474408 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -63,6 +63,10 @@ class UngleichMatrixClient: self.access_token = self.login_response.json()['access_token'] def get_room_keys(self): + """ + We assume version == 1 is correct because that's what's seen in reality + In theory we need to query the current version on the server first. + """ self._ensure_logged_in() params = { @@ -139,6 +143,8 @@ class UngleichMatrixClient: # use + b'==') to expand padding https://stackoverflow.com/questions/2941995/python-ignore-incorrect-padding-error-when-base64-decoding ephemeral_key_bytes = base64.b64decode(ephemeral_key + '==') ephemeral_public_key = X25519PublicKey.from_public_bytes(ephemeral_key_bytes) + + # This is effectively ECDH provided by cryptography library shared_key = self.security_private_key.exchange(ephemeral_public_key) # when we have shared secret, use HDKF to get the AES part @@ -152,6 +158,7 @@ class UngleichMatrixClient: # The first 32 bytes are used as the AES key, # the next 32 bytes are used as the MAC key, # and the last 16 bytes are used as the AES initialization vector." + # Using a key derivation function derived_key = HKDF( algorithm=hashes.SHA256(), length=80, @@ -171,6 +178,8 @@ class UngleichMatrixClient: # Pass an empty string through HMAC-SHA-256 using the MAC key generated above. The first 8 bytes of the resulting MAC are base64-encoded, and become the mac property of the session_data. + # hashed message authentication code = HMAC + # This basically allows us to check if we derived the correct key mac = HMAC(mac_key, hashes.SHA256()) mac.update(b'') @@ -195,6 +204,7 @@ class UngleichMatrixClient: session_key_bytes = decryptor.update(encrypted_session_key_bytes) + decryptor.finalize() # Remove PKCS7 padding - block size 128 was guessed / tested to be correct + # Needs to be verified - it should in theory be 256 unpadder = padding.PKCS7(128).unpadder() data = unpadder.update(session_key_bytes) data += unpadder.finalize() From 8c4abba69d360e543001cd199e4b44edfe99f2be Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 31 Jan 2025 15:25:40 +0100 Subject: [PATCH 529/543] remove unused scripts --- matrix/matrix-client.py | 16 --------------- matrix/matrix-client.sh | 45 ----------------------------------------- 2 files changed, 61 deletions(-) delete mode 100755 matrix/matrix-client.py delete mode 100755 matrix/matrix-client.sh diff --git a/matrix/matrix-client.py b/matrix/matrix-client.py deleted file mode 100755 index f4d0097..0000000 --- a/matrix/matrix-client.py +++ /dev/null @@ -1,16 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import urllib -from mautrix.client import ClientAPI - -matrix_server="ungleich.matrix.ungleich.cloud" -matrix_user="@nico:ungleich.ch" -matrix_password=sys.argv[1] - -client = ClientAPI(matrixuser, - base_url=f"https://{matrix_server}" - token="syt_123_456") - - -await print(client.whoami()) diff --git a/matrix/matrix-client.sh b/matrix/matrix-client.sh deleted file mode 100755 index af5f3ae..0000000 --- a/matrix/matrix-client.sh +++ /dev/null @@ -1,45 +0,0 @@ -#!/bin/sh - -matrix_server=$1; shift -# matrix_user=$1; shift -# matrix_user=$(echo $matrix_user | sed 's/^@/\@/') -# matrix_password=$1; shift - - -# Login -# http -j -p "HBhb" \ -# https://${matrix_server}/_matrix/client/v3/login \ -# identifier:="{ \"type\":\"m.id.user\", \"user\":\"$matrix_user\" }" \ -# device_id=nicoshell initial_device_display_name="matrixclientsh" \ -# type=m.login.password \ -# password=$matrix_password - -# { -# "access_token": "..", -# "device_id": "nicoshell", -# "home_server": "ungleich.ch", -# "user_id": "@nico:ungleich.ch", -# "well_known": { -# "m.homeserver": { -# "base_url": "https://ungleich.matrix.ungleich.cloud/" -# } -# } -# } - -key_url=https://${matrix_server}/_matrix/client/v3/room_keys/keys -token=$1; shift -http -j -p "HBhb" $key_url \ - identifier:="{ \"type\":\"m.id.user\", \"user\":\"$matrix_user\" }" \ - device_id=nicoshell initial_device_display_name="matrixclientsh" \ - type=m.login.password \ - password=$matrix_password - - -# get device keys + master_keys of Nico -http https://ungleich.matrix.ungleich.cloud/_matrix/client/v3/keys/query access_token=="$token" device_keys:='{ "@nico:ungleich.ch": [] }' - -# Get messages -room_id="!..:..." -http 'https://ungleich.matrix.ungleich.cloud/_matrix/client/v3/rooms/!pkPjzozYKUNEbcZKKA:unchat.cat/messages' access_token=="$token" - -# Get room keys From f96e1ea2304a430463c3652967d80e4a70427faf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 31 Jan 2025 15:26:12 +0100 Subject: [PATCH 530/543] change padding to 256 bits --- matrix/matrix-get-messages-from-encrypted-room-via-backup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py index f474408..f46b1a0 100755 --- a/matrix/matrix-get-messages-from-encrypted-room-via-backup.py +++ b/matrix/matrix-get-messages-from-encrypted-room-via-backup.py @@ -205,7 +205,7 @@ class UngleichMatrixClient: # Remove PKCS7 padding - block size 128 was guessed / tested to be correct # Needs to be verified - it should in theory be 256 - unpadder = padding.PKCS7(128).unpadder() + unpadder = padding.PKCS7(256).unpadder() data = unpadder.update(session_key_bytes) data += unpadder.finalize() From bd1a5288efe88850cec29ef9a06a4f0da47e02b7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 5 Feb 2025 23:40:07 +0100 Subject: [PATCH 531/543] openwrt/vigir: bump firmware to 23.05.5 --- openwrt/vigir-1-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index 2b5579e..0ff0731 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,7 +12,7 @@ set -x vigir_ip=$1; shift # openwrt -version=23.05.4 +version=23.05.5 filename=openwrt-${version}-ramips-mt7621-zbtlink_zbt-wg3526-16m-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} From f67374f5cc00f21a3a1b92bc0dcac1938ef46481 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 20 Mar 2025 10:02:53 +0100 Subject: [PATCH 532/543] alpine-install: update to 3.21.3 --- alpine-install-on-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 4b8efb7..e6a26d0 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -17,7 +17,7 @@ SSH_KEYS=$1; shift BOOT_VIA=$1; shift MAJOR_VERSION=3.21 -MINOR_VERSION=0 +MINOR_VERSION=3 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz From 5031fe8aca4743988125070448660b40641c7908 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 2 May 2025 07:42:05 +0200 Subject: [PATCH 533/543] debian installer: add note on what is required for install --- debian-devuan-install-on-disk.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/debian-devuan-install-on-disk.sh b/debian-devuan-install-on-disk.sh index a816120..de78a44 100755 --- a/debian-devuan-install-on-disk.sh +++ b/debian-devuan-install-on-disk.sh @@ -9,6 +9,7 @@ if [ $# -ne 3 ]; then echo suite: beowulf or similar echo keyfile: file containing the ssh keys echo disk: the block device + echo "Need to install before: debootstrap parted sudo" exit 1 fi From 5f1dedf89cc2ce0d85c13d38b797cd19f8852147 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 26 Oct 2025 17:46:04 +0100 Subject: [PATCH 534/543] update viwib-1 for 24.10.4 Signed-off-by: Nico Schottelius --- openwrt/viwib-1-black-firmware-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/viwib-1-black-firmware-upgrade.sh b/openwrt/viwib-1-black-firmware-upgrade.sh index 306c435..22952d1 100755 --- a/openwrt/viwib-1-black-firmware-upgrade.sh +++ b/openwrt/viwib-1-black-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=23.05.4 +version=24.10.4 filename=openwrt-${version}-ath79-nand-glinet_gl-ar300m-nor-squashfs-sysupgrade.bin url=https://downloads.openwrt.org/releases/${version}/targets/ath79/nand From 64b0ae77be2bf28167a9e44d92f9aac6f24f5c4a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 26 Oct 2025 18:37:47 +0100 Subject: [PATCH 535/543] openwrt/ipv4: update for wireguard --- openwrt/openwrt-add-ipv4-vpn.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/openwrt/openwrt-add-ipv4-vpn.sh b/openwrt/openwrt-add-ipv4-vpn.sh index e81c98d..a43b548 100755 --- a/openwrt/openwrt-add-ipv4-vpn.sh +++ b/openwrt/openwrt-add-ipv4-vpn.sh @@ -31,6 +31,9 @@ case $my_wireguard_ip in 185.155.30.*) vpn_endpoint_pubkey="5ach7pUQ57aa402LHz1MYh7lyBZS0GvBEw2PC6dMHW4=" ;; + 147.78.195.*) + vpn_endpoint_pubkey="f4ssV9bO/Gv9oWIXDUTEY6lOCopwNvVI0++u7AgNTCs=" + ;; *) echo "Unknown VPN host for IP $my_wireguard_ip" >&2 exit 1 @@ -45,8 +48,8 @@ set -x opkg update opkg install libustream-openssl ca-bundle ca-certificates -opkg install wireguard -opkg install luci-app-wireguard +opkg install wireguard-tools +opkg install luci-proto-wireguard uci set network.${interface}=interface uci set network.${interface}.proto='wireguard' From bf8cb20462c9db5887d1fc6dd7297bdeb454a5b0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 26 Oct 2025 14:04:46 +0100 Subject: [PATCH 536/543] [openwrt] bump version to 24.10.4 Signed-off-by: Nico Schottelius --- openwrt/microuter-n300-firmware-upgrade.sh | 2 +- openwrt/vigir-1-firmware-upgrade.sh | 2 +- openwrt/viirb-1-firmware-upgrade.sh | 2 +- openwrt/viwib-1-yellow-firmware-upgrade.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/openwrt/microuter-n300-firmware-upgrade.sh b/openwrt/microuter-n300-firmware-upgrade.sh index 6b317bb..751b808 100755 --- a/openwrt/microuter-n300-firmware-upgrade.sh +++ b/openwrt/microuter-n300-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x device_ip=$1; shift # openwrt -version=21.02.1 +version=24.10.4 filename=openwrt-${version}-ramips-mt76x8-glinet_microuter-n300-squashfs-sysupgrade.bin # don't care about other/old known_host entries diff --git a/openwrt/vigir-1-firmware-upgrade.sh b/openwrt/vigir-1-firmware-upgrade.sh index 0ff0731..385d14e 100755 --- a/openwrt/vigir-1-firmware-upgrade.sh +++ b/openwrt/vigir-1-firmware-upgrade.sh @@ -12,7 +12,7 @@ set -x vigir_ip=$1; shift # openwrt -version=23.05.5 +version=24.10.4 filename=openwrt-${version}-ramips-mt7621-zbtlink_zbt-wg3526-16m-squashfs-sysupgrade.bin # don't care about other/old known_host entries ssh-keygen -R ${vigir_ip} diff --git a/openwrt/viirb-1-firmware-upgrade.sh b/openwrt/viirb-1-firmware-upgrade.sh index 07a309a..ce2cef4 100755 --- a/openwrt/viirb-1-firmware-upgrade.sh +++ b/openwrt/viirb-1-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viirb_ip=$1; shift # openwrt -version=23.05.4 +version=24.10.4 filename=openwrt-${version}-ramips-mt76x8-vocore_vocore2-squashfs-sysupgrade.bin # don't care about other/old known_host entries diff --git a/openwrt/viwib-1-yellow-firmware-upgrade.sh b/openwrt/viwib-1-yellow-firmware-upgrade.sh index e1b3cdf..2edb3b3 100755 --- a/openwrt/viwib-1-yellow-firmware-upgrade.sh +++ b/openwrt/viwib-1-yellow-firmware-upgrade.sh @@ -13,7 +13,7 @@ set -x viwib_ip=$1; shift # openwrt -version=23.05.4 +version=24.10.4 filename=openwrt-${version}-ramips-mt76x8-glinet_gl-mt300n-v2-squashfs-sysupgrade.bin # don't care about other/old known_host entries From 21e18e863132aeecc5f4d8a81477a41783f43f4a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 27 Oct 2025 15:21:39 +0100 Subject: [PATCH 537/543] openwrt: add script to setup remote management --- openwrt/openwrt-add-ipv6-remote-management.sh | 80 +++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100755 openwrt/openwrt-add-ipv6-remote-management.sh diff --git a/openwrt/openwrt-add-ipv6-remote-management.sh b/openwrt/openwrt-add-ipv6-remote-management.sh new file mode 100755 index 0000000..97edbf9 --- /dev/null +++ b/openwrt/openwrt-add-ipv6-remote-management.sh @@ -0,0 +1,80 @@ +#!/bin/sh +# 2025-10-27 +# Add 1 (!) IPv6 address for remotely accessing a device +# Not intended to be shared, just for the router/device itself + +values_yaml=~/vcs/k8s-config/apps/prod/bird-router/values.yaml + +if [ $# -lt 2 ]; then + echo "$0 host ipv6-address [private-key]" + echo " host: where to find the OpenWRT device" + echo " ipv4-address: which ipv4 address to use" + echo " private-key: Use this wireguard key instead of generating one" + echo "" + echo "f.i.: $0 fe80::9683:c4ff:fe0f:%eth0 2a0a:e5c1:b00:100::42" + exit 1 +fi + +my_ip=$1; shift +my_wireguard_ip=$1; shift + +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi +public_key=$(echo $private_key | wg pubkey) + +case $my_wireguard_ip in + 2a0a:e5c1:b00:*) + vpn_endpoint_host="vpn-2a0ae5c1b00.ungleich.ch" + vpn_endpoint_pubkey="6XKJ4en+FqQg9rrHBDE8tf1q6NMuruHG4R34jdX2nTE=" + ;; + *) + echo "Unknown VPN host for IP $my_wireguard_ip" >&2 + exit 1 + ;; +esac + +interface="ipv6ra" + +cat < Date: Mon, 27 Oct 2025 15:24:17 +0100 Subject: [PATCH 538/543] openwrt: update package list --- openwrt/openwrt-add-ipv6-remote-management.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openwrt/openwrt-add-ipv6-remote-management.sh b/openwrt/openwrt-add-ipv6-remote-management.sh index 97edbf9..af6131f 100755 --- a/openwrt/openwrt-add-ipv6-remote-management.sh +++ b/openwrt/openwrt-add-ipv6-remote-management.sh @@ -43,8 +43,7 @@ set -x opkg update opkg install libustream-openssl ca-bundle ca-certificates -opkg install wireguard -opkg install luci-app-wireguard +opkg install luci-proto-wireguard wireguard-tools uci set network.${interface}=interface uci set network.${interface}.proto='wireguard' From d9c0ea50099382ad2e21bdf132599bcd86fc500c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Oct 2025 10:56:29 +0100 Subject: [PATCH 539/543] openwrt/wifi: add minimal script for setting up a wifi+ssh accessible device --- openwrt/openwrt-wifi-with-ssh.sh | 55 ++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100755 openwrt/openwrt-wifi-with-ssh.sh diff --git a/openwrt/openwrt-wifi-with-ssh.sh b/openwrt/openwrt-wifi-with-ssh.sh new file mode 100755 index 0000000..7f168ab --- /dev/null +++ b/openwrt/openwrt-wifi-with-ssh.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# 2025-10-30, Nico Schottelius +# Configure a device with a WAN port to allow ssh access and be IPv6 client + +if [ $# -ne 2 ]; then + echo "$0 ip-address ssid wifi-psk" + echo " ip-address: where to find the OpenWRT device" + echo " hostname: which hostname to set" + echo " ssid for the wifi" + echo " psk for the wifi" + exit 1 +fi + +my_ip=$1; shift +new_hostname=$1; shift +ssid=$1; shift +psk=$1; shift + +cat < Date: Thu, 30 Oct 2025 10:58:27 +0100 Subject: [PATCH 540/543] [openwrt] wifi script correct argc --- openwrt/openwrt-wifi-with-ssh.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-wifi-with-ssh.sh b/openwrt/openwrt-wifi-with-ssh.sh index 7f168ab..cdd502e 100755 --- a/openwrt/openwrt-wifi-with-ssh.sh +++ b/openwrt/openwrt-wifi-with-ssh.sh @@ -2,7 +2,7 @@ # 2025-10-30, Nico Schottelius # Configure a device with a WAN port to allow ssh access and be IPv6 client -if [ $# -ne 2 ]; then +if [ $# -ne 4 ]; then echo "$0 ip-address ssid wifi-psk" echo " ip-address: where to find the OpenWRT device" echo " hostname: which hostname to set" From 0b022506e868b56f70845c20352e7bfb72b5093e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 4 Dec 2025 13:14:35 +0100 Subject: [PATCH 541/543] update alpine install on disk to use 3.23.0 --- alpine-install-on-disk.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index e6a26d0..033fc54 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -16,8 +16,8 @@ DISK=$1; shift SSH_KEYS=$1; shift BOOT_VIA=$1; shift -MAJOR_VERSION=3.21 -MINOR_VERSION=3 +MAJOR_VERSION=3.23 +MINOR_VERSION=0 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz From cb516cb8c6bff17709042c6a4f53fef65f8c833c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 4 Dec 2025 13:14:51 +0100 Subject: [PATCH 542/543] openwrt typo --- openwrt/openwrt-add-ipv6-remote-management.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openwrt/openwrt-add-ipv6-remote-management.sh b/openwrt/openwrt-add-ipv6-remote-management.sh index af6131f..120694a 100755 --- a/openwrt/openwrt-add-ipv6-remote-management.sh +++ b/openwrt/openwrt-add-ipv6-remote-management.sh @@ -76,4 +76,4 @@ uci commit EOF echo "Host ${my_ip} uses ip ${my_wireguard_ip}" -echo "Public key = ${public_key}:" +echo "Public key = ${public_key}" From 8343902ef2d19c5fc4948aabbb75b9f4ed22d12a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 10 Dec 2025 19:49:46 +0100 Subject: [PATCH 543/543] change uuid grepper --- alpine-install-on-disk.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/alpine-install-on-disk.sh b/alpine-install-on-disk.sh index 033fc54..0211a4e 100755 --- a/alpine-install-on-disk.sh +++ b/alpine-install-on-disk.sh @@ -112,7 +112,7 @@ sudo cp "$RESOLVCONF" $rootfs_tmpdir/etc/resolv.conf # without -t ext4 for mounting and returns "No such file or directory" # nb2:~# blkid| grep ^${DISK}1 | awk '{ print $2 }' # UUID="fecf4182-f6dd-4d2c-9af7-8f36444ee25c" -eval $(blkid | grep ^${DISK}1 | awk '{ print $2 }') +eval $(blkid | grep ^${partition1} | awk '{ print $2 }') UUID_1=$UUID