From fd319a934220130af57da3da3eda94071ef63889 Mon Sep 17 00:00:00 2001 From: llnu Date: Fri, 6 Dec 2019 18:06:37 +0100 Subject: [PATCH 001/164] ipxe script updated for alpine --- rebuild-ipxe.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rebuild-ipxe.sh b/rebuild-ipxe.sh index 561cd46..2d1ad5a 100755 --- a/rebuild-ipxe.sh +++ b/rebuild-ipxe.sh @@ -1,7 +1,8 @@ #!/bin/sh # Nico Schottelius, 2019-09-20, Seoul, Coffebean, 23:56 # Copying: GPLv3 - +echo "If you are running alpine, these packages are needed:" +echo "apk add alpine-sdk xz-dev" set -x set -e From a666916a72609551f1c6e39dd4546d5a928741f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 08:48:31 +0100 Subject: [PATCH 002/164] Fedora image: initialize /etc/hosts --- fedora-build-opennebula-image.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 77d6a0b..40f5d12 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -110,6 +110,13 @@ mount --bind /sys /mnt/sys # TODO: use non-$BIGCORP DNS service. echo 'nameserver 1.1.1.1' >> /mnt/etc/resolv.conf +# Initialize /etc/hosts. +cat > /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + # See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. # network-scripts.x86_64 : Legacy scripts for manipulating of network devices run_root dnf -y install network-scripts From 157379235c42f7bce8672a8122bd023ce99511c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 09:53:14 +0100 Subject: [PATCH 003/164] Fedora image: reset systemd machine-id and random-seed, clean dnf database --- fedora-build-opennebula-image.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 40f5d12..6222efc 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -134,6 +134,8 @@ run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service # Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... run_root dnf -y install kernel grub2 run_root grub2-install --target=i386-pc "${NBD_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg @@ -150,5 +152,14 @@ UUID=$boot_uuid /boot ext4 rw,relatime,data=ordered 0 2 UUID=$root_uuid / ext4 rw,relatime,data=ordered 0 1 EOF +# Reset systemd's environment. +run_root rm -f /etc/machine-id +run_root touch /etc/machine-id +rm -f /var/lib/systemd/random-seed + +# Remove temporary files and reclaim freed disk space. +# Note: build logs could be removed as well. +run_root dnf clean all + # Make sure everything is written to disk before exiting. sync From 7e01fa5f06ba53c7d19d25a3ab83ab18ab598328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 12:08:11 +0100 Subject: [PATCH 004/164] Fedora image: add virtio-blk driver to initramfs --- fedora-build-opennebula-image.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 6222efc..ac26d7d 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -137,6 +137,14 @@ run_root systemctl enable systemd-timesyncd.service # Note: linux-firmware is not required our environment and takes almost 200M # uncompressed but is a direct dependency of kernel-core... run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf < Date: Mon, 9 Dec 2019 14:57:42 +0100 Subject: [PATCH 005/164] Fedora image: mention build-time depency on e2fsprogs --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index ac26d7d..b120dc6 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -9,7 +9,7 @@ # definitely opinionated. # Depends on the following packages (as of Fedora 31): -# qemu-img util-linux coreutils dnf curl +# qemu-img util-linux coreutils dnf curl e2fsprogs # Run locally (without network) with: # qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 From aa01350594f48a88dc82236f60f87a8dca8eefb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 18:58:41 +0100 Subject: [PATCH 006/164] Fedora image: use `date +%+F` instead of `date --iso-8601` --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index b120dc6..4d22522 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -20,7 +20,7 @@ set -x # XXX: Handle command-line arguments? RELEASE=31 ARCH=x86_64 -IMAGE_PATH=fedora-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd1 From 366c9c65b2d2ca05329494254915ae3ee124d0a8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:39:03 +0100 Subject: [PATCH 007/164] begin devuan netboo script --- devuan-netboot.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh new file mode 100644 index 0000000..6e7f39c --- /dev/null +++ b/devuan-netboot.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +date=$(date +%F) +suite=ascii + +dir=${suit}-${date} + +debootstrap ${suite} From be9616a17f25e263656021bab4183de1f3a90f2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:43:41 +0100 Subject: [PATCH 008/164] ++ netboot devuan script base --- devuan-netboot.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh new file mode 100644 index 0000000..c47c92b --- /dev/null +++ b/devuan-netboot.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Nico Schottelius, 2019-12-09 + + +date=$(date +%F) +suite=ascii + +dir=${suit}-${date} + +debootstrap "${suite}" "${dir}" +chroot ${dir} apt install -y openssh-server rdnssd + +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet6 auto +EOF From 5e5e37a7d737f6743572656f5e24a7d3c83026eb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:57:57 +0100 Subject: [PATCH 009/164] [netboot] add ssh keys --- devuan-netboot.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index c47c92b..066472e 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -5,10 +5,12 @@ date=$(date +%F) suite=ascii -dir=${suit}-${date} +dir=${suite}-${date} +kernel=${dir}-kernel +keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 cat > ${dir}/etc/network/interfaces << EOF auto lo @@ -17,3 +19,13 @@ iface lo inet loopback auto eth0 iface eth0 inet6 auto EOF + +mv ${dir}/boot/vmlinuz-* ${kernel} +rm ${dir}/boot/initrd* +mkdir -p ${dir}/root/.ssh + +for key in balazs dominique jinguk nico; do + curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys +done + +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 2970833df53656ffb1fb0b726c3e7b8529125b8f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:01:46 +0100 Subject: [PATCH 010/164] pass in suite from outside --- devuan-netboot.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index 066472e..c627ec1 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -1,9 +1,14 @@ #!/bin/sh # Nico Schottelius, 2019-12-09 +if [ $# -ne 1 ]; then + echo $0 suite + echo suite is for instance ascii, beowulf, etc +fi + +suite=$1; shift date=$(date +%F) -suite=ascii dir=${suite}-${date} kernel=${dir}-kernel From 1870aeca15f8777bd23f18fb1dbb36d2748c159e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:02:21 +0100 Subject: [PATCH 011/164] +exit error --- devuan-netboot.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index c627ec1..3dbb17f 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -4,6 +4,7 @@ if [ $# -ne 1 ]; then echo $0 suite echo suite is for instance ascii, beowulf, etc + exit 1 fi suite=$1; shift From caca8ea43f7cabdaaeebc5b29c2d2c527b0ae844 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:02:50 +0100 Subject: [PATCH 012/164] +perms --- build-alpine-chroot.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build-alpine-chroot.sh diff --git a/build-alpine-chroot.sh b/build-alpine-chroot.sh old mode 100644 new mode 100755 From 6cfdf9d40ee30f74f0017e020b27ad9bfe42a77e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:03:30 +0100 Subject: [PATCH 013/164] +perms --- devuan-netboot.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh old mode 100644 new mode 100755 From 8ed587eb0ad3c15d1477bb2805f93f04f3d3044e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:04:46 +0100 Subject: [PATCH 014/164] rename - script is suitable for debian and devuan --- devuan-netboot.sh | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100755 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh deleted file mode 100755 index 3dbb17f..0000000 --- a/devuan-netboot.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Nico Schottelius, 2019-12-09 - -if [ $# -ne 1 ]; then - echo $0 suite - echo suite is for instance ascii, beowulf, etc - exit 1 -fi - -suite=$1; shift - -date=$(date +%F) - -dir=${suite}-${date} -kernel=${dir}-kernel -keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files - -debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 - -cat > ${dir}/etc/network/interfaces << EOF -auto lo -iface lo inet loopback - -auto eth0 -iface eth0 inet6 auto -EOF - -mv ${dir}/boot/vmlinuz-* ${kernel} -rm ${dir}/boot/initrd* -mkdir -p ${dir}/root/.ssh - -for key in balazs dominique jinguk nico; do - curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys -done - -( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 0c56febefe7dd40d0f801d5af9111c8746ad0df9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 21:40:42 +0100 Subject: [PATCH 015/164] [fedora] use resolv.conf from the host --- fedora-build-opennebula-image.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 4d22522..f078042 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -107,8 +107,7 @@ mount --bind /sys /mnt/sys # systemd-networkd. # Required to resolve package mirror in chroot. -# TODO: use non-$BIGCORP DNS service. -echo 'nameserver 1.1.1.1' >> /mnt/etc/resolv.conf +cp /etc/resolv.conf /mnt/etc/resolv.conf # Initialize /etc/hosts. cat > /mnt/etc/hosts << EOF From 7bcdd02c9ec619dd450e6f0e171442e94127e835 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:11:48 +0100 Subject: [PATCH 016/164] Add new file for getting VM List --- vm_list | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 vm_list diff --git a/vm_list b/vm_list new file mode 100644 index 0000000..7d77eee --- /dev/null +++ b/vm_list @@ -0,0 +1,19 @@ +#!/bin/bash -e +#option $1 is ldap password +#option $2 is ou + + +uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) + +for ((i=0; i<${#uid_list[@]}; i++)) do + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=customer,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_vmid=() + list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) + for ((j=0; j<${#list_vmid[@]}; j++)) do + temp=$(onevm show ${list_vmid[$j]} | grep PORT) + temp1="${temp#*\"}" + port="${temp1%%\"*}" + host=$(onevm show ${list_vmid[$j]} | grep HOST | grep ungleich | awk '{print $3}') + echo ${uid_list[$i]} ${list_vmid[$j]} $port $host >> ~/vm_vnc_list + done +done \ No newline at end of file From cdd7779e2ce2654a089e20ad09deab5e68d0f2ab Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:12:52 +0100 Subject: [PATCH 017/164] Update vm_list --- vm_list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_list b/vm_list index 7d77eee..95f64be 100644 --- a/vm_list +++ b/vm_list @@ -6,7 +6,7 @@ uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) for ((i=0; i<${#uid_list[@]}; i++)) do - list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=customer,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=$2,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) list_vmid=() list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) for ((j=0; j<${#list_vmid[@]}; j++)) do From c7f02bce5c9d0307a7e1d07006d9a87c490b5bb4 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:23:28 +0100 Subject: [PATCH 018/164] Add new file --- create-guacamole-session-ldap-DB | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 create-guacamole-session-ldap-DB diff --git a/create-guacamole-session-ldap-DB b/create-guacamole-session-ldap-DB new file mode 100644 index 0000000..25819e2 --- /dev/null +++ b/create-guacamole-session-ldap-DB @@ -0,0 +1,39 @@ +#!/bin/bash +#option $1 is vm_list file name +#option $2 id DB location +#option $3 is DB user +#option $4 is DB name + +host='localhost' + +user_arr=( $(cat $1 | awk '{print $1}' )) +vmid_arr=( $(cat $1 | awk '{print $2}' )) +port_arr=( $(cat $1 | awk '{print $3}' )) +place_arr=( $(cat $1 | awk '{print $4}' )) + +for ((i=0; i<${#user_arr[@]}; i++)) do + #create user + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_entity (name, type) VALUES ('${user_arr[i]}','USER');" + en_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT entity_id FROM guacamole_entity WHERE name = '${user_arr[i]}';") + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('$en_id', '\x74657374', now());" + + #create connection + cn=${user_arr[i]}${vmid_arr[i]} + echo $cn + if [ 0 -eq $(psql -h $2 -U $3 -d $4 -tAc "SELECT connection_id FROM guacamole_connection WHERE connection_name = '$cn';" | wc -l) ]; then + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');" + cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") + + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');" + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');" + + #connection permission + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');" + + else + cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';" + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';" + fi + +done \ No newline at end of file From 36cc9b3c50ace54519c62bae43f32c2deeda4f1d Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:29:47 +0100 Subject: [PATCH 019/164] Add new file --- create-guacamole-session-ldap-file | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 create-guacamole-session-ldap-file diff --git a/create-guacamole-session-ldap-file b/create-guacamole-session-ldap-file new file mode 100644 index 0000000..c11b4bc --- /dev/null +++ b/create-guacamole-session-ldap-file @@ -0,0 +1,38 @@ +#!/bin/bash +#option $1 is vm_list file name +#option $2 is DB name +#this script should be run on guacamole server + + +host='localhost' +user_arr=( $(cat $1 | awk '{print $1}' )) +vmid_arr=( $(cat $1 | awk '{print $2}' )) +port_arr=( $(cat $1 | awk '{print $3}' )) +place_arr=( $(cat $1 | awk '{print $4}' )) + +for ((i=0; i<${#user_arr[@]}; i++)) do + #create user + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_entity (name, type) VALUES ('${user_arr[i]}','USER');\"" + en_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT entity_id FROM guacamole_entity WHERE name = '${user_arr[i]}';\"") + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('$en_id', '\x74657374', now());\"" + + #create connection + cn=${user_arr[i]}${vmid_arr[i]} + + if [ 0 -eq $(su - postgres -c "psql postgres -d $2 -tAc \"SELECT connection_id FROM guacamole_connection WHERE connection_name = '$cn';\"" | wc -l) ]; then + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');\"" + cn_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;\"") + + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');\"" + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');\"" + + #connection permission + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');\"" + + else + cn_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;\"") + su - postgres -c "psql postgres -d $2 -tAc \"UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';\"" + su - postgres -c "psql postgres -d $2 -tAc \"UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';\"" + fi + +done \ No newline at end of file From 7d37b7d5a6960c35ac787aea753b5d98144c84fd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 11 Dec 2019 15:33:24 +0100 Subject: [PATCH 020/164] ++ script for debian netboot --- debian-devuan-netboot.sh | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 debian-devuan-netboot.sh diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh new file mode 100755 index 0000000..9585709 --- /dev/null +++ b/debian-devuan-netboot.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Nico Schottelius, 2019-12-09 + +if [ $# -ne 1 ]; then + echo $0 suite + echo suite is for instance ascii, beowulf, etc + exit 1 +fi + +suite=$1; shift + +date=$(date +%F) + +dir=${suite}-${date} +kernel=${dir}-kernel +keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files + +debootstrap "${suite}" "${dir}" +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 + +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet6 auto +EOF + +mv ${dir}/boot/vmlinuz-* ${kernel} +rm ${dir}/boot/initrd* +mkdir -p ${dir}/root/.ssh + +for key in balazs dominique jinguk nico; do + curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys +done + +# ensure there is /init in the initramfs -> otherwise there is a kernel panic +# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init +# in our case, they are just the same +ln -s /sbin/init ${dir}/init + +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 96f0f94cfd4b0bd13058be8febba73c2b9443095 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 11 Dec 2019 22:56:54 +0100 Subject: [PATCH 021/164] not working state --- debian-devuan-netboot.sh | 31 ++++++++++++++++++++++++++++--- magiccommand | 2 ++ 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100755 magiccommand diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 9585709..799ee11 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -1,12 +1,16 @@ #!/bin/sh # Nico Schottelius, 2019-12-09 +# the ugly code is llnu + +#this can only run in the ungleich-tools directory because of the cat magiccommand........ if [ $# -ne 1 ]; then - echo $0 suite + echo $0 suite # rootpw echo suite is for instance ascii, beowulf, etc +# echo rootpw: set root password exit 1 fi - +#pw=$2 suite=$1; shift date=$(date +%F) @@ -16,7 +20,8 @@ kernel=${dir}-kernel keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 # ; \ +# echo "root:${pw}" | chgpasswd cat > ${dir}/etc/network/interfaces << EOF auto lo @@ -26,6 +31,23 @@ auto eth0 iface eth0 inet6 auto EOF + +#set hostname +echo "unconfigured-host" > ${dir}/etc/hostname + + +#add script to display eth0's ip +# the pretty but ugly code and not working +#echo 'echo $(ip -o -6 addr show | grep -E -v "lo|one" | awk '{print $1" " $2": "$4}') >> /etc/issue' > ${dir}/etc/rc.local + +#the also not working +#echo '* * * * * root ip -o -6 addr show | grep -E -v "lo|one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr + +# +#echo '* * * * * root ' > ${dir}/etc/cron.d/ipv6addr +cp ./magiccommand ${dir}/etc/cron.d/ipv6addr + + mv ${dir}/boot/vmlinuz-* ${kernel} rm ${dir}/boot/initrd* mkdir -p ${dir}/root/.ssh @@ -40,3 +62,6 @@ done ln -s /sbin/init ${dir}/init ( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) + +# tried to clear history but doesnt work :/ +# history -c diff --git a/magiccommand b/magiccommand new file mode 100755 index 0000000..e724d8e --- /dev/null +++ b/magiccommand @@ -0,0 +1,2 @@ +* * * * * root ip -o -6 addr show | grep -E -v "lo |one" | awk '{print $1" " $2": "$4}' >> /dev/tty1 + From fbcecc1bf2fe46cce7f7051fac4c60d258486b98 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 16:47:35 +0100 Subject: [PATCH 022/164] make cp instead of rm --- debian-devuan-netboot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 9585709..8937872 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -26,8 +26,8 @@ auto eth0 iface eth0 inet6 auto EOF -mv ${dir}/boot/vmlinuz-* ${kernel} -rm ${dir}/boot/initrd* +cp ${dir}/boot/vmlinuz-* ${kernel} +#rm ${dir}/boot/initrd* mkdir -p ${dir}/root/.ssh for key in balazs dominique jinguk nico; do From b549e1059dc7dfe0c88cf213c1d85e11426de3dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 16:57:25 +0100 Subject: [PATCH 023/164] prepend out-dir for building image --- debian-devuan-netboot.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 8eb9817..fdc43f4 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -4,19 +4,21 @@ #this can only run in the ungleich-tools directory because of the cat magiccommand........ -if [ $# -ne 1 ]; then - echo $0 suite # rootpw +if [ $# -ne 2 ]; then + echo $0 suite out-directory + echo out-directory: into which directory to place resulting files echo suite is for instance ascii, beowulf, etc -# echo rootpw: set root password exit 1 fi -#pw=$2 + suite=$1; shift +outdir=$1; shift date=$(date +%F) -dir=${suite}-${date} +dir=${outdir}/${suite}-${date} kernel=${dir}-kernel +initramfs=${dir}-initramfs keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" @@ -33,7 +35,7 @@ EOF cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v "lo|one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v 'lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh @@ -46,4 +48,4 @@ done # in our case, they are just the same ln -s /sbin/init ${dir}/init -( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From 3824bb6d94a6a7841b31debb92a28621a28dffdf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:02:42 +0100 Subject: [PATCH 024/164] prefix kernel + initramfs --- debian-devuan-netboot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index fdc43f4..c5d254a 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -17,8 +17,8 @@ outdir=$1; shift date=$(date +%F) dir=${outdir}/${suite}-${date} -kernel=${dir}-kernel -initramfs=${dir}-initramfs +kernel=kernel-${dir} +initramfs=initramfs-${dir} keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" From 279f86a1f0f042e8d5feeb513916aa7807dc58c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:14:13 +0100 Subject: [PATCH 025/164] read boot interface at boot --- debian-devuan-netboot.sh | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index c5d254a..6fdeecb 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -23,15 +23,11 @@ keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/ debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 +# need non-free for firmware-bnx2 +echo 'deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free' > ${dir}/etc/apt/sources.list -cat > ${dir}/etc/network/interfaces << EOF -auto lo -iface lo inet loopback +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 -auto eth0 -iface eth0 inet6 auto -EOF cp ${dir}/boot/vmlinuz-* ${kernel} @@ -43,6 +39,33 @@ for key in balazs dominique jinguk nico; do curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys done +################################################################################ +# networking + +# always lo +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +EOF + +# find the boot interfaces at boot +cat > ${dir}/etc/rc.local < /etc/network/interfaces.d/bootinterface << eof +auto \$dev +iface \$dev inet6 auto +eof + +ifup "\${dev}" + +exit 0 + + +EOF + # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same From 3f2dece3f5bf566712a084ccee766427e6d7c1c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:24:25 +0100 Subject: [PATCH 026/164] ++update --- debian-devuan-netboot.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 6fdeecb..0d366fa 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -16,22 +16,24 @@ outdir=$1; shift date=$(date +%F) -dir=${outdir}/${suite}-${date} -kernel=kernel-${dir} -initramfs=initramfs-${dir} +basename=${suite}-${date} +dir=${outdir}/${basename} +kernel=${outdir}/kernel-${basename} +initramfs=${outdir}/initramfs-${basename} keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" # need non-free for firmware-bnx2 -echo 'deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free' > ${dir}/etc/apt/sources.list +echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${dir}/etc/apt/sources.list +chroot ${dir} apt update chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v 'lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v ' lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh @@ -62,8 +64,6 @@ eof ifup "\${dev}" exit 0 - - EOF # ensure there is /init in the initramfs -> otherwise there is a kernel panic @@ -71,4 +71,5 @@ EOF # in our case, they are just the same ln -s /sbin/init ${dir}/init +# Finally building the initramfs ( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From d1ded3265ca4aeb7ca38224c0113bf731d65ded2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:26:49 +0100 Subject: [PATCH 027/164] do not nest single quotes --- debian-devuan-netboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 0d366fa..a9e0228 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -33,7 +33,7 @@ chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-b cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v ' lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh From b97d667753a2a0f5a427bf53e933d281039b8692 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:42:12 +0100 Subject: [PATCH 028/164] include network config cfiles --- debian-devuan-netboot.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index a9e0228..d64de5e 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -49,6 +49,7 @@ cat > ${dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback +source-directory /etc/network/interfaces.d EOF # find the boot interfaces at boot @@ -66,6 +67,8 @@ ifup "\${dev}" exit 0 EOF +chmod a+rx ${dir}/etc/rc.local" + # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same From 09a05b6a56f55fe27a37d26aaab13c65aa82a9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 15 Dec 2019 14:42:56 +0100 Subject: [PATCH 029/164] Add e2fsprogs to fedora image (used by one-context to resize fs) --- fedora-build-opennebula-image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index f078042..3dddae1 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -125,6 +125,9 @@ curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + # Initalize base services. run_root systemd-machine-id-setup run_root systemctl enable systemd-networkd.service From 65f5bfb99699328e5bfc5d3d28399f27512aac7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 16 Dec 2019 14:51:14 +0100 Subject: [PATCH 030/164] Do not enable unused systemd-networkd on Fedora image --- fedora-build-opennebula-image.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 3dddae1..219ad6e 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -130,7 +130,6 @@ run_root dnf -y install e2fsprogs # Initalize base services. run_root systemd-machine-id-setup -run_root systemctl enable systemd-networkd.service run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service From d11620a9fb7a1550df37d221711413cd2960704b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 17 Dec 2019 10:18:40 +0100 Subject: [PATCH 031/164] Fedora image: be sure to use right kernel version when generating initramfs --- fedora-build-opennebula-image.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 219ad6e..837ba6f 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -143,7 +143,8 @@ run_root dnf -y install kernel grub2 cat > /mnt/etc/dracut.conf.d/virtio-blk.conf < Date: Tue, 17 Dec 2019 11:21:54 +0100 Subject: [PATCH 032/164] Remove now irrelevant comment from Fedora image definition --- fedora-build-opennebula-image.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 837ba6f..5197acc 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -83,8 +83,6 @@ mount "${NBD_DEVICE}p2" /mnt mkdir /mnt/boot mount "${NBD_DEVICE}p1" /mnt/boot -# XXX: dnf has a lot a weird (libX11?) dependencies, use microdnf instead? - dnf -y \ --releasever=$RELEASE \ --installroot=/mnt \ From 5e438aafe83a10444fc68f35b4074748df4905c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 17 Dec 2019 13:56:05 +0100 Subject: [PATCH 033/164] Fedora image: increase /boot from 100M to 500M --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 5197acc..45c1629 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -68,7 +68,7 @@ trap cleanup EXIT # Create partition table, format partitions. sfdisk --no-reread "$NBD_DEVICE" < Date: Mon, 16 Dec 2019 13:46:34 +0100 Subject: [PATCH 034/164] Initial CentOS image build script --- centos-build-opennebula-image.sh | 170 +++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100755 centos-build-opennebula-image.sh diff --git a/centos-build-opennebula-image.sh b/centos-build-opennebula-image.sh new file mode 100755 index 0000000..6a8fe31 --- /dev/null +++ b/centos-build-opennebula-image.sh @@ -0,0 +1,170 @@ +#!/bin/sh + +# This script generates CentOS images for OpenNebula. + +# Depends on the following packages (as of CentOS 8): +# qemu-img util-linux coreutils dnf curl e2fsprogs + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=8 +ARCH=x86_64 +IMAGE_PATH=centos-$RELEASE-$(date --iso-8601).img +IMAGE_SIZE=10G +LOOPBACK_DEVICE=/dev/loop0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + losetup -d "$LOOPBACK_DEVICE" +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/centos-release' ]; then + echo "WARNING: this script has been designed to run on a CentOS system." >&2 + echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base RAW image (no LOOPBACK support in RHEL/CentOS). +qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" +losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +{ +sfdisk --no-reread "$LOOPBACK_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. +# network-scripts.x86_64 : Legacy scripts for manipulating of network devices +run_root dnf -y install network-scripts + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install and configure NTP client. +run_root dnf install -y chrony +run_root systemctl enable chronyd.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <>/mnt/etc/fstab < Date: Thu, 26 Dec 2019 12:16:38 +0100 Subject: [PATCH 035/164] [vpn stats] added number of configured/active VPNs --- vpn-statistics.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vpn-statistics.sh b/vpn-statistics.sh index a1e7960..c721cf9 100755 --- a/vpn-statistics.sh +++ b/vpn-statistics.sh @@ -9,3 +9,10 @@ done # countries with counter ( for ip in $(wg | grep endpoint | sed -e 's/endpoint: //' -e 's/\(.*\):[0-9]*/\1/' -e 's/\[//' -e 's/\]//'); do curl -s ipinfo.io/$ip | grep -e country ; done ) | sort | uniq -c | sort -g + +# Get number of configured VPNs +configured_vpns=$(wg show | grep ^peer | wc -l) +active_vpns=$(wg show | grep endpoint | wc -l) + +echo "Configured VPNs: ${configured_vpns}" +echo "Active VPNs: ${active_vpns}" From 1d88aac14ee6a53288aa95fd6de4ad552b68a411 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Fri, 27 Dec 2019 11:06:45 +0100 Subject: [PATCH 036/164] [script]Add new file for ceph pg repair --- pg_repair | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 pg_repair diff --git a/pg_repair b/pg_repair new file mode 100644 index 0000000..428923f --- /dev/null +++ b/pg_repair @@ -0,0 +1,14 @@ +#!/bin/bash -e + +pglist_arr=( $(ceph health detail | grep pg | grep active | awk '{print $2}' )) + +echo ${pglist_arr[*]} + +for ((i=0; i<${#pglist_arr[@]}; i++)) do + if [ 1 -eq $(ceph pg repair ${pglist_arr[$i]} | grep repair | grep instructing | wc -l) ]; then + echo repair error + break + fi + echo ${pglist_arr[$i]} repair ok + sleep 10 +done \ No newline at end of file From 693b1bed0063ebd97586746d08cb4a9fb386b9dc Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Fri, 27 Dec 2019 11:08:24 +0100 Subject: [PATCH 037/164] [script]Update pg_repair for error message --- pg_repair | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_repair b/pg_repair index 428923f..caa9953 100644 --- a/pg_repair +++ b/pg_repair @@ -6,9 +6,9 @@ echo ${pglist_arr[*]} for ((i=0; i<${#pglist_arr[@]}; i++)) do if [ 1 -eq $(ceph pg repair ${pglist_arr[$i]} | grep repair | grep instructing | wc -l) ]; then - echo repair error + echo repair script error break fi - echo ${pglist_arr[$i]} repair ok + echo ${pglist_arr[$i]} repair script done sleep 10 done \ No newline at end of file From 54e4bc1d49f951a57576b86f9a9620d7b3df5a8a Mon Sep 17 00:00:00 2001 From: llnu Date: Thu, 2 Jan 2020 20:46:15 +0100 Subject: [PATCH 038/164] added alpine compatible version --- ceph-osd-create-start-alpine | 107 +++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 ceph-osd-create-start-alpine diff --git a/ceph-osd-create-start-alpine b/ceph-osd-create-start-alpine new file mode 100755 index 0000000..a19e1da --- /dev/null +++ b/ceph-osd-create-start-alpine @@ -0,0 +1,107 @@ +#!/bin/sh +# 17:19, 2018-02-09 +# Nico Schottelius + +# Based on ceph-disk -v prepare --bluestore /dev/sdc --osd-id ${ID} --osd-uuid $(uuidgen) --crush-device-class "ssd" + +# Create: +# - block -> link to partuuid +# - block_uuid -e> uuid if the block +# - ceph_fsid -> get from ceph-conf +# crush_device_class -> ssd, hdd +# fsid -> uuidgen! +# magic -> string "ceph osd volume v026" +# type -> bluestore + +fsid=$(ceph-conf --cluster=ceph --name=osd. --lookup fsid) +fs_uuid=$(uuidgen) +magic="ceph osd volume v026" + +set -x +set -e + +if [ $# -lt 2 ]; then + echo "$0 disk class [osdweight]" + echo "class = hdd or ssd" + exit 1 +fi + +export DEV=$1;shift +export CLASS=$1; shift + + +uuid_metadata=$(uuidgen) +uuid_block=$(uuidgen) + +osd_id=$(ceph osd create) + +dev_metadata="/dev/disk/by-partuuid/$uuid_metadata" +dev_block="/dev/disk/by-partuuid/$uuid_block" + +/usr/bin/sgdisk --new=0:0:+100M --change-name="0:ceph data" \ + --partition-guid="0:$uuid_metadata" \ + --typecode=0:4fbd7e29-9d25-41b8-afd0-062c0ceff05d \ + --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + +# Using gdisk --largest-new does not change the name or set guid; +# So use 2 steps instead +/usr/bin/sgdisk --largest-new=0 --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + + +lastpart=$(gdisk -l $DEV | tail -n1 | awk '{ print $1 }') +/usr/bin/sgdisk --change-name="${lastpart}:ceph block" \ + --partition-guid="${lastpart}:$uuid_block" \ + --typecode="${lastpart}:cafecafe-9b03-4f30-b4c6-b4b80ceff106" \ + --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + +#echo $1 +#echo $(blkid | grep $1"2") + +#cblock=$(blkid | grep $1"2" | cut -d'"' -f4) +#echo $cblock + +/sbin/mkfs -t xfs -f -i size=2048 -- "$dev_metadata" + +mountpath=/var/lib/ceph/osd/ceph-${osd_id} + +mkdir -p "$mountpath" +mount "$dev_metadata" "$mountpath" + +ln -s $dev_block "$mountpath/block" + +echo "$uuid_block" > "$mountpath/block_uuid" +echo "$fsid" > "$mountpath/ceph_fsid" +echo "$magic" > "$mountpath/magic" +echo "$CLASS" > "$mountpath/crush_device_class" +echo $(echo $dev_block | cut -c23-) > "$mountpath/fsid" + + +# Important, otherwise --mkfs later will try to create filestore +echo bluestore > "$mountpath/type" + +ceph auth get-or-create "osd.${osd_id}" osd \ + 'allow *' mon 'allow profile osd' > $mountpath/keyring + +echo ${osd_id} > "$mountpath/whoami" +touch "$mountpath/openrc" + +ceph-osd --cluster ceph -i "${osd_id}" --mkfs +chown -R ceph:ceph "$mountpath" + +if [ $# -eq 1 ]; then + WEIGHT=$1; shift +else + devname=$(readlink -f $dev_block) + nodev=$(echo $devname | sed 's,/dev/,,') + WEIGHT=$(lsblk -l -b | awk "/^$nodev/ { print \$4/(1024^4) }") +fi + +ceph osd crush add osd.${osd_id} ${WEIGHT} host=$(hostname) + +echo "$metadata_dev /var/lib/ceph/osd/ceph-${osd_id} xfs noatime 0 0" >> /etc/fstab + +# Starting with monit, if available +ceph-osd -i ${osd_id} From fe27d294b1ca5884586a8713c169780030d3b7bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 7 Jan 2020 18:07:34 +0100 Subject: [PATCH 039/164] Add script to detect DNS64 prefix --- detect-dns64-prefix.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 detect-dns64-prefix.py diff --git a/detect-dns64-prefix.py b/detect-dns64-prefix.py new file mode 100644 index 0000000..e5bd179 --- /dev/null +++ b/detect-dns64-prefix.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# Nico Schottelius, 2020-01-07 +# Detect the DNS64 prefix +# Based on https://tools.ietf.org/html/draft-ietf-behave-nat64-discovery-heuristic-05 + +import dns.resolver +import ipaddress + + +if __name__ == '__main__': + dns64_prefix = None + answers = dns.resolver.query('ipv4only.arpa', 'AAAA') + + for rdata in answers: + address = str(rdata) + network = ipaddress.IPv6Network("{}/96".format(address), + strict=False) + # print("{}: {}".format(rdata, network)) + print("{}".format(network)) From e56e1d587800cf31f7aa9d3f8f0023fb710fb8bf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 7 Jan 2020 18:08:53 +0100 Subject: [PATCH 040/164] + explanation --- detect-dns64-prefix.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/detect-dns64-prefix.py b/detect-dns64-prefix.py index e5bd179..1179ca4 100644 --- a/detect-dns64-prefix.py +++ b/detect-dns64-prefix.py @@ -2,6 +2,12 @@ # Nico Schottelius, 2020-01-07 # Detect the DNS64 prefix # Based on https://tools.ietf.org/html/draft-ietf-behave-nat64-discovery-heuristic-05 +# +# How it works: +# - ipv4only.arpa only has A records. +# - a DNS64 server will add AAAA records +# - we take this response (if any) and derive the IPv6 prefix from it +# import dns.resolver import ipaddress From 86ad491bbc404d3ec02fda72d798c8b3dbadf065 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 00:27:03 +0100 Subject: [PATCH 041/164] [alpine] version bump --- alpine-rebuild-initramfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 643cc3f..b56454b 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -3,8 +3,8 @@ set -e set -x -MAJOR_VERSION=3.10 -MINOR_VERSION=3 +MAJOR_VERSION=3.11 +MINOR_VERSION=2 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz SSH_KEYS=$(cat ~/.ssh/id_rsa.pub) RESOLVCONF=/etc/resolv.conf From edb8a359e8caa81333ae37145f085608189c4e59 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 16 Jan 2020 15:47:50 +0100 Subject: [PATCH 042/164] Update create-guacamole-session-ldap-DB --- create-guacamole-session-ldap-DB | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/create-guacamole-session-ldap-DB b/create-guacamole-session-ldap-DB index 25819e2..ce1e5cd 100644 --- a/create-guacamole-session-ldap-DB +++ b/create-guacamole-session-ldap-DB @@ -4,7 +4,7 @@ #option $3 is DB user #option $4 is DB name -host='localhost' +#host='localhost' user_arr=( $(cat $1 | awk '{print $1}' )) vmid_arr=( $(cat $1 | awk '{print $2}' )) @@ -24,15 +24,17 @@ for ((i=0; i<${#user_arr[@]}; i++)) do psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');" cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") - psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');" + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','${place_arr[i]}');" psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');" #connection permission psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');" + #clipboard-encoding + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','clipboard-encoding','UTF-8');" else cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") - psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';" + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${place_arr[i]}' where connection_id='$cn_id' and parameter_name='hostname';" psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';" fi From ddc9ebaeaf509525c4f38b7079efa3c9ae1a0776 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 16 Jan 2020 15:59:26 +0100 Subject: [PATCH 043/164] Add new file --- vm_list_dual_uid | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 vm_list_dual_uid diff --git a/vm_list_dual_uid b/vm_list_dual_uid new file mode 100644 index 0000000..bf21c0b --- /dev/null +++ b/vm_list_dual_uid @@ -0,0 +1,20 @@ +#!/bin/bash -e +#option $1 is ldap password +#option $2 is ou + + +uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) + +for ((i=0; i<${#uid_list[@]}; i++)) do + uid_temp=$(echo ${uid_list[i]} | sed "s/b'//g" | sed "s/'//g") + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=$2,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_vmid=() + list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) + for ((j=0; j<${#list_vmid[@]}; j++)) do + temp=$(onevm show ${list_vmid[$j]} | grep PORT) + temp1="${temp#*\"}" + port="${temp1%%\"*}" + host=$(onevm show ${list_vmid[$j]} | grep HOST | grep ungleich | awk '{print $3}') + echo $uid_temp ${list_vmid[$j]} $port $host >> ~/vm_vnc_list + done +done From 0e91be0d69b3fd47425921bb3a2667f825efbb91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:22:56 +0100 Subject: [PATCH 044/164] Add ldap-get-emails script --- ldap-get-emails | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 ldap-get-emails diff --git a/ldap-get-emails b/ldap-get-emails new file mode 100755 index 0000000..733811a --- /dev/null +++ b/ldap-get-emails @@ -0,0 +1,31 @@ +#!/bin/sh +# +# List mail addresses found under base DN $1 (defaults to dc=ungleich,dc=ch) + +set -e + +# Hardcoded parameters. +LDAP_SERVER="ldaps://ldap1.ungleich.ch" +LDAP_BIND_DN="cn=manager,dc=ungleich,dc=ch" + +if [ "$1" != "" ]; then + LDAP_SEARCH_BASE="$1" +else + LDAP_SEARCH_BASE="dc=ungleich,dc=ch" +fi + +# Read secrets from environment. +if [ "$LDAP_BIND_PASSWD" = "" ]; then + echo "You have to define LDAP_BIND_PASSWD before launching this script." >&2 + exit 1 +fi + +# Extract mail addresses from LDAP directory. +ldap_search_result="$( + ldapsearch -x -H "$LDAP_SERVER" \ + -D "$LDAP_BIND_DN" \ + -w "$LDAP_BIND_PASSWD" \ + -b "$LDAP_SEARCH_BASE" mail + )" + +echo "$ldap_search_result" | grep 'mail:' | cut -d ' ' -f 2 - From f3a219755c382d81d7dc385bce237932401c5bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:25:39 +0100 Subject: [PATCH 045/164] Add one-get-instances scripts --- one-get-instances | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 one-get-instances diff --git a/one-get-instances b/one-get-instances new file mode 100755 index 0000000..653fed6 --- /dev/null +++ b/one-get-instances @@ -0,0 +1,18 @@ +#!/bin/sh +# +# This script extract VM IDs and filter them if a pattern is provided as first +# argument. + +set -e + +# Extract instances from ONE. +instances=$(onevm list --csv | tail -n +2) + +# Filter them is a pattern has been provided. +if [ "$1" != "" ]; then + filtered_instances="$(echo "$instances" | grep -E "$1")" + instances="$filtered_instances" +fi + +# Outputs instance IDs. +echo "$instances" | cut -d ',' -f 1 - From 1f2ea6305f43c3a817f027fb218a872710175a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:40:54 +0100 Subject: [PATCH 046/164] Add one-inspect-instance-network script --- one-inspect-instance-network | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 one-inspect-instance-network diff --git a/one-inspect-instance-network b/one-inspect-instance-network new file mode 100755 index 0000000..70e5795 --- /dev/null +++ b/one-inspect-instance-network @@ -0,0 +1,18 @@ +#!/bin/sh +# +# This script is expected to run on the ONE server (i.e. +# opennebula.ungleich.ch). + +set -e + +# Fetch instance list from STDIN. +instances=$(cat -) + +# For every instance, extract relevant information: +for id in $instances; do + nics_raw="$(onevm show --xml $id | xml_grep 'NIC')" + networks="$(echo $nics_raw | xml_grep --text_only 'NETWORK' | tr '\n' ',' | sed 's/,$//')" + ip="$(echo $nics_raw | xml_grep --text_only 'IP' | tr '\n' ',' | sed 's/,$//')" + ip6="$(echo $nics_raw | xml_grep --text_only 'IP6_GLOBAL' | tr '\n' ',' | sed 's/,$//')" + echo "$id,$networks,$ip,$ip6" +done From fea52e0295056d5e5f9606c4cba027eca5ad567c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 18 Jan 2020 19:56:06 +0100 Subject: [PATCH 047/164] Add ubuntu ONE image definition --- ubuntu-build-opennebula-image.sh | 153 +++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 ubuntu-build-opennebula-image.sh diff --git a/ubuntu-build-opennebula-image.sh b/ubuntu-build-opennebula-image.sh new file mode 100644 index 0000000..6535f66 --- /dev/null +++ b/ubuntu-build-opennebula-image.sh @@ -0,0 +1,153 @@ +#!/bin/sh + +# This script generates Ubuntu images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=eoan # 19.10 +ARCH=amd64 +IMAGE_PATH=ubuntu-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Ubuntu" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat >/mnt/etc/apt/sources.list < "/mnt$ONE_CONTEXT_DEB_PATH" +run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" +run_root rm "$ONE_CONTEXT_DEB_PATH" + +# Manually install legacy network scripts used by one-context. +run_root apt-get -y install ifupdown + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and bootloader. Do not autoconfigure grub. +run_root echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections +run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-base linux-image-generic grub-pc + +# Configure grub. +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root apt-get -y install openssh-server + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Sat, 25 Jan 2020 10:17:49 +0100 Subject: [PATCH 048/164] Add hacking script for import vm info into etcd --- etcd_import_opennebula_vm.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 etcd_import_opennebula_vm.py diff --git a/etcd_import_opennebula_vm.py b/etcd_import_opennebula_vm.py new file mode 100644 index 0000000..d2c94c9 --- /dev/null +++ b/etcd_import_opennebula_vm.py @@ -0,0 +1,28 @@ +import json +import pprint +#import etcd3 + +with open("nico-vm-one.json", "r") as fd: + vmcontent = fd.read() + +#vm = json.loads(vmcontent.decode('utf-8')) +vm = json.loads(vmcontent) +pprint.pprint(vm['TEMPLATE']['DISK']) + +# storing info + +for_etcd={} +for_etcd['data_version'] = "1" +for_etcd['vm_id'] = vm['ID'] +for_etcd['owner'] = vm['UNAME'] + +for_etcd['disks'] = [] +for disk in vm['TEMPLATE']['DISK']: + disk_etcd = {} + disk_etcd['image_name'] = disk['IMAGE'] + disk_etcd['image_id'] = disk['IMAGE_ID'] + disk_etcd['datastore_name'] = disk['DATASTORE'] + disk_etcd['datastore_id'] = disk['DATASTORE_ID'] + for_etcd['disks'].append(disk_etcd) + +pprint.pprint(for_etcd) From 1ffc6f57681db8cc1eae76038be324afcceca97d Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 15:34:09 +0500 Subject: [PATCH 049/164] Added scripts related to getting info of OpenNebula VM into etcd and related queries --- .gitignore | 6 + opennebula-vm-etcd/config-and-secrets.conf | 18 +++ opennebula-vm-etcd/config.py | 12 ++ opennebula-vm-etcd/etcd_wrapper.py | 75 ++++++++++++ opennebula-vm-etcd/put-vm-info-into-etcd.py | 126 ++++++++++++++++++++ opennebula-vm-etcd/vm-queries.py | 55 +++++++++ 6 files changed, 292 insertions(+) create mode 100644 .gitignore create mode 100644 opennebula-vm-etcd/config-and-secrets.conf create mode 100644 opennebula-vm-etcd/config.py create mode 100644 opennebula-vm-etcd/etcd_wrapper.py create mode 100644 opennebula-vm-etcd/put-vm-info-into-etcd.py create mode 100644 opennebula-vm-etcd/vm-queries.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c093faa --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +opennebula-snapshot/config-and-secrets.conf + +*.pyc + +.idea/ +.vscode/ \ No newline at end of file diff --git a/opennebula-vm-etcd/config-and-secrets.conf b/opennebula-vm-etcd/config-and-secrets.conf new file mode 100644 index 0000000..b7faa08 --- /dev/null +++ b/opennebula-vm-etcd/config-and-secrets.conf @@ -0,0 +1,18 @@ +# Do not put single/double quotation mark for string as they are +# aslo considered as normal character. + +[oca] +client_secrets = ahmedbilal96@gmail.com:d00359fa33a74fcb5ea40bb088e299fd2ab85126 + +[etcd] +# url = localhost +# port = 2379 +# ca_cert +# cert_cert +# cert_key + +url = etcd1.ungleich.ch +port = 2379 +ca_cert = /home/meow/.cdist/files/etcd/ca.pem +cert_cert = /home/meow/.cdist/files/etcd/developer.pem +cert_key = /home/meow/.cdist/files/etcd/developer-key.pem \ No newline at end of file diff --git a/opennebula-vm-etcd/config.py b/opennebula-vm-etcd/config.py new file mode 100644 index 0000000..b329f4f --- /dev/null +++ b/opennebula-vm-etcd/config.py @@ -0,0 +1,12 @@ +import configparser + +from etcd_wrapper import EtcdWrapper + +config = configparser.ConfigParser(allow_no_value=True) +config.read('config-and-secrets.conf') + +etcd_client = EtcdWrapper( + host=config['etcd']['url'], port=config['etcd']['port'], + ca_cert=config['etcd']['ca_cert'], cert_key=config['etcd']['cert_key'], + cert_cert=config['etcd']['cert_cert'] +) diff --git a/opennebula-vm-etcd/etcd_wrapper.py b/opennebula-vm-etcd/etcd_wrapper.py new file mode 100644 index 0000000..9624677 --- /dev/null +++ b/opennebula-vm-etcd/etcd_wrapper.py @@ -0,0 +1,75 @@ +import etcd3 +import json + +from functools import wraps + +from uncloud import UncloudException +from uncloud.common import logger + + +class EtcdEntry: + def __init__(self, meta_or_key, value, value_in_json=True): + if hasattr(meta_or_key, 'key'): + # if meta has attr 'key' then get it + self.key = meta_or_key.key.decode('utf-8') + else: + # otherwise meta is the 'key' + self.key = meta_or_key + self.value = value.decode('utf-8') + + if value_in_json: + self.value = json.loads(self.value) + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError: + raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') + except etcd3.exceptions.ConnectionTimeoutError as err: + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err + except Exception as err: + logger.exception('Some etcd error occured. See syslog for details.', err) + + return wrapper + + +class EtcdWrapper: + @readable_errors + def __init__(self, *args, **kwargs): + self.client = etcd3.client(*args, **kwargs) + + @readable_errors + def get(self, *args, value_in_json=True, **kwargs): + _value, _key = self.client.get(*args, **kwargs) + if _key is None or _value is None: + return None + return EtcdEntry(_key, _value, value_in_json=value_in_json) + + @readable_errors + def put(self, *args, value_in_json=True, **kwargs): + _key, _value = args + if value_in_json: + _value = json.dumps(_value) + + if not isinstance(_key, str): + _key = _key.decode('utf-8') + + return self.client.put(_key, _value, **kwargs) + + @readable_errors + def get_prefix(self, *args, value_in_json=True, **kwargs): + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) + + @readable_errors + def watch_prefix(self, key, value_in_json=True): + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = getattr('e', '_event') + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py new file mode 100644 index 0000000..70a1b09 --- /dev/null +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -0,0 +1,126 @@ +import pyone + +from enum import IntEnum +from config import config, etcd_client + +# How to get client secrets? +# 1. Login to OpenNebula +# 2. Go to Settings then Auth +# 3. Click on "Manage login tokens" button +# 4. Click on "Get a new token" button + +one_client = pyone.OneServer( + uri='https://opennebula.ungleich.ch:2634/RPC2', + session=config['oca']['client_secrets'] +) + + +def get_hostname_of_vm(vm_id): + host_pool = { + host.NAME: { + 'name': host.NAME, + 'id': host.ID, + 'cluster': { + 'name': host.CLUSTER, + 'id': host.CLUSTER_ID + }, + 'vms': host.VMS.ID + } + for host in one_client.hostpool.info().HOST + } + for hostname, host in host_pool.items(): + if vm_id in host['vms']: + return host + + return None + + +def put_under_list(obj): + if not isinstance(obj, list): + return [obj] + return obj + + +class Snapshot: + def __init__(self, disk_id, snapshot): + self.active = bool(snapshot.ACTIVE) + self.date = snapshot.DATE + self.id = snapshot.ID + self.name = snapshot.NAME + self.size = snapshot.SIZE + self.disk_id = disk_id + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + +class VM: + def __init__(self, vm): + self.name = vm.get_NAME() + self.id = vm.get_ID() + self.owner = { + 'name': vm.get_UNAME(), + 'id': vm.get_UID(), + } + + template = vm.get_TEMPLATE() + host = get_hostname_of_vm(self.id) + + self.vcpu = template.get('VCPU', None) + self.memory = template.get('MEMORY', None) + self.disks = [dict(disk) for disk in put_under_list(template.get('DISK', []))] + self.graphics = [dict(graphics) for graphics in put_under_list(template.get('GRAPHICS', []))] + self.nics = [dict(nic) for nic in put_under_list(template.get('NIC', []))] + self.status = pyone.VM_STATE(vm.get_STATE()).name.lower() + self.snapshots = [] + + for disk in one_client.vm.info(self.id).SNAPSHOTS: + disk_id = disk.DISK_ID + for snapshot in disk.SNAPSHOT: + self.snapshots.append(Snapshot(disk_id, snapshot).get_data()) + + if host: + self.host = { + 'name': host['name'], + 'id': host['id'] + } + else: + self.host = host + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + def __repr__(self): + return str(self.get_data()) + + +class VmFilterFlag(IntEnum): + UIDUserResources = 0 # UID User’s Resources + UserAndItsGroupsResources = -1 # Resources belonging to the user and any of his groups + AllResources = -2 # All resources + UserResources = -3 # Resources belonging to the user + UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group + + +def main(): + VM_STATES = list(pyone.VM_STATE) + START_ID = -1 # First id whatever it is + END_ID = -1 # Last id whatever it is + + for VM_STATE in VM_STATES: + vm_pool = one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, VM_STATE) + for i, vm in enumerate(vm_pool.VM): + vm = VM(vm) + etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + + +if __name__ == "__main__": + main() diff --git a/opennebula-vm-etcd/vm-queries.py b/opennebula-vm-etcd/vm-queries.py new file mode 100644 index 0000000..e92ef14 --- /dev/null +++ b/opennebula-vm-etcd/vm-queries.py @@ -0,0 +1,55 @@ +from pprint import pprint + +from config import config, etcd_client + + +def get_vm_by_ip(vms, ip, status='active'): + vms_by_status = { + vm_id: vm + for vm_id, vm in vms.items() + if vm['status'] == status + } + for vm_id, vm in vms_by_status.items(): + vm_ips = [] + for nic in vm.get('nics', []): + global_ipv6 = nic.get('IP6_GLOBAL', None) + local_ipv6 = nic.get('IP6_LINK', None) + ipv4 = nic.get('IP', None) + vm_ips += [global_ipv6, local_ipv6, ipv4] + + if ip in vm_ips: + return {vm_id: vm} + return None + + +def main(): + vm_prefix = '/opennebula/vm/' + + vms = { + int(vm.key.split('/')[-1]): vm.value + for vm in etcd_client.get_prefix(vm_prefix) + } + + VM_ID = 10761 # One of nico's VM + + # Get all data related to a VM + pprint(vms.get(VM_ID)) + + # Get host of a VM + print(vms.get(VM_ID).get('host').get('name')) + + # Get VNC Port of a VM + print(vms.get(VM_ID).get('graphics')[0].get('PORT')) + + # Get all disks attached with VM + pprint(vms.get(VM_ID).get('disks')) + + # Who is owner of a VM? + print(vms.get(VM_ID).get('owner').get('name')) + + # Get VM who has 2a0a:e5c0:0:5:0:78ff:fe11:d75f + search_ungleich_ch = get_vm_by_ip(vms, '2a0a:e5c0:0:5:0:78ff:fe11:d75f') + pprint(search_ungleich_ch) + +if __name__ == '__main__': + main() From 89244dc4ee89414db01621965e29a070512359f5 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 15:51:31 +0500 Subject: [PATCH 050/164] remove uncloud specific things --- opennebula-vm-etcd/etcd_wrapper.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/opennebula-vm-etcd/etcd_wrapper.py b/opennebula-vm-etcd/etcd_wrapper.py index 9624677..f448dcf 100644 --- a/opennebula-vm-etcd/etcd_wrapper.py +++ b/opennebula-vm-etcd/etcd_wrapper.py @@ -1,11 +1,9 @@ import etcd3 import json +import logging from functools import wraps -from uncloud import UncloudException -from uncloud.common import logger - class EtcdEntry: def __init__(self, meta_or_key, value, value_in_json=True): @@ -27,11 +25,11 @@ def readable_errors(func): try: return func(*args, **kwargs) except etcd3.exceptions.ConnectionFailedError: - raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') + raise etcd3.exceptions.ConnectionFailedError('Cannot connect to etcd: is etcd running as configured?') except etcd3.exceptions.ConnectionTimeoutError as err: raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err except Exception as err: - logger.exception('Some etcd error occured. See syslog for details.', err) + logging.exception('Some etcd error occured. See syslog for details.', err) return wrapper From e34abc449d5216715cb81dbe88f3eafc8c44522a Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 18:32:03 +0500 Subject: [PATCH 051/164] Removed config-and-secrets.conf --- .gitignore | 4 ++-- opennebula-vm-etcd/config-and-secrets.conf | 18 ------------------ 2 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 opennebula-vm-etcd/config-and-secrets.conf diff --git a/.gitignore b/.gitignore index c093faa..359653f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -opennebula-snapshot/config-and-secrets.conf +opennebula-vm-etcd/config-and-secrets.conf *.pyc .idea/ -.vscode/ \ No newline at end of file +.vscode/ diff --git a/opennebula-vm-etcd/config-and-secrets.conf b/opennebula-vm-etcd/config-and-secrets.conf deleted file mode 100644 index b7faa08..0000000 --- a/opennebula-vm-etcd/config-and-secrets.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Do not put single/double quotation mark for string as they are -# aslo considered as normal character. - -[oca] -client_secrets = ahmedbilal96@gmail.com:d00359fa33a74fcb5ea40bb088e299fd2ab85126 - -[etcd] -# url = localhost -# port = 2379 -# ca_cert -# cert_cert -# cert_key - -url = etcd1.ungleich.ch -port = 2379 -ca_cert = /home/meow/.cdist/files/etcd/ca.pem -cert_cert = /home/meow/.cdist/files/etcd/developer.pem -cert_key = /home/meow/.cdist/files/etcd/developer-key.pem \ No newline at end of file From 55a686800640af68f541fede1b3815e28289b586 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 23:28:00 +0500 Subject: [PATCH 052/164] Some optimizations to improve throughput --- opennebula-vm-etcd/put-vm-info-into-etcd.py | 47 +++++++++++++-------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py index 70a1b09..5b5495f 100644 --- a/opennebula-vm-etcd/put-vm-info-into-etcd.py +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -2,6 +2,7 @@ import pyone from enum import IntEnum from config import config, etcd_client +from functools import reduce # How to get client secrets? # 1. Login to OpenNebula @@ -14,24 +15,24 @@ one_client = pyone.OneServer( session=config['oca']['client_secrets'] ) +host_pool = { + host.NAME: { + 'name': host.NAME, + 'id': host.ID, + 'cluster': { + 'name': host.CLUSTER, + 'id': host.CLUSTER_ID + }, + 'vms': host.VMS.ID + } + for host in one_client.hostpool.info().HOST +} + def get_hostname_of_vm(vm_id): - host_pool = { - host.NAME: { - 'name': host.NAME, - 'id': host.ID, - 'cluster': { - 'name': host.CLUSTER, - 'id': host.CLUSTER_ID - }, - 'vms': host.VMS.ID - } - for host in one_client.hostpool.info().HOST - } for hostname, host in host_pool.items(): if vm_id in host['vms']: return host - return None @@ -115,11 +116,21 @@ def main(): START_ID = -1 # First id whatever it is END_ID = -1 # Last id whatever it is - for VM_STATE in VM_STATES: - vm_pool = one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, VM_STATE) - for i, vm in enumerate(vm_pool.VM): - vm = VM(vm) - etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + # Get VMs in all kind of states + + # vms is a list of lists + vms = [ + one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, vm_state).VM + for vm_state in VM_STATES + ] + # Take out elements from nested lists and put them into the original list + # forming a nice flat list + vms = list(reduce(lambda n, n_1: n + n_1, vms)) + print('Total VMs:', len(vms)) + for i, _vm in enumerate(vms): + vm = VM(_vm) + etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + print(i, end=' ') if __name__ == "__main__": From 78470501dd7c79d6a8094197cf6d9aa03e621591 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 31 Jan 2020 23:05:25 +0500 Subject: [PATCH 053/164] Directly use builtin XMLRPC client as opposed to pyone library --- .gitignore | 4 +- opennebula-vm-etcd/put-vm-info-into-etcd.py | 177 ++++++++------------ 2 files changed, 71 insertions(+), 110 deletions(-) diff --git a/.gitignore b/.gitignore index 359653f..f8835d9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ opennebula-vm-etcd/config-and-secrets.conf *.pyc -.idea/ -.vscode/ +.idea +.vscode diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py index 5b5495f..0ba2275 100644 --- a/opennebula-vm-etcd/put-vm-info-into-etcd.py +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -1,39 +1,17 @@ -import pyone +import json from enum import IntEnum +from xmlrpc.client import ServerProxy as RPCClient + +from xmltodict import parse + from config import config, etcd_client -from functools import reduce - -# How to get client secrets? -# 1. Login to OpenNebula -# 2. Go to Settings then Auth -# 3. Click on "Manage login tokens" button -# 4. Click on "Get a new token" button - -one_client = pyone.OneServer( - uri='https://opennebula.ungleich.ch:2634/RPC2', - session=config['oca']['client_secrets'] -) - -host_pool = { - host.NAME: { - 'name': host.NAME, - 'id': host.ID, - 'cluster': { - 'name': host.CLUSTER, - 'id': host.CLUSTER_ID - }, - 'vms': host.VMS.ID - } - for host in one_client.hostpool.info().HOST -} -def get_hostname_of_vm(vm_id): - for hostname, host in host_pool.items(): - if vm_id in host['vms']: - return host - return None +# Constants +ALL_VM_STATES = -1 +START_ID = -1 # First id whatever it is +END_ID = -1 # Last id whatever it is def put_under_list(obj): @@ -42,65 +20,19 @@ def put_under_list(obj): return obj -class Snapshot: - def __init__(self, disk_id, snapshot): - self.active = bool(snapshot.ACTIVE) - self.date = snapshot.DATE - self.id = snapshot.ID - self.name = snapshot.NAME - self.size = snapshot.SIZE - self.disk_id = disk_id - - def get_data(self): - return { - attr: getattr(self, attr) - for attr in dir(self) - if not attr.startswith('__') and not callable(getattr(self, attr)) - } - - -class VM: - def __init__(self, vm): - self.name = vm.get_NAME() - self.id = vm.get_ID() - self.owner = { - 'name': vm.get_UNAME(), - 'id': vm.get_UID(), - } - - template = vm.get_TEMPLATE() - host = get_hostname_of_vm(self.id) - - self.vcpu = template.get('VCPU', None) - self.memory = template.get('MEMORY', None) - self.disks = [dict(disk) for disk in put_under_list(template.get('DISK', []))] - self.graphics = [dict(graphics) for graphics in put_under_list(template.get('GRAPHICS', []))] - self.nics = [dict(nic) for nic in put_under_list(template.get('NIC', []))] - self.status = pyone.VM_STATE(vm.get_STATE()).name.lower() - self.snapshots = [] - - for disk in one_client.vm.info(self.id).SNAPSHOTS: - disk_id = disk.DISK_ID - for snapshot in disk.SNAPSHOT: - self.snapshots.append(Snapshot(disk_id, snapshot).get_data()) - - if host: - self.host = { - 'name': host['name'], - 'id': host['id'] - } - else: - self.host = host - - def get_data(self): - return { - attr: getattr(self, attr) - for attr in dir(self) - if not attr.startswith('__') and not callable(getattr(self, attr)) - } - - def __repr__(self): - return str(self.get_data()) +class VMState(IntEnum): + INIT = 0 + PENDING = 1 + HOLD = 2 + ACTIVE = 3 + STOPPED = 4 + SUSPENDED = 5 + DONE = 6 + FAILED = 7 + POWEROFF = 8 + UNDEPLOYED = 9 + CLONING = 10 + CLONING_FAILURE = 11 class VmFilterFlag(IntEnum): @@ -111,26 +43,55 @@ class VmFilterFlag(IntEnum): UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group +class VM: + def __init__(self, vm: dict): + self.id = vm.get('ID', None) + self.owner = { + 'id': vm.get('UID', None), + 'name': vm.get('UNAME', None), + 'gname': vm.get('GNAME', None) + } + self.name = vm.get('NAME', None) + self.status = vm.get('STATE', None) + if self.status: + self.status = VMState(int(self.status)).name.lower() + + template = vm['TEMPLATE'] + + self.disk = put_under_list(template.get('DISK', [])) + self.graphics = template.get('GRAPHICS', {}) + self.memory = template.get('MEMORY', None) + self.nic = put_under_list(template.get('NIC', [])) + self.vcpu = template.get('VCPU', None) + self.host = { + 'name': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None), + 'id': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HID', None), + } + self.snapshots = put_under_list(vm.get('SNAPSHOTS', [])) + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + def main(): - VM_STATES = list(pyone.VM_STATE) - START_ID = -1 # First id whatever it is - END_ID = -1 # Last id whatever it is + with RPCClient('https://opennebula.ungleich.ch:2634/RPC2') as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + config['oca']['client_secrets'], VmFilterFlag.AllResources.value, START_ID, END_ID, ALL_VM_STATES + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + for i, vm in enumerate(vms): + vm_id = vm['ID'] + etcd_client.put(f'/opennebula/vm/{vm_id}', vm) - # Get VMs in all kind of states - - # vms is a list of lists - vms = [ - one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, vm_state).VM - for vm_state in VM_STATES - ] - # Take out elements from nested lists and put them into the original list - # forming a nice flat list - vms = list(reduce(lambda n, n_1: n + n_1, vms)) - print('Total VMs:', len(vms)) - for i, _vm in enumerate(vms): - vm = VM(_vm) - etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) - print(i, end=' ') + parsed_vm = VM(vm) + etcd_client.put(f'/opennebula/parsed_vm/{parsed_vm.id}', parsed_vm.get_data()) + else: + print(response) if __name__ == "__main__": From 46fc35d3c3ad3e9028920e5257ee80b6d98af35c Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 31 Jan 2020 23:10:16 +0500 Subject: [PATCH 054/164] Fix vm-queries.py --- opennebula-vm-etcd/vm-queries.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/opennebula-vm-etcd/vm-queries.py b/opennebula-vm-etcd/vm-queries.py index e92ef14..e1da013 100644 --- a/opennebula-vm-etcd/vm-queries.py +++ b/opennebula-vm-etcd/vm-queries.py @@ -1,6 +1,6 @@ from pprint import pprint -from config import config, etcd_client +from config import etcd_client def get_vm_by_ip(vms, ip, status='active'): @@ -11,7 +11,7 @@ def get_vm_by_ip(vms, ip, status='active'): } for vm_id, vm in vms_by_status.items(): vm_ips = [] - for nic in vm.get('nics', []): + for nic in vm.get('nic', []): global_ipv6 = nic.get('IP6_GLOBAL', None) local_ipv6 = nic.get('IP6_LINK', None) ipv4 = nic.get('IP', None) @@ -23,7 +23,7 @@ def get_vm_by_ip(vms, ip, status='active'): def main(): - vm_prefix = '/opennebula/vm/' + vm_prefix = '/opennebula/parsed_vm/' vms = { int(vm.key.split('/')[-1]): vm.value @@ -39,10 +39,10 @@ def main(): print(vms.get(VM_ID).get('host').get('name')) # Get VNC Port of a VM - print(vms.get(VM_ID).get('graphics')[0].get('PORT')) + print(vms.get(VM_ID).get('graphics').get('PORT')) # Get all disks attached with VM - pprint(vms.get(VM_ID).get('disks')) + pprint(vms.get(VM_ID).get('disk')) # Who is owner of a VM? print(vms.get(VM_ID).get('owner').get('name')) @@ -51,5 +51,6 @@ def main(): search_ungleich_ch = get_vm_by_ip(vms, '2a0a:e5c0:0:5:0:78ff:fe11:d75f') pprint(search_ungleich_ch) + if __name__ == '__main__': main() From fb68ce7c1c07b422ab7362a93c936cdd50a8c2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 14 Feb 2020 15:23:54 +0100 Subject: [PATCH 055/164] Add OpenNebula image definition for Alpine Linux --- alpine-build-opennebula-image.sh | 179 +++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100755 alpine-build-opennebula-image.sh diff --git a/alpine-build-opennebula-image.sh b/alpine-build-opennebula-image.sh new file mode 100755 index 0000000..0a074b4 --- /dev/null +++ b/alpine-build-opennebula-image.sh @@ -0,0 +1,179 @@ +#!/bin/sh + +# This script generates Alpine images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=v3.11 +ARCH=x86_64 +IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 +APK_MIRROR=http://dl-2.alpinelinux.org/alpine/ # Mind the trailing / + +ONE_CONTEXT_APK_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-r1.apk" +ONE_CONTEXT_APK_PATH=/root/one-context.apk + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ "$(lsb_release --short --id)" != "Alpine" ]; then + echo "WARNING: this script has been designed to run on an Alpine system." >&2 + echo "WARNING: Not running Alpine. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +run_root setup-timezone -z UTC +if [ "$RELEASE" = "edge" ] +then + cat >/mnt/etc/apk/repositories </mnt/etc/apk/repositories <>/mnt/etc/fstab </mnt/boot/extlinux.conf < "/mnt$ONE_CONTEXT_APK_PATH" +run_root apk add --allow-untrusted "$ONE_CONTEXT_APK_PATH" +run_root rm "$ONE_CONTEXT_APK_PATH" + +# Remove resolvconf: handled by uncloud-init. +run_root rm /etc/resolv.conf + +# Make sure everything is written to disk before exiting. +sync From b6cf60b83b8aae1dacbca0d9feb33d99d2283bfa Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Tue, 3 Mar 2020 08:43:52 +0100 Subject: [PATCH 056/164] Add new directory for vnc_console --- vnc_console_connection/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 vnc_console_connection/.gitkeep diff --git a/vnc_console_connection/.gitkeep b/vnc_console_connection/.gitkeep new file mode 100644 index 0000000..e69de29 From 9774225f958eee50bb6edcd6d8cb1280a125b344 Mon Sep 17 00:00:00 2001 From: kjg Date: Tue, 3 Mar 2020 08:55:30 +0100 Subject: [PATCH 057/164] [file]update vnc_console --- vnc_console_connection/config.py | 5 ++ vnc_console_connection/db_export.py | 55 ++++++++++++++++++ vnc_console_connection/get_info.py | 88 +++++++++++++++++++++++++++++ vnc_console_connection/ldap_list.py | 30 ++++++++++ 4 files changed, 178 insertions(+) create mode 100755 vnc_console_connection/config.py create mode 100755 vnc_console_connection/db_export.py create mode 100755 vnc_console_connection/get_info.py create mode 100755 vnc_console_connection/ldap_list.py diff --git a/vnc_console_connection/config.py b/vnc_console_connection/config.py new file mode 100755 index 0000000..2a36c96 --- /dev/null +++ b/vnc_console_connection/config.py @@ -0,0 +1,5 @@ +import configparser + +config = configparser.ConfigParser(allow_no_value=True) +config.read('config-and-secrets.conf') + diff --git a/vnc_console_connection/db_export.py b/vnc_console_connection/db_export.py new file mode 100755 index 0000000..d283eb4 --- /dev/null +++ b/vnc_console_connection/db_export.py @@ -0,0 +1,55 @@ +import psycopg2 as pg2 +from config import config + +db_name = config['db']['db_name'] +db_user = config['db']['db_user'] +db_password = config['db']['db_password'] +db_port = config['db']['db_port'] + + +def setconn(u_id, vm_num, vm_port,vm_host): + conn = pg2.connect("host = localhost dbname={} user={} password={} port={}".format(db_name,db_user,db_password,db_port)) + conn.autocommit = True + cur = conn.cursor() + cur.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) + row = cur.fetchone() + if row == None: + cur.execute("INSERT INTO guacamole_entity (name, type) VALUES ('{}','USER')".format(u_id)) + cur.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) + row = cur.fetchone() + en_id = row[0] + cur.execute("INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('{}', '\x74657374', now())".format(en_id)) + print("create user : " , u_id) + else: + en_id = row[0] + cur.execute("SELECT password_hash FROM guacamole_user WHERE entity_id = '{}'".format(en_id)) + row = cur.fetchone() + if row == None: + cur.execute("INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('{}', '\x74657374', now())".format(en_id)) + print("user exsit") + cn = "{}{}".format(u_id,vm_num) + cur.execute("SELECT connection_id FROM guacamole_connection WHERE connection_name = '{}'".format(cn)) + row = cur.fetchone() + if row == None: + #create connection + cur.execute("INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('{}', 'vnc')".format(cn)) + cur.execute("SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '{}' AND parent_id IS NULL".format(cn)) + temp_cn_id = cur.fetchone() + cn_id = temp_cn_id[0] + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','hostname','{}')".format(cn_id, vm_host)) + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','port','{}')".format(cn_id,vm_port)) + #connection permission + cur.execute("INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('{}', '{}', 'READ')".format(en_id,cn_id)) + #clipboard-encoding + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','clipboard-encoding','UTF-8')".format(cn_id)) + print("create connection") + else: + cur.execute("SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '{}' AND parent_id IS NULL".format(cn)) + temp_cn_id = cur.fetchone() + cn_id = temp_cn_id[0] + cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='{}' where connection_id='{}' and parameter_name='hostname'".format(vm_host,cn_id)) + cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='{}' where connection_id='{}' and parameter_name='port'".format(vm_port,cn_id)) + #cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='UTF-8' where connection_id='{}' and parameter_name='clipboard-encoding'".format(cn_id)) + print("no connection") + conn.close() + return None \ No newline at end of file diff --git a/vnc_console_connection/get_info.py b/vnc_console_connection/get_info.py new file mode 100755 index 0000000..ac09633 --- /dev/null +++ b/vnc_console_connection/get_info.py @@ -0,0 +1,88 @@ +import json + +from enum import IntEnum +from xmlrpc.client import ServerProxy as RPCClient +from xmltodict import parse +from config import config +from ldap_list import vm_list +from db_export import setconn + +# Constants +ALL_VM_STATES = -1 +START_ID = -1 # First id whatever it is +END_ID = -1 # Last id whatever it is +session_string = config['oca']['client_secrets'] +opnserver = config['oca']['opn_server'] + +class VMState(IntEnum): + INIT = 0 + PENDING = 1 + HOLD = 2 + ACTIVE = 3 + STOPPED = 4 + SUSPENDED = 5 + DONE = 6 + FAILED = 7 + POWEROFF = 8 + UNDEPLOYED = 9 + CLONING = 10 + CLONING_FAILURE = 11 + + +class VmFilterFlag(IntEnum): + UIDUserResources = 0 # UID User’s Resources + UserAndItsGroupsResources = -1 # Resources belonging to the user and any of his groups + AllResources = -2 # All resources + UserResources = -3 # Resources belonging to the user + UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group + + +class VM: + def __init__(self, vm: dict): + self.id = vm.get('ID', None) + self.owner = { + 'id': vm.get('UID', None), + 'name': vm.get('UNAME', None), + 'gname': vm.get('GNAME', None) + } + self.name = vm.get('NAME', None) + self.status = vm.get('STATE', None) + if self.status: + self.status = VMState(int(self.status)).name.lower() + + template = vm['TEMPLATE'] + + self.graphics = template.get('GRAPHICS', {}) + self.memory = template.get('MEMORY', None) + self.vcpu = template.get('VCPU', None) + self.host = { + 'name': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None), + 'id': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HID', None), + } + + +def main(): + with RPCClient(opnserver) as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + session_string , VmFilterFlag.AllResources.value, START_ID, END_ID, VMState.ACTIVE.value + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + for entry in vm_list.entries: + temp_uname = entry.mail + for i, vm in enumerate(vms): + vm_user = vm['UNAME'] + vm_id = vm['ID'] + vm_port = vm['TEMPLATE']['GRAPHICS'].get('PORT') + vm_host = vm['HISTORY_RECORDS']['HISTORY']['HOSTNAME'] + if vm['UNAME'] == temp_uname: + #print(entry.uid, vm_id, vm_port, vm_host) + setconn(entry.uid, vm_id, vm_port, vm_host) + + else: + print(response) + + +if __name__ == "__main__": + main() + diff --git a/vnc_console_connection/ldap_list.py b/vnc_console_connection/ldap_list.py new file mode 100755 index 0000000..a9e322f --- /dev/null +++ b/vnc_console_connection/ldap_list.py @@ -0,0 +1,30 @@ +import ldap3 +import sys +from config import config +from ldap3 import Server, Connection, ObjectDef, Reader, ALL, SUBTREE, ALL_ATTRIBUTES +from ldap3.core import exceptions + + +LDAP_SERVER = config['ldap']['server'] +LDAP_PASSWORD = config['ldap']['admin_password'] +LDAP_USER = config['ldap']['admin_dn'] +LDAP_PORT = int(config['ldap']['ldap_port']) + +# Create the Server object with the given address. +server = Server(LDAP_SERVER, LDAP_PORT, get_info=ALL) +#Create a connection object, and bind with the given DN and password. +try: + conn = Connection(server, LDAP_USER, LDAP_PASSWORD, auto_bind=True) + print('LDAP Bind Successful.') + # Perform a search for a pre-defined criteria. + # Mention the search filter / filter type and attributes. + conn.search('ou=customer,dc=ungleich,dc=ch', '(&(!({}={})))'.format('mail','*@ungleich.ch') , attributes=['uid','mail']) + #conn.search('ou=customer,dc=ungleich,dc=ch', '(objectClass=*)' , attributes=['uid','mail']) + # Print the resulting entriesn. + #for entry in conn.entries: + #print(entry.uid, entry.mail) + vm_list = conn +except exceptions.LDAPException as err: + sys.exit(f'LDAP Error: {err}') + + From 873eca64673e075b37caacc1257d9d1d922a4ef2 Mon Sep 17 00:00:00 2001 From: "jinguk.kwon" Date: Fri, 20 Mar 2020 16:13:06 +0900 Subject: [PATCH 058/164] new file for vm map --- vm_map.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 vm_map.sh diff --git a/vm_map.sh b/vm_map.sh new file mode 100755 index 0000000..15c80dc --- /dev/null +++ b/vm_map.sh @@ -0,0 +1,5 @@ +vm_list=( $(virsh list | awk '{print $2}') ) + +for ((i=0; i<${#vm_list[@]}; i++)) do + ceph osd map hdd ${vm_list[i]} +done From ad8b7b2c6d5c30d1739d838320dcaf9b8589c2d7 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 8 Apr 2020 07:18:56 +0200 Subject: [PATCH 059/164] [tools]Update get_info.py for uid --- vnc_console_connection/get_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnc_console_connection/get_info.py b/vnc_console_connection/get_info.py index ac09633..e98ae72 100755 --- a/vnc_console_connection/get_info.py +++ b/vnc_console_connection/get_info.py @@ -69,7 +69,7 @@ def main(): if success: vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] for entry in vm_list.entries: - temp_uname = entry.mail + temp_uname = entry.uid for i, vm in enumerate(vms): vm_user = vm['UNAME'] vm_id = vm['ID'] From 4fc8995e074495f9fe9e495083fe0652749b8b97 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 15 Apr 2020 12:14:18 +0200 Subject: [PATCH 060/164] [ungleich-tool]Update config.py for path --- vnc_console_connection/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnc_console_connection/config.py b/vnc_console_connection/config.py index 2a36c96..7b7acc7 100755 --- a/vnc_console_connection/config.py +++ b/vnc_console_connection/config.py @@ -1,5 +1,5 @@ import configparser config = configparser.ConfigParser(allow_no_value=True) -config.read('config-and-secrets.conf') +config.read('/opt/ungleich-tools/vnc_console_connection/config-and-secrets.conf') From 6b51feb1fdfe2f0735c716e35bb3570dff37d669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 7 May 2020 08:25:00 +0200 Subject: [PATCH 061/164] Bump Fedora image (F31 -> F32) --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 45c1629..a84a960 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -18,7 +18,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=31 +RELEASE=32 ARCH=x86_64 IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G From 8970127c246a052d6aa24825f8db453719cbbca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 7 May 2020 08:30:08 +0200 Subject: [PATCH 062/164] Add haveged to Fedora image --- fedora-build-opennebula-image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index a84a960..b4bc5c6 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -132,6 +132,10 @@ run_root systemd-machine-id-setup run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service +# Install haveged due to lack of entropy in ONE environment. +run_root dnf -y install haveged +run_root systemctl enable haveged.service + # Install kernel and bootloader. # Note: linux-firmware is not required our environment and takes almost 200M # uncompressed but is a direct dependency of kernel-core... From 6ea31faa476f9b516b4bdc51c214e79b1b024f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:18:46 +0200 Subject: [PATCH 063/164] Move ONE image defitions to their own folder --- .../alpine-build-opennebula-image.sh | 0 .../fedora-build-opennebula-image.sh | 0 .../ubuntu-build-opennebula-image.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename alpine-build-opennebula-image.sh => opennebula-images/alpine-build-opennebula-image.sh (100%) rename fedora-build-opennebula-image.sh => opennebula-images/fedora-build-opennebula-image.sh (100%) rename ubuntu-build-opennebula-image.sh => opennebula-images/ubuntu-build-opennebula-image.sh (100%) diff --git a/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh similarity index 100% rename from alpine-build-opennebula-image.sh rename to opennebula-images/alpine-build-opennebula-image.sh diff --git a/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh similarity index 100% rename from fedora-build-opennebula-image.sh rename to opennebula-images/fedora-build-opennebula-image.sh diff --git a/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh similarity index 100% rename from ubuntu-build-opennebula-image.sh rename to opennebula-images/ubuntu-build-opennebula-image.sh From 43b52b264df75d8b223d40ed532cfb7c33a42857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:20:00 +0200 Subject: [PATCH 064/164] Add forgotten centos to opennebula-images dir, +x ubuntu-build-one-... --- .../centos-build-opennebula-image.sh | 0 opennebula-images/ubuntu-build-opennebula-image.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename centos-build-opennebula-image.sh => opennebula-images/centos-build-opennebula-image.sh (100%) mode change 100644 => 100755 opennebula-images/ubuntu-build-opennebula-image.sh diff --git a/centos-build-opennebula-image.sh b/opennebula-images/centos-build-opennebula-image.sh similarity index 100% rename from centos-build-opennebula-image.sh rename to opennebula-images/centos-build-opennebula-image.sh diff --git a/opennebula-images/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh old mode 100644 new mode 100755 From b3f79de311110baa532e81c9b3437db7093a05e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:51:26 +0200 Subject: [PATCH 065/164] Add debian image build definition --- .../debian-build-opennebula-image.sh | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 opennebula-images/debian-build-opennebula-image.sh diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh new file mode 100755 index 0000000..17dc1e3 --- /dev/null +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -0,0 +1,164 @@ +#!/bin/sh + +# This script generates Debian images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=buster # 10.X +ARCH=amd64 +IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Ubuntu" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat >/mnt/etc/apt/sources.list < "/mnt$ONE_CONTEXT_DEB_PATH" +run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" +run_root rm "$ONE_CONTEXT_DEB_PATH" + +# Manually install legacy network scripts used by one-context. +run_root apt-get -y install ifupdown + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and bootloader. Do not autoconfigure grub. +run_root 'echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections' +run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-image-amd64 grub-pc + +# Configure grub. +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root apt-get -y install openssh-server + +# Install haveged due to lack of entropy in ONE environment. +run_root apt-get -y install haveged +run_root systemctl enable haveged.service + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Mon, 11 May 2020 12:29:52 +0200 Subject: [PATCH 066/164] [OpenNebula-images] Add scripts to generate FreeBSD images. Tested for 12.1-RELEASE, 11.3-STABLE should be tested as well. Two scripts are added, FreeBSD-build-opennebula-image-generic.sh takes information from the environment and generates the specified image with either dualstack or IPv6-only kernel and base applications. FreeBSD-build-opennebula-image.sh has the versions supported in DCL and generates all neeeded images (combination of Version x Network support). This should run in a DCL VM with an OK amount of cores (4/8 minimum), 2-4G RAM, and storage of roughly 20G + 5G * #resulting_images. This is because there is the base system, a 'pristine chroot', and during the build there can be 2 copies of the resulting system written to the system. Since there are 4 combinations of images: {STABLE,RELEASE} x {dualstack, IPv6ONLY} That means we'll need to assign about 40G storage to be on the safe side. --- .../FreeBSD-build-opennebula-image-generic.sh | 243 ++++++++++++++++++ .../FreeBSD-build-opennebula-image.sh | 31 +++ 2 files changed, 274 insertions(+) create mode 100644 opennebula-images/FreeBSD-build-opennebula-image-generic.sh create mode 100755 opennebula-images/FreeBSD-build-opennebula-image.sh diff --git a/opennebula-images/FreeBSD-build-opennebula-image-generic.sh b/opennebula-images/FreeBSD-build-opennebula-image-generic.sh new file mode 100644 index 0000000..d251f56 --- /dev/null +++ b/opennebula-images/FreeBSD-build-opennebula-image-generic.sh @@ -0,0 +1,243 @@ +#!/bin/sh +# +# Copyright 2020 -- Evilham +# This is BSD licensed as it's based on BSD-licensed code +# +# We could have used e.g. something like: +# - https://git.sr.ht/~sircmpwn/builds.sr.ht/tree/master/images/freebsd/genimg +# +# But we actually do want to compile the kernel, so that the IPv6-only images +# are different and don't support INET. + +# Explode if something goes wrong +set -e + +# What are we building? +# These are the only configuration options. +# They default to current environment. +# RELEASE: should be 'CURRENT' for current or 'X.Y' Defaults to 'CURRENT'. +# ARCH: probably amd64 for DCL +# VMFORMATS: defaults to qcow2, can also be raw. See man mkimg. +# OPENNEBULA_CONTEXT_VERSION: For DCL's OpenNebula that'd be 5.10.0 (default) +# OPENNEBULA_CONTEXT_REVISION: Defaults to 1. +RELEASE=${RELEASE:-CURRENT} +if [ "${RELEASE}" == "CURRENT" ]; then + SRCBRANCH="master" +else + SRCBRANCH="releng/${RELEASE}" +fi +ARCH=${ARCH:-amd64} +VMFORMATS=${VMFORMATS:-qcow2} +OPENNEBULA_CONTEXT_VERSION=${OPENNEBULA_CONTEXT_VERSION:-5.10.0} +OPENNEBULA_CONTEXT_REVISION=${OPENNEBULA_CONTEXT_REVISION:-1} + +# Didn't see a need to make these configurable. +CHROOTDIR="/scratch" +SRCDIR="${CHROOTDIR}/usr/src" +OUR_DIR="$(realpath $(dirname "${0}"))" +OUR_SRCCONF="${SRCDIR}/release/src.conf" +OUR_RELEASE_CONF="${SRCDIR}/release/release.conf" +# Shorthand for the package file name. +OPENNEBULA_CONTEXT="one-context-${OPENNEBULA_CONTEXT_VERSION}_${OPENNEBULA_CONTEXT_REVISION}.txz" + +setup_sources() { + # Let's use git, we might need to install it + if ! which git 2>&1 > /dev/null; then + pkg install -y git + fi + + if [ ! -d "$(dirname ${SRCDIR})" ]; then + mkdir -p "$(dirname ${SRCDIR})" + fi + + # Checkout needed branch + if [ ! -d "${SRCDIR}" ]; then + git clone "https://github.com/freebsd/freebsd" \ + --branch "${SRCBRANCH}" "${SRCDIR}" + else + GIT_CMD="git -C ${SRCDIR}" + ${GIT_CMD} clean -df + ${GIT_CMD} reset --hard + ${GIT_CMD} fetch + ${GIT_CMD} checkout "${SRCBRANCH}" + ${GIT_CMD} pull + fi + + # Add settings for IPv6-only kernel + cat > "${SRCDIR}/sys/${ARCH}/conf/GENERIC-IPV6ONLY" << EOF +include GENERIC +ident GENERIC-IPV6ONLY +makeoptions MKMODULESENV+="WITHOUT_INET_SUPPORT=" +nooptions INET +nodevice gre +EOF + # Fix vmimage.subr to install custom package and fix other things + cat >> "${SRCDIR}/release/tools/vmimage.subr" << EOF +vm_extra_install_ports() { + # Make sure we install the opennbula context package + cp "/${OPENNEBULA_CONTEXT}" "\${DESTDIR}/tmp/${OPENNEBULA_CONTEXT}" + chroot \${DESTDIR} \${EMULATOR} env ASSUME_ALWAYS_YES=yes \\ + /usr/sbin/pkg add '/tmp/${OPENNEBULA_CONTEXT}' + + # Now make sure the system has better defaults + cat >> "\${DESTDIR}/etc/rc.conf" << eof +# Update to latest patch on first boot +firstboot_freebsd_update_enable="YES" +# Enable OpenNebula's service. +one_context_enable="YES" +# Enable SSH for customers +sshd_enable="YES" +# Clear tmp on boot +clear_tmp_enable="YES" +# Disable sendmail by default +sendmail_enable="NONE" +# Disable crash dumps +dumpdev="NO" +eof + # Enable root access with SSH key. + # It is user's responsibility to further secure their system. + sed -i '' -E \ + 's/(^#[ ]*|^)PermitRootLogin .*/PermitRootLogin without-password/' \ + "\${DESTDIR}/etc/ssh/sshd_config" +} +EOF + # Skip building iso images + rm "${SRCDIR}/release/${ARCH}/mkisoimages.sh" + # This is a hack to not build the memstick + cat > "${SRCDIR}/release/${ARCH}/make-memstick.sh" < \${CHROOTDIR}/etc/src-env.conf +} + +## Set the directory within which the release will be built. +CHROOTDIR="${CHROOTDIR}" + +## Set to override the default target architecture and kernel +TARGET="${ARCH}" +TARGET_ARCH="${ARCH}" +KERNEL="${KERNEL_CONFIG}" + +## Set to specify a custom make.conf and/or src.conf +SRC_CONF="${OUR_SRCCONF}" + +# Since these are VMs, users should add other components if they want to. +NODOC=YES +NOPORTS=YES +NOSRC=YES + +# We manage sources manually +SRC_UPDATE_SKIP=YES + +## Set to pass additional flags to make(1) for the build chroot setup, such +## as TARGET/TARGET_ARCH. +# This was necessary for "cross-compiling" +CHROOT_MAKEENV="MK_LLVM_TARGET_X86=yes" + +WITH_VMIMAGES=YES + +# VM image size, see man 1 truncate +VMSIZE="10G" + +# List of disk image formats, see man mkgimg. +VMFORMATS="${VMFORMATS}" + +# These variables have to be exported because they are needed in subprocesses. +export NOSWAP=YES +# Custom ports +# - firstboot-freebsd-update helps us not have to create an image for each +# patch level. We still will have to do it for each minor version update. +# - bash is apparently needed for one-context +export VM_EXTRA_PACKAGES="firstboot-freebsd-update bash" +EOF +} + +_do_run_release() { + . "${SRCDIR}/release/release.sh" +} +run_release() { + _do_run_release -c "${OUR_RELEASE_CONF}" +} + + +build_image() { + # Generate configuration + echo "${2}" > "${OUR_SRCCONF}" + KERNEL_CONFIG="${1}" + gen_releaseconf > "${OUR_RELEASE_CONF}" + # Be paranoid about files and stuff + sync + # Continue with the release script + run_release + # Be paranoid about files and stuff + sync + + mv "${CHROOTDIR}/R/vmimages" "${OUR_DIR}/FreeBSD-${RELEASE}-${1}" + + # Be paranoid about files and stuff + sync +} + +our_main() { + case "$1" in + --dualstack) + BUILD_DUALSTACK=yes + ;; + --ipv6only) + BUILD_IPV6ONLY=yes + ;; + *) + cat << EOF +Run with --dualstack or --ipv6only depending on the image you want. +EOF + exit 1 + ;; + esac + setup_sources + setup_our_env + # Fetch OpenNebula's context package + fetch "https://github.com/OpenNebula/addon-context-linux/releases/download/v${OPENNEBULA_CONTEXT_VERSION}/${OPENNEBULA_CONTEXT}" \ + -o "${CHROOTDIR}/${OPENNEBULA_CONTEXT}" + # Do run + if [ -n "${BUILD_DUALSTACK}" ]; then + build_image "GENERIC" + fi + if [ -n "${BUILD_IPV6ONLY}" ]; then + build_image "GENERIC-IPV6ONLY" "$(cat << EOF +WITHOUT_INET=yes +WITHOUT_INET_SUPPORT=yes +EOF +)" + fi + + cat << EOF + +*************** DONE *************** +You will find the images under "${OUR_DIR}". +************************************ +EOF +} + +our_main "${@}" diff --git a/opennebula-images/FreeBSD-build-opennebula-image.sh b/opennebula-images/FreeBSD-build-opennebula-image.sh new file mode 100755 index 0000000..c72a2b0 --- /dev/null +++ b/opennebula-images/FreeBSD-build-opennebula-image.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Copyright 2020 -- Evilham +# This is BSD licensed as it's based on BSD-licensed code +# +# +# This builds all needed FreeBSD images for ungleich's Data Center Light +# When there are new releases, they should be updated here and the script +# should run. +# 11.4 is scheduled end of June 2020 +# 12.2 is scheduled end of October 2020 +# + +SUPPORTED_RELEASES="11.3 12.1" + +# This should run in a DCL VM with an OK amount of cores (4/8 minimum), +# 4G RAM, and storage of roughly 20G + 5G * #resulting_images. +# +# This is because there is the base system, a 'pristine chroot', and during the +# build there can be 2 copies of the resulting system written to the system. +# Since there are 4 combinations of images: +# {STABLE,RELEASE} x {dualstack, IPv6ONLY} +# +# That means we'll need to assign about 40G storage to be on the safe side. + +for release in ${SUPPORTED_RELEASES}; do + for build in dualstack ipv6only; do + env RELEASE=${release} sh FreeBSD-build-opennebula-image-generic.sh --${build} \ + | tee "FreeBSD-${release}-${build}.log" + done +done From 4eb3bc55239e3439f126ba9cc7c0993abb15b1e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 12 May 2020 07:15:50 +0200 Subject: [PATCH 067/164] Add WIP archlinux image definition script --- .../arch-build-opennebula-image.sh | 154 ++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100755 opennebula-images/arch-build-opennebula-image.sh diff --git a/opennebula-images/arch-build-opennebula-image.sh b/opennebula-images/arch-build-opennebula-image.sh new file mode 100755 index 0000000..00c454f --- /dev/null +++ b/opennebula-images/arch-build-opennebula-image.sh @@ -0,0 +1,154 @@ +#!/bin/sh + +# This script generates Debian images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +ARCH=amd64 +IMAGE_PATH=arch-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Arch" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat > /mnt/etc/pacman.d/mirrorlist << EOF +## +## Arch Linux repository mirrorlist +## Generated on 2020-05-12 +## + +## Switzerland +Server = http://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch +Server = https://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch +Server = http://mirror.init7.net/archlinux/$repo/os/$arch +Server = https://mirror.init7.net/archlinux/$repo/os/$arch +Server = http://mirror.puzzle.ch/archlinux/$repo/os/$arch +Server = https://mirror.puzzle.ch/archlinux/$repo/os/$arch +Server = https://mirror.ungleich.ch/mirror/packages/archlinux/$repo/os/$arch +EOF +run_root pacman --sync --refresh --upgrade + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and generate initramfs. +run_root pacman --sync linux mkinitcpio +run_rot mkinitcpio -P + +# Install and configure bootloader. +run_root pacman --sync grub +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root pacman --sync openssh-server + +# Install haveged due to lack of entropy in ONE environment. +run_root pacman --sync haveged +run_root systemctl enable haveged.service + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Wed, 13 May 2020 11:41:34 +0200 Subject: [PATCH 068/164] Rename freebsd build scripts for consistency --- ...image-generic.sh => freebsd-build-opennebula-image-generic.sh} | 0 ...uild-opennebula-image.sh => freebsd-build-opennebula-image.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename opennebula-images/{FreeBSD-build-opennebula-image-generic.sh => freebsd-build-opennebula-image-generic.sh} (100%) mode change 100644 => 100755 rename opennebula-images/{FreeBSD-build-opennebula-image.sh => freebsd-build-opennebula-image.sh} (100%) diff --git a/opennebula-images/FreeBSD-build-opennebula-image-generic.sh b/opennebula-images/freebsd-build-opennebula-image-generic.sh old mode 100644 new mode 100755 similarity index 100% rename from opennebula-images/FreeBSD-build-opennebula-image-generic.sh rename to opennebula-images/freebsd-build-opennebula-image-generic.sh diff --git a/opennebula-images/FreeBSD-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh similarity index 100% rename from opennebula-images/FreeBSD-build-opennebula-image.sh rename to opennebula-images/freebsd-build-opennebula-image.sh From 5af488b9d6a470f112740087529fffa97adce217 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 13 May 2020 12:09:26 +0200 Subject: [PATCH 069/164] Fix script call in FreeBSD build following renaming --- opennebula-images/freebsd-build-opennebula-image.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index c72a2b0..266f639 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -23,9 +23,10 @@ SUPPORTED_RELEASES="11.3 12.1" # # That means we'll need to assign about 40G storage to be on the safe side. +date=$(date -I) for release in ${SUPPORTED_RELEASES}; do for build in dualstack ipv6only; do - env RELEASE=${release} sh FreeBSD-build-opennebula-image-generic.sh --${build} \ - | tee "FreeBSD-${release}-${build}.log" + env RELEASE=${release} sh freebsd-build-opennebula-image-generic.sh --${build} \ + | tee "freebsd-${release}-${build}-${date}.log" done done From 0e7aa75959cdc8673c6c62c7abf02d686770cd8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 13 May 2020 14:25:17 +0200 Subject: [PATCH 070/164] Fix locale issue in debian image --- opennebula-images/debian-build-opennebula-image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 17dc1e3..9a51f1e 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -144,6 +144,10 @@ run_root apt-get -y install openssh-server run_root apt-get -y install haveged run_root systemctl enable haveged.service +# Generate locales. +run_root sed -i 's/^# *\(en_GB.UTF-8\)/\1/' /etc/locale.gen +run_root locale-gen + # Generate fstab file. boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") From 5a2e01d0edfebc45e081067f259dc8fbe66d820d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 13 May 2020 14:27:58 +0200 Subject: [PATCH 071/164] s/Ubuntu/Debian/ in debian build script --- opennebula-images/debian-build-opennebula-image.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 9a51f1e..67714bf 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -43,9 +43,9 @@ if [ "$(whoami)" != 'root' ]; then exit 1 fi -if [ $(lsb_release --short --id) != "Ubuntu" ]; then - echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 - echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 +if [ $(lsb_release --short --id) != "Debian" ]; then + echo "WARNING: this script has been designed to run on an Debian system." >&2 + echo "WARNING: Not running Debian. Giving you 5 seconds to abort." >&2 sleep 5 fi From 1d3b786a0f635f533116c024efee9ca2664fac77 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 15 May 2020 20:51:28 +0200 Subject: [PATCH 072/164] add shell script to update server to ceph nautilus --- ceph-upgrade-server-to-nautilus.sh | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 ceph-upgrade-server-to-nautilus.sh diff --git a/ceph-upgrade-server-to-nautilus.sh b/ceph-upgrade-server-to-nautilus.sh new file mode 100644 index 0000000..058e40d --- /dev/null +++ b/ceph-upgrade-server-to-nautilus.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +rm -f /etc/apt/sources.list.d/ceph.list + +cat > /etc/apt/sources.list < /etc/apt/sources.list.d/backports.list +apt update +apt dist-upgrade -y +apt install -t buster-backports -y ceph From a2558df86804e403c1a2273fdf863fbad7349555 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 15 May 2020 20:52:45 +0200 Subject: [PATCH 073/164] Mode update --- ceph-upgrade-server-to-nautilus.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 ceph-upgrade-server-to-nautilus.sh diff --git a/ceph-upgrade-server-to-nautilus.sh b/ceph-upgrade-server-to-nautilus.sh old mode 100644 new mode 100755 From b2d33e624ac99781af5395cc3b672fc3ba5693c0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 21 May 2020 17:09:39 +0200 Subject: [PATCH 074/164] mikrotik-update: add notes for crs326 --- mikrotik-update.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mikrotik-update.sh b/mikrotik-update.sh index b2c0fc6..934ea47 100755 --- a/mikrotik-update.sh +++ b/mikrotik-update.sh @@ -5,9 +5,15 @@ if [ $# -lt 2 ]; then echo "$0 router [router...]" cat < Date: Sun, 24 May 2020 08:26:18 +0200 Subject: [PATCH 075/164] Debian ONE image: set hostname --- opennebula-images/debian-build-opennebula-image.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 67714bf..577f573 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -14,6 +14,7 @@ ARCH=amd64 IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd0 +HOSTNAME=debian # TODO: find the package definition and built ourself, publish in some RPM repository. ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" @@ -94,11 +95,13 @@ cp /etc/resolv.conf /mnt/etc/resolv.conf # Initialize /etc/hosts. cat > /mnt/etc/hosts << EOF -127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 -::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 +127.0.0.1 $HOSTNAME localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 $HOSTNAME localhost localhost.localdomain localhost6 localhost6.localdomain6 EOF +run_root hostnamectl set-hostname $HOSTNAME + # Configure package sources and update package index. cat >/mnt/etc/apt/sources.list < Date: Sun, 24 May 2020 08:33:59 +0200 Subject: [PATCH 076/164] Move EvilHam's FreeBSD scripts to legacy directory --- legacy/README.md | 2 ++ .../freebsd-build-opennebula-image-generic.sh | 0 {opennebula-images => legacy}/freebsd-build-opennebula-image.sh | 0 3 files changed, 2 insertions(+) create mode 100644 legacy/README.md rename {opennebula-images => legacy}/freebsd-build-opennebula-image-generic.sh (100%) rename {opennebula-images => legacy}/freebsd-build-opennebula-image.sh (100%) diff --git a/legacy/README.md b/legacy/README.md new file mode 100644 index 0000000..97eae65 --- /dev/null +++ b/legacy/README.md @@ -0,0 +1,2 @@ +This directory contains old scripts that are not used anymore but might still +be useful. diff --git a/opennebula-images/freebsd-build-opennebula-image-generic.sh b/legacy/freebsd-build-opennebula-image-generic.sh similarity index 100% rename from opennebula-images/freebsd-build-opennebula-image-generic.sh rename to legacy/freebsd-build-opennebula-image-generic.sh diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/legacy/freebsd-build-opennebula-image.sh similarity index 100% rename from opennebula-images/freebsd-build-opennebula-image.sh rename to legacy/freebsd-build-opennebula-image.sh From 0d554866c82ba9d6e3135265a59bfb4198a9f5f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 24 May 2020 08:49:41 +0200 Subject: [PATCH 077/164] Initial sourcehut-inspired freebsd image build script --- .../freebsd-build-opennebula-image.sh | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 opennebula-images/freebsd-build-opennebula-image.sh diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh new file mode 100755 index 0000000..18722ab --- /dev/null +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -0,0 +1,109 @@ +#!/bin/sh + +# This script generates FreeBSD images for OpenNebula, being heavily inspired +# from srht's FreeBSD build image definition. It assumes running on a FreeBSD host. + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=12.1-RELEASE +IMAGE_PATH=freebsd-$RELEASE-$(date +%+F).img.qcow2 +IMAGE_SIZE=10G + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + sync || true + umount /mnt/dev || true + umount /mnt || true + mdconfig -du md0 || true +} +trap cleanup EXIT + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +# Allocate and partition/format disk image. +truncate -s 6G disk.img +mdconfig -a -t vnode -f disk.img -u md0 +gpart create -s gpt /dev/md0 +gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 +gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md0 +gpart add -t freebsd-ufs -l rootfs -b 1M -s 5G md0 +newfs -U /dev/md0p2 + +# Mount allocated image. +mount /dev/md0p2 /mnt +mkdir -p /mnt/dev +mount -t devfs devfs /mnt/dev + +# Download and extract base system. +dist_files="kernel.txz base.txz" +dist_dir="/usr/freebsd-dist/$arch/$release" + +mkdir -p "$dist_dir" +for f in $dist_files +do + fetch -m -o "$dist_dir/$f" "$dist_base/$f" + tar -C /mnt -xJf "$dist_dir/$f" +done + +# Configure new system. +echo "/dev/gpt/rootfs / ufs rw,noatime 1 1" >/mnt/etc/fstab +touch /mnt/firstboot +echo 'autoboot_delay="-1"' >>/mnt/boot/loader.conf + +cat >>/mnt/etc/rc.conf < /mnt/etc/resolv.conf +tzsetup -s -C /mnt UTC + +cat >>/mnt/etc/ssh/sshd_config </mnt/usr/local/etc/pkg/repos/FreeBSD.conf < Date: Sun, 24 May 2020 08:59:19 +0200 Subject: [PATCH 078/164] FreeBSD image: fix environment, mktemp disk image --- .../freebsd-build-opennebula-image.sh | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index 18722ab..ec64a56 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -8,9 +8,13 @@ set -x # XXX: Handle command-line arguments? RELEASE=12.1-RELEASE +ARCH=amd64 IMAGE_PATH=freebsd-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G +DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" +PORTS_BASE="https://download.freebsd.org/ftp/snapshots/$ARCH/12.1-STABLE" + # TODO: find the package definition and built ourself, publish in some RPM repository. ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm @@ -29,8 +33,9 @@ if [ "$(whoami)" != 'root' ]; then fi # Allocate and partition/format disk image. -truncate -s 6G disk.img -mdconfig -a -t vnode -f disk.img -u md0 +disk=$(mktemp) +truncate -s 6G $disk +mdconfig -a -t vnode -f $disk -u md0 gpart create -s gpt /dev/md0 gpart add -t freebsd-boot -l bootfs -b 40 -s 512K md0 gpart bootcode -b /boot/pmbr -p /boot/gptboot -i 1 md0 @@ -44,12 +49,12 @@ mount -t devfs devfs /mnt/dev # Download and extract base system. dist_files="kernel.txz base.txz" -dist_dir="/usr/freebsd-dist/$arch/$release" +dist_dir="/usr/freebsd-dist/$ARCH/$RELEASE" mkdir -p "$dist_dir" for f in $dist_files do - fetch -m -o "$dist_dir/$f" "$dist_base/$f" + fetch -m -o "$dist_dir/$f" "$DIST_BASE/$f" tar -C /mnt -xJf "$dist_dir/$f" done @@ -83,11 +88,11 @@ FreeBSD: { EOF # freebsd-update is only supported for RELEASE -if [ "${release%-RELEASE}" != "$release" ] +if [ "${release%-RELEASE}" != "$RELEASE" ] then env PAGER=true /usr/sbin/freebsd-update \ -b /mnt \ - --currently-running "$release" \ + --currently-running "$RELEASE" \ --not-running-from-cron -F \ fetch install fi @@ -95,15 +100,15 @@ fi env ASSUME_ALWAYS_YES=YES pkg -c /mnt bootstrap -f env ASSUME_ALWAYS_YES=YES pkg -c /mnt install bash -fetch -m -o "$dist_dir/ports.txz" "$ports_base/ports.txz" +fetch -m -o "$dist_dir/ports.txz" "$PORTS_BASE/ports.txz" tar -C /mnt -xJf "$dist_dir/ports.txz" cleanup trap : EXIT -mkdir -p "$arch" -qemu-img convert -f raw -O qcow2 disk.img "$arch"/root.img.qcow2 -rm disk.img +mkdir -p "$ARCH" +qemu-img convert -f raw -O qcow2 $disk "$ARCH"/root.img.qcow2 +rm $disk # Filesystem will be enlarged by growfs(7) on next startup qemu-img resize $IMAGE_PATH $IMAGE_SIZE From 9c17dad41ded4db2029ce0c39bfd9d1c85f2e3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 24 May 2020 09:22:29 +0200 Subject: [PATCH 079/164] Fix local generation for debian image --- opennebula-images/debian-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh index 577f573..d18e349 100755 --- a/opennebula-images/debian-build-opennebula-image.sh +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -148,7 +148,7 @@ run_root apt-get -y install haveged run_root systemctl enable haveged.service # Generate locales. -run_root sed -i 's/^# *\(en_GB.UTF-8\)/\1/' /etc/locale.gen +run_root 'sed -i "s/^# *\(en_GB.UTF-8\)/\1/" etc/locale.gen' run_root locale-gen # Generate fstab file. From 95b1922a3663be95b09401929537ed7df3b7adea Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:15:58 +0200 Subject: [PATCH 080/164] Added WIP script for mapping osd to disk types, and helper script (megaclisas-status) that provides a nicely formatted output for megacli --- map-osd-to-disktype | 13 + megaclisas-status | 875 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 888 insertions(+) create mode 100644 map-osd-to-disktype create mode 100755 megaclisas-status diff --git a/map-osd-to-disktype b/map-osd-to-disktype new file mode 100644 index 0000000..f1a2ff8 --- /dev/null +++ b/map-osd-to-disktype @@ -0,0 +1,13 @@ +# Assumes, osd is up and + +# Tempfile +./megacli-status > /tmp/megaclisas-status.out + +# Gets osd numbers of a particular device class, and gets their mountpoints on the host, and puts them into a tempfile +for osd in $(ceph osd tree | grep $OSD | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep $osd| cut -c 24- >> /tmp/list_osd_mountpoint; done + +# Gets the Megacli mappings for the mountpoints +for MOUNT in $(cat /tmp/list_osd_mountpoint | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done + +# Gets the hardware types for the Megacli mappings +for i in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $i | cut -d '|' -f 3 >> /tmp/disk_types; done diff --git a/megaclisas-status b/megaclisas-status new file mode 100755 index 0000000..3495502 --- /dev/null +++ b/megaclisas-status @@ -0,0 +1,875 @@ +#!/usr/bin/python +# $Id: megaclisas-status,v 1.68 2016/10/21 14:38:56 root Exp root $ +# +# Written by Adam Cecile +# Modified by Vincent S. Cojot +# + +import os +import re +import sys +import pdb +if sys.platform == 'win32': + import ctypes + +def_megaclipath = "/opt/MegaRAID/MegaCli/MegaCli64" + +# Non-Nagios Mode defaults +nagiosmode = False +nagiosoutput='' +nagiosgoodarray = 0 +nagiosbadarray = 0 +nagiosgooddisk = 0 +nagiosbaddisk = 0 + +# Sane defaults +printarray = True +printcontroller = True +debugmode = False +notempmode = False +totaldrivenumber = 0 + +# Hardcode a max of 16 HBA and 128 LDs for now. LDTable must be initialized to accept populating list of LD's into each ctlr's list. +MaxNumHBA = 16 +MaxNumLD = 128 +LDTable = [ [] * MaxNumHBA for i in range(MaxNumLD) ] +NestedLDTable = [[False for i in range(MaxNumHBA)] for j in range(MaxNumLD)] + +# Outputs is a 'dict' of all MegaCLI outputs so we can re-use them during loops.. +Outputs = {} + +# Startup +def print_usage(): + print 'Usage: megaraid-status [--nagios|--debug|--notemp]' + +# We need root access to query +if __name__ == '__main__': + try: + root_or_admin = os.geteuid() == 0 + except AttributeError: + root_or_admin = ctypes.windll.shell32.IsUserAnAdmin() !=0 + if not root_or_admin: + print '# This script requires Administrator privileges' + sys.exit(5) + +# Check command line arguments to enable nagios or not +if len(sys.argv) > 2: + print_usage() + sys.exit(1) + +if len(sys.argv) > 1: + if sys.argv[1] == '--nagios': + nagiosmode = True + elif sys.argv[1] == '--debug': + debugmode = True + elif sys.argv[1] == '--notemp': + notempmode = True + else: + print_usage() + sys.exit(1) +# Functions +def dbgprint(msg): + if (debugmode): + sys.stderr.write ( str('# DEBUG : '+msg+'\n')) + +def is_exe(fpath): + return os.path.isfile(fpath) and os.access(fpath, os.X_OK) + +def which(program): + import os + fpath, fname = os.path.split(program) + if fpath: + if is_exe(program): + return program + else: + # Add some defaults + os.environ["PATH"] += os.pathsep + '/opt/MegaRAID/MegaCli' + os.environ["PATH"] += os.pathsep + '/ms/dist/hwmgmt/bin' + os.environ["PATH"] += os.pathsep + os.path.dirname(os.path.realpath(sys.argv[0])) + for path in os.environ["PATH"].split(os.pathsep): + dbgprint ('Looking in PATH '+str(path)) + path = path.strip('"') + exe_file = os.path.join(path, program) + if is_exe(exe_file): + dbgprint ('Found "'+program+'" at '+exe_file) + return exe_file + return None + +# Find MegaCli +for megabin in "MegaCli64","MegaCli","megacli", "MegaCli.exe": + dbgprint ('Looking for '+str(megabin)+' in PATH next..') + megaclipath = which(megabin) + if (megaclipath != None): + dbgprint ('Will use MegaCLI from here: '+str(megaclipath)) + break + +# Check binary exists (and +x), if not print an error message +if (megaclipath != None): + if os.path.exists(megaclipath) and os.access(megaclipath, os.X_OK): + pass + else: + if nagiosmode: + print 'UNKNOWN - Cannot find '+megaclipath + else: + print 'Cannot find ' + megaclipath + 'in your PATH. Please install it.' + sys.exit(3) +else: + print 'Cannot find "MegaCli64","MegaCli" or "megacli" or "MegaCli.exe" in your PATH. Please install it.' + sys.exit(3) + + +#### pdb.set_trace() + +def returnWdthFromArrayCol(glarray,idx): + maxwdth = 0 + for glrow in glarray: + if ( len(glrow[idx]) > maxwdth): + maxwdth = len(glrow[idx]) + return maxwdth + +# Get command output +def getOutput(cmd): + lines = [] + if ( Outputs.has_key(cmd) ): + dbgprint ("Got Cached value: "+str(cmd)) + lines = Outputs[cmd] + else: + dbgprint ("Not a Cached value: "+str(cmd)) + output = os.popen(cmd) + for line in output: + if not re.match(r'^$',line.strip()): + lines.append(line.strip()) + Outputs[cmd] = lines + return lines + +def returnControllerNumber(output): + for line in output: + if re.match(r'^Controller Count.*$',line.strip()): + return int(line.split(':')[1].strip().strip('.')) + +def returnTotalDriveNumber(output): + for line in output: + if re.match(r'Number of Physical Drives on Adapter.*$',line.strip()): + return int(line.split(':')[1].strip()) + +def returnRebuildProgress(output): + percent = 0 + tmpstr = '' + for line in output: + if re.match(r'^Rebuild Progress on Device at Enclosure.*, Slot .* Completed ',line.strip()): + tmpstr = line.split('Completed')[1].strip() + percent = int(tmpstr.split('%')[0].strip()) + return percent + +def returnConfDriveNumber(output): + # Count the configured drives + confdrives = 0 + for line in output: + if re.match(r'.*Number of PDs:.*$',line.strip()): + confdrives += int(line.split(':')[2].strip()) + return int(confdrives) + +def returnUnConfDriveNumber(output): + # Count the configured drives + confdrives = 0 + for line in output: + if re.match(r'^Firmware state: Unconfigured.*$',line.strip()): + confdrives += 1 + return int(confdrives) + +def returnControllerModel(output): + for line in output: + if re.match(r'^Product Name.*$',line.strip()): + return line.split(':')[1].strip() + +def returnMemorySize(output): + for line in output: + if re.match(r'^Memory Size.*$',line.strip()): + return line.split(':')[1].strip() + +def returnFirmwareVersion(output): + for line in output: + if re.match(r'^FW Package Build.*$',line.strip()): + return line.split(':')[1].strip() + +def returnROCTemp(output): + ROCtemp = '' + tmpstr = '' + if (notempmode): + return str('N/A') + else: + for line in output: + if re.match(r'^ROC temperature :.*$',line.strip()): + tmpstr = line.split(':')[1].strip() + ROCtemp = re.sub(' +.*$', '', tmpstr) + if ( ROCtemp != '' ): + return str(str(ROCtemp)+'C') + else: + return str('N/A') + +def returnBBUPresence(output): + BBU = '' + tmpstr = '' + for line in output: + if re.match(r'^BBU +:.*$',line.strip()): + tmpstr = line.split(':')[1].strip() + BBU = re.sub(' +.*$', '', tmpstr) + break + if ( BBU != '' ): + return str(BBU) + else: + return str('N/A') + +def returnBBUStatus(output): + BBUStatus = '' + tmpstr = '' + for line in output: + if re.match(r'^ *Battery Replacement required +:.*$',line.strip()): + tmpstr = line.split(':')[1].strip() + BBUStatus = re.sub(' +.*$', '', tmpstr) + break + if ( BBUStatus == 'Yes' ): + return str('REPL') + else: + return str('Good') + +def returnArrayNumber(output): + i = 0 + for line in output: + if re.match(r'^(CacheCade )?Virtual Drive:.*$',line.strip()): + i += 1 + return i + +def returnHBAPCIInfo(output): + busprefix = '0000' + busid = '' + devid = '' + functionid = '' + pcipath = '' + for line in output: + if re.match(r'^Bus Number.*:.*$',line.strip()): + busid = str(line.strip().split(':')[1].strip()).zfill(2) + if re.match(r'^Device Number.*:.*$',line.strip()): + devid = str(line.strip().split(':')[1].strip()).zfill(2) + if re.match(r'^Function Number.*:.*$',line.strip()): + functionid = str(line.strip().split(':')[1].strip()).zfill(1) + if busid: + pcipath = str(busprefix + ':' + busid + ':' + devid + '.' + functionid) + dbgprint("Array PCI path : "+pcipath) + return str(pcipath) + else: + return None + +def returnHBAInfo(table,output,controllerid): + controllermodel = 'Unknown' + controllerram = 'Unknown' + controllerrev = 'Unknown' + controllertemp = '' + controllermodel = returnControllerModel(output) + controllerram = returnMemorySize(output) + controllerrev = returnFirmwareVersion(output) + controllertemp = returnROCTemp(output) + controllerbbu = returnBBUPresence(output) + if controllerbbu == 'Present': + cmd = '%s -AdpBbuCmd -GetBbuStatus -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + controllerbbu = returnBBUStatus(output) + + if controllermodel != 'Unknown': + table.append([ 'c'+str(controllerid), controllermodel, controllerram, str(controllertemp), str(controllerbbu), str('FW: '+controllerrev) ]) + +def returnArrayInfo(output,controllerid,arrayid,arrayindex): + id = 'c'+str(controllerid)+'u'+str(arrayid) + operationlinennumber = False + linenumber = 0 + targetid = '' + raidtype = '' + raidlvl = '' + size = '' + state = 'N/A' + strpsz = '' + dskcache = 'N/A' + properties = '' + spandepth = 0 + diskperspan = 0 + cachecade_info = 'None' + + for line in output: + if re.match(r'^(CacheCade )?Virtual Drive:.*(Target Id: [0-9]+).*$',line.strip()): + # Extract the SCSI Target ID + targetid = line.strip().split(':')[2].split(')')[0].strip() + elif re.match(r'^RAID Level.*?:.*$',line.strip()): + # Extract the primary raid type, decide on X0 RAID level later when we hit Span Depth + raidlvl = int(line.strip().split(':')[1].split(',')[0].split('-')[1].strip()) + elif re.match(r'^Size.*?:.*$',line.strip()): + # Size reported in MB + if re.match(r'^.*MB$',line.strip().split(':')[1]): + size = line.strip().split(':')[1].strip('MB').strip() + if ( float(size) > 1000): + size = str(int(round((float(size) / 1000))))+'G' + else: + size = str(int(round(float(size))))+'M' + # Size reported in TB + elif re.match(r'^.*TB$',line.strip().split(':')[1]): + size = line.strip().split(':')[1].strip('TB').strip() + size = str(int(round((float(size) * 1000))))+'G' + # Size reported in GB (default) + else: + size = line.strip().split(':')[1].strip('GB').strip() + size = str(int(round((float(size)))))+'G' + elif re.match(r'^Span Depth.*?:.*$',line.strip()): + # If Span Depth is greater than 1 chances are we have a RAID 10, 50 or 60 + spandepth = line.strip().split(':')[1].strip() + elif re.match(r'^State.*?:.*$',line.strip()): + state = line.strip().split(':')[1].strip() + elif re.match(r'^Strip Size.*?:.*$',line.strip()): + strpsz = line.strip().split(':')[1].strip() + elif re.match(r'^Number Of Drives per span.*:.*$',line.strip()): + diskperspan = int(line.strip().split(':')[1].strip()) + elif re.match(r'^Current Cache Policy.*?:.*$',line.strip()): + props = line.strip().split(':')[1].strip() + if re.search('ReadAdaptive', props): + properties += 'ADRA' + if re.search('ReadAhead', props): + properties += 'RA' + if re.match('ReadAheadNone', props): + properties += 'NORA' + if re.search('WriteBack', props): + properties += ',WB' + if re.match('WriteThrough', props): + properties += ',WT' + elif re.match(r'^Disk Cache Policy.*?:.*$',line.strip()): + props = line.strip().split(':')[1].strip() + if re.search('Disabled', props): + dskcache = 'Disabled' + if re.search('Disk.s Default', props): + dskcache = 'Default' + if re.search('Enabled', props): + dskcache = 'Enabled' + elif re.match(r'^Ongoing Progresses.*?:.*$',line.strip()): + operationlinennumber = linenumber + elif re.match(r'Cache Cade Type\s*:.*$', line): + cachecade_info = "Type : " + line.strip().split(':')[1].strip() + elif re.match(r'^Target Id of the Associated LDs\s*:.*$', line): + associated=[] + for array in line.split(':')[1].strip().split(','): + if array.isdigit(): + associated.append('c%du%d' % (controllerid, int(array))) + if len(associated) >= 1: + cachecade_info = "Associated : %s" %(', '.join(associated)) + linenumber += 1 + + # If there was an ongoing operation, find the relevant line in the previous output + if operationlinennumber: + inprogress = output[operationlinennumber + 1] + else: + inprogress = 'None' + + # Compute the RAID level + NestedLDTable[int(controllerid)][int(arrayindex)] = False + if raidlvl == '': + raidtype = str('N/A') + else: + if (int(spandepth) >= 2): + raidtype = str('RAID-' + str(raidlvl) + '0') + NestedLDTable[controllerid][int(arrayindex)] = True + else: + if(raidlvl == 1): + if(diskperspan > 2): + raidtype = str('RAID-10') + NestedLDTable[controllerid][int(arrayindex)] = True + else: + raidtype = str('RAID-' + str(raidlvl)) + else: + raidtype = str('RAID-' + str(raidlvl)) + + dbgprint('RAID Level: ' + str(raidlvl) + + ' Span Depth: ' + str(spandepth) + + ' Disk Per Span: ' + str(diskperspan) + + ' Raid Type: ' + str(raidtype)) + return [id,raidtype,size,strpsz,properties,dskcache,state,targetid,cachecade_info,inprogress] + +def returnDiskInfo(output,controllerid): + arrayid = False + arrayindex = -1 + sarrayid = 'Unknown' + diskid = False + oldenclid = False + enclid = False + spanid = False + slotid = False + lsidid = 'Unknown' + table = [] + fstate = 'Offline' + substate = 'Unknown' + model = 'Unknown' + speed = 'Unknown' + dsize = 'Unknown' + temp = 'Unk0C' + percent = 0 + for line in output: + if re.match(r'^Span: [0-9]+ - Number of PDs:',line.strip()): + spanid = line.split(':')[1].strip() + spanid = re.sub(' - Number of PDs.*', '', spanid) + elif re.match(r'Enclosure Device ID: .*$',line.strip()): + # We match here early in the analysis so reset the vars if this is a new disk we're reading.. + oldenclid = enclid + enclid = line.split(':')[1].strip() + if oldenclid != False: + fstate = 'Offline' + model = 'Unknown' + speed = 'Unknown' + temp = 'Unk0C' + slotid = False + lsidid = 'Unknown' + elif re.match(r'^Coerced Size: ',line.strip()): + dsize = line.split(':')[1].strip() + dsize = re.sub(' \[.*\.*$', '', dsize) + dsize = re.sub('[0-9][0-9] GB', ' Gb', dsize) + elif re.match(r'^(CacheCade )?Virtual (Disk|Drive): [0-9]+.*$',line.strip()): + arrayindex += 1 + arrayid = line.split('(')[0].split(':')[1].strip() + elif re.match(r'PD: [0-9]+ Information.*$',line.strip()): + diskid = line.split()[1].strip() + elif re.match(r'^Device Id: .*$',line.strip()): + lsidid = line.split(':')[1].strip() + elif re.match(r'Slot Number: .*$',line.strip()): + slotid = line.split(':')[1].strip() + elif re.match(r'Firmware state: .*$',line.strip()): + fstate = line.split(':')[1].strip() + subfstate = re.sub('\(.*', '', fstate) + dbgprint('Firmware State: '+str(fstate)+' '+str(subfstate)) + elif re.match(r'Inquiry Data: .*$',line.strip()): + model = line.split(':')[1].strip() + model = re.sub(' +', ' ', model) + # Sub code + manuf = re.sub(' .*', '', model) + dtype = re.sub(manuf+' ', '', model) + dtype = re.sub(' .*', '', dtype) + hwserial = re.sub('.*'+dtype+' *', '', model) + elif re.match(r'^Media Type: .*$',line.strip()): + mtype = line.split(':')[1].strip() + if mtype == 'Hard Disk Device': + mtype = 'HDD' + else: + if mtype == 'Solid State Device': + mtype = 'SSD' + else: + mtype = 'N/A' + elif re.match(r'Device Speed: .*$',line.strip()): + speed = line.split(':')[1].strip() + elif re.match(r'Drive Temperature :.*$',line.strip()): + if (notempmode): + temp = 'N/A' + else: + # Drive temp is amongst the last few lines matched, decide here if we add information to the table.. + temp = line.split(':')[1].strip() + temp = re.sub(' \(.*\)', '', temp) + if model != 'Unknown': + dbgprint('Disk Info: '+str(arrayid)+' '+str(diskid)+' '+str(oldenclid)) + if subfstate == 'Rebuild': + cmd = '%s pdrbld -showprog -physdrv\[%s:%s\] -a%d -NoLog' % (megaclipath, enclid, slotid, controllerid) + output = getOutput(cmd) + percent = returnRebuildProgress(output) + fstate = str('Rebuilding (%d%%)' % (percent)) + + if (( NestedLDTable[controllerid][int(arrayindex)] == True) and (spanid != False)): + sarrayid = str(arrayid)+"s"+spanid + else: + sarrayid = str(arrayid) + table.append([sarrayid, str(diskid), mtype, model, dsize, fstate , speed, temp, enclid, slotid, lsidid]) + return table + + +def returnUnconfDiskInfo(output,controllerid): + arrayid = False + diskid = False + olddiskid = False + enclid = False + slotid = False + lsidid = 'Unknown' + table = [] + fstate = 'Offline' + substate = 'Unknown' + model = 'Unknown' + speed = 'Unknown' + mtype = 'Unknown' + dsize = 'Unknown' + temp = 'Unk0C' + for line in output: + if re.match(r'Enclosure Device ID: .*$',line.strip()): + # We match here early in the analysis so reset the vars if this is a new disk we're reading.. + oldenclid = enclid + enclid = line.split(':')[1].strip() + if oldenclid != False: + arrayid = False + fstate = 'Offline' + model = 'Unknown' + speed = 'Unknown' + temp = 'Unk0C' + slotid = False + lsidid = 'Unknown' + + elif re.match(r'^Coerced Size: ',line.strip()): + dsize = line.split(':')[1].strip() + dsize = re.sub(' \[.*\.*$', '', dsize) + dsize = re.sub('[0-9][0-9] GB', ' Gb', dsize) + elif re.match(r'^Drive.s position: DiskGroup: [0-9]+,.*$',line.strip()): + arrayid = line.split(',')[1].split(':')[1].strip() + elif re.match(r'^Device Id: [0-9]+.*$',line.strip()): + diskid = line.split(':')[1].strip() + elif re.match(r'^Device Id: .*$',line.strip()): + lsidid = line.split(':')[1].strip() + elif re.match(r'Slot Number: .*$',line.strip()): + slotid = line.split(':')[1].strip() + elif re.match(r'Firmware state: .*$',line.strip()): + fstate = line.split(':')[1].strip() + subfstate = re.sub('\(.*', '', fstate) + dbgprint('Firmware State: '+str(fstate)+' '+str(subfstate)) + elif re.match(r'Inquiry Data: .*$',line.strip()): + model = line.split(':')[1].strip() + model = re.sub(' +', ' ', model) + manuf = re.sub(' .*', '', model) + dtype = re.sub(manuf+' ', '', model) + dtype = re.sub(' .*', '', dtype) + hwserial = re.sub('.*'+dtype+' *', '', model) + elif re.match(r'^Media Type: .*$',line.strip()): + mtype = line.split(':')[1].strip() + if mtype == 'Hard Disk Device': + mtype = 'HDD' + else: + if mtype == 'Solid State Device': + mtype = 'SSD' + else: + mtype = 'N/A' + elif re.match(r'Device Speed: .*$',line.strip()): + speed = line.split(':')[1].strip() + elif re.match(r'Drive Temperature :.*$',line.strip()): + temp = line.split(':')[1].strip() + temp = re.sub('\(.*\)', '', temp) + # Drive temp is amongst the last few lines matched, decide here if we add information to the table.. + if arrayid == False: + if subfstate == 'Unconfigured': + dbgprint('Unconfigured Disk: Arrayid: '+str(arrayid)+' DiskId: '+str(diskid)+' '+str(olddiskid)+' '+str(fstate)) + elif subfstate == 'Online, Spun Up': + dbgprint('Online Disk: Arrayid: '+str(arrayid)+' DiskId: '+str(diskid)+' '+str(olddiskid)+' '+str(fstate)) + table.append([ mtype, model, dsize, fstate, speed, temp, enclid, slotid, lsidid]) + return table + +cmd = '%s -adpCount -NoLog' % (megaclipath) +output = getOutput(cmd) +controllernumber = returnControllerNumber(output) + +bad = False + +# List available controller +if printcontroller: + if controllernumber: + if not nagiosmode: + print '-- Controller information --' + + i = 0 + controllerid = 0 + mlen = 0 + hbainfo = [] + while controllerid < controllernumber: + cmd = '%s -AdpAllInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + returnHBAInfo(hbainfo, output,controllerid) + controllerid += 1 + mlen = returnWdthFromArrayCol(hbainfo,1) + + controllerid = 0 + for hba in hbainfo: + hbafmt = str('%-5s | %-'+str(mlen)+'s | %-6s | %-4s | %-6s | %-12s ') + # Header + if ( i == 0 ): + if not nagiosmode: + print hbafmt % ("-- ID","H/W Model","RAM","Temp","BBU", "Firmware") + if not nagiosmode: + print hbafmt % ( + hba[0], + hba[1], + hba[2], + hba[3], + hba[4], + hba[5]) + i += 1 + if not nagiosmode: + print '' + else: + print "No MegaRAID or PERC adapter detected on your system!" + exit(1) + +if printarray: + if not nagiosmode: + print '-- Array information --' + + controllerid = 0 + pcipath = '' + diskpath = '' + i = 0 ; j = 0 + mlen = 0 ; rlen = 0 ; clen = 0 + while controllerid < controllernumber: + arrayindex = 0 + + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + # We need to explore each HBA to look for gaps in LD's + ldid = 0 ; ldcount = 0 + while ldcount < arraynumber: + cmd = '%s -LDInfo -l%d -a%d -NoLog' % (megaclipath, ldid, controllerid) + output = getOutput(cmd) + for line in output: + if re.match(r'^Adapter.*Virtual Drive .* Does not Exist',line.strip()): + ldid += 1 + elif re.match(r'^(CacheCade )?Virtual Drive:',line.strip()): + LDTable[controllerid].append ( ldid ) + #NestedLDTable[controllerid][int(arrayindex)] = False + ldcount += 1 + ldid += 1 + + while arrayindex < arraynumber: + ldid = LDTable[controllerid][arrayindex] + cmd = '%s -LDInfo -l%d -a%d -NoLog' % (megaclipath, ldid, controllerid) + output = getOutput(cmd) + arrayinfo = returnArrayInfo(output, controllerid, ldid, arrayindex) + if ( len(arrayinfo[1]) > rlen): + rlen = len(arrayinfo[1]) + if ( len(arrayinfo[4]) > mlen): + mlen = len(arrayinfo[4]) + if ( len(arrayinfo[8]) > clen): + clen = len(arrayinfo[8]) + arrayindex += 1 + controllerid += 1 + + controllerid = 0 + while controllerid < controllernumber: + arrayindex = 0 + + cmd = '%s -AdpGetPciInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + pcipath = returnHBAPCIInfo(output) + + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + while arrayindex < arraynumber: + ldid = LDTable[controllerid][arrayindex] + cmd = '%s -LDInfo -l%d -a%d -NoLog' % (megaclipath, ldid, controllerid) + output = getOutput(cmd) + arrayinfo = returnArrayInfo(output,controllerid, ldid, arrayindex) + + if pcipath: + diskprefix = str('/dev/disk/by-path/pci-' + pcipath + '-scsi-0:') + for j in range (8): + diskpath = diskprefix + str(j) + ':' + str(arrayinfo[7]) + ':0' + if os.path.exists(diskpath): + arrayinfo[7] = os.path.realpath(diskpath) + else: + arrayinfo[7] = 'N/A' + + # Pad the string length, just to make sure it's aligned with the headers... + if (rlen < len("Type")): + rlen = len("Type") + if (mlen < len("Flags")): + mlen = len("Flags") + if (clen < len("CacheCade")): + clen = len("CacheCade") + + ldfmt = str('%-5s | %-'+str(rlen)+'s | %7s | %7s | %'+str(mlen)+'s | %8s | %8s | %8s | %-'+str(clen)+'s |%-12s ') + # Header + if ( i == 0 ): + if not nagiosmode: + print ldfmt % ("-- ID", "Type", "Size", "Strpsz", "Flags", "DskCache", "Status", "OS Path", "CacheCade", "InProgress" ) + if not nagiosmode: + print ldfmt % ( + arrayinfo[0], + arrayinfo[1], + arrayinfo[2], + arrayinfo[3], + arrayinfo[4], + arrayinfo[5], + arrayinfo[6], + arrayinfo[7], + arrayinfo[8], + arrayinfo[9]) + dbgprint("Array state : "+arrayinfo[6]) + if arrayinfo[6] not in [ 'Optimal', 'N/A' ]: + bad = True + nagiosbadarray=nagiosbadarray+1 + else: + nagiosgoodarray=nagiosgoodarray+1 + arrayindex += 1 + i += 1 + controllerid += 1 + if not nagiosmode: + print '' + +controllerid = 0 +while controllerid < controllernumber: + cmd = '%s -PDGetNum -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + totaldrivenumber += returnTotalDriveNumber(output) + controllerid += 1 + +if totaldrivenumber: + if not nagiosmode: + print '-- Disk information --' + + i = 0 + dlen = 0 ; mlen = 0 ; flen = 0 + controllerid = 0 + while controllerid < controllernumber: + arrayid = 0 + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + #### BUG: -LdPdInfo shows all PD on the adapter, not just for said LD.. + #### while arrayid <= arraynumber: + cmd = '%s -LdPdInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraydisk = returnDiskInfo(output,controllerid) + for array in arraydisk: + dbgprint('Disk c'+str(controllerid)+'u'+array[0]+'p'+array[1] + ' status : ' + array[5]) + if array[5] not in [ 'Online', 'Online, Spun Up' ]: + bad = True + nagiosbaddisk=nagiosbaddisk+1 + else: + nagiosgooddisk=nagiosgooddisk+1 + + if ( returnWdthFromArrayCol(arraydisk,0) > dlen): + dlen = returnWdthFromArrayCol(arraydisk,0) + if ( returnWdthFromArrayCol(arraydisk,3) > mlen): + mlen = returnWdthFromArrayCol(arraydisk,3) + if ( returnWdthFromArrayCol(arraydisk,5) > flen): + flen = returnWdthFromArrayCol(arraydisk,5) + controllerid += 1 + + controllerid = 0 + while controllerid < controllernumber: + arrayid = 0 + + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + #### BUG: -LdPdInfo shows all PD on the adapter, not just for said LD.. + #### while arrayid <= arraynumber: + + cmd = '%s -LdPdInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraydisk = returnDiskInfo(output,controllerid) + + # Adjust print format with width computed above + drvfmt = "%-"+str(dlen+5)+"s | %-4s | %-"+str(mlen)+"s | %-8s | %-"+str(flen)+"s | %-8s | %-4s | %-8s | %-8s" + for array in arraydisk: + # Header + if ( i == 0 ): + if not nagiosmode: + print drvfmt % ( + "-- ID", "Type", "Drive Model", "Size", "Status", "Speed", "Temp", "Slot ID", "LSI Device ID") + # Drive information + if not nagiosmode: + print drvfmt % ( + str('c'+str(controllerid)+'u'+array[0]+'p'+array[1]), # c0p0 + array[2], # HDD/SDD + array[3], # Model Information (Variable len) + array[4], # Size + array[5], # Status (Variable len) + array[6], # Speed + array[7], # Temp + str('['+array[8]+':'+array[9]+']'), # Slot ID + array[10]) # LSI ID + i = i + 1 + controllerid += 1 + if not nagiosmode: + print '' + +controllerid = 0 +totalconfdrivenumber = 0 +totalunconfdrivenumber = 0 +totaldrivenumber = 0 +while controllerid < controllernumber: + cmd = '%s -LdPdInfo -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + totalconfdrivenumber += returnConfDriveNumber(output) + + cmd = '%s -PDGetNum -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + totaldrivenumber += returnTotalDriveNumber(output) + + cmd = '%s -PDList -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + totalunconfdrivenumber += returnUnConfDriveNumber(output) + + controllerid += 1 + +dbgprint('Total Drives in system : ' + str(totaldrivenumber)) +dbgprint('Total Configured Drives : ' + str(totalconfdrivenumber)) +dbgprint('Total Unconfigured Drives : ' + str(totalunconfdrivenumber)) + +if totalunconfdrivenumber: + if not nagiosmode: + print '-- Unconfigured Disk information --' + + controllerid = 0 + while controllerid < controllernumber: + arrayid = 0 + + cmd = '%s -LDInfo -lall -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraynumber = returnArrayNumber(output) + #### BUG: -LdPdInfo shows all PD on the adapter, not just for given LD.. + #### while arrayid <= arraynumber: + + cmd = '%s -PDList -a%d -NoLog' % (megaclipath, controllerid) + output = getOutput(cmd) + arraydisk = returnUnconfDiskInfo(output,controllerid) + for array in arraydisk: + dbgprint('Disk c'+str(controllerid)+'uXpY status : ' + array[3]) + if array[3] not in [ 'Online', 'Unconfigured(good), Spun Up', 'Unconfigured(good), Spun down', 'JBOD','Hotspare, Spun Up','Hotspare, Spun down' ]: + bad = True + nagiosbaddisk=nagiosbaddisk+1 + else: + nagiosgooddisk=nagiosgooddisk+1 + + mlen = returnWdthFromArrayCol(arraydisk,1) + flen = returnWdthFromArrayCol(arraydisk,3) + + # Adjust print format with widths computed above + drvfmt = "%-7s | %-4s | %-"+str(mlen)+"s | %-8s | %-"+str(flen+2)+"s | %-8s | %-4s | %-8s | %-8s" + i = 0 + for array in arraydisk: + # Header + if ( i == 0 ): + if not nagiosmode: + print drvfmt % ( + "-- ID", "Type", "Drive Model", "Size", "Status", "Speed", "Temp", "Slot ID", "LSI Device ID") + # Drive information + if not nagiosmode: + print drvfmt % ( + str('c'+str(controllerid)+'uXpY'), # cXpY + array[0], # HDD/SDD + array[1], # Model Information (Variable len) + array[2], # Size + array[3], # Status (Variable len) + array[4], # Speed + array[5], # Temp + str('['+array[6]+':'+array[7]+']'), # Slot ID + array[8]) # LSI ID + i = i + 1 + controllerid += 1 + if not nagiosmode: + print '' + +if nagiosmode: + if bad: + print 'RAID ERROR - Arrays: OK:'+str(nagiosgoodarray)+' Bad:'+str(nagiosbadarray)+' - Disks: OK:'+str(nagiosgooddisk)+' Bad:'+str(nagiosbaddisk) + sys.exit(2) + else: + print 'RAID OK - Arrays: OK:'+str(nagiosgoodarray)+' Bad:'+str(nagiosbadarray)+' - Disks: OK:'+str(nagiosgooddisk)+' Bad:'+str(nagiosbaddisk) +else: + if bad: + print '\nThere is at least one disk/array in a NOT OPTIMAL state.' + sys.exit(1) From fbf4618afe503101db92665f4ac4e4f6248d5a8c Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:27:26 +0200 Subject: [PATCH 081/164] Fixing bugs in map disk types to osd script --- map-osd-to-disktype | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) mode change 100644 => 100755 map-osd-to-disktype diff --git a/map-osd-to-disktype b/map-osd-to-disktype old mode 100644 new mode 100755 index f1a2ff8..0a77dae --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -1,13 +1,17 @@ # Assumes, osd is up and +# +OSDTYPE=ssd + + # Tempfile -./megacli-status > /tmp/megaclisas-status.out +/opt/ungleich-tools/megaclisas-status > /tmp/megaclisas-status.out # Gets osd numbers of a particular device class, and gets their mountpoints on the host, and puts them into a tempfile -for osd in $(ceph osd tree | grep $OSD | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep $osd| cut -c 24- >> /tmp/list_osd_mountpoint; done +for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep $osd | cut -c 24- >> /tmp/list_osd_mountpoint.out; done # Gets the Megacli mappings for the mountpoints -for MOUNT in $(cat /tmp/list_osd_mountpoint | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done +for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done # Gets the hardware types for the Megacli mappings -for i in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $i | cut -d '|' -f 3 >> /tmp/disk_types; done +for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 3 >> /tmp/disk_types.out; done From 1b30ab71b626e5cd4446227ef7f5b91f81705c32 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:36:20 +0200 Subject: [PATCH 082/164] Fixing bugs in map disk types to osd script. Combining the outputs is needed, then it's complete --- map-osd-to-disktype | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 0a77dae..51f850e 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -8,7 +8,7 @@ OSDTYPE=ssd /opt/ungleich-tools/megaclisas-status > /tmp/megaclisas-status.out # Gets osd numbers of a particular device class, and gets their mountpoints on the host, and puts them into a tempfile -for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep $osd | cut -c 24- >> /tmp/list_osd_mountpoint.out; done +for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep "ceph-$osd " | cut -c 24- >> /tmp/list_osd_mountpoint.out; done # Gets the Megacli mappings for the mountpoints for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done From 6ecea763f9300dfb63e9b6824f14065f1308073b Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:52:02 +0200 Subject: [PATCH 083/164] map-osd-to-disktype v0.1, it works but haven't been tested extensively --- map-osd-to-disktype | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 51f850e..66e0f28 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -15,3 +15,13 @@ for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9 # Gets the hardware types for the Megacli mappings for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 3 >> /tmp/disk_types.out; done + +# Formatting, to get the local $OSDTYPE osd-s +cat /tmp/list_osd_mountpoint.out | awk '{print $1}' > /tmp/local_osds + +# Combine and display the outputs +paste /tmp/local_osds /tmp/disk_types.out + + +# Cleanup *.out files in the temp dir +rm /tmp/*.out From 9c567bb057fd7712fd1f9835cbe171dd0b4b87a0 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 12:56:25 +0200 Subject: [PATCH 084/164] map-osd-to-disktype v0.2, formatting output --- map-osd-to-disktype | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 66e0f28..552804a 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -17,10 +17,10 @@ for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9 for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 3 >> /tmp/disk_types.out; done # Formatting, to get the local $OSDTYPE osd-s -cat /tmp/list_osd_mountpoint.out | awk '{print $1}' > /tmp/local_osds +for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}'); do echo osd-$osd_num >> /tmp/local_osds.out; done # Combine and display the outputs -paste /tmp/local_osds /tmp/disk_types.out +paste /tmp/local_osds.out /tmp/disk_types.out # Cleanup *.out files in the temp dir From 058a1d0baa8f24dc1f1519c297be0653c51b952e Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 19:21:33 +0200 Subject: [PATCH 085/164] map-osd-to-disktype v1, changed from findmnt to lsblk, added more info to the output --- map-osd-to-disktype | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 552804a..4150bda 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -1,23 +1,24 @@ # Assumes, osd is up and # -OSDTYPE=ssd +#OSDTYPE=ssd # Tempfile /opt/ungleich-tools/megaclisas-status > /tmp/megaclisas-status.out # Gets osd numbers of a particular device class, and gets their mountpoints on the host, and puts them into a tempfile -for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep "ceph-$osd " | cut -c 24- >> /tmp/list_osd_mountpoint.out; done +#for osd in $(ceph osd tree | grep $OSDTYPE | grep -v down | cut -b 1-3); do findmnt -t xfs -n -o TARGET,SOURCE | grep "ceph-$osd " | cut -c 24- >> /tmp/list_osd_mountpoint.out; done +for osd in $(ceph osd tree | grep 'ssd\|hdd' | grep -v down | cut -b 1-3); do lsblk -p -o NAME,MOUNTPOINT | grep -w "/var/lib/ceph/osd/ceph-$osd" | cut -c 3- >> /tmp/list_osd_mountpoint.out ; done # Gets the Megacli mappings for the mountpoints -for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done +for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done # Gets the hardware types for the Megacli mappings -for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 3 >> /tmp/disk_types.out; done +for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 2-6,8 >> /tmp/disk_types.out; done # Formatting, to get the local $OSDTYPE osd-s -for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}'); do echo osd-$osd_num >> /tmp/local_osds.out; done +for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24- ); do echo osd-$osd_num >> /tmp/local_osds.out; done # Combine and display the outputs paste /tmp/local_osds.out /tmp/disk_types.out From 4309c1639d97532edb5431cca63364c6c3f34aef Mon Sep 17 00:00:00 2001 From: root Date: Wed, 27 May 2020 19:42:27 +0200 Subject: [PATCH 086/164] [map-osd-to-disktype] fixing bug: grep matched on c0u1 and c0u10 at the same time, causing an extra entry to appear, and messing up the output --- map-osd-to-disktype | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 4150bda..43a7a29 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -15,7 +15,7 @@ for osd in $(ceph osd tree | grep 'ssd\|hdd' | grep -v down | cut -b 1-3); do ls for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}' | sed 's/[0-9]*//g') ; do cat /tmp/megaclisas-status.out | grep $MOUNT | awk '{print $1}' >> /tmp/megacli-mappings.out; done # Gets the hardware types for the Megacli mappings -for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep $megacli_mappings | cut -d '|' -f 2-6,8 >> /tmp/disk_types.out; done +for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep -w "$megacli_mappings"p0 | cut -d '|' -f 2-6,8 >> /tmp/disk_types.out; done # Formatting, to get the local $OSDTYPE osd-s for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24- ); do echo osd-$osd_num >> /tmp/local_osds.out; done From 97c1893dce92d8bf823aa58b37ec1a476b9ad302 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 20:13:39 +0200 Subject: [PATCH 087/164] [map-osd-to-disktype] improved formatting, printf fixed size, so the columns match every time! (probably will have a bug if we reach 99+ osds) --- map-osd-to-disktype | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 43a7a29..7bafb75 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -18,11 +18,11 @@ for MOUNT in $(cat /tmp/list_osd_mountpoint.out | awk '{print $1}' | sed 's/[0-9 for megacli_mappings in $(cat /tmp/megacli-mappings.out); do awk '/Disk info/,0' /tmp/megaclisas-status.out | grep -w "$megacli_mappings"p0 | cut -d '|' -f 2-6,8 >> /tmp/disk_types.out; done # Formatting, to get the local $OSDTYPE osd-s -for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24- ); do echo osd-$osd_num >> /tmp/local_osds.out; done +for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24- ); do printf "%-7s%s\n" "osd-$osd_num" >> /tmp/local_osds.out; done # Combine and display the outputs -paste /tmp/local_osds.out /tmp/disk_types.out +paste /tmp/local_osds.out /tmp/disk_types.out -d '|' # Cleanup *.out files in the temp dir -rm /tmp/*.out +#rm /tmp/*.out From 2dfaedc85875f96bd5662a48bc9326c6edffe51d Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 20:16:58 +0200 Subject: [PATCH 088/164] [map-osd-to-disktype] bugfix: debug feature was added accidentally (no cleanup) --- map-osd-to-disktype | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 7bafb75..54d01fa 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -25,4 +25,4 @@ paste /tmp/local_osds.out /tmp/disk_types.out -d '|' # Cleanup *.out files in the temp dir -#rm /tmp/*.out +rm /tmp/*.out From a78649c6abdc95dce38cba2c7f2baa1d5853ee2b Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 27 May 2020 20:41:21 +0200 Subject: [PATCH 089/164] [map-osd-to-disktype] added header, checked for bashism, should be posix compatible --- map-osd-to-disktype | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/map-osd-to-disktype b/map-osd-to-disktype index 54d01fa..c609c87 100755 --- a/map-osd-to-disktype +++ b/map-osd-to-disktype @@ -1,6 +1,15 @@ -# Assumes, osd is up and +#!/bin/sh +# This script will find the locally active osd-s and display their information with the help of Megacli tools. +# Assumes that you run it on a host which has at least 1 osd that matches hdd or ssd disk device class in ceph osd df tree output. +# +# An example for usage: +# for NUM in 14 2 3 4 6 ; do printf "server$NUM\n" >> /tmp/osd_infos; ssh root@server"$NUM".place6.ungleich.ch "/opt/ungleich-tools/map-osd-to-disktype" >> /tmp/osd_infos ; printf "\n \n" >> /tmp/osd_infos; done # +# llnu's most hacky/advanced script at the time of writing (2020-05-27) + + +# Future functionality (arguments, and filtering): #OSDTYPE=ssd @@ -23,6 +32,5 @@ for osd_num in $(cat /tmp/list_osd_mountpoint.out | awk '{print $2}' | cut -c 24 # Combine and display the outputs paste /tmp/local_osds.out /tmp/disk_types.out -d '|' - # Cleanup *.out files in the temp dir rm /tmp/*.out From e4d3062d45c6240d227bda4c00c5b80864581f35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 30 May 2020 07:50:27 +0200 Subject: [PATCH 090/164] Bump alpine script to 3.12 --- opennebula-images/alpine-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh index 0a074b4..1ec6f3f 100755 --- a/opennebula-images/alpine-build-opennebula-image.sh +++ b/opennebula-images/alpine-build-opennebula-image.sh @@ -9,7 +9,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=v3.11 +RELEASE=v3.12 ARCH=x86_64 IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G From b96c39ccb6fe6deb45003747025a2c8d61f4a2c4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 16:46:44 +0200 Subject: [PATCH 091/164] Setup hostname for debian/devuan --- debian-devuan-netboot.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index d64de5e..5bc2d66 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -30,6 +30,7 @@ echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > $ chroot ${dir} apt update chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 +echo "unconfigured-host" > ${dir}/etc/hostname cp ${dir}/boot/vmlinuz-* ${kernel} @@ -67,7 +68,7 @@ ifup "\${dev}" exit 0 EOF -chmod a+rx ${dir}/etc/rc.local" +chmod a+rx "${dir}/etc/rc.local" # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init From 2a624e84422ebd51c3d2b49392156425e7e688a6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 17:20:12 +0200 Subject: [PATCH 092/164] Fix ssh dir and init link --- debian-devuan-netboot.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 5bc2d66..0dd9d97 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -42,6 +42,9 @@ for key in balazs dominique jinguk nico; do curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys done +# Fix possible permission issue from above +chown -R root:root ${dir}/root/ + ################################################################################ # networking @@ -73,7 +76,7 @@ chmod a+rx "${dir}/etc/rc.local" # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same -ln -s /sbin/init ${dir}/init +ln -fs /sbin/init ${dir}/init # Finally building the initramfs ( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From 2cfcca567959371ad6a0a51507dd276fa715b7c6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 18:31:06 +0200 Subject: [PATCH 093/164] cleanup the debian/devuan build script --- debian-devuan-netboot.sh | 43 ++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 0dd9d97..db56d64 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -2,7 +2,8 @@ # Nico Schottelius, 2019-12-09 # the ugly code is llnu -#this can only run in the ungleich-tools directory because of the cat magiccommand........ +set -e +set -x if [ $# -ne 2 ]; then echo $0 suite out-directory @@ -15,41 +16,45 @@ suite=$1; shift outdir=$1; shift date=$(date +%F) +mkdir -p ${outdir} basename=${suite}-${date} -dir=${outdir}/${basename} -kernel=${outdir}/kernel-${basename} -initramfs=${outdir}/initramfs-${basename} +abs_outdir=$(cd ${outdir} && pwd -P) + +chroot_dir=${abs_outdir}/${basename} +kernel=${abs_outdir}/kernel-${basename} +initramfs=${abs_outdir}/initramfs-${basename} + keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files -debootstrap "${suite}" "${dir}" +debootstrap "${suite}" "${chroot_dir}" # need non-free for firmware-bnx2 -echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${dir}/etc/apt/sources.list +echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${chroot_dir}/etc/apt/sources.list -chroot ${dir} apt update -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 +chroot ${chroot_dir} apt update +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 inetutils-ping -echo "unconfigured-host" > ${dir}/etc/hostname +echo "unconfigured-host" > ${chroot_dir}/etc/hostname -cp ${dir}/boot/vmlinuz-* ${kernel} +cp ${chroot_dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${chroot_dir}/etc/cron.d/ipv6addr -mkdir -p ${dir}/root/.ssh +mkdir -p ${chroot_dir}/root/.ssh for key in balazs dominique jinguk nico; do - curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys + curl -s ${keyurl}/${key}.pub >> ${chroot_dir}/root/.ssh/authorized_keys done # Fix possible permission issue from above -chown -R root:root ${dir}/root/ +chown -R root:root ${chroot_dir}/root/ ################################################################################ # networking # always lo -cat > ${dir}/etc/network/interfaces << EOF +cat > ${chroot_dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback @@ -57,7 +62,7 @@ source-directory /etc/network/interfaces.d EOF # find the boot interfaces at boot -cat > ${dir}/etc/rc.local < ${chroot_dir}/etc/rc.local < otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same -ln -fs /sbin/init ${dir}/init +ln -fs /sbin/init ${chroot_dir}/init # Finally building the initramfs -( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) +( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From e23de2ca3b603fdebb9752d168f959f1af25c97d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 18:31:24 +0200 Subject: [PATCH 094/164] ignore ipxe build directory --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index f8835d9..f4be182 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,5 @@ opennebula-vm-etcd/config-and-secrets.conf .idea .vscode + +ipxe/ From a7d53cae1bf14f6ccc55a7261f13e01eb4a701fc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 20:14:13 +0200 Subject: [PATCH 095/164] add script to revert to old style iptables --- debian-use-old-iptables | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 debian-use-old-iptables diff --git a/debian-use-old-iptables b/debian-use-old-iptables new file mode 100644 index 0000000..50d47c1 --- /dev/null +++ b/debian-use-old-iptables @@ -0,0 +1,9 @@ +#!/bin/sh + +# reverting for a running system that still needs access to old style +# rules + +update-alternatives --set iptables /usr/sbin/iptables-legacy +update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy +update-alternatives --set arptables /usr/sbin/arptables-legacy +update-alternatives --set ebtables /usr/sbin/ebtables-legacy From 4cad7eb28c4d15d0480d0124c4dd8ab76e3b6f20 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 20:14:28 +0200 Subject: [PATCH 096/164] + correct permissions --- debian-devuan-netboot.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index db56d64..5111f08 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -85,3 +85,7 @@ ln -fs /sbin/init ${chroot_dir}/init # Finally building the initramfs ( cd ${chroot_dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) + +# Fix paranoid permissions +chmod a+rx ${abs_outdir} +chmod a+r ${kernel} ${initramfs} From 47d15a2349d150f1677d6d9dee3f6da93f0ce3a3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 22:44:50 +0200 Subject: [PATCH 097/164] do not install broken ping --- debian-devuan-netboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 5111f08..b703e60 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -33,7 +33,7 @@ debootstrap "${suite}" "${chroot_dir}" echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${chroot_dir}/etc/apt/sources.list chroot ${chroot_dir} apt update -chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 inetutils-ping +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 echo "unconfigured-host" > ${chroot_dir}/etc/hostname From 6a3750a6cf93e3c5fabcff0746731774201a70bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 2 Jun 2020 22:44:57 +0200 Subject: [PATCH 098/164] + correct mode --- debian-use-old-iptables | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 debian-use-old-iptables diff --git a/debian-use-old-iptables b/debian-use-old-iptables old mode 100644 new mode 100755 From 8dfb2c95e62f407c4524008aff3497656fcadeab Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 13 Jun 2020 21:40:10 +0200 Subject: [PATCH 099/164] phase in script for setting up viirbs --- .gitignore | 2 + setup-viirb.sh | 213 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 215 insertions(+) create mode 100755 setup-viirb.sh diff --git a/.gitignore b/.gitignore index f4be182..1de0e42 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ opennebula-vm-etcd/config-and-secrets.conf .vscode ipxe/ + +openwrt-*-*.bin diff --git a/setup-viirb.sh b/setup-viirb.sh new file mode 100755 index 0000000..79483c6 --- /dev/null +++ b/setup-viirb.sh @@ -0,0 +1,213 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 4 ]; then + echo "$0 interface viirb-id your-dot-cdist" + echo " interface to add the config ip address to" + echo " viirb-id: number in decimal format" + echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" + echo " owner-mail-reference: How to identify the owner" + echo "" + echo "Example to configure viirb02:" + echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342'" + exit 1 +fi + +set -x + +set -x +dev=$1; shift +id=$1; shift +dot_cdist=$1; shift +owner=$1; shift + +hex_id=$(printf "%0.2x\n" "$id") +viirb_hostname=viirb${hex_id} + +prefix_base=2a0a:e5c1:3 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +# openwrt +version=19.07.3 +filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin + +# IP address for setting it up initially +viirb_ip=192.168.61.1 + +# wireguard +private_key=$(wg genkey) +private_key=EL76tScnk84v8TGSSD3tPDhUjjYVPrfmNMBE3zbuRXg= +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch +vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= + +# cdist +dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files +peerfile=${dot_cdist_files}/${vpn_endpoint_host}.peer${hex_id} +vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} + +# Configure VPN server / update cdist +echo Updating VPNserver +cat < ${peerfile} +# ${viirb_hostname} ${owner} +[Peer] +PublicKey = ${public_key} +AllowedIPs = ${my_network} + +EOF + +# Generate real config +cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} +cd ${dot_cdist_files} +git add ${vpn_endpoint_host} +git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" +git pull +git push + +cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} + +exit 0 + +# System +cat </dev/null || true +sudo ip addr add 192.168.61.2/24 dev "$dev" + +# don't care about other/old known_host entries +ssh-keygen -R 192.168.61.1 + +ping -c2 ${viirb_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach any VIIRB - exiting" + exit 1 +fi + +set -e + + +# Don't re-download if we already have it +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} +scp ${filename} root@${viirb_ip}:/tmp +ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" + +wait=0 +found="" + +while [ $wait -lt 180 ]; do + ping -c1 ${viirb_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + fi +done + +if [ ! "$found" ]; then + echo "Did not find updated viirb - debug / restart it" + exit 1 +fi + +exit 0 From 4ca2a762fdcfcfe5c881794c7036625d6966722e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 00:07:47 +0200 Subject: [PATCH 100/164] update script to have stages --- setup-viirb.sh | 270 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 205 insertions(+), 65 deletions(-) diff --git a/setup-viirb.sh b/setup-viirb.sh index 79483c6..2f8ba82 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -2,18 +2,28 @@ # 2020-06-13, Nico Schottelius # See https://ungleich.ch/u/products/viirb-ipv6-box/ -if [ $# -ne 4 ]; then - echo "$0 interface viirb-id your-dot-cdist" +if [ $# -lt 4 ]; then + echo "$0 interface viirb-id your-dot-cdist [stages]" echo " interface to add the config ip address to" echo " viirb-id: number in decimal format" echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" echo " owner-mail-reference: How to identify the owner" + echo " stages: define which stages to execute" + echo "" + echo " stage1: setup your host, check connection to VIIRB" + echo " stage2: flash latest openwrt onto the VIIRB" + echo " stage3: configure the vpn endpoint" + echo " stage4: configure the VIIRB with wireguard + settings" + echo " stage5: Verify VIIRB on VPN, cleanup VIIRB" echo "" echo "Example to configure viirb02:" echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342'" + echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342' '1 3 4'" exit 1 fi +echo "FIXME: missing IPv6 announcements on LAN" + set -x set -x @@ -22,6 +32,12 @@ id=$1; shift dot_cdist=$1; shift owner=$1; shift +if [ $# -ge 1 ]; then + stages=$1; shift +else + stages="1 2 3 4 5" +fi + hex_id=$(printf "%0.2x\n" "$id") viirb_hostname=viirb${hex_id} @@ -37,12 +53,14 @@ my_wifi_ip=${my_prefix}:7ea::42 version=19.07.3 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin +# root password +root_password=$(pwgen -1 32) + # IP address for setting it up initially viirb_ip=192.168.61.1 # wireguard private_key=$(wg genkey) -private_key=EL76tScnk84v8TGSSD3tPDhUjjYVPrfmNMBE3zbuRXg= public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch @@ -50,12 +68,81 @@ vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= # cdist dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files -peerfile=${dot_cdist_files}/${vpn_endpoint_host}.peer${hex_id} +peerfilename=${vpn_endpoint_host}.peer${hex_id} +peerfile=${dot_cdist_files}/${peerfilename} vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} -# Configure VPN server / update cdist -echo Updating VPNserver -cat < ${peerfile} +################################################################################ +# Stage 1: test / connect to the new VIIRB +# +# We delete so that we can run idempotent +stage1() +{ + sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true + sudo ip addr add 192.168.61.2/24 dev "$dev" + + # don't care about other/old known_host entries + ssh-keygen -R ${viirb_ip} + + ping -c2 ${viirb_ip} + if [ $? -ne 0 ]; then + echo "Cannot reach any VIIRB - exiting" + exit 1 + fi + + cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_keys" +} + + + +################################################################################ +# Get latest OpenWRT & flash it +stage2() +{ + # Don't re-download if we already have it + wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} + scp ${filename} root@${viirb_ip}:/tmp + ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" + + # It still pings for some time - wait for the reboot to happen + echo "Waiting for VIIRB to disappear" + sleep 15 + + wait=0 + found="" + + while [ $wait -lt 180 ]; do + ping -c1 ${viirb_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + break + fi + + sleep 1 + wait=$((wait+1)) + done + + if [ ! "$found" ]; then + echo "Did not find updated viirb - debug / restart it" + exit 1 + fi + +} + + +################################################################################ +# Stage 3: prepare VPN endpoint +# + +stage3() +{ + + # Configure VPN server / update cdist + echo Updating VPNserver + cat < ${peerfile} # ${viirb_hostname} ${owner} [Peer] PublicKey = ${public_key} @@ -63,28 +150,32 @@ AllowedIPs = ${my_network} EOF -# Generate real config -cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} -cd ${dot_cdist_files} -git add ${vpn_endpoint_host} -git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" -git pull -git push + # Generate real config + cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} + cd ${dot_cdist_files} + git add ${vpn_endpoint_host} ${peerfilename} + git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" + git pull + git push -cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} + cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} +} -exit 0 +################################################################################ +# Stage 4: configure the VIIRB +# +stage4() +{ -# System -cat </dev/null || true -sudo ip addr add 192.168.61.2/24 dev "$dev" + while [ $wait -lt 180 ]; do + ping -c1 ${my_wireguard_ip} >/dev/null -# don't care about other/old known_host entries -ssh-keygen -R 192.168.61.1 + if [ $? -eq 0 ]; then + found=yes + break + fi + sleep 1 + wait=$((wait+1)) + done -ping -c2 ${viirb_ip} -if [ $? -ne 0 ]; then - echo "Cannot reach any VIIRB - exiting" - exit 1 -fi + if [ ! "$found" ]; then + echo "Cannot reach VIIRB via VPN - check manually" + exit 1 + fi -set -e + echo "Cleanup process." + echo "Set the root password when prompted to: ${root_password}" + # VPN works, remove artefacts, set correct DNS servers that support DNS64 + cat </dev/null +# Remove our ssh keys +rm -f /etc/dropbear/authorized_keys - if [ $? -eq 0 ]; then - found=yes - # wait for ssh to come up - sleep 10 +# Setup root password +printf "${root_password}\n${root_password}\n" | passwd +EOF + + echo "Submit to user the root password = ${root_password}" + +} + +for stage in $(seq 1 5);do + if echo $stages | grep -q $stage; then + eval stage${stage} fi done - -if [ ! "$found" ]; then - echo "Did not find updated viirb - debug / restart it" - exit 1 -fi - -exit 0 From 345905e70ee46dac7a4ad66f6df66d1d0181a547 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 00:52:07 +0200 Subject: [PATCH 101/164] [viirb] fix the dns problem --- setup-viirb.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/setup-viirb.sh b/setup-viirb.sh index 2f8ba82..b8d34ea 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -225,6 +225,14 @@ uci set dhcp.wifi.interface='wifi' uci set dhcp.wifi.ra='server' uci set dhcp.wifi.dynamicdhcp='0' +# Fix DNS: make dnsmasq NOT use a resolv.conf +# so that it only reads from our servers with DNS64 enabled +uci set dhcp.@dnsmasq[0].noresolv='1' + +# Fix DNS: make the OS use the locally provided DNS servers +# otherwise the VPN tunnel cannot be established +dhcp.@dnsmasq[0].localuse='0' + # VPN / Wireguard uci set network.wg0=interface From d7c1e5263972436f39de90810245466e50ced213 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 21:20:43 +0200 Subject: [PATCH 102/164] ++debian stuff --- debian-devuan-netboot.sh | 52 ++++++++++++++++++++++++++++------------ 1 file changed, 37 insertions(+), 15 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index b703e60..eb0d381 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -33,7 +33,7 @@ debootstrap "${suite}" "${chroot_dir}" echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${chroot_dir}/etc/apt/sources.list chroot ${chroot_dir} apt update -chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 +chroot ${chroot_dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 ifenslave vlan echo "unconfigured-host" > ${chroot_dir}/etc/hostname @@ -53,30 +53,52 @@ chown -R root:root ${chroot_dir}/root/ ################################################################################ # networking -# always lo +# echo bonding + cat > ${chroot_dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback -source-directory /etc/network/interfaces.d +auto bond0 +iface bond0 inet manual + bond-miimon 500 + bond-mode 4 + post-up /sbin/ip link set \$IFACE mtu 9000 + bond-slaves none + +auto eth0 +iface eth0 inet manual + bond-master bond0 + post-up /sbin/ip link set \$IFACE mtu 9000 + +auto eth1 +iface eth1 inet manual + bond-master bond0 + post-up /sbin/ip link set \$IFACE mtu 9000 + +# server network +auto bond0.11 +iface bond0.11 inet6 auto + post-up /sbin/ip link set \$IFACE mtu 9000 + vlan-raw-device bond0 EOF -# find the boot interfaces at boot -cat > ${chroot_dir}/etc/rc.local < ${chroot_dir}/etc/rc.local < /etc/network/interfaces.d/bootinterface << eof -auto \$dev -iface \$dev inet6 auto -eof +# cat > /etc/network/interfaces.d/bootinterface << eof +# auto \$dev +# iface \$dev inet6 auto +# eof -ifup "\${dev}" +# ifup "\${dev}" -exit 0 -EOF +# exit 0 +# EOF -chmod a+rx "${chroot_dir}/etc/rc.local" +# chmod a+rx "${chroot_dir}/etc/rc.local" # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init From 061d67377ad89544f8ac1abc7d483731cbe3cb24 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 21:21:01 +0200 Subject: [PATCH 103/164] virb: +dns --- setup-viirb.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/setup-viirb.sh b/setup-viirb.sh index 2f8ba82..e6e2638 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -23,6 +23,7 @@ if [ $# -lt 4 ]; then fi echo "FIXME: missing IPv6 announcements on LAN" +echo "FIXME: DNS situation: upstream, non upstream, ungleich, how to resolve tunnel endpoint" set -x From 9b57f031ce050053019ed8d35fb84f1e71027fb5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 22:22:11 +0200 Subject: [PATCH 104/164] Make stage4 output the wireguard public key --- setup-viirb.sh | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/setup-viirb.sh b/setup-viirb.sh index b8e7ca6..3e60825 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -195,6 +195,12 @@ opkg update opkg install wireguard opkg install luci-app-wireguard +# The IPv6 lan configuration +uci set network.lanv6=interface +uci set network.lanv6.proto='static' +uci set network.lanv6.ip6addr='${my_lan_ip}/64' +uci set network.lanv6.ifname='br-lan' + # wifi ip address uci set network.wifi=interface uci set network.wifi.proto='static' @@ -226,6 +232,23 @@ uci set dhcp.wifi.interface='wifi' uci set dhcp.wifi.ra='server' uci set dhcp.wifi.dynamicdhcp='0' +# LAN / Router advertisements / DHCP +# DHCP: we are not authoratative +uci delete dhcp.@dnsmasq[0].authoritative +uci delete dhcp.lan.dhcpv6 +uci delete dhcp.lan.start +uci delete dhcp.lan.limit +uci delete dhcp.lan.leasetime + +# Do not announce ULA - we have GUA +uci delete network.globals.ula_prefix + +uci set dhcp.lan=dhcp +uci set dhcp.lan.interface='lanv6' +uci set dhcp.lan.ra='server' +uci set dhcp.lan.dynamicdhcp='0' + + # Fix DNS: make dnsmasq NOT use a resolv.conf # so that it only reads from our servers with DNS64 enabled uci set dhcp.@dnsmasq[0].noresolv='1' @@ -234,6 +257,10 @@ uci set dhcp.@dnsmasq[0].noresolv='1' # otherwise the VPN tunnel cannot be established dhcp.@dnsmasq[0].localuse='0' +# DNS upstream over VPN gives DNS64 +uci delete dhcp.@dnsmasq[0].server +uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:0:a::a' +uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:2:a::a' # VPN / Wireguard uci set network.wg0=interface @@ -292,7 +319,7 @@ uci set firewall.@rule[-1].target='ACCEPT' fi # Add interfaces to the right network zone -uci set firewall.@zone[0].network='lan lanv4 wifi' +uci set firewall.@zone[0].network='lan lanv6 wifi' uci set firewall.@zone[1].network='wg0' uci commit firewall @@ -301,6 +328,7 @@ uci commit firewall reboot EOF + echo "Wireguard public key: ${public_key}" } ################################################################################ @@ -334,14 +362,8 @@ stage5() # VPN works, remove artefacts, set correct DNS servers that support DNS64 cat < Date: Sun, 14 Jun 2020 22:31:03 +0200 Subject: [PATCH 105/164] ensure viirb has internet before updating pkg db --- setup-viirb.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup-viirb.sh b/setup-viirb.sh index 3e60825..d5337de 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -188,6 +188,10 @@ uci set network.lanv4temp.netmask='255.255.255.0' uci commit network /etc/init.d/network restart +# ensure internet is up and running +sleep 3 +ping -c5 ungleich.ch + # update the sources opkg update From 8921f6ac7d5002e9fb1cbcce73f09186bfe0bfd9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 22:39:49 +0200 Subject: [PATCH 106/164] add new single use script for the viirb --- setup-viirb.sh | 3 +++ viirb-cleanup-final-stage.sh | 29 +++++++++++++++++++++++++++++ 2 files changed, 32 insertions(+) create mode 100644 viirb-cleanup-final-stage.sh diff --git a/setup-viirb.sh b/setup-viirb.sh index d5337de..63286cb 100755 --- a/setup-viirb.sh +++ b/setup-viirb.sh @@ -368,6 +368,9 @@ stage5() cat < Date: Sun, 14 Jun 2020 22:50:43 +0200 Subject: [PATCH 107/164] ++viirb --- viirb-cleanup-final-stage.sh | 0 viirb-configure-vpnendpoint.sh | 52 ++++++++++++++++++++++++++++ setup-viirb.sh => viirb-setup-all.sh | 0 3 files changed, 52 insertions(+) mode change 100644 => 100755 viirb-cleanup-final-stage.sh create mode 100755 viirb-configure-vpnendpoint.sh rename setup-viirb.sh => viirb-setup-all.sh (100%) diff --git a/viirb-cleanup-final-stage.sh b/viirb-cleanup-final-stage.sh old mode 100644 new mode 100755 diff --git a/viirb-configure-vpnendpoint.sh b/viirb-configure-vpnendpoint.sh new file mode 100755 index 0000000..96186fc --- /dev/null +++ b/viirb-configure-vpnendpoint.sh @@ -0,0 +1,52 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 3 ]; then + echo "$0 viirb-id public-key your-dot-cdist [stages]" + echo " viirb-id: number in decimal format" + echo " wireguard public key" + echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" + exit 1 +fi + +set -x +id=$1; shift +public_key=$1; shift +dot_cdist=$1; shift + +hex_id=$(printf "%0.2x\n" "$id") +viirb_hostname=viirb${hex_id} + +prefix_base=2a0a:e5c1:3 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 + +vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch + +# cdist +dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files +peerfilename=${vpn_endpoint_host}.peer${hex_id} +peerfile=${dot_cdist_files}/${peerfilename} +vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} + + +# Configure VPN server / update cdist +echo Updating VPNserver +cat < ${peerfile} +# ${viirb_hostname}, $(date +%F) +[Peer] +PublicKey = ${public_key} +AllowedIPs = ${my_network} + +EOF + +# Generate real config +cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} +cd ${dot_cdist_files} +git add ${vpn_endpoint_host} ${peerfilename} +git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" +git pull +git push + +cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} diff --git a/setup-viirb.sh b/viirb-setup-all.sh similarity index 100% rename from setup-viirb.sh rename to viirb-setup-all.sh From c8036796101fd77e89da7a87bd220ff56093efe7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 22:54:40 +0200 Subject: [PATCH 108/164] ++viirb cleanup --- viirb-cleanup-final-stage.sh | 18 +++++++++++++++--- viirb-configure-vpnendpoint.sh | 2 +- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/viirb-cleanup-final-stage.sh b/viirb-cleanup-final-stage.sh index ee7c0f6..a9a116a 100755 --- a/viirb-cleanup-final-stage.sh +++ b/viirb-cleanup-final-stage.sh @@ -5,9 +5,23 @@ set -e set -x +if [ $# -ne 2 ]; then + echo "$0 viirb-ip-address viirb-id" + echo " viirb-ip-address: where to find the viirb" + echo " viirb-id: number in decimal format" + exit 1 +fi + viirb_ip=$1; shift + +id=$1; shift +viirb_hostname=viirb${id} + root_password=$(pwgen -1 32) +# Save for sending to user +# FIXME: future make this more easy / better to transfer +echo $root_password > ${viirb_hostname}.rootpw cat < Date: Sun, 14 Jun 2020 23:01:16 +0200 Subject: [PATCH 109/164] viirb: flash latest firmware --- viirb-connect-flash-latest-openwrt.sh | 69 +++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100755 viirb-connect-flash-latest-openwrt.sh diff --git a/viirb-connect-flash-latest-openwrt.sh b/viirb-connect-flash-latest-openwrt.sh new file mode 100755 index 0000000..e6f794e --- /dev/null +++ b/viirb-connect-flash-latest-openwrt.sh @@ -0,0 +1,69 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 1 ]; then + echo "$0 interface" + echo " interface to add the config ip address to" + exit 1 +fi + +set -x +dev=$1; shift +id=$1; shift +dot_cdist=$1; shift +owner=$1; shift + +# openwrt +version=19.07.3 +filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin + +# IP address for setting it up initially +viirb_ip=192.168.61.1 + +sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true +sudo ip addr add 192.168.61.2/24 dev "$dev" + +# don't care about other/old known_host entries +ssh-keygen -R ${viirb_ip} + +ping -c2 ${viirb_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach any VIIRB - exiting" + exit 1 +fi + +cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_keys" + +# Don't re-download if we already have it +wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} +scp ${filename} root@${viirb_ip}:/tmp +ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" + +# It still pings for some time - wait for the reboot to happen +echo "Waiting for VIIRB to disappear" +sleep 15 + +wait=0 +found="" + +while [ $wait -lt 180 ]; do + ping -c1 ${viirb_ip} >/dev/null + + if [ $? -eq 0 ]; then + found=yes + # wait for ssh to come up + sleep 10 + break + fi + + sleep 1 + wait=$((wait+1)) + done + +if [ ! "$found" ]; then + echo "Did not find updated viirb - debug / restart it" + exit 1 +fi + +echo "VIIRB successfully updated to ${version}" From a2ef38267af28c01ad733156c9115bbefbcacf84 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 23:01:59 +0200 Subject: [PATCH 110/164] remove unused vars --- viirb-connect-flash-latest-openwrt.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/viirb-connect-flash-latest-openwrt.sh b/viirb-connect-flash-latest-openwrt.sh index e6f794e..28ba121 100755 --- a/viirb-connect-flash-latest-openwrt.sh +++ b/viirb-connect-flash-latest-openwrt.sh @@ -10,9 +10,6 @@ fi set -x dev=$1; shift -id=$1; shift -dot_cdist=$1; shift -owner=$1; shift # openwrt version=19.07.3 From 53e827ba43aefbd4e9399c2a5900b88247b15f39 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 23:09:06 +0200 Subject: [PATCH 111/164] viirb: script to fully configure --- viirb-configure-fully-after-upgrade.sh | 199 +++++++++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100755 viirb-configure-fully-after-upgrade.sh diff --git a/viirb-configure-fully-after-upgrade.sh b/viirb-configure-fully-after-upgrade.sh new file mode 100755 index 0000000..bd86c25 --- /dev/null +++ b/viirb-configure-fully-after-upgrade.sh @@ -0,0 +1,199 @@ +#!/bin/sh +# 2020-06-13, Nico Schottelius +# See https://ungleich.ch/u/products/viirb-ipv6-box/ + +if [ $# -ne 2 ]; then + echo "$0 viirb-ip-address viirb-id" + echo " viirb-ip-address: where to find the viirb" + echo " viirb-id: number in decimal format" + exit 1 +fi + + +set -x +viirb_ip=$1; shift +id=$1; shift +hex_id=$(printf "%0.2x\n" "$id") +viirb_hostname=viirb${id} + +prefix_base=2a0a:e5c1:3 +my_prefix=${prefix_base}${hex_id} +my_network=${my_prefix}::/48 + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 + +# wireguard +private_key=$(wg genkey) +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch +vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= + +cat < ${viirb_hostname}.public_key From b7f84307b2b487c2ab6ac064a30ae3db225e11c6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 23:21:38 +0200 Subject: [PATCH 112/164] Change configuration to drop the conflicting ipv4 address --- viirb-configure-fully-after-upgrade.sh | 120 +++++++++++++------------ 1 file changed, 62 insertions(+), 58 deletions(-) diff --git a/viirb-configure-fully-after-upgrade.sh b/viirb-configure-fully-after-upgrade.sh index bd86c25..1868cd9 100755 --- a/viirb-configure-fully-after-upgrade.sh +++ b/viirb-configure-fully-after-upgrade.sh @@ -33,7 +33,6 @@ vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= cat < Date: Sun, 14 Jun 2020 23:32:38 +0200 Subject: [PATCH 113/164] add debug for reaching new v6 address --- ...latest-openwrt.sh => viirb-1-connect-flash-latest-openwrt.sh | 0 ...after-upgrade.sh => viirb-2-configure-fully-after-upgrade.sh | 2 ++ ...configure-vpnendpoint.sh => viirb-3-configure-vpnendpoint.sh | 0 3 files changed, 2 insertions(+) rename viirb-connect-flash-latest-openwrt.sh => viirb-1-connect-flash-latest-openwrt.sh (100%) rename viirb-configure-fully-after-upgrade.sh => viirb-2-configure-fully-after-upgrade.sh (99%) rename viirb-configure-vpnendpoint.sh => viirb-3-configure-vpnendpoint.sh (100%) diff --git a/viirb-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh similarity index 100% rename from viirb-connect-flash-latest-openwrt.sh rename to viirb-1-connect-flash-latest-openwrt.sh diff --git a/viirb-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh similarity index 99% rename from viirb-configure-fully-after-upgrade.sh rename to viirb-2-configure-fully-after-upgrade.sh index 1868cd9..3a4da33 100755 --- a/viirb-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -126,6 +126,8 @@ sleep 10 # change to ipv6 viirb_ip=${my_lan_ip} +ping -c5 ${viirb_ip} + cat < Date: Sun, 14 Jun 2020 23:43:20 +0200 Subject: [PATCH 114/164] [viirb] include reboot to reset resolv.conf --- viirb-2-configure-fully-after-upgrade.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 3a4da33..b96b967 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -62,7 +62,6 @@ uci set dhcp.lan.interface='lanv6' uci set dhcp.lan.ra='server' uci set dhcp.lan.dynamicdhcp='0' - # Fix DNS: make dnsmasq NOT use a resolv.conf # so that it only reads from our servers with DNS64 enabled uci set dhcp.@dnsmasq[0].noresolv='1' @@ -117,7 +116,10 @@ uci delete wireless.radio0.disabled uci commit -/etc/init.d/network restart +# Need to reboot to restore /etc/resolv.conf +reboot + +#/etc/init.d/network restart EOF # ensure network is up and running - bridge takes a bit of time @@ -129,7 +131,6 @@ viirb_ip=${my_lan_ip} ping -c5 ${viirb_ip} cat < Date: Sun, 14 Jun 2020 23:43:30 +0200 Subject: [PATCH 115/164] [viirb] ping ipv6 address in vpn config for verification --- viirb-3-configure-vpnendpoint.sh | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-configure-vpnendpoint.sh index 4c45fdd..d03b90e 100755 --- a/viirb-3-configure-vpnendpoint.sh +++ b/viirb-3-configure-vpnendpoint.sh @@ -3,17 +3,17 @@ # See https://ungleich.ch/u/products/viirb-ipv6-box/ if [ $# -ne 3 ]; then - echo "$0 viirb-id public-key your-dot-cdist [stages]" + echo "$0 your-dot-cdist viirb-id public-key" + echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" echo " viirb-id: number in decimal format" echo " wireguard public key" - echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" exit 1 fi set -x +dot_cdist=$1; shift id=$1; shift public_key=$1; shift -dot_cdist=$1; shift hex_id=$(printf "%0.2x\n" "$id") viirb_hostname=viirb${id} @@ -21,6 +21,9 @@ viirb_hostname=viirb${id} prefix_base=2a0a:e5c1:3 my_prefix=${prefix_base}${hex_id} my_network=${my_prefix}::/48 +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_wifi_ip=${my_prefix}:7ea::42 vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch @@ -50,3 +53,10 @@ git pull git push cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} + +# Test that the VPN connection is established +sleep 10 + +ping -c3 ${my_wireguard_ip} +ping -c3 ${my_lan_ip} +ping -c3 ${my_wifi_ip} From 13257ad00b6e61aec166131a54a14b6c60bf5be3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Jun 2020 23:59:33 +0200 Subject: [PATCH 116/164] sleep longer for waiting for network and reboot --- viirb-2-configure-fully-after-upgrade.sh | 6 ++++-- ...cleanup-final-stage.sh => viirb-4-cleanup-final-stage.sh | 0 2 files changed, 4 insertions(+), 2 deletions(-) rename viirb-cleanup-final-stage.sh => viirb-4-cleanup-final-stage.sh (100%) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index b96b967..b190e94 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -122,8 +122,10 @@ reboot #/etc/init.d/network restart EOF -# ensure network is up and running - bridge takes a bit of time -sleep 10 +# ensure viirb is back +# ensure viirb network is up and running - bridge takes a bit of time +# ensure viirb gives out ipv6 address +sleep 30 # change to ipv6 viirb_ip=${my_lan_ip} diff --git a/viirb-cleanup-final-stage.sh b/viirb-4-cleanup-final-stage.sh similarity index 100% rename from viirb-cleanup-final-stage.sh rename to viirb-4-cleanup-final-stage.sh From 7472aeebf33c816615da51e6067e0b551de8bfb1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 15 Jun 2020 00:17:10 +0200 Subject: [PATCH 117/164] ++delay --- viirb-2-configure-fully-after-upgrade.sh | 3 ++- viirb-3-configure-vpnendpoint.sh | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index b190e94..c7dc10a 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -125,7 +125,8 @@ EOF # ensure viirb is back # ensure viirb network is up and running - bridge takes a bit of time # ensure viirb gives out ipv6 address -sleep 30 +# Found: 30s is not enough for full reconfiguration +sleep 60 # change to ipv6 viirb_ip=${my_lan_ip} diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-configure-vpnendpoint.sh index d03b90e..cd21131 100755 --- a/viirb-3-configure-vpnendpoint.sh +++ b/viirb-3-configure-vpnendpoint.sh @@ -55,8 +55,9 @@ git push cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} # Test that the VPN connection is established +# Might take longer due to reboot sleep 10 -ping -c3 ${my_wireguard_ip} -ping -c3 ${my_lan_ip} -ping -c3 ${my_wifi_ip} +ping -c10 ${my_wireguard_ip} +ping -c10 ${my_lan_ip} +ping -c10 ${my_wifi_ip} From 5b64fd461ee41b2a261dccb279b3a3a1d469d0ca Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 15 Jun 2020 00:27:53 +0200 Subject: [PATCH 118/164] Abort if config cannot find viirb --- viirb-2-configure-fully-after-upgrade.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index c7dc10a..161b39d 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -31,6 +31,12 @@ public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= +ping -c3 ${viirb_ip} +if [ $? -ne 0 ]; then + echo "Cannot reach ${viirb_ip}, aborting" + exit 1 +fi + cat < Date: Mon, 22 Jun 2020 23:05:13 +0200 Subject: [PATCH 119/164] Allow to use ip address instead of device --- viirb-1-connect-flash-latest-openwrt.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index 28ba121..c551ef1 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -2,24 +2,33 @@ # 2020-06-13, Nico Schottelius # See https://ungleich.ch/u/products/viirb-ipv6-box/ -if [ $# -ne 1 ]; then - echo "$0 interface" +if [ $# -lt 1 ]; then + echo "$0 interface [address]" echo " interface to add the config ip address to" + echo " address: connect to this address, ignore the interface" exit 1 fi set -x dev=$1; shift +if [ $# -ge 1 ]; then + viirb_ip=$1; shift + dev="" +else + viirb_ip=192.168.61.1 +fi + # openwrt version=19.07.3 filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin # IP address for setting it up initially -viirb_ip=192.168.61.1 -sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true -sudo ip addr add 192.168.61.2/24 dev "$dev" +if [ "$dev" ]; then + sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true + sudo ip addr add 192.168.61.2/24 dev "$dev" +fi # don't care about other/old known_host entries ssh-keygen -R ${viirb_ip} From 8e73c603c8f4ccf32a05594b990568262f221651 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Wed, 24 Jun 2020 08:53:37 +0200 Subject: [PATCH 120/164] Add fnux's key to devuan netboot image --- debian-devuan-netboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index eb0d381..2858ae4 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -43,7 +43,7 @@ echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > mkdir -p ${chroot_dir}/root/.ssh -for key in balazs dominique jinguk nico; do +for key in fnux balazs dominique jinguk nico; do curl -s ${keyurl}/${key}.pub >> ${chroot_dir}/root/.ssh/authorized_keys done From afaa9ef712a536af0a314e28b0e71877c9fec85f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 29 Jun 2020 08:49:24 +0200 Subject: [PATCH 121/164] Create initial structure for LUKS-encrypted CentOS images --- .../centos-build-luks-opennebula-image.sh | 175 ++++++++++++++++++ 1 file changed, 175 insertions(+) create mode 100755 opennebula-images/centos-build-luks-opennebula-image.sh diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh new file mode 100755 index 0000000..e438df8 --- /dev/null +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -0,0 +1,175 @@ +#!/bin/sh + +# This script generates CentOS images for OpenNebula. + +# Depends on the following packages (as of CentOS 8): +# qemu-img util-linux coreutils dnf curl e2fsprogs cryptsetup + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=7 +ARCH=x86_64 +IMAGE_PATH=centos-$RELEASE-$(date --iso-8601).img +IMAGE_SIZE=10G +LOOPBACK_DEVICE=/dev/loop0 +LUKS_PASSPHRASE=secretsecretsecret +LUKS_DEVICE_NAME=cryptroot +LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + losetup -d "$LOOPBACK_DEVICE" +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/centos-release' ]; then + echo "WARNING: this script has been designed to run on a CentOS system." >&2 + echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base RAW image (no LOOPBACK support in RHEL/CentOS). +qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" +losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +{ +sfdisk --no-reread "$LOOPBACK_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. +# network-scripts.x86_64 : Legacy scripts for manipulating of network devices +run_root dnf -y install network-scripts + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install and configure NTP client. +run_root dnf install -y chrony +run_root systemctl enable chronyd.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <>/mnt/etc/fstab < Date: Mon, 29 Jun 2020 12:25:39 +0200 Subject: [PATCH 122/164] Tune centos image for 7 --- .../centos-build-luks-opennebula-image.sh | 23 +++++++++++-------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh index e438df8..fcd34c3 100755 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -6,23 +6,21 @@ # qemu-img util-linux coreutils dnf curl e2fsprogs cryptsetup # Run locally (without network) with: -# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=raw set -e set -x -# XXX: Handle command-line arguments? RELEASE=7 ARCH=x86_64 -IMAGE_PATH=centos-$RELEASE-$(date --iso-8601).img +IMAGE_PATH=centos-luks-$RELEASE-$(date --iso-8601).img IMAGE_SIZE=10G LOOPBACK_DEVICE=/dev/loop0 -LUKS_PASSPHRASE=secretsecretsecret +LUKS_PASSPHRASE=secret LUKS_DEVICE_NAME=cryptroot LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" -# TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el7.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm cleanup() { @@ -82,13 +80,18 @@ mount "${LUKS_DEVICE}" /mnt mkdir /mnt/boot mount "${LOOPBACK_DEVICE}p1" /mnt/boot +# AppStream landed in CentOS 8. +if [ $RELEASE >= 8 ]; then + enabled_repos="--enablerepo=BaseOS --enablerepo=AppStream --enablerepo=extras" +else + enabled_repos="--enablerepo=Base --enablerepo=extras" +fi + dnf -y \ --releasever=$RELEASE \ --installroot=/mnt \ --disablerepo='*' \ - --enablerepo=BaseOS \ - --enablerepo=AppStream \ - --enablerepo=extras \ + $enabled_repos \ --setopt=install_weak_deps=False install \ bash basesystem systemd systemd-udev dnf centos-release @@ -147,6 +150,8 @@ kernel_version=$(ls /mnt/boot | grep "vmlinuz.*.$ARCH" | cut -d- -f2-) run_root dracut --force --kver $kernel_version # Configure grub2. +luks_uuid=$(blkid --match-tag UUID --output value "${LOOPBACK_DEVICE}p2") +echo "GRUB_CMDLINE_LINUX='rd.luks.uuid=$luks_uuid'" >> /mnt/etc/default/grub run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg From 4682ed9a3aa35f714a876ae840ddf904fb01a922 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 29 Jun 2020 17:46:48 +0200 Subject: [PATCH 123/164] More tuning to CentOS LUKS image --- .../centos-build-luks-opennebula-image.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh index fcd34c3..63effac 100755 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -20,7 +20,7 @@ LUKS_PASSPHRASE=secret LUKS_DEVICE_NAME=cryptroot LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el7.noarch.rpm" +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm cleanup() { @@ -81,10 +81,12 @@ mkdir /mnt/boot mount "${LOOPBACK_DEVICE}p1" /mnt/boot # AppStream landed in CentOS 8. -if [ $RELEASE >= 8 ]; then +if [ $RELEASE -ge 8 ]; then enabled_repos="--enablerepo=BaseOS --enablerepo=AppStream --enablerepo=extras" + release_specific_packages="systemd-udev" else - enabled_repos="--enablerepo=Base --enablerepo=extras" + enabled_repos="--enablerepo=base --enablerepo=extras" + release_specific_packages="dnf" fi dnf -y \ @@ -93,7 +95,7 @@ dnf -y \ --disablerepo='*' \ $enabled_repos \ --setopt=install_weak_deps=False install \ - bash basesystem systemd systemd-udev dnf centos-release + bash basesystem systemd dnf centos-release cryptsetup $release_specific_packages mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts @@ -119,7 +121,9 @@ EOF # See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. # network-scripts.x86_64 : Legacy scripts for manipulating of network devices -run_root dnf -y install network-scripts +if [ $RELEASE -ge 8 ]; then + run_root dnf -y install network-scripts +fi # Install (magic?) one-context RPM and hope things works as expected. curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" @@ -147,7 +151,7 @@ cat > /mnt/etc/dracut.conf.d/virtio-blk.conf < Date: Thu, 2 Jul 2020 21:42:46 +0200 Subject: [PATCH 124/164] adjust monit for broken init script / ceph osds --- monit-ceph-create-start | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/monit-ceph-create-start b/monit-ceph-create-start index 9b9cb58..afcac60 100755 --- a/monit-ceph-create-start +++ b/monit-ceph-create-start @@ -20,13 +20,26 @@ if echo $to_monitor | grep ^osd; then depends="${depends}, ${to_monitor}-whoami" osd="yes" osdid=$(echo $to_monitor | cut -d. -f2) -fi + cat > "$conf" < "$conf" < "$conf" <> "$conf" < Date: Thu, 2 Jul 2020 21:55:15 +0200 Subject: [PATCH 125/164] use pkill for ceph monitors --- monit-ceph-create-start | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monit-ceph-create-start b/monit-ceph-create-start index afcac60..a44efa0 100755 --- a/monit-ceph-create-start +++ b/monit-ceph-create-start @@ -24,7 +24,7 @@ if echo $to_monitor | grep ^osd; then # Generated by $0 check process ${to_monitor} with pidfile /var/run/ceph/${to_monitor}.pid start program = "/usr/bin/ceph-osd -i ${osdid} --pid-file /var/run/ceph/osd.${osdid}.pid -c /etc/ceph/ceph.conf --cluster ceph --setuser ceph --setgroup ceph" with timeout 3600 seconds - + stop program = "/usr/bin/pkill -f '/usr/bin/ceph-osd -i ${osdid}'" EOF else @@ -33,6 +33,7 @@ else # Generated by $0 check process ${to_monitor} with pidfile /var/run/ceph/${to_monitor}.pid start program = "/etc/init.d/ceph start ${to_monitor}" with timeout 60 seconds + stop program = "/etc/init.d/ceph stop ${to_monitor}" EOF @@ -40,7 +41,6 @@ fi # final clause same for both cat >> "$conf" < Date: Mon, 6 Jul 2020 09:35:47 +0200 Subject: [PATCH 126/164] CentOS luks image -> replace sfdisk by parted --- .../centos-build-luks-opennebula-image.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh index 63effac..d577e9f 100755 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -61,12 +61,10 @@ losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" trap cleanup EXIT # Create partition table, format partitions. -{ -sfdisk --no-reread "$LOOPBACK_DEVICE" < Date: Mon, 6 Jul 2020 09:36:10 +0200 Subject: [PATCH 127/164] centos luks image: use local repo dir, crypttab instead of kernel args --- .../centos-build-luks-opennebula-image.sh | 10 +++++----- .../rpm-repositories/centos-7-minus.repo | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 opennebula-images/rpm-repositories/centos-7-minus.repo diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh index d577e9f..f39d937 100755 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ b/opennebula-images/centos-build-luks-opennebula-image.sh @@ -93,6 +93,7 @@ dnf -y \ --disablerepo='*' \ $enabled_repos \ --setopt=install_weak_deps=False install \ + --setopt=reposdir=rpm-repositories \ bash basesystem systemd dnf centos-release cryptsetup $release_specific_packages mount --bind /dev /mnt/dev @@ -144,16 +145,15 @@ run_root systemctl enable chronyd.service # uncompressed but is a direct dependency of kernel-core... run_root dnf -y install kernel grub2 -# Add support for virtio block devices at boot time. +# Add support for virtio block devices at boot time, configure bootloader. cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <> /mnt/etc/default/grub +echo "cryptroot UUID=$luks_uuid luks,timeout=30" >> /mnt/etc/crypttab + +run_root dracut -v --force --kver $kernel_version run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg diff --git a/opennebula-images/rpm-repositories/centos-7-minus.repo b/opennebula-images/rpm-repositories/centos-7-minus.repo new file mode 100644 index 0000000..53bae51 --- /dev/null +++ b/opennebula-images/rpm-repositories/centos-7-minus.repo @@ -0,0 +1,16 @@ +[base] +name=CentOS-$releasever - Base +mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=os&infra=$infra +#baseurl=http://mirror.centos.org/centos/$releasever/os/$basearch/ +gpgcheck=0 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 + +#released updates +[updates] +name=CentOS-$releasever - Updates +mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=updates&infra=$infra +#baseurl=http://mirror.centos.org/centos/$releasever/updates/$basearch/ +gpgcheck=0 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-7 From c9d1ab6e13521601c0d5ba807871f2b3dde86e07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 09:44:41 +0200 Subject: [PATCH 128/164] Add missing centos-extras repo --- opennebula-images/rpm-repositories/centos-extras.repo | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 opennebula-images/rpm-repositories/centos-extras.repo diff --git a/opennebula-images/rpm-repositories/centos-extras.repo b/opennebula-images/rpm-repositories/centos-extras.repo new file mode 100644 index 0000000..c53d52a --- /dev/null +++ b/opennebula-images/rpm-repositories/centos-extras.repo @@ -0,0 +1,7 @@ +[extras] +name=CentOS-$releasever - Extras +mirrorlist=http://mirrorlist.centos.org/?release=$releasever&arch=$basearch&repo=extras&infra=$infra +#baseurl=http://mirror.centos.org/$contentdir/$releasever/extras/$basearch/os/ +gpgcheck=0 +enabled=0 +gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-centosofficial From 45f48f3e05ad7ab87c37c07706d723342d124cf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 09:59:52 +0200 Subject: [PATCH 129/164] centos-build-luks -> centos7-build-luks, fix blkid calls --- .../centos7-build-luks-opennebula-image.sh | 182 ++++++++++++++++++ 1 file changed, 182 insertions(+) create mode 100755 opennebula-images/centos7-build-luks-opennebula-image.sh diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh new file mode 100755 index 0000000..d5696fc --- /dev/null +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -0,0 +1,182 @@ +#!/bin/sh + +# This script generates CentOS images for OpenNebula. Expected to run on CentOS 7. + +# Depends on the following packages: +# qemu-img util-linux coreutils dnf curl e2fsprogs cryptsetup parted + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=raw + +set -e +set -x + +RELEASE=7 +ARCH=x86_64 +IMAGE_PATH=centos-luks-$RELEASE-$(date --iso-8601).img +IMAGE_SIZE=10G +LOOPBACK_DEVICE=/dev/loop2 +LUKS_PASSPHRASE=secret +LUKS_DEVICE_NAME=cryptroot2 +LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" + +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + losetup -d "$LOOPBACK_DEVICE" +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/centos-release' ]; then + echo "WARNING: this script has been designed to run on a CentOS system." >&2 + echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base RAW image (no LOOPBACK support in RHEL/CentOS). +qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" +losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +parted --script "$LOOPBACK_DEVICE" \ + mklabel msdos \ + mkpart primary ext4 1M 500M \ + mkpart primary ext4 500M 100% + +partprobe "$LOOPBACK_DEVICE" + +mkfs.ext4 "${LOOPBACK_DEVICE}p1" +echo -n "$LUKS_PASSPHRASE" | cryptsetup luksFormat -v -d - "${LOOPBACK_DEVICE}p2" +echo -n "$LUKS_PASSPHRASE" | cryptsetup open -v -d - "${LOOPBACK_DEVICE}p2" "$LUKS_DEVICE_NAME" +mkfs.ext4 "$LUKS_DEVICE" + +# Mount partitions, install base OS. +mount "${LUKS_DEVICE}" /mnt +mkdir /mnt/boot +mount "${LOOPBACK_DEVICE}p1" /mnt/boot + +# AppStream landed in CentOS 8. +if [ $RELEASE -ge 8 ]; then + enabled_repos="--enablerepo=BaseOS --enablerepo=AppStream --enablerepo=extras" + release_specific_packages="systemd-udev" +else + enabled_repos="--enablerepo=base --enablerepo=extras" + release_specific_packages="dnf" +fi + +dnf -y \ + --releasever=$RELEASE \ + --installroot=/mnt \ + --disablerepo='*' \ + $enabled_repos \ + --setopt=install_weak_deps=False install \ + --setopt=reposdir=rpm-repositories \ + bash basesystem systemd dnf centos-release cryptsetup $release_specific_packages + +mount --bind /dev /mnt/dev +mount --bind /dev/pts /mnt/dev/pts +mount --bind /dev/shm /mnt/dev/shm +mount --bind /proc /mnt/proc +mount --bind /run /mnt/run +mount --bind /sys /mnt/sys + +# Guest networking is to be handled by the one-context package. +# See https://github.com/OpenNebula/addon-context-linux for details. +# Note: as of writing, one-context does not support NetworkManager or +# systemd-networkd. + +# Required to resolve package mirror in chroot. +cp /etc/resolv.conf /mnt/etc/resolv.conf + +# Initialize /etc/hosts. +cat > /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. +# network-scripts.x86_64 : Legacy scripts for manipulating of network devices +if [ $RELEASE -ge 8 ]; then + run_root dnf -y install network-scripts +fi + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install and configure NTP client. +run_root dnf install -y chrony +run_root systemctl enable chronyd.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time, configure bootloader. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <> /mnt/etc/crypttab + +run_root dracut -v --force --kver $kernel_version +run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}" +run_root grub2-mkconfig -o /boot/grub2/grub.cfg + +# Install en configure SSH daemon. +run_root dnf -y install openssh-server +run_root systemctl enable sshd + +# Generate fstab file. +boot_uuid=$(blkid -o value "${LOOPBACK_DEVICE}p1" | head -n 1) +root_uuid=$(blkid -o value "$LUKS_DEVICE" | head -n 1) +cat >>/mnt/etc/fstab < Date: Mon, 6 Jul 2020 10:38:17 +0200 Subject: [PATCH 130/164] Remove incomplete centos-8 support centos-luks ONE image --- .../centos-build-luks-opennebula-image.sh | 182 ------------------ .../centos7-build-luks-opennebula-image.sh | 30 +-- 2 files changed, 9 insertions(+), 203 deletions(-) delete mode 100755 opennebula-images/centos-build-luks-opennebula-image.sh diff --git a/opennebula-images/centos-build-luks-opennebula-image.sh b/opennebula-images/centos-build-luks-opennebula-image.sh deleted file mode 100755 index f39d937..0000000 --- a/opennebula-images/centos-build-luks-opennebula-image.sh +++ /dev/null @@ -1,182 +0,0 @@ -#!/bin/sh - -# This script generates CentOS images for OpenNebula. - -# Depends on the following packages (as of CentOS 8): -# qemu-img util-linux coreutils dnf curl e2fsprogs cryptsetup - -# Run locally (without network) with: -# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=raw - -set -e -set -x - -RELEASE=7 -ARCH=x86_64 -IMAGE_PATH=centos-luks-$RELEASE-$(date --iso-8601).img -IMAGE_SIZE=10G -LOOPBACK_DEVICE=/dev/loop0 -LUKS_PASSPHRASE=secret -LUKS_DEVICE_NAME=cryptroot -LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" - -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" -ONE_CONTEXT_RPM_PATH=/root/one-context.rpm - -cleanup() { - # The order here is important. - umount /mnt/dev/pts 2>/dev/null || true - umount /mnt/dev/shm 2>/dev/null || true - umount /mnt/dev 2>/dev/null || true - umount /mnt/proc 2>/dev/null || true - umount /mnt/run 2>/dev/null || true - umount /mnt/sys 2>/dev/null || true - umount /mnt/boot 2>/dev/null || true - umount /mnt 2>/dev/null || true - losetup -d "$LOOPBACK_DEVICE" -} - -run_root() { - chroot /mnt /usr/bin/env \ - PATH=/sbin:/usr/sbin:/bin:/usr/bin \ - sh -c "$*" -} - -if [ "$(whoami)" != 'root' ]; then - echo "This script must be run as root." >&2 - exit 1 -fi - -if [ ! -f '/etc/centos-release' ]; then - echo "WARNING: this script has been designed to run on a CentOS system." >&2 - echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 - sleep 5 -fi - -# Create base RAW image (no LOOPBACK support in RHEL/CentOS). -qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" -losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" - -# Don't forget to cleanup, even if the script crash. -trap cleanup EXIT - -# Create partition table, format partitions. -parted --script "$LOOPBACK_DEVICE" \ - mklabel msdos \ - mkpart primary ext4 1M 500M \ - mkpart primary ext4 500M 100% - -partprobe "$LOOPBACK_DEVICE" - -mkfs.ext4 "${LOOPBACK_DEVICE}p1" -echo -n "$LUKS_PASSPHRASE" | cryptsetup luksFormat -v -d - "${LOOPBACK_DEVICE}p2" -echo -n "$LUKS_PASSPHRASE" | cryptsetup open -v -d - "${LOOPBACK_DEVICE}p2" "$LUKS_DEVICE_NAME" -mkfs.ext4 "$LUKS_DEVICE" - -# Mount partitions, install base OS. -mount "${LUKS_DEVICE}" /mnt -mkdir /mnt/boot -mount "${LOOPBACK_DEVICE}p1" /mnt/boot - -# AppStream landed in CentOS 8. -if [ $RELEASE -ge 8 ]; then - enabled_repos="--enablerepo=BaseOS --enablerepo=AppStream --enablerepo=extras" - release_specific_packages="systemd-udev" -else - enabled_repos="--enablerepo=base --enablerepo=extras" - release_specific_packages="dnf" -fi - -dnf -y \ - --releasever=$RELEASE \ - --installroot=/mnt \ - --disablerepo='*' \ - $enabled_repos \ - --setopt=install_weak_deps=False install \ - --setopt=reposdir=rpm-repositories \ - bash basesystem systemd dnf centos-release cryptsetup $release_specific_packages - -mount --bind /dev /mnt/dev -mount --bind /dev/pts /mnt/dev/pts -mount --bind /dev/shm /mnt/dev/shm -mount --bind /proc /mnt/proc -mount --bind /run /mnt/run -mount --bind /sys /mnt/sys - -# Guest networking is to be handled by the one-context package. -# See https://github.com/OpenNebula/addon-context-linux for details. -# Note: as of writing, one-context does not support NetworkManager or -# systemd-networkd. - -# Required to resolve package mirror in chroot. -cp /etc/resolv.conf /mnt/etc/resolv.conf - -# Initialize /etc/hosts. -cat > /mnt/etc/hosts << EOF -127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 -::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 - -EOF - -# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. -# network-scripts.x86_64 : Legacy scripts for manipulating of network devices -if [ $RELEASE -ge 8 ]; then - run_root dnf -y install network-scripts -fi - -# Install (magic?) one-context RPM and hope things works as expected. -curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" -run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" -run_root rm "$ONE_CONTEXT_RPM_PATH" - -# Install resize2fs, which is required to resize the root file-system. -run_root dnf -y install e2fsprogs - -# Initalize base services. -run_root systemd-machine-id-setup -run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime - -# Install and configure NTP client. -run_root dnf install -y chrony -run_root systemctl enable chronyd.service - -# Install kernel and bootloader. -# Note: linux-firmware is not required our environment and takes almost 200M -# uncompressed but is a direct dependency of kernel-core... -run_root dnf -y install kernel grub2 - -# Add support for virtio block devices at boot time, configure bootloader. -cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <> /mnt/etc/crypttab - -run_root dracut -v --force --kver $kernel_version -run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}" -run_root grub2-mkconfig -o /boot/grub2/grub.cfg - -# Install en configure SSH daemon. -run_root dnf -y install openssh-server -run_root systemctl enable sshd - -# Generate fstab file. -boot_uuid=$(blkid --match-tag UUID --output value "${LOOPBACK_DEVICE}p1") -root_uuid=$(blkid --match-tag UUID --output value "$LUKS_DEVICE") -cat >>/mnt/etc/fstab < /mnt/etc/hosts << EOF 127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 - EOF -# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. -# network-scripts.x86_64 : Legacy scripts for manipulating of network devices -if [ $RELEASE -ge 8 ]; then - run_root dnf -y install network-scripts -fi - -# Install (magic?) one-context RPM and hope things works as expected. +# Install one-context RPM and hope things works as expected. curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" @@ -180,3 +165,6 @@ run_root dnf clean all # Make sure everything is written to disk before exiting. sync + +# Cleanup! +cleanup From c0f32f78e3059c9a0a9264e37f15a6e39d425c74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 10:43:27 +0200 Subject: [PATCH 131/164] CentOS/luks: disable SSH/user+password ONE scripts --- opennebula-images/centos7-build-luks-opennebula-image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 6bc6eca..6ea75e9 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -19,6 +19,7 @@ LOOPBACK_DEVICE=/dev/loop0 LUKS_PASSPHRASE=secret LUKS_DEVICE_NAME=cryptroot LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" +DISABLED_ONE_SCRIPTS="loc-20-set-username-password loc-22-ssh_public_key" ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm @@ -113,6 +114,9 @@ EOF curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" +for script in "$DISABLED_ONE_SCRIPTS"; do + run_root rm "/etc/one-context.d/$script" +done # Install resize2fs, which is required to resize the root file-system. run_root dnf -y install e2fsprogs From 9ba788e75b4d47483c3495e64597bf8ad6df39e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 10:52:55 +0200 Subject: [PATCH 132/164] Centos 7 / LUKS: fix ONE script removal --- opennebula-images/centos7-build-luks-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 6ea75e9..ddb7189 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -114,7 +114,7 @@ EOF curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" -for script in "$DISABLED_ONE_SCRIPTS"; do +for script in $DISABLED_ONE_SCRIPTS; do run_root rm "/etc/one-context.d/$script" done From 1e39b0aa84d70a9264ba5e766c3c81c954998fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 11:36:09 +0200 Subject: [PATCH 133/164] Do not use custom reposdir for centos7/LUKS image --- opennebula-images/centos7-build-luks-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index ddb7189..367211e 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -79,6 +79,7 @@ mount "${LUKS_DEVICE}" /mnt mkdir /mnt/boot mount "${LOOPBACK_DEVICE}p1" /mnt/boot +# Add --setopt=reposdir=rpm-repositories if you do not run on CentOS 7. dnf -y \ --releasever=$RELEASE \ --installroot=/mnt \ @@ -86,7 +87,6 @@ dnf -y \ --enablerepo=base \ --enablerepo=extras \ --setopt=install_weak_deps=False install \ - --setopt=reposdir=rpm-repositories \ bash basesystem systemd dnf centos-release cryptsetup dnf mount --bind /dev /mnt/dev From 8db3786b5b291a002c780acc0ea6a74e161a5c8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 6 Jul 2020 12:37:18 +0200 Subject: [PATCH 134/164] centos7/LUKS script: take LUKS passphrase as command-line argument --- opennebula-images/centos7-build-luks-opennebula-image.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 367211e..3be4b61 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -16,7 +16,6 @@ ARCH=x86_64 IMAGE_PATH=centos-luks-$RELEASE-$(date --iso-8601).img IMAGE_SIZE=10G LOOPBACK_DEVICE=/dev/loop0 -LUKS_PASSPHRASE=secret LUKS_DEVICE_NAME=cryptroot LUKS_DEVICE="/dev/mapper/$LUKS_DEVICE_NAME" DISABLED_ONE_SCRIPTS="loc-20-set-username-password loc-22-ssh_public_key" @@ -24,6 +23,13 @@ DISABLED_ONE_SCRIPTS="loc-20-set-username-password loc-22-ssh_public_key" ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el$RELEASE.noarch.rpm" ONE_CONTEXT_RPM_PATH=/root/one-context.rpm +# Get LUKS passphrase. +if [ -z "$1" ]; then + echo "Usage: centos7-build-luks-opennebula-image.sh LUKS_PASSPHRASE" + exit 1 +fi +LUKS_PASSPHRASE="$1" + cleanup() { # The order here is important. umount /mnt/dev/pts 2>/dev/null || true From 3ff2b138cb1b2d36d6bd4c0f1ee96fd5adc071cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 7 Jul 2020 21:00:49 +0200 Subject: [PATCH 135/164] in viirb2, output the commandline for next command --- viirb-2-configure-fully-after-upgrade.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 161b39d..284e783 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -211,5 +211,5 @@ uci commit reboot EOF -echo "Wireguard public key: ${public_key}" +echo "Wireguard public key and id: ${id} ${public_key}" echo ${public_key} > ${viirb_hostname}.public_key From 7e55e76b14d696724c3d9595ab05c4bc1ca5ff2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 21 Jul 2020 10:55:49 +0200 Subject: [PATCH 136/164] Fix FreeBSD ONE image build script --- .../freebsd-build-opennebula-image.sh | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh index ec64a56..535d2a6 100755 --- a/opennebula-images/freebsd-build-opennebula-image.sh +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -9,15 +9,13 @@ set -x # XXX: Handle command-line arguments? RELEASE=12.1-RELEASE ARCH=amd64 -IMAGE_PATH=freebsd-$RELEASE-$(date +%+F).img.qcow2 +IMAGE_PATH=freebsd-$RELEASE-$(date -I).img.qcow2 IMAGE_SIZE=10G DIST_BASE="https://download.freebsd.org/ftp/releases/$ARCH/$RELEASE" PORTS_BASE="https://download.freebsd.org/ftp/snapshots/$ARCH/12.1-STABLE" -# TODO: find the package definition and built ourself, publish in some RPM repository. -ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" -ONE_CONTEXT_RPM_PATH=/root/one-context.rpm +ONE_CONTEXT_PKG_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.12.0/one-context-5.12.0_1.txz" cleanup() { sync || true @@ -70,7 +68,7 @@ growfs_enable=YES hostname="freebsd" EOF -cp /etc/resolv.conf > /mnt/etc/resolv.conf +cp /etc/resolv.conf /mnt/etc/resolv.conf tzsetup -s -C /mnt UTC cat >>/mnt/etc/ssh/sshd_config < Date: Tue, 21 Jul 2020 12:08:38 +0200 Subject: [PATCH 137/164] Install requirements for plain centos7 to build centos7 VM --- opennebula-images/centos7-build-luks-opennebula-image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 3be4b61..f3d34c8 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -60,6 +60,9 @@ if [ ! -f '/etc/centos-release' ]; then sleep 5 fi +# Install requirements +yum install -y qemu cryptsetup + # Create base RAW image (no LOOPBACK support in RHEL/CentOS). qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" From 9e8e39b2ee5ea37b169a2418308eda3b15f8d521 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 21 Jul 2020 12:18:13 +0200 Subject: [PATCH 138/164] Also install dnf for centos --- opennebula-images/centos7-build-luks-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index f3d34c8..69a5c64 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -61,7 +61,7 @@ if [ ! -f '/etc/centos-release' ]; then fi # Install requirements -yum install -y qemu cryptsetup +yum install -y qemu cryptsetup dnf # Create base RAW image (no LOOPBACK support in RHEL/CentOS). qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" From a109bb16f8fbfd8ff16dcaeb464446f319416dbc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 28 Jul 2020 15:40:15 +0200 Subject: [PATCH 139/164] [mikrotik] + permissions +shebang --- mikrotik-setup.sh | 0 mikrotik-update.sh | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) mode change 100644 => 100755 mikrotik-setup.sh diff --git a/mikrotik-setup.sh b/mikrotik-setup.sh old mode 100644 new mode 100755 diff --git a/mikrotik-update.sh b/mikrotik-update.sh index 934ea47..5d1c281 100755 --- a/mikrotik-update.sh +++ b/mikrotik-update.sh @@ -1,4 +1,4 @@ -#!/bin +#!/bin/sh # Nico Schottelius, 2019-12-02 # Update mikrotik routers to the latest package From 6d8c93b54cb5d7c1928331351ce2a2c84c8d743c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 12:26:17 +0200 Subject: [PATCH 140/164] iHmm? Signed-off-by: Nico Schottelius --- viirb-2-configure-fully-after-upgrade.sh | 4 + viirb-setup-all.sh | 391 ----------------------- 2 files changed, 4 insertions(+), 391 deletions(-) delete mode 100755 viirb-setup-all.sh diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 161b39d..f92a13c 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -63,7 +63,10 @@ uci delete dhcp.lan.leasetime # Do not announce ULA - we have GUA uci delete network.globals.ula_prefix +# This is configuring the dhcp IPv4 client uci set dhcp.lan=dhcp + +# Setup Router Advertisements uci set dhcp.lan.interface='lanv6' uci set dhcp.lan.ra='server' uci set dhcp.lan.dynamicdhcp='0' @@ -114,6 +117,7 @@ uci delete wireless.radio0.disabled # This is temporary - keeping it until the config process is through # Probably not needet - we can connect to the final IPv6 address! +# This code commented out == The address vanishes due to above reconfiguration # uci set network.lanv4temp=interface # uci set network.lanv4temp.proto='static' # uci set network.lanv4temp.ifname='br-lan' diff --git a/viirb-setup-all.sh b/viirb-setup-all.sh deleted file mode 100755 index 63286cb..0000000 --- a/viirb-setup-all.sh +++ /dev/null @@ -1,391 +0,0 @@ -#!/bin/sh -# 2020-06-13, Nico Schottelius -# See https://ungleich.ch/u/products/viirb-ipv6-box/ - -if [ $# -lt 4 ]; then - echo "$0 interface viirb-id your-dot-cdist [stages]" - echo " interface to add the config ip address to" - echo " viirb-id: number in decimal format" - echo " your-dot-cdist: path to YOUR ungleich-dot-cdist repo" - echo " owner-mail-reference: How to identify the owner" - echo " stages: define which stages to execute" - echo "" - echo " stage1: setup your host, check connection to VIIRB" - echo " stage2: flash latest openwrt onto the VIIRB" - echo " stage3: configure the vpn endpoint" - echo " stage4: configure the VIIRB with wireguard + settings" - echo " stage5: Verify VIIRB on VPN, cleanup VIIRB" - echo "" - echo "Example to configure viirb02:" - echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342'" - echo "$0 wlan0 2 ~/vcs/ungleich-dot-cdist 'Nico Schottelius, nico.schottelius@ungleich.ch, Ticket 2342' '1 3 4'" - exit 1 -fi - -echo "FIXME: missing IPv6 announcements on LAN" -echo "FIXME: DNS situation: upstream, non upstream, ungleich, how to resolve tunnel endpoint" - -set -x - -set -x -dev=$1; shift -id=$1; shift -dot_cdist=$1; shift -owner=$1; shift - -if [ $# -ge 1 ]; then - stages=$1; shift -else - stages="1 2 3 4 5" -fi - -hex_id=$(printf "%0.2x\n" "$id") -viirb_hostname=viirb${hex_id} - -prefix_base=2a0a:e5c1:3 -my_prefix=${prefix_base}${hex_id} -my_network=${my_prefix}::/48 - -my_wireguard_ip=${my_prefix}::42 -my_lan_ip=${my_prefix}:cafe::42 -my_wifi_ip=${my_prefix}:7ea::42 - -# openwrt -version=19.07.3 -filename=openwrt-${version}-ramips-mt76x8-vocore2-squashfs-sysupgrade.bin - -# root password -root_password=$(pwgen -1 32) - -# IP address for setting it up initially -viirb_ip=192.168.61.1 - -# wireguard -private_key=$(wg genkey) -public_key=$(echo $private_key | wg pubkey) - -vpn_endpoint_host=vpn-2a0ae5c1300.ungleich.ch -vpn_endpoint_pubkey=ft68G2RID7gZ6PXjFCSCOdJ9yspRg+tUw0YrNK9cTxE= - -# cdist -dot_cdist_files=${dot_cdist}/type/__ungleich_wireguard/files -peerfilename=${vpn_endpoint_host}.peer${hex_id} -peerfile=${dot_cdist_files}/${peerfilename} -vpnconfig=${dot_cdist_files}/${vpn_endpoint_host} - -################################################################################ -# Stage 1: test / connect to the new VIIRB -# -# We delete so that we can run idempotent -stage1() -{ - sudo ip addr del 192.168.61.2/24 dev "$dev" 2>/dev/null || true - sudo ip addr add 192.168.61.2/24 dev "$dev" - - # don't care about other/old known_host entries - ssh-keygen -R ${viirb_ip} - - ping -c2 ${viirb_ip} - if [ $? -ne 0 ]; then - echo "Cannot reach any VIIRB - exiting" - exit 1 - fi - - cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_keys" -} - - - -################################################################################ -# Get latest OpenWRT & flash it -stage2() -{ - # Don't re-download if we already have it - wget -c http://downloads.openwrt.org/releases/${version}/targets/ramips/mt76x8/${filename} - scp ${filename} root@${viirb_ip}:/tmp - ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" - - # It still pings for some time - wait for the reboot to happen - echo "Waiting for VIIRB to disappear" - sleep 15 - - wait=0 - found="" - - while [ $wait -lt 180 ]; do - ping -c1 ${viirb_ip} >/dev/null - - if [ $? -eq 0 ]; then - found=yes - # wait for ssh to come up - sleep 10 - break - fi - - sleep 1 - wait=$((wait+1)) - done - - if [ ! "$found" ]; then - echo "Did not find updated viirb - debug / restart it" - exit 1 - fi - -} - - -################################################################################ -# Stage 3: prepare VPN endpoint -# - -stage3() -{ - - # Configure VPN server / update cdist - echo Updating VPNserver - cat < ${peerfile} -# ${viirb_hostname} ${owner} -[Peer] -PublicKey = ${public_key} -AllowedIPs = ${my_network} - -EOF - - # Generate real config - cat ${dot_cdist_files}/${vpn_endpoint_host}.* > ${vpnconfig} - cd ${dot_cdist_files} - git add ${vpn_endpoint_host} ${peerfilename} - git commit -m "[vpn] Updated config for peer ${viirb_hostname} ${my_network}" - git pull - git push - - cdist config -vv -j8 ${vpn_endpoint_host} -c ${dot_cdist} -} - -################################################################################ -# Stage 4: configure the VIIRB -# -stage4() -{ - - cat </dev/null - - if [ $? -eq 0 ]; then - found=yes - break - fi - sleep 1 - wait=$((wait+1)) - done - - if [ ! "$found" ]; then - echo "Cannot reach VIIRB via VPN - check manually" - exit 1 - fi - - echo "Cleanup process." - echo "Set the root password when prompted to: ${root_password}" - - # VPN works, remove artefacts, set correct DNS servers that support DNS64 - cat < Date: Sat, 1 Aug 2020 12:37:51 +0200 Subject: [PATCH 141/164] [viirb] add script to do step1+2 together --- viirb1-2-together.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 viirb1-2-together.sh diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh new file mode 100644 index 0000000..b18c4d8 --- /dev/null +++ b/viirb1-2-together.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 viirb-id" + echo " viirb-id: number in decimal format" + exit 1 +fi + +id=$1; shift + +./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 +./viirb-2-configure-fully-after-upgrade.sh From b7649a60bca84babf8d22616f331f42cc9eafd29 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 12:38:27 +0200 Subject: [PATCH 142/164] +sleep between 1+2 --- viirb1-2-together.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh index b18c4d8..5d1d736 100644 --- a/viirb1-2-together.sh +++ b/viirb1-2-together.sh @@ -9,4 +9,6 @@ fi id=$1; shift ./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 +# reboot may take longer +sleep 60 ./viirb-2-configure-fully-after-upgrade.sh From 389e6417f1f2694d3389b7552995ddda4743af94 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 13:42:51 +0200 Subject: [PATCH 143/164] [viirb12] pass args to viirb2 --- viirb1-2-together.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh index 5d1d736..3edc0bf 100644 --- a/viirb1-2-together.sh +++ b/viirb1-2-together.sh @@ -11,4 +11,4 @@ id=$1; shift ./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 # reboot may take longer sleep 60 -./viirb-2-configure-fully-after-upgrade.sh +./viirb-2-configure-fully-after-upgrade.sh 192.168.61.1 "$id" From d68242713a703debbf88d841f8cd3663b77782ab Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 13:46:00 +0200 Subject: [PATCH 144/164] [viirb] + pubkey + args --- viirb-2-configure-fully-after-upgrade.sh | 1 + viirb1-2-together.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/viirb-2-configure-fully-after-upgrade.sh b/viirb-2-configure-fully-after-upgrade.sh index 16f704e..94fc628 100755 --- a/viirb-2-configure-fully-after-upgrade.sh +++ b/viirb-2-configure-fully-after-upgrade.sh @@ -136,6 +136,7 @@ EOF # ensure viirb network is up and running - bridge takes a bit of time # ensure viirb gives out ipv6 address # Found: 30s is not enough for full reconfiguration +echo "Public VPN key: ${id} ${public_key}" sleep 60 # change to ipv6 diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh index 3edc0bf..2d4378b 100644 --- a/viirb1-2-together.sh +++ b/viirb1-2-together.sh @@ -1,6 +1,6 @@ #!/bin/sh -if [ $# -ne 2 ]; then +if [ $# -ne 1 ]; then echo "$0 viirb-id" echo " viirb-id: number in decimal format" exit 1 From e0a313c7097b7f56c1a89cdbc071cd1b8d788163 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 14:00:49 +0200 Subject: [PATCH 145/164] increase timeout for flash writing --- viirb1-2-together.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/viirb1-2-together.sh b/viirb1-2-together.sh index 2d4378b..e1a4206 100644 --- a/viirb1-2-together.sh +++ b/viirb1-2-together.sh @@ -9,6 +9,6 @@ fi id=$1; shift ./viirb-1-connect-flash-latest-openwrt.sh eth0 192.168.61.1 -# reboot may take longer -sleep 60 +# reboot may take longer / rewriting the flash +sleep 120 ./viirb-2-configure-fully-after-upgrade.sh 192.168.61.1 "$id" From 7bbac98e3b6e28f32d6f56edc9136c9512fac621 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 16:21:33 +0200 Subject: [PATCH 146/164] [viirb] wait until it comes online --- viirb-1-connect-flash-latest-openwrt.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index c551ef1..f6b5a34 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -33,11 +33,10 @@ fi # don't care about other/old known_host entries ssh-keygen -R ${viirb_ip} -ping -c2 ${viirb_ip} -if [ $? -ne 0 ]; then - echo "Cannot reach any VIIRB - exiting" - exit 1 -fi +while ! ping -c1 ${viirb_ip}; do + echo "Cannot ping $viirb_ip yet - waiting" + sleep 1 +done cat ~/.ssh/id_rsa.pub | ssh root@${viirb_ip} "cat > /etc/dropbear/authorized_keys" From 40cb3be77683a288f494de501a092cfb0507f521 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 17:08:34 +0200 Subject: [PATCH 147/164] [viirb] ping it longer --- viirb-3-configure-vpnendpoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-3-configure-vpnendpoint.sh b/viirb-3-configure-vpnendpoint.sh index cd21131..ac3b742 100755 --- a/viirb-3-configure-vpnendpoint.sh +++ b/viirb-3-configure-vpnendpoint.sh @@ -60,4 +60,4 @@ sleep 10 ping -c10 ${my_wireguard_ip} ping -c10 ${my_lan_ip} -ping -c10 ${my_wifi_ip} +ping -c90 ${my_wifi_ip} From 23139cfe9d4d328c5df7fe472a7e8bea056fcff7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 1 Aug 2020 17:08:44 +0200 Subject: [PATCH 148/164] [viirb] update comment --- viirb-1-connect-flash-latest-openwrt.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/viirb-1-connect-flash-latest-openwrt.sh b/viirb-1-connect-flash-latest-openwrt.sh index f6b5a34..4cbb393 100755 --- a/viirb-1-connect-flash-latest-openwrt.sh +++ b/viirb-1-connect-flash-latest-openwrt.sh @@ -46,7 +46,7 @@ scp ${filename} root@${viirb_ip}:/tmp ssh root@${viirb_ip} "sysupgrade /tmp/*.bin" # It still pings for some time - wait for the reboot to happen -echo "Waiting for VIIRB to disappear" +echo "Waiting for VIIRB to really disappear" sleep 15 wait=0 From 7b24804ac998ac3a0a81b910fca8c0c6bfd502e6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 30 Jul 2020 18:56:24 +0200 Subject: [PATCH 149/164] [mikrotik] update / minus bugs --- mikrotik-setup.sh | 7 ++++++- mikrotik-update.sh | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mikrotik-setup.sh b/mikrotik-setup.sh index dbe2c4f..e3654e5 100755 --- a/mikrotik-setup.sh +++ b/mikrotik-setup.sh @@ -16,7 +16,12 @@ conf() { ssh admin@${target} "$@" } copy() { - scp "$1" admin@${target}: + if echo ${target} | grep -q :; then + ltarget="[$target]" + else + ltarget="$target" + fi + scp "$1" admin@${ltarget}: } # store ssh key in the admin user! diff --git a/mikrotik-update.sh b/mikrotik-update.sh index 5d1c281..027bdba 100755 --- a/mikrotik-update.sh +++ b/mikrotik-update.sh @@ -29,7 +29,7 @@ cd "$tmp" wget "${url}" unzip "${file}" -pkg_list="dhcp ipv6 lcd lte multicast ppp routing security system user-manager wireless"! +pkg_list="dhcp ipv6 lcd lte multicast ppp routing security system user-manager wireless" while [ $# -ge 1 ]; do target=$1; shift From 91d6eac9c7ce4025185ce4a7c62e9a112ce4a114 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 3 Aug 2020 20:51:29 +0200 Subject: [PATCH 150/164] add script to "standard" configure a mikrotik-crs326 --- mikrotik-configure-crs326.sh | 103 +++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 mikrotik-configure-crs326.sh diff --git a/mikrotik-configure-crs326.sh b/mikrotik-configure-crs326.sh new file mode 100755 index 0000000..5f3b330 --- /dev/null +++ b/mikrotik-configure-crs326.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Nico Schottelius, 2020-08-03 +# Setup a standard crs326 + + + +if [ $# -ne 3 ]; then + echo "$0 " + echo "Example:" + echo "$0 fe80::764d:28ff:fe09:9355%eth1 mikrotik-crs326-8 $(pass place6-linthal/mikrotik)" + exit 1 +fi + +ip=$1; shift +hostname=$1; shift +password=$1; shift + +target=$ip +bridge=bridgevlans + +internal=10 +coworking=15 +server=11 +other="8 16 18 33 34" + +tagged="ether23 ether24 sfp-sfpplus1" + +net_internal=2a0a:e5c0:2::/64 + +conf() { + echo $@ + ssh admin@${target} "$@" +} + +commastring() { + echo $@ | sed 's/ /,/g' +} + +set -x + + +# do this out of band -- see mikrotik-setup.sh +#conf "/password new-password=$password confirm-new-password=$password old-password=\"\"" + +conf "/system identity set name=$hostname" +conf "/interface bridge add name=$bridge" + +################################################################################ +# MTU + +for i in $(seq 1 24); do + conf "/interface ethernet set ether$i mtu=9200 l2mtu=9204" +done + + +for i in $(seq 1 2); do + conf "/interface ethernet set sfp-sfpplus$i mtu=9200 l2mtu=9204" +done + + +################################################################################ +# VLANs + +# Internal ports 1-16 +ifaces="" +for i in $(seq 1 16); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$internal" + ifaces="ether$i ${ifaces}" +done + +# also tag the bridge for the vlan interface we need later +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged),$bridge untagged=$(commastring $ifaces) vlan-ids=$internal" + +# Coworking 17-18 +ifaces="" +for i in $(seq 17 18); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$coworking" + ifaces="ether$i ${ifaces}" +done +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) untagged=$(commastring $ifaces) vlan-ids=$coworking" + +# Server 19-20 +ifaces="" +for i in $(seq 19 20); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$server" + ifaces="ether$i ${ifaces}" +done +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) untagged=$(commastring $ifaces) vlan-ids=$server" + +# Not modified 21-22 + +# Tagged 23-24, sfp-sfpplus1 +for iface in ; do + conf "/interface bridge port add bridge=$bridge interface=$iface hw=yes" +done + +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) lan-ids=$(commastring $other)" +conf "/interface vlan add interface=$bridge vlan-id=$internal name=MGMT" +conf "/ipv6 address add eui-64=yes advertise=no address=$net_internal interface=MGMT" +conf "/interface bridge set $bridge vlan-filtering=yes" + +# Show neigh +conf "/interface bridge host print where !local" From 5e966b852d30ce9a7b2354033d7157810e84288a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Aug 2020 15:01:56 +0200 Subject: [PATCH 151/164] setup password in opennebula --- opennebula-images/centos7-build-luks-opennebula-image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 69a5c64..91df92b 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -119,6 +119,9 @@ cat > /mnt/etc/hosts << EOF ::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 EOF +# Setup root password +run_root passwd + # Install one-context RPM and hope things works as expected. curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" From 8429201d0df3c9b5d52956bc25fdebef2cf955dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 7 Aug 2020 15:03:09 +0200 Subject: [PATCH 152/164] + mikrotik scripts --- mikrotik-configure-crs326-dumb.sh | 59 ++++++++++++++ mikrotik-configure-crs326-with-vlans.sh | 103 ++++++++++++++++++++++++ 2 files changed, 162 insertions(+) create mode 100755 mikrotik-configure-crs326-dumb.sh create mode 100755 mikrotik-configure-crs326-with-vlans.sh diff --git a/mikrotik-configure-crs326-dumb.sh b/mikrotik-configure-crs326-dumb.sh new file mode 100755 index 0000000..a2689d7 --- /dev/null +++ b/mikrotik-configure-crs326-dumb.sh @@ -0,0 +1,59 @@ +#!/bin/sh +# Nico Schottelius, 2020-08-03 +# Setup a standard crs326 + + + +if [ $# -ne 4 ]; then + echo "$0 " + echo "Example:" + echo "$0 fe80::764d:28ff:fe09:9355%eth1 2a0a:e5c0:2::/64 mikrotik-crs326-8 \$(pass ...)" + exit 1 +fi + +ip=$1; shift +newip=$1; shift +hostname=$1; shift +password=$1; shift + +target=$ip +bridge=bridge + +conf() { + echo $@ + ssh admin@${target} "$@" +} + +commastring() { + echo $@ | sed 's/ /,/g' +} + +conf "/system identity set name=$hostname" +conf "/interface bridge add name=$bridge" + +################################################################################ +# MTU + +for i in $(seq 1 24); do + conf "/interface ethernet set ether$i mtu=9200 l2mtu=9204" + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes" +done + + +for i in $(seq 1 2); do + conf "/interface ethernet set sfp-sfpplus$i mtu=9200 l2mtu=9204" + conf "/interface bridge port add bridge=$bridge interface=sfp-sfpplus$i hw=yes" +done + + +################################################################################ +# IPv6 address, password + +conf "/ipv6 address add eui-64=yes advertise=no address=$newip interface=$bridge" +conf "/ipv6 address print" +conf "/password old-password=\"\" new-password=$password confirm-new-password=$password" + +# Show neigh +conf "/interface bridge host print where !local" + +echo "do not forget to set a password" diff --git a/mikrotik-configure-crs326-with-vlans.sh b/mikrotik-configure-crs326-with-vlans.sh new file mode 100755 index 0000000..4ae1b62 --- /dev/null +++ b/mikrotik-configure-crs326-with-vlans.sh @@ -0,0 +1,103 @@ +#!/bin/sh +# Nico Schottelius, 2020-08-03 +# Setup a standard crs326 + + + +if [ $# -ne 2 ]; then + echo "$0 " + echo "Example:" + echo "$0 fe80::764d:28ff:fe09:9355%eth1 mikrotik-crs326-8" + exit 1 +fi + +ip=$1; shift +hostname=$1; shift +password=$1; shift + +target=$ip +bridge=bridgevlans + +internal=10 +coworking=15 +server=11 +other="8 16 18 33 34" + +tagged="ether23 ether24 sfp-sfpplus1" + +net_internal=2a0a:e5c0:2::/64 + +conf() { + echo $@ + ssh admin@${target} "$@" +} + +commastring() { + echo $@ | sed 's/ /,/g' +} + +#set -x + + +# do this out of band -- see mikrotik-setup.sh +#conf "/password new-password=$password confirm-new-password=$password old-password=\"\"" + +conf "/system identity set name=$hostname" +conf "/interface bridge add name=$bridge" + +################################################################################ +# MTU + +for i in $(seq 1 24); do + conf "/interface ethernet set ether$i mtu=9200 l2mtu=9204" +done + + +for i in $(seq 1 2); do + conf "/interface ethernet set sfp-sfpplus$i mtu=9200 l2mtu=9204" +done + + +################################################################################ +# VLANs + +# Internal ports 1-16 +ifaces="" +for i in $(seq 1 16); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$internal" + ifaces="ether$i ${ifaces}" +done + +# also tag the bridge for the vlan interface we need later +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged),$bridge untagged=$(commastring $ifaces) vlan-ids=$internal" + +# Coworking 17-18 +ifaces="" +for i in $(seq 17 18); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$coworking" + ifaces="ether$i ${ifaces}" +done +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) untagged=$(commastring $ifaces) vlan-ids=$coworking" + +# Server 19-20 +ifaces="" +for i in $(seq 19 20); do + conf "/interface bridge port add bridge=$bridge interface=ether$i hw=yes pvid=$server" + ifaces="ether$i ${ifaces}" +done +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) untagged=$(commastring $ifaces) vlan-ids=$server" + +# Not modified 21-22 + +# Tagged 23-24, sfp-sfpplus1 +for iface in $tagged; do + conf "/interface bridge port add bridge=$bridge interface=$iface hw=yes" +done + +conf "/interface bridge vlan add bridge=$bridge tagged=$(commastring $tagged) vlan-ids=$(commastring $other)" +conf "/interface vlan add interface=$bridge vlan-id=$internal mtu=9200 name=internal" +conf "/ipv6 address add eui-64=yes advertise=no address=$net_internal interface=internal" +conf "/interface bridge set $bridge vlan-filtering=yes" + +# Show neigh +conf "/interface bridge host print where !local" From 7f07870d7f58d0342d052b5aadb66cd9f87e3156 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 24 Aug 2020 09:48:10 +0200 Subject: [PATCH 153/164] phase in pib setup --- pib-setup.sh | 168 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 168 insertions(+) create mode 100755 pib-setup.sh diff --git a/pib-setup.sh b/pib-setup.sh new file mode 100755 index 0000000..b0e4438 --- /dev/null +++ b/pib-setup.sh @@ -0,0 +1,168 @@ +#!/bin/sh + +if [ $# -ne 2 ]; then + echo "$0 ip-address vpn-network" + echo " ip-address: where to find the PIB" + echo " network: 2a0a:e5c0:123::/48" + exit 1 +fi + + + +my_ip=$1; shift +my_network=$1; shift + +my_prefix=$(echo $my_network | sed 's,::/.*,,') +my_hostname=pib-$(echo ${my_prefix} | sed 's/:/-/g') + +my_wireguard_ip=${my_prefix}::42 +my_lan_ip=${my_prefix}:cafe::42 +my_lanv6_ip=${my_prefix}:7ea::42 + +private_key=$(wg genkey) +public_key=$(echo $private_key | wg pubkey) + +vpn_endpoint_host=vpn-2a0ae5c1.ungleich.ch +vpn_endpoint_pubkey=hi60lGP+xEUQ+kVnqA7PlJAO1SVqTS1W36g0LhFP0xQ= + +cat < ${my_hostname}.public_key + +exit 0 + + +# wget -O - https://downloads.openwrt.org/releases/19.07.3/targets/x86/64/openwrt-19.07.3-x86-64-rootfs-ext4.img.gz | gunzip > /dev/sda From 912dcabe9feb16230705d1cdb78210f948bd4824 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 24 Aug 2020 10:00:02 +0200 Subject: [PATCH 154/164] +passwd Signed-off-by: Nico Schottelius --- opennebula-images/centos7-build-luks-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opennebula-images/centos7-build-luks-opennebula-image.sh b/opennebula-images/centos7-build-luks-opennebula-image.sh index 91df92b..6b7fe1d 100755 --- a/opennebula-images/centos7-build-luks-opennebula-image.sh +++ b/opennebula-images/centos7-build-luks-opennebula-image.sh @@ -96,7 +96,7 @@ dnf -y \ --enablerepo=base \ --enablerepo=extras \ --setopt=install_weak_deps=False install \ - bash basesystem systemd dnf centos-release cryptsetup dnf + bash basesystem systemd dnf centos-release cryptsetup dnf passwd mount --bind /dev /mnt/dev mount --bind /dev/pts /mnt/dev/pts From 3470ca62df000f5013fa818c7a9071d5c835cfd3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 7 Sep 2020 16:20:36 +0200 Subject: [PATCH 155/164] [pib] update setup script --- pib-setup.sh | 94 +++++++++++++++++++++++++++------------------------- 1 file changed, 49 insertions(+), 45 deletions(-) diff --git a/pib-setup.sh b/pib-setup.sh index b0e4438..f95486e 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -1,25 +1,33 @@ #!/bin/sh +# Assumptions: +# - pib is factory reset +# - WAN port is connected with an active upstream +# - You are connected via LAN/can ssh into it + if [ $# -ne 2 ]; then - echo "$0 ip-address vpn-network" + echo "$0 ip-address vpn-network [wireguard-private-key]" echo " ip-address: where to find the PIB" echo " network: 2a0a:e5c0:123::/48" + echo " private-key: specify if you already have a private key" exit 1 fi - - my_ip=$1; shift my_network=$1; shift +if [ $# -eq 1 ]; then + private_key=$1; shift +else + private_key=$(wg genkey) +fi + my_prefix=$(echo $my_network | sed 's,::/.*,,') my_hostname=pib-$(echo ${my_prefix} | sed 's/:/-/g') my_wireguard_ip=${my_prefix}::42 my_lan_ip=${my_prefix}:cafe::42 -my_lanv6_ip=${my_prefix}:7ea::42 -private_key=$(wg genkey) public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1.ungleich.ch @@ -28,7 +36,16 @@ vpn_endpoint_pubkey=hi60lGP+xEUQ+kVnqA7PlJAO1SVqTS1W36g0LhFP0xQ= cat < ${my_hostname}.public_key -exit 0 +cat < /dev/sda +- Remove your ssh key(s) from the device +- Setup a secure root password and forward it to the customer +EOF From caabcc2f47c88dfa38911cf0ab4e993a2b1ddb12 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 7 Sep 2020 16:52:14 +0200 Subject: [PATCH 156/164] [pib] cleanup / document the script --- pib-setup.sh | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/pib-setup.sh b/pib-setup.sh index f95486e..39b7c0b 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -1,11 +1,14 @@ #!/bin/sh # Assumptions: -# - pib is factory reset -# - WAN port is connected with an active upstream -# - You are connected via LAN/can ssh into it +# - pib (APU) is factory reset OpenWRT +# - WAN port is connected with an active upstream (pib has internet connectivity via WAN port) +# - You are connected via LAN and you can ssh into it -if [ $# -ne 2 ]; then +# How it works +# + +if [ $# -lt 2 ]; then echo "$0 ip-address vpn-network [wireguard-private-key]" echo " ip-address: where to find the PIB" echo " network: 2a0a:e5c0:123::/48" @@ -44,8 +47,7 @@ opkg update opkg install libustream-openssl ca-bundle ca-certificates # install wireguard + gui -opkg install wireguard -opkg install luci-app-wireguard +opkg install wireguard luci-app-wireguard # We are never authoritative for IPv4 uci delete dhcp.@dnsmasq[0].authoritative @@ -53,6 +55,9 @@ uci delete dhcp.@dnsmasq[0].authoritative # Do not announce ULA - we have GUA uci delete network.globals.ula_prefix +# Setup hostname +uci set system.@system[0].hostname="${my_hostname}" + # Do not set/get? Was necessary, don't recall why uci set dhcp.@dnsmasq[0].noresolv='1' @@ -107,8 +112,6 @@ uci set network.@wireguard_wg0[0].endpoint_host='${vpn_endpoint_host}' uci set network.@wireguard_wg0[0].endpoint_port='51820' uci set network.@wireguard_wg0[0].route_allowed_ips='1' -uci set system.@system[0].hostname="${my_hostname}" - # Firewall configuration if ! uci show firewall | grep "name='Allow-SSH'"; then uci add firewall rule @@ -149,16 +152,19 @@ uci delete dhcp.@dnsmasq[0].server uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:0:a::a' uci add_list dhcp.@dnsmasq[0].server='2a0a:e5c0:2:a::a' +# This is the save & apply button in LUCI (or just save button) uci commit + reboot EOF my_ip=$my_lan_ip + echo "Waiting for it to come back..." while ! ping -c1 ${my_ip}; do echo "Cannot ping $my_ip yet - waiting" - sleep 1 + sleep 2 done echo "Wireguard public key and id: ${id} ${public_key}" @@ -169,4 +175,5 @@ Open steps: - Remove your ssh key(s) from the device - Setup a secure root password and forward it to the customer + EOF From ddbc95f6f01f4de9ffaa01d34691244f9e7e2ef0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 8 Sep 2020 17:19:51 +0200 Subject: [PATCH 157/164] exit if ssh breaks --- pib-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pib-setup.sh b/pib-setup.sh index 39b7c0b..1bbdd71 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -36,7 +36,7 @@ public_key=$(echo $private_key | wg pubkey) vpn_endpoint_host=vpn-2a0ae5c1.ungleich.ch vpn_endpoint_pubkey=hi60lGP+xEUQ+kVnqA7PlJAO1SVqTS1W36g0LhFP0xQ= -cat < Date: Tue, 8 Sep 2020 17:22:52 +0200 Subject: [PATCH 158/164] [pib setup] ping6 alternatively before --- pib-setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pib-setup.sh b/pib-setup.sh index 1bbdd71..13db223 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -40,7 +40,7 @@ cat < Date: Tue, 8 Sep 2020 17:36:32 +0200 Subject: [PATCH 159/164] ++comment phrasing --- pib-setup.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pib-setup.sh b/pib-setup.sh index 13db223..04efb70 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -173,7 +173,8 @@ echo ${public_key} > ${my_hostname}.public_key cat < Date: Tue, 8 Sep 2020 17:37:20 +0200 Subject: [PATCH 160/164] ++notes --- pib-setup.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/pib-setup.sh b/pib-setup.sh index 04efb70..649542a 100755 --- a/pib-setup.sh +++ b/pib-setup.sh @@ -176,5 +176,6 @@ Open steps: - Remove your ssh key(s) from the device (if any are present) - Setup a secure root password and forward it to the customer - Ensure that the VPN works +- Connect to the LAN port and surf in the Internet IPv6 only! EOF From 55f2843ce30b2644113e28adf9884d28bf5bd873 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 11 Sep 2020 13:23:14 +0200 Subject: [PATCH 161/164] [openwrt] add new script to configure LTE uplink --- openwrt-add-lte.sh | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 openwrt-add-lte.sh diff --git a/openwrt-add-lte.sh b/openwrt-add-lte.sh new file mode 100644 index 0000000..5e3a2c7 --- /dev/null +++ b/openwrt-add-lte.sh @@ -0,0 +1,43 @@ +#!/bin/sh +# Based on work of Samuel Hailu, 2020-09-10 + +if [ $# -ne 2 ]; then + echo "$0 ip-address interface" + echo " ip-address: where to find the OpenWRT device" + echo " interface: which interface (eth3 for instance) is the LTE device" + exit 1 +fi + +my_ip=$1; shift +interface=$1; shift + +cat < Date: Thu, 24 Sep 2020 15:13:24 +0200 Subject: [PATCH 162/164] [openwrt] WIP: add new scripts for acme and camera --- openwrt-add-letsencrypt.sh | 6 ++++ openwrt-motion-config.sh | 57 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 openwrt-add-letsencrypt.sh create mode 100644 openwrt-motion-config.sh diff --git a/openwrt-add-letsencrypt.sh b/openwrt-add-letsencrypt.sh new file mode 100644 index 0000000..43e3b4b --- /dev/null +++ b/openwrt-add-letsencrypt.sh @@ -0,0 +1,6 @@ + +opkg install luci-ssl-openssl acme luci-app-acme + +uci set uhttpd.main.redirect_https=1 +uci commit +/etc/init.d/uhttpd restart diff --git a/openwrt-motion-config.sh b/openwrt-motion-config.sh new file mode 100644 index 0000000..b4cb5cc --- /dev/null +++ b/openwrt-motion-config.sh @@ -0,0 +1,57 @@ +#!/bin/sh + + +if [ $# -ne 1 ]; then + echo "$0 ip-address interface" + echo " ip-address: where to find the OpenWRT device" + exit 1 +fi + +my_ip=$1; shift + +cat < Date: Thu, 24 Sep 2020 16:07:25 +0200 Subject: [PATCH 163/164] ++camera setting --- openwrt-motion-config.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/openwrt-motion-config.sh b/openwrt-motion-config.sh index b4cb5cc..1648f90 100644 --- a/openwrt-motion-config.sh +++ b/openwrt-motion-config.sh @@ -55,3 +55,6 @@ webcontrol_parms 0 stream_port 8081 stream_localhost off + +# use native mjpeg +v4l2_palette 8 From ea78ea8f67b71741e76d5b355abaf437542644c9 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Sat, 26 Sep 2020 14:47:59 +0200 Subject: [PATCH 164/164] Add new file --- get_ceph_recovery_config | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 get_ceph_recovery_config diff --git a/get_ceph_recovery_config b/get_ceph_recovery_config new file mode 100644 index 0000000..5c18269 --- /dev/null +++ b/get_ceph_recovery_config @@ -0,0 +1,18 @@ +#!/bin/bash -e + +rm -rf ~/rc_test_result + +osd_list=( $(ceph tell 'osd.*' config get osd_recovery_sleep_hdd | awk '{print $1}') ) +osd_recovery_sleep_hdd=( $(ceph tell 'osd.*' config get osd_recovery_sleep_hdd | awk '{print $2}') ) +osd_recovery_op_priority=( $(ceph tell 'osd.*' config get osd_recovery_op_priority | awk '{print $2}') ) +osd_recovery_max_single_start=( $(ceph tell 'osd.*' config get osd_recovery_max_single_start | awk '{print $2}') ) +osd_recovery_sleep_hybrid=( $(ceph tell 'osd.*' config get osd_recovery_sleep_hybrid | awk '{print $2}') ) +osd_max_backfills=( $(ceph tell 'osd.*' config get osd_max_backfills | awk '{print $2}') ) +osd_recovery_max_active=( $(ceph tell 'osd.*' config get osd_recovery_max_active | awk '{print $2}') ) +osd_recovery_priority=( $(ceph tell 'osd.*' config get osd_recovery_priority |awk '{print $2}') ) +osd_client_op_priority=( $(ceph tell 'osd.*' config get osd_client_op_priority | awk '{print $2}') ) + +echo "|osd|osd_recovery_sleep_hdd|osd_recovery_op_priority|osd_recovery_max_single_start|osd_recovery_sleep_hybrid|osd_max_backfills|osd_recovery_max_active|osd_recovery_priority|osd_client_op_priority|" >> ~/rc_test_result +for ((i=0; i<${#osd_max_backfills[@]}; i++)) do + echo "| "${osd_list[$i]}" |"${osd_recovery_sleep_hdd[$i]}" | "${osd_recovery_op_priority[$i]}" | "${osd_recovery_max_single_start}" | "${osd_recovery_sleep_hybrid[$i]}" | "${osd_max_backfills[$i]}" | "${osd_recovery_max_active[$i]}" | "${osd_recovery_priority[$i]}" | "${osd_client_op_priority[$i]}" |" >> ~/rc_test_result +done \ No newline at end of file