From fd319a934220130af57da3da3eda94071ef63889 Mon Sep 17 00:00:00 2001 From: llnu Date: Fri, 6 Dec 2019 18:06:37 +0100 Subject: [PATCH 01/66] ipxe script updated for alpine --- rebuild-ipxe.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rebuild-ipxe.sh b/rebuild-ipxe.sh index 561cd46..2d1ad5a 100755 --- a/rebuild-ipxe.sh +++ b/rebuild-ipxe.sh @@ -1,7 +1,8 @@ #!/bin/sh # Nico Schottelius, 2019-09-20, Seoul, Coffebean, 23:56 # Copying: GPLv3 - +echo "If you are running alpine, these packages are needed:" +echo "apk add alpine-sdk xz-dev" set -x set -e From a666916a72609551f1c6e39dd4546d5a928741f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 08:48:31 +0100 Subject: [PATCH 02/66] Fedora image: initialize /etc/hosts --- fedora-build-opennebula-image.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 77d6a0b..40f5d12 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -110,6 +110,13 @@ mount --bind /sys /mnt/sys # TODO: use non-$BIGCORP DNS service. echo 'nameserver 1.1.1.1' >> /mnt/etc/resolv.conf +# Initialize /etc/hosts. +cat > /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + # See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. # network-scripts.x86_64 : Legacy scripts for manipulating of network devices run_root dnf -y install network-scripts From 157379235c42f7bce8672a8122bd023ce99511c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 09:53:14 +0100 Subject: [PATCH 03/66] Fedora image: reset systemd machine-id and random-seed, clean dnf database --- fedora-build-opennebula-image.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 40f5d12..6222efc 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -134,6 +134,8 @@ run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service # Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... run_root dnf -y install kernel grub2 run_root grub2-install --target=i386-pc "${NBD_DEVICE}" run_root grub2-mkconfig -o /boot/grub2/grub.cfg @@ -150,5 +152,14 @@ UUID=$boot_uuid /boot ext4 rw,relatime,data=ordered 0 2 UUID=$root_uuid / ext4 rw,relatime,data=ordered 0 1 EOF +# Reset systemd's environment. +run_root rm -f /etc/machine-id +run_root touch /etc/machine-id +rm -f /var/lib/systemd/random-seed + +# Remove temporary files and reclaim freed disk space. +# Note: build logs could be removed as well. +run_root dnf clean all + # Make sure everything is written to disk before exiting. sync From 7e01fa5f06ba53c7d19d25a3ab83ab18ab598328 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 12:08:11 +0100 Subject: [PATCH 04/66] Fedora image: add virtio-blk driver to initramfs --- fedora-build-opennebula-image.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 6222efc..ac26d7d 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -137,6 +137,14 @@ run_root systemctl enable systemd-timesyncd.service # Note: linux-firmware is not required our environment and takes almost 200M # uncompressed but is a direct dependency of kernel-core... run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf < Date: Mon, 9 Dec 2019 14:57:42 +0100 Subject: [PATCH 05/66] Fedora image: mention build-time depency on e2fsprogs --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index ac26d7d..b120dc6 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -9,7 +9,7 @@ # definitely opinionated. # Depends on the following packages (as of Fedora 31): -# qemu-img util-linux coreutils dnf curl +# qemu-img util-linux coreutils dnf curl e2fsprogs # Run locally (without network) with: # qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 From aa01350594f48a88dc82236f60f87a8dca8eefb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 9 Dec 2019 18:58:41 +0100 Subject: [PATCH 06/66] Fedora image: use `date +%+F` instead of `date --iso-8601` --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index b120dc6..4d22522 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -20,7 +20,7 @@ set -x # XXX: Handle command-line arguments? RELEASE=31 ARCH=x86_64 -IMAGE_PATH=fedora-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G NBD_DEVICE=/dev/nbd1 From 366c9c65b2d2ca05329494254915ae3ee124d0a8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:39:03 +0100 Subject: [PATCH 07/66] begin devuan netboo script --- devuan-netboot.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh new file mode 100644 index 0000000..6e7f39c --- /dev/null +++ b/devuan-netboot.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +date=$(date +%F) +suite=ascii + +dir=${suit}-${date} + +debootstrap ${suite} From be9616a17f25e263656021bab4183de1f3a90f2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:43:41 +0100 Subject: [PATCH 08/66] ++ netboot devuan script base --- devuan-netboot.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh new file mode 100644 index 0000000..c47c92b --- /dev/null +++ b/devuan-netboot.sh @@ -0,0 +1,19 @@ +#!/bin/sh +# Nico Schottelius, 2019-12-09 + + +date=$(date +%F) +suite=ascii + +dir=${suit}-${date} + +debootstrap "${suite}" "${dir}" +chroot ${dir} apt install -y openssh-server rdnssd + +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet6 auto +EOF From 5e5e37a7d737f6743572656f5e24a7d3c83026eb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 19:57:57 +0100 Subject: [PATCH 09/66] [netboot] add ssh keys --- devuan-netboot.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index c47c92b..066472e 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -5,10 +5,12 @@ date=$(date +%F) suite=ascii -dir=${suit}-${date} +dir=${suite}-${date} +kernel=${dir}-kernel +keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 cat > ${dir}/etc/network/interfaces << EOF auto lo @@ -17,3 +19,13 @@ iface lo inet loopback auto eth0 iface eth0 inet6 auto EOF + +mv ${dir}/boot/vmlinuz-* ${kernel} +rm ${dir}/boot/initrd* +mkdir -p ${dir}/root/.ssh + +for key in balazs dominique jinguk nico; do + curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys +done + +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 2970833df53656ffb1fb0b726c3e7b8529125b8f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:01:46 +0100 Subject: [PATCH 10/66] pass in suite from outside --- devuan-netboot.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index 066472e..c627ec1 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -1,9 +1,14 @@ #!/bin/sh # Nico Schottelius, 2019-12-09 +if [ $# -ne 1 ]; then + echo $0 suite + echo suite is for instance ascii, beowulf, etc +fi + +suite=$1; shift date=$(date +%F) -suite=ascii dir=${suite}-${date} kernel=${dir}-kernel From 1870aeca15f8777bd23f18fb1dbb36d2748c159e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:02:21 +0100 Subject: [PATCH 11/66] +exit error --- devuan-netboot.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/devuan-netboot.sh b/devuan-netboot.sh index c627ec1..3dbb17f 100644 --- a/devuan-netboot.sh +++ b/devuan-netboot.sh @@ -4,6 +4,7 @@ if [ $# -ne 1 ]; then echo $0 suite echo suite is for instance ascii, beowulf, etc + exit 1 fi suite=$1; shift From caca8ea43f7cabdaaeebc5b29c2d2c527b0ae844 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:02:50 +0100 Subject: [PATCH 12/66] +perms --- build-alpine-chroot.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 build-alpine-chroot.sh diff --git a/build-alpine-chroot.sh b/build-alpine-chroot.sh old mode 100644 new mode 100755 From 6cfdf9d40ee30f74f0017e020b27ad9bfe42a77e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:03:30 +0100 Subject: [PATCH 13/66] +perms --- devuan-netboot.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh old mode 100644 new mode 100755 From 8ed587eb0ad3c15d1477bb2805f93f04f3d3044e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 20:04:46 +0100 Subject: [PATCH 14/66] rename - script is suitable for debian and devuan --- devuan-netboot.sh | 37 ------------------------------------- 1 file changed, 37 deletions(-) delete mode 100755 devuan-netboot.sh diff --git a/devuan-netboot.sh b/devuan-netboot.sh deleted file mode 100755 index 3dbb17f..0000000 --- a/devuan-netboot.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/sh -# Nico Schottelius, 2019-12-09 - -if [ $# -ne 1 ]; then - echo $0 suite - echo suite is for instance ascii, beowulf, etc - exit 1 -fi - -suite=$1; shift - -date=$(date +%F) - -dir=${suite}-${date} -kernel=${dir}-kernel -keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files - -debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 - -cat > ${dir}/etc/network/interfaces << EOF -auto lo -iface lo inet loopback - -auto eth0 -iface eth0 inet6 auto -EOF - -mv ${dir}/boot/vmlinuz-* ${kernel} -rm ${dir}/boot/initrd* -mkdir -p ${dir}/root/.ssh - -for key in balazs dominique jinguk nico; do - curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys -done - -( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 0c56febefe7dd40d0f801d5af9111c8746ad0df9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 9 Dec 2019 21:40:42 +0100 Subject: [PATCH 15/66] [fedora] use resolv.conf from the host --- fedora-build-opennebula-image.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 4d22522..f078042 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -107,8 +107,7 @@ mount --bind /sys /mnt/sys # systemd-networkd. # Required to resolve package mirror in chroot. -# TODO: use non-$BIGCORP DNS service. -echo 'nameserver 1.1.1.1' >> /mnt/etc/resolv.conf +cp /etc/resolv.conf /mnt/etc/resolv.conf # Initialize /etc/hosts. cat > /mnt/etc/hosts << EOF From 7bcdd02c9ec619dd450e6f0e171442e94127e835 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:11:48 +0100 Subject: [PATCH 16/66] Add new file for getting VM List --- vm_list | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 vm_list diff --git a/vm_list b/vm_list new file mode 100644 index 0000000..7d77eee --- /dev/null +++ b/vm_list @@ -0,0 +1,19 @@ +#!/bin/bash -e +#option $1 is ldap password +#option $2 is ou + + +uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) + +for ((i=0; i<${#uid_list[@]}; i++)) do + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=customer,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_vmid=() + list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) + for ((j=0; j<${#list_vmid[@]}; j++)) do + temp=$(onevm show ${list_vmid[$j]} | grep PORT) + temp1="${temp#*\"}" + port="${temp1%%\"*}" + host=$(onevm show ${list_vmid[$j]} | grep HOST | grep ungleich | awk '{print $3}') + echo ${uid_list[$i]} ${list_vmid[$j]} $port $host >> ~/vm_vnc_list + done +done \ No newline at end of file From cdd7779e2ce2654a089e20ad09deab5e68d0f2ab Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:12:52 +0100 Subject: [PATCH 17/66] Update vm_list --- vm_list | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vm_list b/vm_list index 7d77eee..95f64be 100644 --- a/vm_list +++ b/vm_list @@ -6,7 +6,7 @@ uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) for ((i=0; i<${#uid_list[@]}; i++)) do - list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=customer,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=$2,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) list_vmid=() list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) for ((j=0; j<${#list_vmid[@]}; j++)) do From c7f02bce5c9d0307a7e1d07006d9a87c490b5bb4 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:23:28 +0100 Subject: [PATCH 18/66] Add new file --- create-guacamole-session-ldap-DB | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 create-guacamole-session-ldap-DB diff --git a/create-guacamole-session-ldap-DB b/create-guacamole-session-ldap-DB new file mode 100644 index 0000000..25819e2 --- /dev/null +++ b/create-guacamole-session-ldap-DB @@ -0,0 +1,39 @@ +#!/bin/bash +#option $1 is vm_list file name +#option $2 id DB location +#option $3 is DB user +#option $4 is DB name + +host='localhost' + +user_arr=( $(cat $1 | awk '{print $1}' )) +vmid_arr=( $(cat $1 | awk '{print $2}' )) +port_arr=( $(cat $1 | awk '{print $3}' )) +place_arr=( $(cat $1 | awk '{print $4}' )) + +for ((i=0; i<${#user_arr[@]}; i++)) do + #create user + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_entity (name, type) VALUES ('${user_arr[i]}','USER');" + en_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT entity_id FROM guacamole_entity WHERE name = '${user_arr[i]}';") + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('$en_id', '\x74657374', now());" + + #create connection + cn=${user_arr[i]}${vmid_arr[i]} + echo $cn + if [ 0 -eq $(psql -h $2 -U $3 -d $4 -tAc "SELECT connection_id FROM guacamole_connection WHERE connection_name = '$cn';" | wc -l) ]; then + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');" + cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") + + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');" + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');" + + #connection permission + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');" + + else + cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';" + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';" + fi + +done \ No newline at end of file From 36cc9b3c50ace54519c62bae43f32c2deeda4f1d Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 11 Dec 2019 05:29:47 +0100 Subject: [PATCH 19/66] Add new file --- create-guacamole-session-ldap-file | 38 ++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 create-guacamole-session-ldap-file diff --git a/create-guacamole-session-ldap-file b/create-guacamole-session-ldap-file new file mode 100644 index 0000000..c11b4bc --- /dev/null +++ b/create-guacamole-session-ldap-file @@ -0,0 +1,38 @@ +#!/bin/bash +#option $1 is vm_list file name +#option $2 is DB name +#this script should be run on guacamole server + + +host='localhost' +user_arr=( $(cat $1 | awk '{print $1}' )) +vmid_arr=( $(cat $1 | awk '{print $2}' )) +port_arr=( $(cat $1 | awk '{print $3}' )) +place_arr=( $(cat $1 | awk '{print $4}' )) + +for ((i=0; i<${#user_arr[@]}; i++)) do + #create user + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_entity (name, type) VALUES ('${user_arr[i]}','USER');\"" + en_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT entity_id FROM guacamole_entity WHERE name = '${user_arr[i]}';\"") + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('$en_id', '\x74657374', now());\"" + + #create connection + cn=${user_arr[i]}${vmid_arr[i]} + + if [ 0 -eq $(su - postgres -c "psql postgres -d $2 -tAc \"SELECT connection_id FROM guacamole_connection WHERE connection_name = '$cn';\"" | wc -l) ]; then + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');\"" + cn_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;\"") + + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');\"" + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');\"" + + #connection permission + su - postgres -c "psql postgres -d $2 -tAc \"INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');\"" + + else + cn_id=$(su - postgres -c "psql postgres -d $2 -tAc \"SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;\"") + su - postgres -c "psql postgres -d $2 -tAc \"UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';\"" + su - postgres -c "psql postgres -d $2 -tAc \"UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';\"" + fi + +done \ No newline at end of file From 7d37b7d5a6960c35ac787aea753b5d98144c84fd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 11 Dec 2019 15:33:24 +0100 Subject: [PATCH 20/66] ++ script for debian netboot --- debian-devuan-netboot.sh | 42 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100755 debian-devuan-netboot.sh diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh new file mode 100755 index 0000000..9585709 --- /dev/null +++ b/debian-devuan-netboot.sh @@ -0,0 +1,42 @@ +#!/bin/sh +# Nico Schottelius, 2019-12-09 + +if [ $# -ne 1 ]; then + echo $0 suite + echo suite is for instance ascii, beowulf, etc + exit 1 +fi + +suite=$1; shift + +date=$(date +%F) + +dir=${suite}-${date} +kernel=${dir}-kernel +keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files + +debootstrap "${suite}" "${dir}" +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 + +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +auto eth0 +iface eth0 inet6 auto +EOF + +mv ${dir}/boot/vmlinuz-* ${kernel} +rm ${dir}/boot/initrd* +mkdir -p ${dir}/root/.ssh + +for key in balazs dominique jinguk nico; do + curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys +done + +# ensure there is /init in the initramfs -> otherwise there is a kernel panic +# reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init +# in our case, they are just the same +ln -s /sbin/init ${dir}/init + +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) From 96f0f94cfd4b0bd13058be8febba73c2b9443095 Mon Sep 17 00:00:00 2001 From: llnu Date: Wed, 11 Dec 2019 22:56:54 +0100 Subject: [PATCH 21/66] not working state --- debian-devuan-netboot.sh | 31 ++++++++++++++++++++++++++++--- magiccommand | 2 ++ 2 files changed, 30 insertions(+), 3 deletions(-) create mode 100755 magiccommand diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 9585709..799ee11 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -1,12 +1,16 @@ #!/bin/sh # Nico Schottelius, 2019-12-09 +# the ugly code is llnu + +#this can only run in the ungleich-tools directory because of the cat magiccommand........ if [ $# -ne 1 ]; then - echo $0 suite + echo $0 suite # rootpw echo suite is for instance ascii, beowulf, etc +# echo rootpw: set root password exit 1 fi - +#pw=$2 suite=$1; shift date=$(date +%F) @@ -16,7 +20,8 @@ kernel=${dir}-kernel keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 # ; \ +# echo "root:${pw}" | chgpasswd cat > ${dir}/etc/network/interfaces << EOF auto lo @@ -26,6 +31,23 @@ auto eth0 iface eth0 inet6 auto EOF + +#set hostname +echo "unconfigured-host" > ${dir}/etc/hostname + + +#add script to display eth0's ip +# the pretty but ugly code and not working +#echo 'echo $(ip -o -6 addr show | grep -E -v "lo|one" | awk '{print $1" " $2": "$4}') >> /etc/issue' > ${dir}/etc/rc.local + +#the also not working +#echo '* * * * * root ip -o -6 addr show | grep -E -v "lo|one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr + +# +#echo '* * * * * root ' > ${dir}/etc/cron.d/ipv6addr +cp ./magiccommand ${dir}/etc/cron.d/ipv6addr + + mv ${dir}/boot/vmlinuz-* ${kernel} rm ${dir}/boot/initrd* mkdir -p ${dir}/root/.ssh @@ -40,3 +62,6 @@ done ln -s /sbin/init ${dir}/init ( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) + +# tried to clear history but doesnt work :/ +# history -c diff --git a/magiccommand b/magiccommand new file mode 100755 index 0000000..e724d8e --- /dev/null +++ b/magiccommand @@ -0,0 +1,2 @@ +* * * * * root ip -o -6 addr show | grep -E -v "lo |one" | awk '{print $1" " $2": "$4}' >> /dev/tty1 + From fbcecc1bf2fe46cce7f7051fac4c60d258486b98 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 16:47:35 +0100 Subject: [PATCH 22/66] make cp instead of rm --- debian-devuan-netboot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 9585709..8937872 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -26,8 +26,8 @@ auto eth0 iface eth0 inet6 auto EOF -mv ${dir}/boot/vmlinuz-* ${kernel} -rm ${dir}/boot/initrd* +cp ${dir}/boot/vmlinuz-* ${kernel} +#rm ${dir}/boot/initrd* mkdir -p ${dir}/root/.ssh for key in balazs dominique jinguk nico; do From b549e1059dc7dfe0c88cf213c1d85e11426de3dd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 16:57:25 +0100 Subject: [PATCH 23/66] prepend out-dir for building image --- debian-devuan-netboot.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 8eb9817..fdc43f4 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -4,19 +4,21 @@ #this can only run in the ungleich-tools directory because of the cat magiccommand........ -if [ $# -ne 1 ]; then - echo $0 suite # rootpw +if [ $# -ne 2 ]; then + echo $0 suite out-directory + echo out-directory: into which directory to place resulting files echo suite is for instance ascii, beowulf, etc -# echo rootpw: set root password exit 1 fi -#pw=$2 + suite=$1; shift +outdir=$1; shift date=$(date +%F) -dir=${suite}-${date} +dir=${outdir}/${suite}-${date} kernel=${dir}-kernel +initramfs=${dir}-initramfs keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" @@ -33,7 +35,7 @@ EOF cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v "lo|one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v 'lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh @@ -46,4 +48,4 @@ done # in our case, they are just the same ln -s /sbin/init ${dir}/init -( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ../${dir}-initramfs ) +( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From 3824bb6d94a6a7841b31debb92a28621a28dffdf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:02:42 +0100 Subject: [PATCH 24/66] prefix kernel + initramfs --- debian-devuan-netboot.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index fdc43f4..c5d254a 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -17,8 +17,8 @@ outdir=$1; shift date=$(date +%F) dir=${outdir}/${suite}-${date} -kernel=${dir}-kernel -initramfs=${dir}-initramfs +kernel=kernel-${dir} +initramfs=initramfs-${dir} keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" From 279f86a1f0f042e8d5feeb513916aa7807dc58c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:14:13 +0100 Subject: [PATCH 25/66] read boot interface at boot --- debian-devuan-netboot.sh | 37 ++++++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index c5d254a..6fdeecb 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -23,15 +23,11 @@ keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/ debootstrap "${suite}" "${dir}" -chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 +# need non-free for firmware-bnx2 +echo 'deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free' > ${dir}/etc/apt/sources.list -cat > ${dir}/etc/network/interfaces << EOF -auto lo -iface lo inet loopback +chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 -auto eth0 -iface eth0 inet6 auto -EOF cp ${dir}/boot/vmlinuz-* ${kernel} @@ -43,6 +39,33 @@ for key in balazs dominique jinguk nico; do curl -s ${keyurl}/${key}.pub >> ${dir}/root/.ssh/authorized_keys done +################################################################################ +# networking + +# always lo +cat > ${dir}/etc/network/interfaces << EOF +auto lo +iface lo inet loopback + +EOF + +# find the boot interfaces at boot +cat > ${dir}/etc/rc.local < /etc/network/interfaces.d/bootinterface << eof +auto \$dev +iface \$dev inet6 auto +eof + +ifup "\${dev}" + +exit 0 + + +EOF + # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same From 3f2dece3f5bf566712a084ccee766427e6d7c1c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:24:25 +0100 Subject: [PATCH 26/66] ++update --- debian-devuan-netboot.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 6fdeecb..0d366fa 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -16,22 +16,24 @@ outdir=$1; shift date=$(date +%F) -dir=${outdir}/${suite}-${date} -kernel=kernel-${dir} -initramfs=initramfs-${dir} +basename=${suite}-${date} +dir=${outdir}/${basename} +kernel=${outdir}/kernel-${basename} +initramfs=${outdir}/initramfs-${basename} keyurl=https://code.ungleich.ch/ungleich-public/__ungleich_staff_ssh_access/raw/master/files debootstrap "${suite}" "${dir}" # need non-free for firmware-bnx2 -echo 'deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free' > ${dir}/etc/apt/sources.list +echo "deb http://pkgmaster.devuan.org/merged ${suite} main contrib non-free" > ${dir}/etc/apt/sources.list +chroot ${dir} apt update chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-bnx2 cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v 'lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v ' lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh @@ -62,8 +64,6 @@ eof ifup "\${dev}" exit 0 - - EOF # ensure there is /init in the initramfs -> otherwise there is a kernel panic @@ -71,4 +71,5 @@ EOF # in our case, they are just the same ln -s /sbin/init ${dir}/init +# Finally building the initramfs ( cd ${dir} ; find . | cpio -H newc -o | gzip -9 > ${initramfs} ) From d1ded3265ca4aeb7ca38224c0113bf731d65ded2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:26:49 +0100 Subject: [PATCH 27/66] do not nest single quotes --- debian-devuan-netboot.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index 0d366fa..a9e0228 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -33,7 +33,7 @@ chroot ${dir} apt install -y openssh-server rdnssd linux-image-amd64 firmware-b cp ${dir}/boot/vmlinuz-* ${kernel} -echo '* * * * * root ip -o -6 addr show | grep -E -v ' lo |one' > /etc/issue' > ${dir}/etc/cron.d/ipv6addr +echo '* * * * * root ip -o -6 addr show | grep -E -v " lo |one" > /etc/issue' > ${dir}/etc/cron.d/ipv6addr mkdir -p ${dir}/root/.ssh From b97d667753a2a0f5a427bf53e933d281039b8692 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 14 Dec 2019 17:42:12 +0100 Subject: [PATCH 28/66] include network config cfiles --- debian-devuan-netboot.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debian-devuan-netboot.sh b/debian-devuan-netboot.sh index a9e0228..d64de5e 100755 --- a/debian-devuan-netboot.sh +++ b/debian-devuan-netboot.sh @@ -49,6 +49,7 @@ cat > ${dir}/etc/network/interfaces << EOF auto lo iface lo inet loopback +source-directory /etc/network/interfaces.d EOF # find the boot interfaces at boot @@ -66,6 +67,8 @@ ifup "\${dev}" exit 0 EOF +chmod a+rx ${dir}/etc/rc.local" + # ensure there is /init in the initramfs -> otherwise there is a kernel panic # reason: initramfs is designed to be PRE regular os, so /init usually hands over to /sbin/init # in our case, they are just the same From 09a05b6a56f55fe27a37d26aaab13c65aa82a9f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sun, 15 Dec 2019 14:42:56 +0100 Subject: [PATCH 29/66] Add e2fsprogs to fedora image (used by one-context to resize fs) --- fedora-build-opennebula-image.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index f078042..3dddae1 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -125,6 +125,9 @@ curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" run_root rm "$ONE_CONTEXT_RPM_PATH" +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + # Initalize base services. run_root systemd-machine-id-setup run_root systemctl enable systemd-networkd.service From 65f5bfb99699328e5bfc5d3d28399f27512aac7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 16 Dec 2019 14:51:14 +0100 Subject: [PATCH 30/66] Do not enable unused systemd-networkd on Fedora image --- fedora-build-opennebula-image.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 3dddae1..219ad6e 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -130,7 +130,6 @@ run_root dnf -y install e2fsprogs # Initalize base services. run_root systemd-machine-id-setup -run_root systemctl enable systemd-networkd.service run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service From d11620a9fb7a1550df37d221711413cd2960704b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 17 Dec 2019 10:18:40 +0100 Subject: [PATCH 31/66] Fedora image: be sure to use right kernel version when generating initramfs --- fedora-build-opennebula-image.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 219ad6e..837ba6f 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -143,7 +143,8 @@ run_root dnf -y install kernel grub2 cat > /mnt/etc/dracut.conf.d/virtio-blk.conf < Date: Tue, 17 Dec 2019 11:21:54 +0100 Subject: [PATCH 32/66] Remove now irrelevant comment from Fedora image definition --- fedora-build-opennebula-image.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 837ba6f..5197acc 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -83,8 +83,6 @@ mount "${NBD_DEVICE}p2" /mnt mkdir /mnt/boot mount "${NBD_DEVICE}p1" /mnt/boot -# XXX: dnf has a lot a weird (libX11?) dependencies, use microdnf instead? - dnf -y \ --releasever=$RELEASE \ --installroot=/mnt \ From 5e438aafe83a10444fc68f35b4074748df4905c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Tue, 17 Dec 2019 13:56:05 +0100 Subject: [PATCH 33/66] Fedora image: increase /boot from 100M to 500M --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 5197acc..45c1629 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -68,7 +68,7 @@ trap cleanup EXIT # Create partition table, format partitions. sfdisk --no-reread "$NBD_DEVICE" < Date: Mon, 16 Dec 2019 13:46:34 +0100 Subject: [PATCH 34/66] Initial CentOS image build script --- centos-build-opennebula-image.sh | 170 +++++++++++++++++++++++++++++++ 1 file changed, 170 insertions(+) create mode 100755 centos-build-opennebula-image.sh diff --git a/centos-build-opennebula-image.sh b/centos-build-opennebula-image.sh new file mode 100755 index 0000000..6a8fe31 --- /dev/null +++ b/centos-build-opennebula-image.sh @@ -0,0 +1,170 @@ +#!/bin/sh + +# This script generates CentOS images for OpenNebula. + +# Depends on the following packages (as of CentOS 8): +# qemu-img util-linux coreutils dnf curl e2fsprogs + +# Run locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=8 +ARCH=x86_64 +IMAGE_PATH=centos-$RELEASE-$(date --iso-8601).img +IMAGE_SIZE=10G +LOOPBACK_DEVICE=/dev/loop0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm" +ONE_CONTEXT_RPM_PATH=/root/one-context.rpm + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + losetup -d "$LOOPBACK_DEVICE" +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ ! -f '/etc/centos-release' ]; then + echo "WARNING: this script has been designed to run on a CentOS system." >&2 + echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base RAW image (no LOOPBACK support in RHEL/CentOS). +qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE" +losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH" + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +{ +sfdisk --no-reread "$LOOPBACK_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details. +# network-scripts.x86_64 : Legacy scripts for manipulating of network devices +run_root dnf -y install network-scripts + +# Install (magic?) one-context RPM and hope things works as expected. +curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH" +run_root dnf -y install "$ONE_CONTEXT_RPM_PATH" +run_root rm "$ONE_CONTEXT_RPM_PATH" + +# Install resize2fs, which is required to resize the root file-system. +run_root dnf -y install e2fsprogs + +# Initalize base services. +run_root systemd-machine-id-setup +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime + +# Install and configure NTP client. +run_root dnf install -y chrony +run_root systemctl enable chronyd.service + +# Install kernel and bootloader. +# Note: linux-firmware is not required our environment and takes almost 200M +# uncompressed but is a direct dependency of kernel-core... +run_root dnf -y install kernel grub2 + +# Add support for virtio block devices at boot time. +cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <>/mnt/etc/fstab < Date: Thu, 26 Dec 2019 12:16:38 +0100 Subject: [PATCH 35/66] [vpn stats] added number of configured/active VPNs --- vpn-statistics.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vpn-statistics.sh b/vpn-statistics.sh index a1e7960..c721cf9 100755 --- a/vpn-statistics.sh +++ b/vpn-statistics.sh @@ -9,3 +9,10 @@ done # countries with counter ( for ip in $(wg | grep endpoint | sed -e 's/endpoint: //' -e 's/\(.*\):[0-9]*/\1/' -e 's/\[//' -e 's/\]//'); do curl -s ipinfo.io/$ip | grep -e country ; done ) | sort | uniq -c | sort -g + +# Get number of configured VPNs +configured_vpns=$(wg show | grep ^peer | wc -l) +active_vpns=$(wg show | grep endpoint | wc -l) + +echo "Configured VPNs: ${configured_vpns}" +echo "Active VPNs: ${active_vpns}" From 1d88aac14ee6a53288aa95fd6de4ad552b68a411 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Fri, 27 Dec 2019 11:06:45 +0100 Subject: [PATCH 36/66] [script]Add new file for ceph pg repair --- pg_repair | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 pg_repair diff --git a/pg_repair b/pg_repair new file mode 100644 index 0000000..428923f --- /dev/null +++ b/pg_repair @@ -0,0 +1,14 @@ +#!/bin/bash -e + +pglist_arr=( $(ceph health detail | grep pg | grep active | awk '{print $2}' )) + +echo ${pglist_arr[*]} + +for ((i=0; i<${#pglist_arr[@]}; i++)) do + if [ 1 -eq $(ceph pg repair ${pglist_arr[$i]} | grep repair | grep instructing | wc -l) ]; then + echo repair error + break + fi + echo ${pglist_arr[$i]} repair ok + sleep 10 +done \ No newline at end of file From 693b1bed0063ebd97586746d08cb4a9fb386b9dc Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Fri, 27 Dec 2019 11:08:24 +0100 Subject: [PATCH 37/66] [script]Update pg_repair for error message --- pg_repair | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pg_repair b/pg_repair index 428923f..caa9953 100644 --- a/pg_repair +++ b/pg_repair @@ -6,9 +6,9 @@ echo ${pglist_arr[*]} for ((i=0; i<${#pglist_arr[@]}; i++)) do if [ 1 -eq $(ceph pg repair ${pglist_arr[$i]} | grep repair | grep instructing | wc -l) ]; then - echo repair error + echo repair script error break fi - echo ${pglist_arr[$i]} repair ok + echo ${pglist_arr[$i]} repair script done sleep 10 done \ No newline at end of file From 54e4bc1d49f951a57576b86f9a9620d7b3df5a8a Mon Sep 17 00:00:00 2001 From: llnu Date: Thu, 2 Jan 2020 20:46:15 +0100 Subject: [PATCH 38/66] added alpine compatible version --- ceph-osd-create-start-alpine | 107 +++++++++++++++++++++++++++++++++++ 1 file changed, 107 insertions(+) create mode 100755 ceph-osd-create-start-alpine diff --git a/ceph-osd-create-start-alpine b/ceph-osd-create-start-alpine new file mode 100755 index 0000000..a19e1da --- /dev/null +++ b/ceph-osd-create-start-alpine @@ -0,0 +1,107 @@ +#!/bin/sh +# 17:19, 2018-02-09 +# Nico Schottelius + +# Based on ceph-disk -v prepare --bluestore /dev/sdc --osd-id ${ID} --osd-uuid $(uuidgen) --crush-device-class "ssd" + +# Create: +# - block -> link to partuuid +# - block_uuid -e> uuid if the block +# - ceph_fsid -> get from ceph-conf +# crush_device_class -> ssd, hdd +# fsid -> uuidgen! +# magic -> string "ceph osd volume v026" +# type -> bluestore + +fsid=$(ceph-conf --cluster=ceph --name=osd. --lookup fsid) +fs_uuid=$(uuidgen) +magic="ceph osd volume v026" + +set -x +set -e + +if [ $# -lt 2 ]; then + echo "$0 disk class [osdweight]" + echo "class = hdd or ssd" + exit 1 +fi + +export DEV=$1;shift +export CLASS=$1; shift + + +uuid_metadata=$(uuidgen) +uuid_block=$(uuidgen) + +osd_id=$(ceph osd create) + +dev_metadata="/dev/disk/by-partuuid/$uuid_metadata" +dev_block="/dev/disk/by-partuuid/$uuid_block" + +/usr/bin/sgdisk --new=0:0:+100M --change-name="0:ceph data" \ + --partition-guid="0:$uuid_metadata" \ + --typecode=0:4fbd7e29-9d25-41b8-afd0-062c0ceff05d \ + --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + +# Using gdisk --largest-new does not change the name or set guid; +# So use 2 steps instead +/usr/bin/sgdisk --largest-new=0 --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + + +lastpart=$(gdisk -l $DEV | tail -n1 | awk '{ print $1 }') +/usr/bin/sgdisk --change-name="${lastpart}:ceph block" \ + --partition-guid="${lastpart}:$uuid_block" \ + --typecode="${lastpart}:cafecafe-9b03-4f30-b4c6-b4b80ceff106" \ + --mbrtogpt -- $DEV +/sbin/udevadm settle --timeout=600 + +#echo $1 +#echo $(blkid | grep $1"2") + +#cblock=$(blkid | grep $1"2" | cut -d'"' -f4) +#echo $cblock + +/sbin/mkfs -t xfs -f -i size=2048 -- "$dev_metadata" + +mountpath=/var/lib/ceph/osd/ceph-${osd_id} + +mkdir -p "$mountpath" +mount "$dev_metadata" "$mountpath" + +ln -s $dev_block "$mountpath/block" + +echo "$uuid_block" > "$mountpath/block_uuid" +echo "$fsid" > "$mountpath/ceph_fsid" +echo "$magic" > "$mountpath/magic" +echo "$CLASS" > "$mountpath/crush_device_class" +echo $(echo $dev_block | cut -c23-) > "$mountpath/fsid" + + +# Important, otherwise --mkfs later will try to create filestore +echo bluestore > "$mountpath/type" + +ceph auth get-or-create "osd.${osd_id}" osd \ + 'allow *' mon 'allow profile osd' > $mountpath/keyring + +echo ${osd_id} > "$mountpath/whoami" +touch "$mountpath/openrc" + +ceph-osd --cluster ceph -i "${osd_id}" --mkfs +chown -R ceph:ceph "$mountpath" + +if [ $# -eq 1 ]; then + WEIGHT=$1; shift +else + devname=$(readlink -f $dev_block) + nodev=$(echo $devname | sed 's,/dev/,,') + WEIGHT=$(lsblk -l -b | awk "/^$nodev/ { print \$4/(1024^4) }") +fi + +ceph osd crush add osd.${osd_id} ${WEIGHT} host=$(hostname) + +echo "$metadata_dev /var/lib/ceph/osd/ceph-${osd_id} xfs noatime 0 0" >> /etc/fstab + +# Starting with monit, if available +ceph-osd -i ${osd_id} From fe27d294b1ca5884586a8713c169780030d3b7bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 7 Jan 2020 18:07:34 +0100 Subject: [PATCH 39/66] Add script to detect DNS64 prefix --- detect-dns64-prefix.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 detect-dns64-prefix.py diff --git a/detect-dns64-prefix.py b/detect-dns64-prefix.py new file mode 100644 index 0000000..e5bd179 --- /dev/null +++ b/detect-dns64-prefix.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +# Nico Schottelius, 2020-01-07 +# Detect the DNS64 prefix +# Based on https://tools.ietf.org/html/draft-ietf-behave-nat64-discovery-heuristic-05 + +import dns.resolver +import ipaddress + + +if __name__ == '__main__': + dns64_prefix = None + answers = dns.resolver.query('ipv4only.arpa', 'AAAA') + + for rdata in answers: + address = str(rdata) + network = ipaddress.IPv6Network("{}/96".format(address), + strict=False) + # print("{}: {}".format(rdata, network)) + print("{}".format(network)) From e56e1d587800cf31f7aa9d3f8f0023fb710fb8bf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 7 Jan 2020 18:08:53 +0100 Subject: [PATCH 40/66] + explanation --- detect-dns64-prefix.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/detect-dns64-prefix.py b/detect-dns64-prefix.py index e5bd179..1179ca4 100644 --- a/detect-dns64-prefix.py +++ b/detect-dns64-prefix.py @@ -2,6 +2,12 @@ # Nico Schottelius, 2020-01-07 # Detect the DNS64 prefix # Based on https://tools.ietf.org/html/draft-ietf-behave-nat64-discovery-heuristic-05 +# +# How it works: +# - ipv4only.arpa only has A records. +# - a DNS64 server will add AAAA records +# - we take this response (if any) and derive the IPv6 prefix from it +# import dns.resolver import ipaddress From 86ad491bbc404d3ec02fda72d798c8b3dbadf065 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 12 Jan 2020 00:27:03 +0100 Subject: [PATCH 41/66] [alpine] version bump --- alpine-rebuild-initramfs.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/alpine-rebuild-initramfs.sh b/alpine-rebuild-initramfs.sh index 643cc3f..b56454b 100755 --- a/alpine-rebuild-initramfs.sh +++ b/alpine-rebuild-initramfs.sh @@ -3,8 +3,8 @@ set -e set -x -MAJOR_VERSION=3.10 -MINOR_VERSION=3 +MAJOR_VERSION=3.11 +MINOR_VERSION=2 IMAGE=alpine-minirootfs-$MAJOR_VERSION.$MINOR_VERSION-x86_64.tar.gz SSH_KEYS=$(cat ~/.ssh/id_rsa.pub) RESOLVCONF=/etc/resolv.conf From edb8a359e8caa81333ae37145f085608189c4e59 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 16 Jan 2020 15:47:50 +0100 Subject: [PATCH 42/66] Update create-guacamole-session-ldap-DB --- create-guacamole-session-ldap-DB | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/create-guacamole-session-ldap-DB b/create-guacamole-session-ldap-DB index 25819e2..ce1e5cd 100644 --- a/create-guacamole-session-ldap-DB +++ b/create-guacamole-session-ldap-DB @@ -4,7 +4,7 @@ #option $3 is DB user #option $4 is DB name -host='localhost' +#host='localhost' user_arr=( $(cat $1 | awk '{print $1}' )) vmid_arr=( $(cat $1 | awk '{print $2}' )) @@ -24,15 +24,17 @@ for ((i=0; i<${#user_arr[@]}; i++)) do psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('$cn', 'vnc');" cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") - psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','$host');" + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','hostname','${place_arr[i]}');" psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','port','${port_arr[i]}');" #connection permission psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('$en_id', '$cn_id', 'READ');" + #clipboard-encoding + psql -h $2 -U $3 -d $4 -tAc "INSERT INTO guacamole_connection_parameter VALUES ('$cn_id','clipboard-encoding','UTF-8');" else cn_id=$(psql -h $2 -U $3 -d $4 -tAc "SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '$cn' AND parent_id IS NULL;") - psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='$host' where connection_id='$cn_id' and parameter_name='hostname';" + psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${place_arr[i]}' where connection_id='$cn_id' and parameter_name='hostname';" psql -h $2 -U $3 -d $4 -tAc "UPDATE guacamole_connection_parameter SET parameter_value='${port_arr[i]}' where connection_id='$cn_id' and parameter_name='port';" fi From ddc9ebaeaf509525c4f38b7079efa3c9ae1a0776 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Thu, 16 Jan 2020 15:59:26 +0100 Subject: [PATCH 43/66] Add new file --- vm_list_dual_uid | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 vm_list_dual_uid diff --git a/vm_list_dual_uid b/vm_list_dual_uid new file mode 100644 index 0000000..bf21c0b --- /dev/null +++ b/vm_list_dual_uid @@ -0,0 +1,20 @@ +#!/bin/bash -e +#option $1 is ldap password +#option $2 is ou + + +uid_list=( $(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "ou=$2,dc=ungleich,dc=ch" | grep uid: | awk '{print $2}') ) + +for ((i=0; i<${#uid_list[@]}; i++)) do + uid_temp=$(echo ${uid_list[i]} | sed "s/b'//g" | sed "s/'//g") + list_email[$i]=$(ldapsearch -x -H ldaps://ldap1.ungleich.ch:636 -D cn=manager,dc=ungleich,dc=ch -w $1 -b "uid=${uid_list[$i]},ou=$2,dc=ungleich,dc=ch" | grep mail: | awk '{print $2}' ) + list_vmid=() + list_vmid=( $(onevm list | grep ${list_email[$i]} | grep runn | awk '{print $1}' ) ) + for ((j=0; j<${#list_vmid[@]}; j++)) do + temp=$(onevm show ${list_vmid[$j]} | grep PORT) + temp1="${temp#*\"}" + port="${temp1%%\"*}" + host=$(onevm show ${list_vmid[$j]} | grep HOST | grep ungleich | awk '{print $3}') + echo $uid_temp ${list_vmid[$j]} $port $host >> ~/vm_vnc_list + done +done From 0e91be0d69b3fd47425921bb3a2667f825efbb91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:22:56 +0100 Subject: [PATCH 44/66] Add ldap-get-emails script --- ldap-get-emails | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100755 ldap-get-emails diff --git a/ldap-get-emails b/ldap-get-emails new file mode 100755 index 0000000..733811a --- /dev/null +++ b/ldap-get-emails @@ -0,0 +1,31 @@ +#!/bin/sh +# +# List mail addresses found under base DN $1 (defaults to dc=ungleich,dc=ch) + +set -e + +# Hardcoded parameters. +LDAP_SERVER="ldaps://ldap1.ungleich.ch" +LDAP_BIND_DN="cn=manager,dc=ungleich,dc=ch" + +if [ "$1" != "" ]; then + LDAP_SEARCH_BASE="$1" +else + LDAP_SEARCH_BASE="dc=ungleich,dc=ch" +fi + +# Read secrets from environment. +if [ "$LDAP_BIND_PASSWD" = "" ]; then + echo "You have to define LDAP_BIND_PASSWD before launching this script." >&2 + exit 1 +fi + +# Extract mail addresses from LDAP directory. +ldap_search_result="$( + ldapsearch -x -H "$LDAP_SERVER" \ + -D "$LDAP_BIND_DN" \ + -w "$LDAP_BIND_PASSWD" \ + -b "$LDAP_SEARCH_BASE" mail + )" + +echo "$ldap_search_result" | grep 'mail:' | cut -d ' ' -f 2 - From f3a219755c382d81d7dc385bce237932401c5bde Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:25:39 +0100 Subject: [PATCH 45/66] Add one-get-instances scripts --- one-get-instances | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 one-get-instances diff --git a/one-get-instances b/one-get-instances new file mode 100755 index 0000000..653fed6 --- /dev/null +++ b/one-get-instances @@ -0,0 +1,18 @@ +#!/bin/sh +# +# This script extract VM IDs and filter them if a pattern is provided as first +# argument. + +set -e + +# Extract instances from ONE. +instances=$(onevm list --csv | tail -n +2) + +# Filter them is a pattern has been provided. +if [ "$1" != "" ]; then + filtered_instances="$(echo "$instances" | grep -E "$1")" + instances="$filtered_instances" +fi + +# Outputs instance IDs. +echo "$instances" | cut -d ',' -f 1 - From 1f2ea6305f43c3a817f027fb218a872710175a0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 16 Jan 2020 17:40:54 +0100 Subject: [PATCH 46/66] Add one-inspect-instance-network script --- one-inspect-instance-network | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100755 one-inspect-instance-network diff --git a/one-inspect-instance-network b/one-inspect-instance-network new file mode 100755 index 0000000..70e5795 --- /dev/null +++ b/one-inspect-instance-network @@ -0,0 +1,18 @@ +#!/bin/sh +# +# This script is expected to run on the ONE server (i.e. +# opennebula.ungleich.ch). + +set -e + +# Fetch instance list from STDIN. +instances=$(cat -) + +# For every instance, extract relevant information: +for id in $instances; do + nics_raw="$(onevm show --xml $id | xml_grep 'NIC')" + networks="$(echo $nics_raw | xml_grep --text_only 'NETWORK' | tr '\n' ',' | sed 's/,$//')" + ip="$(echo $nics_raw | xml_grep --text_only 'IP' | tr '\n' ',' | sed 's/,$//')" + ip6="$(echo $nics_raw | xml_grep --text_only 'IP6_GLOBAL' | tr '\n' ',' | sed 's/,$//')" + echo "$id,$networks,$ip,$ip6" +done From fea52e0295056d5e5f9606c4cba027eca5ad567c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Sat, 18 Jan 2020 19:56:06 +0100 Subject: [PATCH 47/66] Add ubuntu ONE image definition --- ubuntu-build-opennebula-image.sh | 153 +++++++++++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 ubuntu-build-opennebula-image.sh diff --git a/ubuntu-build-opennebula-image.sh b/ubuntu-build-opennebula-image.sh new file mode 100644 index 0000000..6535f66 --- /dev/null +++ b/ubuntu-build-opennebula-image.sh @@ -0,0 +1,153 @@ +#!/bin/sh + +# This script generates Ubuntu images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=eoan # 19.10 +ARCH=amd64 +IMAGE_PATH=ubuntu-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Ubuntu" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat >/mnt/etc/apt/sources.list < "/mnt$ONE_CONTEXT_DEB_PATH" +run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" +run_root rm "$ONE_CONTEXT_DEB_PATH" + +# Manually install legacy network scripts used by one-context. +run_root apt-get -y install ifupdown + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and bootloader. Do not autoconfigure grub. +run_root echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections +run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-base linux-image-generic grub-pc + +# Configure grub. +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root apt-get -y install openssh-server + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Sat, 25 Jan 2020 10:17:49 +0100 Subject: [PATCH 48/66] Add hacking script for import vm info into etcd --- etcd_import_opennebula_vm.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 etcd_import_opennebula_vm.py diff --git a/etcd_import_opennebula_vm.py b/etcd_import_opennebula_vm.py new file mode 100644 index 0000000..d2c94c9 --- /dev/null +++ b/etcd_import_opennebula_vm.py @@ -0,0 +1,28 @@ +import json +import pprint +#import etcd3 + +with open("nico-vm-one.json", "r") as fd: + vmcontent = fd.read() + +#vm = json.loads(vmcontent.decode('utf-8')) +vm = json.loads(vmcontent) +pprint.pprint(vm['TEMPLATE']['DISK']) + +# storing info + +for_etcd={} +for_etcd['data_version'] = "1" +for_etcd['vm_id'] = vm['ID'] +for_etcd['owner'] = vm['UNAME'] + +for_etcd['disks'] = [] +for disk in vm['TEMPLATE']['DISK']: + disk_etcd = {} + disk_etcd['image_name'] = disk['IMAGE'] + disk_etcd['image_id'] = disk['IMAGE_ID'] + disk_etcd['datastore_name'] = disk['DATASTORE'] + disk_etcd['datastore_id'] = disk['DATASTORE_ID'] + for_etcd['disks'].append(disk_etcd) + +pprint.pprint(for_etcd) From 1ffc6f57681db8cc1eae76038be324afcceca97d Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 15:34:09 +0500 Subject: [PATCH 49/66] Added scripts related to getting info of OpenNebula VM into etcd and related queries --- .gitignore | 6 + opennebula-vm-etcd/config-and-secrets.conf | 18 +++ opennebula-vm-etcd/config.py | 12 ++ opennebula-vm-etcd/etcd_wrapper.py | 75 ++++++++++++ opennebula-vm-etcd/put-vm-info-into-etcd.py | 126 ++++++++++++++++++++ opennebula-vm-etcd/vm-queries.py | 55 +++++++++ 6 files changed, 292 insertions(+) create mode 100644 .gitignore create mode 100644 opennebula-vm-etcd/config-and-secrets.conf create mode 100644 opennebula-vm-etcd/config.py create mode 100644 opennebula-vm-etcd/etcd_wrapper.py create mode 100644 opennebula-vm-etcd/put-vm-info-into-etcd.py create mode 100644 opennebula-vm-etcd/vm-queries.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c093faa --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +opennebula-snapshot/config-and-secrets.conf + +*.pyc + +.idea/ +.vscode/ \ No newline at end of file diff --git a/opennebula-vm-etcd/config-and-secrets.conf b/opennebula-vm-etcd/config-and-secrets.conf new file mode 100644 index 0000000..b7faa08 --- /dev/null +++ b/opennebula-vm-etcd/config-and-secrets.conf @@ -0,0 +1,18 @@ +# Do not put single/double quotation mark for string as they are +# aslo considered as normal character. + +[oca] +client_secrets = ahmedbilal96@gmail.com:d00359fa33a74fcb5ea40bb088e299fd2ab85126 + +[etcd] +# url = localhost +# port = 2379 +# ca_cert +# cert_cert +# cert_key + +url = etcd1.ungleich.ch +port = 2379 +ca_cert = /home/meow/.cdist/files/etcd/ca.pem +cert_cert = /home/meow/.cdist/files/etcd/developer.pem +cert_key = /home/meow/.cdist/files/etcd/developer-key.pem \ No newline at end of file diff --git a/opennebula-vm-etcd/config.py b/opennebula-vm-etcd/config.py new file mode 100644 index 0000000..b329f4f --- /dev/null +++ b/opennebula-vm-etcd/config.py @@ -0,0 +1,12 @@ +import configparser + +from etcd_wrapper import EtcdWrapper + +config = configparser.ConfigParser(allow_no_value=True) +config.read('config-and-secrets.conf') + +etcd_client = EtcdWrapper( + host=config['etcd']['url'], port=config['etcd']['port'], + ca_cert=config['etcd']['ca_cert'], cert_key=config['etcd']['cert_key'], + cert_cert=config['etcd']['cert_cert'] +) diff --git a/opennebula-vm-etcd/etcd_wrapper.py b/opennebula-vm-etcd/etcd_wrapper.py new file mode 100644 index 0000000..9624677 --- /dev/null +++ b/opennebula-vm-etcd/etcd_wrapper.py @@ -0,0 +1,75 @@ +import etcd3 +import json + +from functools import wraps + +from uncloud import UncloudException +from uncloud.common import logger + + +class EtcdEntry: + def __init__(self, meta_or_key, value, value_in_json=True): + if hasattr(meta_or_key, 'key'): + # if meta has attr 'key' then get it + self.key = meta_or_key.key.decode('utf-8') + else: + # otherwise meta is the 'key' + self.key = meta_or_key + self.value = value.decode('utf-8') + + if value_in_json: + self.value = json.loads(self.value) + + +def readable_errors(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except etcd3.exceptions.ConnectionFailedError: + raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') + except etcd3.exceptions.ConnectionTimeoutError as err: + raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err + except Exception as err: + logger.exception('Some etcd error occured. See syslog for details.', err) + + return wrapper + + +class EtcdWrapper: + @readable_errors + def __init__(self, *args, **kwargs): + self.client = etcd3.client(*args, **kwargs) + + @readable_errors + def get(self, *args, value_in_json=True, **kwargs): + _value, _key = self.client.get(*args, **kwargs) + if _key is None or _value is None: + return None + return EtcdEntry(_key, _value, value_in_json=value_in_json) + + @readable_errors + def put(self, *args, value_in_json=True, **kwargs): + _key, _value = args + if value_in_json: + _value = json.dumps(_value) + + if not isinstance(_key, str): + _key = _key.decode('utf-8') + + return self.client.put(_key, _value, **kwargs) + + @readable_errors + def get_prefix(self, *args, value_in_json=True, **kwargs): + event_iterator = self.client.get_prefix(*args, **kwargs) + for e in event_iterator: + yield EtcdEntry(*e[::-1], value_in_json=value_in_json) + + @readable_errors + def watch_prefix(self, key, value_in_json=True): + event_iterator, cancel = self.client.watch_prefix(key) + for e in event_iterator: + if hasattr(e, '_event'): + e = getattr('e', '_event') + if e.type == e.PUT: + yield EtcdEntry(e.kv.key, e.kv.value, value_in_json=value_in_json) diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py new file mode 100644 index 0000000..70a1b09 --- /dev/null +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -0,0 +1,126 @@ +import pyone + +from enum import IntEnum +from config import config, etcd_client + +# How to get client secrets? +# 1. Login to OpenNebula +# 2. Go to Settings then Auth +# 3. Click on "Manage login tokens" button +# 4. Click on "Get a new token" button + +one_client = pyone.OneServer( + uri='https://opennebula.ungleich.ch:2634/RPC2', + session=config['oca']['client_secrets'] +) + + +def get_hostname_of_vm(vm_id): + host_pool = { + host.NAME: { + 'name': host.NAME, + 'id': host.ID, + 'cluster': { + 'name': host.CLUSTER, + 'id': host.CLUSTER_ID + }, + 'vms': host.VMS.ID + } + for host in one_client.hostpool.info().HOST + } + for hostname, host in host_pool.items(): + if vm_id in host['vms']: + return host + + return None + + +def put_under_list(obj): + if not isinstance(obj, list): + return [obj] + return obj + + +class Snapshot: + def __init__(self, disk_id, snapshot): + self.active = bool(snapshot.ACTIVE) + self.date = snapshot.DATE + self.id = snapshot.ID + self.name = snapshot.NAME + self.size = snapshot.SIZE + self.disk_id = disk_id + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + +class VM: + def __init__(self, vm): + self.name = vm.get_NAME() + self.id = vm.get_ID() + self.owner = { + 'name': vm.get_UNAME(), + 'id': vm.get_UID(), + } + + template = vm.get_TEMPLATE() + host = get_hostname_of_vm(self.id) + + self.vcpu = template.get('VCPU', None) + self.memory = template.get('MEMORY', None) + self.disks = [dict(disk) for disk in put_under_list(template.get('DISK', []))] + self.graphics = [dict(graphics) for graphics in put_under_list(template.get('GRAPHICS', []))] + self.nics = [dict(nic) for nic in put_under_list(template.get('NIC', []))] + self.status = pyone.VM_STATE(vm.get_STATE()).name.lower() + self.snapshots = [] + + for disk in one_client.vm.info(self.id).SNAPSHOTS: + disk_id = disk.DISK_ID + for snapshot in disk.SNAPSHOT: + self.snapshots.append(Snapshot(disk_id, snapshot).get_data()) + + if host: + self.host = { + 'name': host['name'], + 'id': host['id'] + } + else: + self.host = host + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + def __repr__(self): + return str(self.get_data()) + + +class VmFilterFlag(IntEnum): + UIDUserResources = 0 # UID User’s Resources + UserAndItsGroupsResources = -1 # Resources belonging to the user and any of his groups + AllResources = -2 # All resources + UserResources = -3 # Resources belonging to the user + UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group + + +def main(): + VM_STATES = list(pyone.VM_STATE) + START_ID = -1 # First id whatever it is + END_ID = -1 # Last id whatever it is + + for VM_STATE in VM_STATES: + vm_pool = one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, VM_STATE) + for i, vm in enumerate(vm_pool.VM): + vm = VM(vm) + etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + + +if __name__ == "__main__": + main() diff --git a/opennebula-vm-etcd/vm-queries.py b/opennebula-vm-etcd/vm-queries.py new file mode 100644 index 0000000..e92ef14 --- /dev/null +++ b/opennebula-vm-etcd/vm-queries.py @@ -0,0 +1,55 @@ +from pprint import pprint + +from config import config, etcd_client + + +def get_vm_by_ip(vms, ip, status='active'): + vms_by_status = { + vm_id: vm + for vm_id, vm in vms.items() + if vm['status'] == status + } + for vm_id, vm in vms_by_status.items(): + vm_ips = [] + for nic in vm.get('nics', []): + global_ipv6 = nic.get('IP6_GLOBAL', None) + local_ipv6 = nic.get('IP6_LINK', None) + ipv4 = nic.get('IP', None) + vm_ips += [global_ipv6, local_ipv6, ipv4] + + if ip in vm_ips: + return {vm_id: vm} + return None + + +def main(): + vm_prefix = '/opennebula/vm/' + + vms = { + int(vm.key.split('/')[-1]): vm.value + for vm in etcd_client.get_prefix(vm_prefix) + } + + VM_ID = 10761 # One of nico's VM + + # Get all data related to a VM + pprint(vms.get(VM_ID)) + + # Get host of a VM + print(vms.get(VM_ID).get('host').get('name')) + + # Get VNC Port of a VM + print(vms.get(VM_ID).get('graphics')[0].get('PORT')) + + # Get all disks attached with VM + pprint(vms.get(VM_ID).get('disks')) + + # Who is owner of a VM? + print(vms.get(VM_ID).get('owner').get('name')) + + # Get VM who has 2a0a:e5c0:0:5:0:78ff:fe11:d75f + search_ungleich_ch = get_vm_by_ip(vms, '2a0a:e5c0:0:5:0:78ff:fe11:d75f') + pprint(search_ungleich_ch) + +if __name__ == '__main__': + main() From 89244dc4ee89414db01621965e29a070512359f5 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 15:51:31 +0500 Subject: [PATCH 50/66] remove uncloud specific things --- opennebula-vm-etcd/etcd_wrapper.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/opennebula-vm-etcd/etcd_wrapper.py b/opennebula-vm-etcd/etcd_wrapper.py index 9624677..f448dcf 100644 --- a/opennebula-vm-etcd/etcd_wrapper.py +++ b/opennebula-vm-etcd/etcd_wrapper.py @@ -1,11 +1,9 @@ import etcd3 import json +import logging from functools import wraps -from uncloud import UncloudException -from uncloud.common import logger - class EtcdEntry: def __init__(self, meta_or_key, value, value_in_json=True): @@ -27,11 +25,11 @@ def readable_errors(func): try: return func(*args, **kwargs) except etcd3.exceptions.ConnectionFailedError: - raise UncloudException('Cannot connect to etcd: is etcd running as configured in uncloud.conf?') + raise etcd3.exceptions.ConnectionFailedError('Cannot connect to etcd: is etcd running as configured?') except etcd3.exceptions.ConnectionTimeoutError as err: raise etcd3.exceptions.ConnectionTimeoutError('etcd connection timeout.') from err except Exception as err: - logger.exception('Some etcd error occured. See syslog for details.', err) + logging.exception('Some etcd error occured. See syslog for details.', err) return wrapper From e34abc449d5216715cb81dbe88f3eafc8c44522a Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 18:32:03 +0500 Subject: [PATCH 51/66] Removed config-and-secrets.conf --- .gitignore | 4 ++-- opennebula-vm-etcd/config-and-secrets.conf | 18 ------------------ 2 files changed, 2 insertions(+), 20 deletions(-) delete mode 100644 opennebula-vm-etcd/config-and-secrets.conf diff --git a/.gitignore b/.gitignore index c093faa..359653f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ -opennebula-snapshot/config-and-secrets.conf +opennebula-vm-etcd/config-and-secrets.conf *.pyc .idea/ -.vscode/ \ No newline at end of file +.vscode/ diff --git a/opennebula-vm-etcd/config-and-secrets.conf b/opennebula-vm-etcd/config-and-secrets.conf deleted file mode 100644 index b7faa08..0000000 --- a/opennebula-vm-etcd/config-and-secrets.conf +++ /dev/null @@ -1,18 +0,0 @@ -# Do not put single/double quotation mark for string as they are -# aslo considered as normal character. - -[oca] -client_secrets = ahmedbilal96@gmail.com:d00359fa33a74fcb5ea40bb088e299fd2ab85126 - -[etcd] -# url = localhost -# port = 2379 -# ca_cert -# cert_cert -# cert_key - -url = etcd1.ungleich.ch -port = 2379 -ca_cert = /home/meow/.cdist/files/etcd/ca.pem -cert_cert = /home/meow/.cdist/files/etcd/developer.pem -cert_key = /home/meow/.cdist/files/etcd/developer-key.pem \ No newline at end of file From 55a686800640af68f541fede1b3815e28289b586 Mon Sep 17 00:00:00 2001 From: meow Date: Tue, 28 Jan 2020 23:28:00 +0500 Subject: [PATCH 52/66] Some optimizations to improve throughput --- opennebula-vm-etcd/put-vm-info-into-etcd.py | 47 +++++++++++++-------- 1 file changed, 29 insertions(+), 18 deletions(-) diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py index 70a1b09..5b5495f 100644 --- a/opennebula-vm-etcd/put-vm-info-into-etcd.py +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -2,6 +2,7 @@ import pyone from enum import IntEnum from config import config, etcd_client +from functools import reduce # How to get client secrets? # 1. Login to OpenNebula @@ -14,24 +15,24 @@ one_client = pyone.OneServer( session=config['oca']['client_secrets'] ) +host_pool = { + host.NAME: { + 'name': host.NAME, + 'id': host.ID, + 'cluster': { + 'name': host.CLUSTER, + 'id': host.CLUSTER_ID + }, + 'vms': host.VMS.ID + } + for host in one_client.hostpool.info().HOST +} + def get_hostname_of_vm(vm_id): - host_pool = { - host.NAME: { - 'name': host.NAME, - 'id': host.ID, - 'cluster': { - 'name': host.CLUSTER, - 'id': host.CLUSTER_ID - }, - 'vms': host.VMS.ID - } - for host in one_client.hostpool.info().HOST - } for hostname, host in host_pool.items(): if vm_id in host['vms']: return host - return None @@ -115,11 +116,21 @@ def main(): START_ID = -1 # First id whatever it is END_ID = -1 # Last id whatever it is - for VM_STATE in VM_STATES: - vm_pool = one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, VM_STATE) - for i, vm in enumerate(vm_pool.VM): - vm = VM(vm) - etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + # Get VMs in all kind of states + + # vms is a list of lists + vms = [ + one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, vm_state).VM + for vm_state in VM_STATES + ] + # Take out elements from nested lists and put them into the original list + # forming a nice flat list + vms = list(reduce(lambda n, n_1: n + n_1, vms)) + print('Total VMs:', len(vms)) + for i, _vm in enumerate(vms): + vm = VM(_vm) + etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) + print(i, end=' ') if __name__ == "__main__": From 78470501dd7c79d6a8094197cf6d9aa03e621591 Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 31 Jan 2020 23:05:25 +0500 Subject: [PATCH 53/66] Directly use builtin XMLRPC client as opposed to pyone library --- .gitignore | 4 +- opennebula-vm-etcd/put-vm-info-into-etcd.py | 177 ++++++++------------ 2 files changed, 71 insertions(+), 110 deletions(-) diff --git a/.gitignore b/.gitignore index 359653f..f8835d9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ opennebula-vm-etcd/config-and-secrets.conf *.pyc -.idea/ -.vscode/ +.idea +.vscode diff --git a/opennebula-vm-etcd/put-vm-info-into-etcd.py b/opennebula-vm-etcd/put-vm-info-into-etcd.py index 5b5495f..0ba2275 100644 --- a/opennebula-vm-etcd/put-vm-info-into-etcd.py +++ b/opennebula-vm-etcd/put-vm-info-into-etcd.py @@ -1,39 +1,17 @@ -import pyone +import json from enum import IntEnum +from xmlrpc.client import ServerProxy as RPCClient + +from xmltodict import parse + from config import config, etcd_client -from functools import reduce - -# How to get client secrets? -# 1. Login to OpenNebula -# 2. Go to Settings then Auth -# 3. Click on "Manage login tokens" button -# 4. Click on "Get a new token" button - -one_client = pyone.OneServer( - uri='https://opennebula.ungleich.ch:2634/RPC2', - session=config['oca']['client_secrets'] -) - -host_pool = { - host.NAME: { - 'name': host.NAME, - 'id': host.ID, - 'cluster': { - 'name': host.CLUSTER, - 'id': host.CLUSTER_ID - }, - 'vms': host.VMS.ID - } - for host in one_client.hostpool.info().HOST -} -def get_hostname_of_vm(vm_id): - for hostname, host in host_pool.items(): - if vm_id in host['vms']: - return host - return None +# Constants +ALL_VM_STATES = -1 +START_ID = -1 # First id whatever it is +END_ID = -1 # Last id whatever it is def put_under_list(obj): @@ -42,65 +20,19 @@ def put_under_list(obj): return obj -class Snapshot: - def __init__(self, disk_id, snapshot): - self.active = bool(snapshot.ACTIVE) - self.date = snapshot.DATE - self.id = snapshot.ID - self.name = snapshot.NAME - self.size = snapshot.SIZE - self.disk_id = disk_id - - def get_data(self): - return { - attr: getattr(self, attr) - for attr in dir(self) - if not attr.startswith('__') and not callable(getattr(self, attr)) - } - - -class VM: - def __init__(self, vm): - self.name = vm.get_NAME() - self.id = vm.get_ID() - self.owner = { - 'name': vm.get_UNAME(), - 'id': vm.get_UID(), - } - - template = vm.get_TEMPLATE() - host = get_hostname_of_vm(self.id) - - self.vcpu = template.get('VCPU', None) - self.memory = template.get('MEMORY', None) - self.disks = [dict(disk) for disk in put_under_list(template.get('DISK', []))] - self.graphics = [dict(graphics) for graphics in put_under_list(template.get('GRAPHICS', []))] - self.nics = [dict(nic) for nic in put_under_list(template.get('NIC', []))] - self.status = pyone.VM_STATE(vm.get_STATE()).name.lower() - self.snapshots = [] - - for disk in one_client.vm.info(self.id).SNAPSHOTS: - disk_id = disk.DISK_ID - for snapshot in disk.SNAPSHOT: - self.snapshots.append(Snapshot(disk_id, snapshot).get_data()) - - if host: - self.host = { - 'name': host['name'], - 'id': host['id'] - } - else: - self.host = host - - def get_data(self): - return { - attr: getattr(self, attr) - for attr in dir(self) - if not attr.startswith('__') and not callable(getattr(self, attr)) - } - - def __repr__(self): - return str(self.get_data()) +class VMState(IntEnum): + INIT = 0 + PENDING = 1 + HOLD = 2 + ACTIVE = 3 + STOPPED = 4 + SUSPENDED = 5 + DONE = 6 + FAILED = 7 + POWEROFF = 8 + UNDEPLOYED = 9 + CLONING = 10 + CLONING_FAILURE = 11 class VmFilterFlag(IntEnum): @@ -111,26 +43,55 @@ class VmFilterFlag(IntEnum): UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group +class VM: + def __init__(self, vm: dict): + self.id = vm.get('ID', None) + self.owner = { + 'id': vm.get('UID', None), + 'name': vm.get('UNAME', None), + 'gname': vm.get('GNAME', None) + } + self.name = vm.get('NAME', None) + self.status = vm.get('STATE', None) + if self.status: + self.status = VMState(int(self.status)).name.lower() + + template = vm['TEMPLATE'] + + self.disk = put_under_list(template.get('DISK', [])) + self.graphics = template.get('GRAPHICS', {}) + self.memory = template.get('MEMORY', None) + self.nic = put_under_list(template.get('NIC', [])) + self.vcpu = template.get('VCPU', None) + self.host = { + 'name': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None), + 'id': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HID', None), + } + self.snapshots = put_under_list(vm.get('SNAPSHOTS', [])) + + def get_data(self): + return { + attr: getattr(self, attr) + for attr in dir(self) + if not attr.startswith('__') and not callable(getattr(self, attr)) + } + + def main(): - VM_STATES = list(pyone.VM_STATE) - START_ID = -1 # First id whatever it is - END_ID = -1 # Last id whatever it is + with RPCClient('https://opennebula.ungleich.ch:2634/RPC2') as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + config['oca']['client_secrets'], VmFilterFlag.AllResources.value, START_ID, END_ID, ALL_VM_STATES + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + for i, vm in enumerate(vms): + vm_id = vm['ID'] + etcd_client.put(f'/opennebula/vm/{vm_id}', vm) - # Get VMs in all kind of states - - # vms is a list of lists - vms = [ - one_client.vmpool.infoextended(VmFilterFlag.AllResources.value, START_ID, END_ID, vm_state).VM - for vm_state in VM_STATES - ] - # Take out elements from nested lists and put them into the original list - # forming a nice flat list - vms = list(reduce(lambda n, n_1: n + n_1, vms)) - print('Total VMs:', len(vms)) - for i, _vm in enumerate(vms): - vm = VM(_vm) - etcd_client.put('/opennebula/vm/{}'.format(vm.id), vm.get_data()) - print(i, end=' ') + parsed_vm = VM(vm) + etcd_client.put(f'/opennebula/parsed_vm/{parsed_vm.id}', parsed_vm.get_data()) + else: + print(response) if __name__ == "__main__": From 46fc35d3c3ad3e9028920e5257ee80b6d98af35c Mon Sep 17 00:00:00 2001 From: meow Date: Fri, 31 Jan 2020 23:10:16 +0500 Subject: [PATCH 54/66] Fix vm-queries.py --- opennebula-vm-etcd/vm-queries.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/opennebula-vm-etcd/vm-queries.py b/opennebula-vm-etcd/vm-queries.py index e92ef14..e1da013 100644 --- a/opennebula-vm-etcd/vm-queries.py +++ b/opennebula-vm-etcd/vm-queries.py @@ -1,6 +1,6 @@ from pprint import pprint -from config import config, etcd_client +from config import etcd_client def get_vm_by_ip(vms, ip, status='active'): @@ -11,7 +11,7 @@ def get_vm_by_ip(vms, ip, status='active'): } for vm_id, vm in vms_by_status.items(): vm_ips = [] - for nic in vm.get('nics', []): + for nic in vm.get('nic', []): global_ipv6 = nic.get('IP6_GLOBAL', None) local_ipv6 = nic.get('IP6_LINK', None) ipv4 = nic.get('IP', None) @@ -23,7 +23,7 @@ def get_vm_by_ip(vms, ip, status='active'): def main(): - vm_prefix = '/opennebula/vm/' + vm_prefix = '/opennebula/parsed_vm/' vms = { int(vm.key.split('/')[-1]): vm.value @@ -39,10 +39,10 @@ def main(): print(vms.get(VM_ID).get('host').get('name')) # Get VNC Port of a VM - print(vms.get(VM_ID).get('graphics')[0].get('PORT')) + print(vms.get(VM_ID).get('graphics').get('PORT')) # Get all disks attached with VM - pprint(vms.get(VM_ID).get('disks')) + pprint(vms.get(VM_ID).get('disk')) # Who is owner of a VM? print(vms.get(VM_ID).get('owner').get('name')) @@ -51,5 +51,6 @@ def main(): search_ungleich_ch = get_vm_by_ip(vms, '2a0a:e5c0:0:5:0:78ff:fe11:d75f') pprint(search_ungleich_ch) + if __name__ == '__main__': main() From fb68ce7c1c07b422ab7362a93c936cdd50a8c2f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Fri, 14 Feb 2020 15:23:54 +0100 Subject: [PATCH 55/66] Add OpenNebula image definition for Alpine Linux --- alpine-build-opennebula-image.sh | 179 +++++++++++++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100755 alpine-build-opennebula-image.sh diff --git a/alpine-build-opennebula-image.sh b/alpine-build-opennebula-image.sh new file mode 100755 index 0000000..0a074b4 --- /dev/null +++ b/alpine-build-opennebula-image.sh @@ -0,0 +1,179 @@ +#!/bin/sh + +# This script generates Alpine images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=v3.11 +ARCH=x86_64 +IMAGE_PATH=alpine-$RELEASE-$(date -I).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 +APK_MIRROR=http://dl-2.alpinelinux.org/alpine/ # Mind the trailing / + +ONE_CONTEXT_APK_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-r1.apk" +ONE_CONTEXT_APK_PATH=/root/one-context.apk + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ "$(lsb_release --short --id)" != "Alpine" ]; then + echo "WARNING: this script has been designed to run on an Alpine system." >&2 + echo "WARNING: Not running Alpine. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +run_root setup-timezone -z UTC +if [ "$RELEASE" = "edge" ] +then + cat >/mnt/etc/apk/repositories </mnt/etc/apk/repositories <>/mnt/etc/fstab </mnt/boot/extlinux.conf < "/mnt$ONE_CONTEXT_APK_PATH" +run_root apk add --allow-untrusted "$ONE_CONTEXT_APK_PATH" +run_root rm "$ONE_CONTEXT_APK_PATH" + +# Remove resolvconf: handled by uncloud-init. +run_root rm /etc/resolv.conf + +# Make sure everything is written to disk before exiting. +sync From b6cf60b83b8aae1dacbca0d9feb33d99d2283bfa Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Tue, 3 Mar 2020 08:43:52 +0100 Subject: [PATCH 56/66] Add new directory for vnc_console --- vnc_console_connection/.gitkeep | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 vnc_console_connection/.gitkeep diff --git a/vnc_console_connection/.gitkeep b/vnc_console_connection/.gitkeep new file mode 100644 index 0000000..e69de29 From 9774225f958eee50bb6edcd6d8cb1280a125b344 Mon Sep 17 00:00:00 2001 From: kjg Date: Tue, 3 Mar 2020 08:55:30 +0100 Subject: [PATCH 57/66] [file]update vnc_console --- vnc_console_connection/config.py | 5 ++ vnc_console_connection/db_export.py | 55 ++++++++++++++++++ vnc_console_connection/get_info.py | 88 +++++++++++++++++++++++++++++ vnc_console_connection/ldap_list.py | 30 ++++++++++ 4 files changed, 178 insertions(+) create mode 100755 vnc_console_connection/config.py create mode 100755 vnc_console_connection/db_export.py create mode 100755 vnc_console_connection/get_info.py create mode 100755 vnc_console_connection/ldap_list.py diff --git a/vnc_console_connection/config.py b/vnc_console_connection/config.py new file mode 100755 index 0000000..2a36c96 --- /dev/null +++ b/vnc_console_connection/config.py @@ -0,0 +1,5 @@ +import configparser + +config = configparser.ConfigParser(allow_no_value=True) +config.read('config-and-secrets.conf') + diff --git a/vnc_console_connection/db_export.py b/vnc_console_connection/db_export.py new file mode 100755 index 0000000..d283eb4 --- /dev/null +++ b/vnc_console_connection/db_export.py @@ -0,0 +1,55 @@ +import psycopg2 as pg2 +from config import config + +db_name = config['db']['db_name'] +db_user = config['db']['db_user'] +db_password = config['db']['db_password'] +db_port = config['db']['db_port'] + + +def setconn(u_id, vm_num, vm_port,vm_host): + conn = pg2.connect("host = localhost dbname={} user={} password={} port={}".format(db_name,db_user,db_password,db_port)) + conn.autocommit = True + cur = conn.cursor() + cur.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) + row = cur.fetchone() + if row == None: + cur.execute("INSERT INTO guacamole_entity (name, type) VALUES ('{}','USER')".format(u_id)) + cur.execute("SELECT entity_id FROM guacamole_entity WHERE name = '{}'".format(u_id)) + row = cur.fetchone() + en_id = row[0] + cur.execute("INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('{}', '\x74657374', now())".format(en_id)) + print("create user : " , u_id) + else: + en_id = row[0] + cur.execute("SELECT password_hash FROM guacamole_user WHERE entity_id = '{}'".format(en_id)) + row = cur.fetchone() + if row == None: + cur.execute("INSERT INTO guacamole_user(entity_id, password_hash, password_date) VALUES ('{}', '\x74657374', now())".format(en_id)) + print("user exsit") + cn = "{}{}".format(u_id,vm_num) + cur.execute("SELECT connection_id FROM guacamole_connection WHERE connection_name = '{}'".format(cn)) + row = cur.fetchone() + if row == None: + #create connection + cur.execute("INSERT INTO guacamole_connection (connection_name, protocol) VALUES ('{}', 'vnc')".format(cn)) + cur.execute("SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '{}' AND parent_id IS NULL".format(cn)) + temp_cn_id = cur.fetchone() + cn_id = temp_cn_id[0] + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','hostname','{}')".format(cn_id, vm_host)) + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','port','{}')".format(cn_id,vm_port)) + #connection permission + cur.execute("INSERT INTO guacamole_connection_permission(entity_id, connection_id, permission) VALUES ('{}', '{}', 'READ')".format(en_id,cn_id)) + #clipboard-encoding + cur.execute("INSERT INTO guacamole_connection_parameter VALUES ('{}','clipboard-encoding','UTF-8')".format(cn_id)) + print("create connection") + else: + cur.execute("SELECT MAX(connection_id) FROM guacamole_connection WHERE connection_name = '{}' AND parent_id IS NULL".format(cn)) + temp_cn_id = cur.fetchone() + cn_id = temp_cn_id[0] + cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='{}' where connection_id='{}' and parameter_name='hostname'".format(vm_host,cn_id)) + cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='{}' where connection_id='{}' and parameter_name='port'".format(vm_port,cn_id)) + #cur.execute("UPDATE guacamole_connection_parameter SET parameter_value='UTF-8' where connection_id='{}' and parameter_name='clipboard-encoding'".format(cn_id)) + print("no connection") + conn.close() + return None \ No newline at end of file diff --git a/vnc_console_connection/get_info.py b/vnc_console_connection/get_info.py new file mode 100755 index 0000000..ac09633 --- /dev/null +++ b/vnc_console_connection/get_info.py @@ -0,0 +1,88 @@ +import json + +from enum import IntEnum +from xmlrpc.client import ServerProxy as RPCClient +from xmltodict import parse +from config import config +from ldap_list import vm_list +from db_export import setconn + +# Constants +ALL_VM_STATES = -1 +START_ID = -1 # First id whatever it is +END_ID = -1 # Last id whatever it is +session_string = config['oca']['client_secrets'] +opnserver = config['oca']['opn_server'] + +class VMState(IntEnum): + INIT = 0 + PENDING = 1 + HOLD = 2 + ACTIVE = 3 + STOPPED = 4 + SUSPENDED = 5 + DONE = 6 + FAILED = 7 + POWEROFF = 8 + UNDEPLOYED = 9 + CLONING = 10 + CLONING_FAILURE = 11 + + +class VmFilterFlag(IntEnum): + UIDUserResources = 0 # UID User’s Resources + UserAndItsGroupsResources = -1 # Resources belonging to the user and any of his groups + AllResources = -2 # All resources + UserResources = -3 # Resources belonging to the user + UserPrimaryGroupResources = -4 # Resources belonging to the user’s primary group + + +class VM: + def __init__(self, vm: dict): + self.id = vm.get('ID', None) + self.owner = { + 'id': vm.get('UID', None), + 'name': vm.get('UNAME', None), + 'gname': vm.get('GNAME', None) + } + self.name = vm.get('NAME', None) + self.status = vm.get('STATE', None) + if self.status: + self.status = VMState(int(self.status)).name.lower() + + template = vm['TEMPLATE'] + + self.graphics = template.get('GRAPHICS', {}) + self.memory = template.get('MEMORY', None) + self.vcpu = template.get('VCPU', None) + self.host = { + 'name': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HOSTNAME', None), + 'id': ((vm.get('HISTORY_RECORDS', {}) or {}).get('HISTORY', {}) or {}).get('HID', None), + } + + +def main(): + with RPCClient(opnserver) as rpc_client: + success, response, *_ = rpc_client.one.vmpool.infoextended( + session_string , VmFilterFlag.AllResources.value, START_ID, END_ID, VMState.ACTIVE.value + ) + if success: + vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] + for entry in vm_list.entries: + temp_uname = entry.mail + for i, vm in enumerate(vms): + vm_user = vm['UNAME'] + vm_id = vm['ID'] + vm_port = vm['TEMPLATE']['GRAPHICS'].get('PORT') + vm_host = vm['HISTORY_RECORDS']['HISTORY']['HOSTNAME'] + if vm['UNAME'] == temp_uname: + #print(entry.uid, vm_id, vm_port, vm_host) + setconn(entry.uid, vm_id, vm_port, vm_host) + + else: + print(response) + + +if __name__ == "__main__": + main() + diff --git a/vnc_console_connection/ldap_list.py b/vnc_console_connection/ldap_list.py new file mode 100755 index 0000000..a9e322f --- /dev/null +++ b/vnc_console_connection/ldap_list.py @@ -0,0 +1,30 @@ +import ldap3 +import sys +from config import config +from ldap3 import Server, Connection, ObjectDef, Reader, ALL, SUBTREE, ALL_ATTRIBUTES +from ldap3.core import exceptions + + +LDAP_SERVER = config['ldap']['server'] +LDAP_PASSWORD = config['ldap']['admin_password'] +LDAP_USER = config['ldap']['admin_dn'] +LDAP_PORT = int(config['ldap']['ldap_port']) + +# Create the Server object with the given address. +server = Server(LDAP_SERVER, LDAP_PORT, get_info=ALL) +#Create a connection object, and bind with the given DN and password. +try: + conn = Connection(server, LDAP_USER, LDAP_PASSWORD, auto_bind=True) + print('LDAP Bind Successful.') + # Perform a search for a pre-defined criteria. + # Mention the search filter / filter type and attributes. + conn.search('ou=customer,dc=ungleich,dc=ch', '(&(!({}={})))'.format('mail','*@ungleich.ch') , attributes=['uid','mail']) + #conn.search('ou=customer,dc=ungleich,dc=ch', '(objectClass=*)' , attributes=['uid','mail']) + # Print the resulting entriesn. + #for entry in conn.entries: + #print(entry.uid, entry.mail) + vm_list = conn +except exceptions.LDAPException as err: + sys.exit(f'LDAP Error: {err}') + + From 873eca64673e075b37caacc1257d9d1d922a4ef2 Mon Sep 17 00:00:00 2001 From: "jinguk.kwon" Date: Fri, 20 Mar 2020 16:13:06 +0900 Subject: [PATCH 58/66] new file for vm map --- vm_map.sh | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 vm_map.sh diff --git a/vm_map.sh b/vm_map.sh new file mode 100755 index 0000000..15c80dc --- /dev/null +++ b/vm_map.sh @@ -0,0 +1,5 @@ +vm_list=( $(virsh list | awk '{print $2}') ) + +for ((i=0; i<${#vm_list[@]}; i++)) do + ceph osd map hdd ${vm_list[i]} +done From ad8b7b2c6d5c30d1739d838320dcaf9b8589c2d7 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 8 Apr 2020 07:18:56 +0200 Subject: [PATCH 59/66] [tools]Update get_info.py for uid --- vnc_console_connection/get_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnc_console_connection/get_info.py b/vnc_console_connection/get_info.py index ac09633..e98ae72 100755 --- a/vnc_console_connection/get_info.py +++ b/vnc_console_connection/get_info.py @@ -69,7 +69,7 @@ def main(): if success: vms = json.loads(json.dumps(parse(response)))['VM_POOL']['VM'] for entry in vm_list.entries: - temp_uname = entry.mail + temp_uname = entry.uid for i, vm in enumerate(vms): vm_user = vm['UNAME'] vm_id = vm['ID'] From 4fc8995e074495f9fe9e495083fe0652749b8b97 Mon Sep 17 00:00:00 2001 From: Jin-Guk Kwon Date: Wed, 15 Apr 2020 12:14:18 +0200 Subject: [PATCH 60/66] [ungleich-tool]Update config.py for path --- vnc_console_connection/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vnc_console_connection/config.py b/vnc_console_connection/config.py index 2a36c96..7b7acc7 100755 --- a/vnc_console_connection/config.py +++ b/vnc_console_connection/config.py @@ -1,5 +1,5 @@ import configparser config = configparser.ConfigParser(allow_no_value=True) -config.read('config-and-secrets.conf') +config.read('/opt/ungleich-tools/vnc_console_connection/config-and-secrets.conf') From 6b51feb1fdfe2f0735c716e35bb3570dff37d669 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 7 May 2020 08:25:00 +0200 Subject: [PATCH 61/66] Bump Fedora image (F31 -> F32) --- fedora-build-opennebula-image.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index 45c1629..a84a960 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -18,7 +18,7 @@ set -e set -x # XXX: Handle command-line arguments? -RELEASE=31 +RELEASE=32 ARCH=x86_64 IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2 IMAGE_SIZE=10G From 8970127c246a052d6aa24825f8db453719cbbca6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Thu, 7 May 2020 08:30:08 +0200 Subject: [PATCH 62/66] Add haveged to Fedora image --- fedora-build-opennebula-image.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/fedora-build-opennebula-image.sh b/fedora-build-opennebula-image.sh index a84a960..b4bc5c6 100755 --- a/fedora-build-opennebula-image.sh +++ b/fedora-build-opennebula-image.sh @@ -132,6 +132,10 @@ run_root systemd-machine-id-setup run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime run_root systemctl enable systemd-timesyncd.service +# Install haveged due to lack of entropy in ONE environment. +run_root dnf -y install haveged +run_root systemctl enable haveged.service + # Install kernel and bootloader. # Note: linux-firmware is not required our environment and takes almost 200M # uncompressed but is a direct dependency of kernel-core... From 6ea31faa476f9b516b4bdc51c214e79b1b024f01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:18:46 +0200 Subject: [PATCH 63/66] Move ONE image defitions to their own folder --- .../alpine-build-opennebula-image.sh | 0 .../fedora-build-opennebula-image.sh | 0 .../ubuntu-build-opennebula-image.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename alpine-build-opennebula-image.sh => opennebula-images/alpine-build-opennebula-image.sh (100%) rename fedora-build-opennebula-image.sh => opennebula-images/fedora-build-opennebula-image.sh (100%) rename ubuntu-build-opennebula-image.sh => opennebula-images/ubuntu-build-opennebula-image.sh (100%) diff --git a/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh similarity index 100% rename from alpine-build-opennebula-image.sh rename to opennebula-images/alpine-build-opennebula-image.sh diff --git a/fedora-build-opennebula-image.sh b/opennebula-images/fedora-build-opennebula-image.sh similarity index 100% rename from fedora-build-opennebula-image.sh rename to opennebula-images/fedora-build-opennebula-image.sh diff --git a/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh similarity index 100% rename from ubuntu-build-opennebula-image.sh rename to opennebula-images/ubuntu-build-opennebula-image.sh From 43b52b264df75d8b223d40ed532cfb7c33a42857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:20:00 +0200 Subject: [PATCH 64/66] Add forgotten centos to opennebula-images dir, +x ubuntu-build-one-... --- .../centos-build-opennebula-image.sh | 0 opennebula-images/ubuntu-build-opennebula-image.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename centos-build-opennebula-image.sh => opennebula-images/centos-build-opennebula-image.sh (100%) mode change 100644 => 100755 opennebula-images/ubuntu-build-opennebula-image.sh diff --git a/centos-build-opennebula-image.sh b/opennebula-images/centos-build-opennebula-image.sh similarity index 100% rename from centos-build-opennebula-image.sh rename to opennebula-images/centos-build-opennebula-image.sh diff --git a/opennebula-images/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh old mode 100644 new mode 100755 From b3f79de311110baa532e81c9b3437db7093a05e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timoth=C3=A9e=20Floure?= Date: Mon, 11 May 2020 10:51:26 +0200 Subject: [PATCH 65/66] Add debian image build definition --- .../debian-build-opennebula-image.sh | 164 ++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 opennebula-images/debian-build-opennebula-image.sh diff --git a/opennebula-images/debian-build-opennebula-image.sh b/opennebula-images/debian-build-opennebula-image.sh new file mode 100755 index 0000000..17dc1e3 --- /dev/null +++ b/opennebula-images/debian-build-opennebula-image.sh @@ -0,0 +1,164 @@ +#!/bin/sh + +# This script generates Debian images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +RELEASE=buster # 10.X +ARCH=amd64 +IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Ubuntu" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat >/mnt/etc/apt/sources.list < "/mnt$ONE_CONTEXT_DEB_PATH" +run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" +run_root rm "$ONE_CONTEXT_DEB_PATH" + +# Manually install legacy network scripts used by one-context. +run_root apt-get -y install ifupdown + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and bootloader. Do not autoconfigure grub. +run_root 'echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections' +run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-image-amd64 grub-pc + +# Configure grub. +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root apt-get -y install openssh-server + +# Install haveged due to lack of entropy in ONE environment. +run_root apt-get -y install haveged +run_root systemctl enable haveged.service + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < Date: Mon, 11 May 2020 12:29:52 +0200 Subject: [PATCH 66/66] [OpenNebula-images] Add scripts to generate FreeBSD images. Tested for 12.1-RELEASE, 11.3-STABLE should be tested as well. Two scripts are added, FreeBSD-build-opennebula-image-generic.sh takes information from the environment and generates the specified image with either dualstack or IPv6-only kernel and base applications. FreeBSD-build-opennebula-image.sh has the versions supported in DCL and generates all neeeded images (combination of Version x Network support). This should run in a DCL VM with an OK amount of cores (4/8 minimum), 2-4G RAM, and storage of roughly 20G + 5G * #resulting_images. This is because there is the base system, a 'pristine chroot', and during the build there can be 2 copies of the resulting system written to the system. Since there are 4 combinations of images: {STABLE,RELEASE} x {dualstack, IPv6ONLY} That means we'll need to assign about 40G storage to be on the safe side. --- .../FreeBSD-build-opennebula-image-generic.sh | 243 ++++++++++++++++++ .../FreeBSD-build-opennebula-image.sh | 31 +++ 2 files changed, 274 insertions(+) create mode 100644 opennebula-images/FreeBSD-build-opennebula-image-generic.sh create mode 100755 opennebula-images/FreeBSD-build-opennebula-image.sh diff --git a/opennebula-images/FreeBSD-build-opennebula-image-generic.sh b/opennebula-images/FreeBSD-build-opennebula-image-generic.sh new file mode 100644 index 0000000..d251f56 --- /dev/null +++ b/opennebula-images/FreeBSD-build-opennebula-image-generic.sh @@ -0,0 +1,243 @@ +#!/bin/sh +# +# Copyright 2020 -- Evilham +# This is BSD licensed as it's based on BSD-licensed code +# +# We could have used e.g. something like: +# - https://git.sr.ht/~sircmpwn/builds.sr.ht/tree/master/images/freebsd/genimg +# +# But we actually do want to compile the kernel, so that the IPv6-only images +# are different and don't support INET. + +# Explode if something goes wrong +set -e + +# What are we building? +# These are the only configuration options. +# They default to current environment. +# RELEASE: should be 'CURRENT' for current or 'X.Y' Defaults to 'CURRENT'. +# ARCH: probably amd64 for DCL +# VMFORMATS: defaults to qcow2, can also be raw. See man mkimg. +# OPENNEBULA_CONTEXT_VERSION: For DCL's OpenNebula that'd be 5.10.0 (default) +# OPENNEBULA_CONTEXT_REVISION: Defaults to 1. +RELEASE=${RELEASE:-CURRENT} +if [ "${RELEASE}" == "CURRENT" ]; then + SRCBRANCH="master" +else + SRCBRANCH="releng/${RELEASE}" +fi +ARCH=${ARCH:-amd64} +VMFORMATS=${VMFORMATS:-qcow2} +OPENNEBULA_CONTEXT_VERSION=${OPENNEBULA_CONTEXT_VERSION:-5.10.0} +OPENNEBULA_CONTEXT_REVISION=${OPENNEBULA_CONTEXT_REVISION:-1} + +# Didn't see a need to make these configurable. +CHROOTDIR="/scratch" +SRCDIR="${CHROOTDIR}/usr/src" +OUR_DIR="$(realpath $(dirname "${0}"))" +OUR_SRCCONF="${SRCDIR}/release/src.conf" +OUR_RELEASE_CONF="${SRCDIR}/release/release.conf" +# Shorthand for the package file name. +OPENNEBULA_CONTEXT="one-context-${OPENNEBULA_CONTEXT_VERSION}_${OPENNEBULA_CONTEXT_REVISION}.txz" + +setup_sources() { + # Let's use git, we might need to install it + if ! which git 2>&1 > /dev/null; then + pkg install -y git + fi + + if [ ! -d "$(dirname ${SRCDIR})" ]; then + mkdir -p "$(dirname ${SRCDIR})" + fi + + # Checkout needed branch + if [ ! -d "${SRCDIR}" ]; then + git clone "https://github.com/freebsd/freebsd" \ + --branch "${SRCBRANCH}" "${SRCDIR}" + else + GIT_CMD="git -C ${SRCDIR}" + ${GIT_CMD} clean -df + ${GIT_CMD} reset --hard + ${GIT_CMD} fetch + ${GIT_CMD} checkout "${SRCBRANCH}" + ${GIT_CMD} pull + fi + + # Add settings for IPv6-only kernel + cat > "${SRCDIR}/sys/${ARCH}/conf/GENERIC-IPV6ONLY" << EOF +include GENERIC +ident GENERIC-IPV6ONLY +makeoptions MKMODULESENV+="WITHOUT_INET_SUPPORT=" +nooptions INET +nodevice gre +EOF + # Fix vmimage.subr to install custom package and fix other things + cat >> "${SRCDIR}/release/tools/vmimage.subr" << EOF +vm_extra_install_ports() { + # Make sure we install the opennbula context package + cp "/${OPENNEBULA_CONTEXT}" "\${DESTDIR}/tmp/${OPENNEBULA_CONTEXT}" + chroot \${DESTDIR} \${EMULATOR} env ASSUME_ALWAYS_YES=yes \\ + /usr/sbin/pkg add '/tmp/${OPENNEBULA_CONTEXT}' + + # Now make sure the system has better defaults + cat >> "\${DESTDIR}/etc/rc.conf" << eof +# Update to latest patch on first boot +firstboot_freebsd_update_enable="YES" +# Enable OpenNebula's service. +one_context_enable="YES" +# Enable SSH for customers +sshd_enable="YES" +# Clear tmp on boot +clear_tmp_enable="YES" +# Disable sendmail by default +sendmail_enable="NONE" +# Disable crash dumps +dumpdev="NO" +eof + # Enable root access with SSH key. + # It is user's responsibility to further secure their system. + sed -i '' -E \ + 's/(^#[ ]*|^)PermitRootLogin .*/PermitRootLogin without-password/' \ + "\${DESTDIR}/etc/ssh/sshd_config" +} +EOF + # Skip building iso images + rm "${SRCDIR}/release/${ARCH}/mkisoimages.sh" + # This is a hack to not build the memstick + cat > "${SRCDIR}/release/${ARCH}/make-memstick.sh" < \${CHROOTDIR}/etc/src-env.conf +} + +## Set the directory within which the release will be built. +CHROOTDIR="${CHROOTDIR}" + +## Set to override the default target architecture and kernel +TARGET="${ARCH}" +TARGET_ARCH="${ARCH}" +KERNEL="${KERNEL_CONFIG}" + +## Set to specify a custom make.conf and/or src.conf +SRC_CONF="${OUR_SRCCONF}" + +# Since these are VMs, users should add other components if they want to. +NODOC=YES +NOPORTS=YES +NOSRC=YES + +# We manage sources manually +SRC_UPDATE_SKIP=YES + +## Set to pass additional flags to make(1) for the build chroot setup, such +## as TARGET/TARGET_ARCH. +# This was necessary for "cross-compiling" +CHROOT_MAKEENV="MK_LLVM_TARGET_X86=yes" + +WITH_VMIMAGES=YES + +# VM image size, see man 1 truncate +VMSIZE="10G" + +# List of disk image formats, see man mkgimg. +VMFORMATS="${VMFORMATS}" + +# These variables have to be exported because they are needed in subprocesses. +export NOSWAP=YES +# Custom ports +# - firstboot-freebsd-update helps us not have to create an image for each +# patch level. We still will have to do it for each minor version update. +# - bash is apparently needed for one-context +export VM_EXTRA_PACKAGES="firstboot-freebsd-update bash" +EOF +} + +_do_run_release() { + . "${SRCDIR}/release/release.sh" +} +run_release() { + _do_run_release -c "${OUR_RELEASE_CONF}" +} + + +build_image() { + # Generate configuration + echo "${2}" > "${OUR_SRCCONF}" + KERNEL_CONFIG="${1}" + gen_releaseconf > "${OUR_RELEASE_CONF}" + # Be paranoid about files and stuff + sync + # Continue with the release script + run_release + # Be paranoid about files and stuff + sync + + mv "${CHROOTDIR}/R/vmimages" "${OUR_DIR}/FreeBSD-${RELEASE}-${1}" + + # Be paranoid about files and stuff + sync +} + +our_main() { + case "$1" in + --dualstack) + BUILD_DUALSTACK=yes + ;; + --ipv6only) + BUILD_IPV6ONLY=yes + ;; + *) + cat << EOF +Run with --dualstack or --ipv6only depending on the image you want. +EOF + exit 1 + ;; + esac + setup_sources + setup_our_env + # Fetch OpenNebula's context package + fetch "https://github.com/OpenNebula/addon-context-linux/releases/download/v${OPENNEBULA_CONTEXT_VERSION}/${OPENNEBULA_CONTEXT}" \ + -o "${CHROOTDIR}/${OPENNEBULA_CONTEXT}" + # Do run + if [ -n "${BUILD_DUALSTACK}" ]; then + build_image "GENERIC" + fi + if [ -n "${BUILD_IPV6ONLY}" ]; then + build_image "GENERIC-IPV6ONLY" "$(cat << EOF +WITHOUT_INET=yes +WITHOUT_INET_SUPPORT=yes +EOF +)" + fi + + cat << EOF + +*************** DONE *************** +You will find the images under "${OUR_DIR}". +************************************ +EOF +} + +our_main "${@}" diff --git a/opennebula-images/FreeBSD-build-opennebula-image.sh b/opennebula-images/FreeBSD-build-opennebula-image.sh new file mode 100755 index 0000000..c72a2b0 --- /dev/null +++ b/opennebula-images/FreeBSD-build-opennebula-image.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Copyright 2020 -- Evilham +# This is BSD licensed as it's based on BSD-licensed code +# +# +# This builds all needed FreeBSD images for ungleich's Data Center Light +# When there are new releases, they should be updated here and the script +# should run. +# 11.4 is scheduled end of June 2020 +# 12.2 is scheduled end of October 2020 +# + +SUPPORTED_RELEASES="11.3 12.1" + +# This should run in a DCL VM with an OK amount of cores (4/8 minimum), +# 4G RAM, and storage of roughly 20G + 5G * #resulting_images. +# +# This is because there is the base system, a 'pristine chroot', and during the +# build there can be 2 copies of the resulting system written to the system. +# Since there are 4 combinations of images: +# {STABLE,RELEASE} x {dualstack, IPv6ONLY} +# +# That means we'll need to assign about 40G storage to be on the safe side. + +for release in ${SUPPORTED_RELEASES}; do + for build in dualstack ipv6only; do + env RELEASE=${release} sh FreeBSD-build-opennebula-image-generic.sh --${build} \ + | tee "FreeBSD-${release}-${build}.log" + done +done