Merge branch 'master' of code.ungleich.ch:ungleich-public/ungleich-tools

centos-build-opennebula-image.sh

@@ -0,0 +1,170 @@
+#!/bin/sh
+
+# This script generates CentOS images for OpenNebula.
+
+# Depends on the following packages (as of CentOS 8):
+#   qemu-img util-linux coreutils dnf curl e2fsprogs
+
+# Run locally (without network) with:
+#   qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2
+
+set -e
+set -x
+
+# XXX: Handle command-line arguments?
+RELEASE=8
+ARCH=x86_64
+IMAGE_PATH=centos-$RELEASE-$(date --iso-8601).img
+IMAGE_SIZE=10G
+LOOPBACK_DEVICE=/dev/loop0
+
+# TODO: find the package definition and built ourself, publish in some RPM repository.
+ONE_CONTEXT_RPM_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context-5.10.0-1.el8.noarch.rpm"
+ONE_CONTEXT_RPM_PATH=/root/one-context.rpm
+
+cleanup() {
+	# The order here is important.
+	umount /mnt/dev/pts 2>/dev/null || true
+	umount /mnt/dev/shm 2>/dev/null || true
+	umount /mnt/dev 2>/dev/null || true
+	umount /mnt/proc 2>/dev/null || true
+	umount /mnt/run 2>/dev/null || true
+	umount /mnt/sys 2>/dev/null || true
+	umount /mnt/boot 2>/dev/null || true
+	umount /mnt 2>/dev/null || true
+	losetup -d "$LOOPBACK_DEVICE"
+}
+
+run_root() {
+	chroot /mnt /usr/bin/env \
+		PATH=/sbin:/usr/sbin:/bin:/usr/bin \
+		sh -c "$*"
+}
+
+if [ "$(whoami)" != 'root' ]; then
+	echo "This script must be run as root." >&2
+	exit 1
+fi
+
+if [ ! -f '/etc/centos-release' ]; then
+	echo "WARNING: this script has been designed to run on a CentOS system." >&2
+	echo "WARNING: Not running CentOS. Giving you 5 seconds to abort." >&2
+	sleep 5
+fi
+
+# Create base RAW image (no LOOPBACK support in RHEL/CentOS).
+qemu-img create -f raw "$IMAGE_PATH" "$IMAGE_SIZE"
+losetup "$LOOPBACK_DEVICE" "$IMAGE_PATH"
+
+# Don't forget to cleanup, even if the script crash.
+trap cleanup EXIT
+
+# Create partition table, format partitions.
+{
+sfdisk --no-reread "$LOOPBACK_DEVICE" <<EOF
+1M,500M,L,*
+,,L
+EOF
+} || true
+
+partprobe "$LOOPBACK_DEVICE"
+
+mkfs.ext4 "${LOOPBACK_DEVICE}p1"
+mkfs.ext4 "${LOOPBACK_DEVICE}p2"
+
+# Mount partitions, install base OS.
+mount "${LOOPBACK_DEVICE}p2" /mnt
+mkdir /mnt/boot
+mount "${LOOPBACK_DEVICE}p1" /mnt/boot
+
+dnf -y \
+	--releasever=$RELEASE \
+	--installroot=/mnt \
+	--disablerepo='*' \
+	--enablerepo=BaseOS \
+	--enablerepo=AppStream \
+	--enablerepo=extras \
+	--setopt=install_weak_deps=False install \
+	bash basesystem systemd systemd-udev dnf centos-release
+
+mount --bind /dev /mnt/dev
+mount --bind /dev/pts /mnt/dev/pts
+mount --bind /dev/shm /mnt/dev/shm
+mount --bind /proc /mnt/proc
+mount --bind /run /mnt/run
+mount --bind /sys /mnt/sys
+
+# Guest networking is to be handled by the one-context package.
+# See https://github.com/OpenNebula/addon-context-linux for details.
+# Note: as of writing, one-context does not support NetworkManager or
+# systemd-networkd.
+
+# Required to resolve package mirror in chroot.
+cp /etc/resolv.conf /mnt/etc/resolv.conf
+
+# Initialize /etc/hosts.
+cat > /mnt/etc/hosts << EOF
+127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
+::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
+
+EOF
+
+# See https://github.com/OpenNebula/addon-context-linux/issues/121 for details.
+# network-scripts.x86_64 : Legacy scripts for manipulating of network devices
+run_root dnf -y install network-scripts
+
+# Install (magic?) one-context RPM and hope things works as expected.
+curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH"
+run_root dnf -y install "$ONE_CONTEXT_RPM_PATH"
+run_root rm "$ONE_CONTEXT_RPM_PATH"
+
+# Install resize2fs, which is required to resize the root file-system.
+run_root dnf -y install e2fsprogs
+
+# Initalize base services.
+run_root systemd-machine-id-setup
+run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime
+
+# Install and configure NTP client.
+run_root dnf install -y chrony
+run_root systemctl enable chronyd.service
+
+# Install kernel and bootloader.
+# Note: linux-firmware is not required our environment and takes almost 200M
+# uncompressed but is a direct dependency of kernel-core...
+run_root dnf -y install kernel grub2
+
+# Add support for virtio block devices at boot time.
+cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <<EOF
+add_drivers="virtio-blk"
+EOF
+kernel_version=$(ls /mnt/boot | grep "vmlinuz.*.$ARCH" | cut -d- -f2-)
+run_root dracut --force --kver $kernel_version
+
+# Configure grub2.
+run_root grub2-install --target=i386-pc "${LOOPBACK_DEVICE}"
+run_root grub2-mkconfig -o /boot/grub2/grub.cfg
+
+# Install en configure SSH daemon.
+run_root dnf -y install openssh-server
+run_root systemctl enable sshd
+
+# Generate fstab file.
+boot_uuid=$(blkid --match-tag UUID --output value "${LOOPBACK_DEVICE}p1")
+root_uuid=$(blkid --match-tag UUID --output value "${LOOPBACK_DEVICE}p2")
+cat >>/mnt/etc/fstab <<EOF
+UUID=$boot_uuid /boot ext4 rw,relatime,data=ordered 0 2
+UUID=$root_uuid /     ext4 rw,relatime,data=ordered 0 1
+EOF
+
+# Reset systemd's environment.
+run_root rm -f /etc/machine-id
+run_root touch /etc/machine-id
+rm -f /var/lib/systemd/random-seed
+
+# Remove temporary files and reclaim freed disk space.
+# Note: build logs could be removed as well.
+run_root dnf clean all
+
+# Make sure everything is written to disk before exiting.
+sync

ceph-osd-create-start-alpine

@@ -0,0 +1,107 @@
+#!/bin/sh
+# 17:19, 2018-02-09
+# Nico Schottelius
+
+# Based on ceph-disk -v prepare --bluestore /dev/sdc --osd-id ${ID} --osd-uuid $(uuidgen) --crush-device-class "ssd"
+
+# Create:
+# - block       -> link to partuuid
+# - block_uuid -e> uuid if the block
+# - ceph_fsid   -> get from ceph-conf
+# crush_device_class -> ssd, hdd
+# fsid          -> uuidgen!
+# magic         -> string "ceph osd volume v026"
+# type          -> bluestore
+
+fsid=$(ceph-conf --cluster=ceph --name=osd. --lookup fsid)
+fs_uuid=$(uuidgen)
+magic="ceph osd volume v026"
+
+set -x
+set -e
+
+if [ $# -lt 2 ]; then
+    echo "$0 disk class [osdweight]"
+    echo "class = hdd or ssd"
+    exit 1
+fi
+
+export DEV=$1;shift
+export CLASS=$1; shift
+
+
+uuid_metadata=$(uuidgen)
+uuid_block=$(uuidgen)
+
+osd_id=$(ceph osd create)
+
+dev_metadata="/dev/disk/by-partuuid/$uuid_metadata"
+dev_block="/dev/disk/by-partuuid/$uuid_block"
+
+/usr/bin/sgdisk --new=0:0:+100M --change-name="0:ceph data" \
+             --partition-guid="0:$uuid_metadata" \
+             --typecode=0:4fbd7e29-9d25-41b8-afd0-062c0ceff05d \
+             --mbrtogpt -- $DEV
+/sbin/udevadm settle --timeout=600
+
+# Using gdisk --largest-new does not change the name or set guid;
+# So use 2 steps instead
+/usr/bin/sgdisk --largest-new=0 --mbrtogpt -- $DEV
+/sbin/udevadm settle --timeout=600
+
+
+lastpart=$(gdisk -l $DEV | tail -n1 | awk '{ print $1 }')
+/usr/bin/sgdisk --change-name="${lastpart}:ceph block" \
+             --partition-guid="${lastpart}:$uuid_block" \
+             --typecode="${lastpart}:cafecafe-9b03-4f30-b4c6-b4b80ceff106" \
+             --mbrtogpt -- $DEV
+/sbin/udevadm settle --timeout=600
+
+#echo $1
+#echo $(blkid | grep $1"2")
+
+#cblock=$(blkid | grep $1"2" | cut -d'"' -f4)
+#echo $cblock
+
+/sbin/mkfs -t xfs -f -i size=2048 -- "$dev_metadata"
+
+mountpath=/var/lib/ceph/osd/ceph-${osd_id}
+
+mkdir -p "$mountpath"
+mount "$dev_metadata" "$mountpath"
+
+ln -s $dev_block        "$mountpath/block"
+
+echo "$uuid_block"    > "$mountpath/block_uuid"
+echo "$fsid"          > "$mountpath/ceph_fsid"
+echo "$magic"         > "$mountpath/magic"
+echo "$CLASS"         > "$mountpath/crush_device_class"
+echo $(echo $dev_block | cut -c23-)    > "$mountpath/fsid"
+
+
+# Important, otherwise --mkfs later will try to create filestore
+echo bluestore        > "$mountpath/type"
+
+ceph auth get-or-create "osd.${osd_id}" osd \
+     'allow *' mon 'allow profile osd' > $mountpath/keyring
+
+echo ${osd_id}         > "$mountpath/whoami"
+touch                   "$mountpath/openrc"
+
+ceph-osd --cluster ceph -i "${osd_id}" --mkfs
+chown -R ceph:ceph      "$mountpath"
+
+if [ $# -eq 1 ]; then
+    WEIGHT=$1; shift
+else
+    devname=$(readlink -f $dev_block)
+    nodev=$(echo $devname | sed 's,/dev/,,')
+    WEIGHT=$(lsblk -l -b | awk "/^$nodev/ { print \$4/(1024^4) }")
+fi
+
+ceph osd crush add osd.${osd_id} ${WEIGHT} host=$(hostname)
+
+echo "$metadata_dev /var/lib/ceph/osd/ceph-${osd_id} xfs noatime 0 0" >> /etc/fstab
+
+# Starting with monit, if available
+ceph-osd -i ${osd_id}

fedora-build-opennebula-image.sh

@@ -68,7 +68,7 @@ trap cleanup EXIT
 
 # Create partition table, format partitions.
 sfdisk --no-reread "$NBD_DEVICE" <<EOF
-1M,100M,L,*
+1M,500M,L,*
 ,,L
 EOF
 
@@ -83,8 +83,6 @@ mount "${NBD_DEVICE}p2" /mnt
 mkdir /mnt/boot
 mount "${NBD_DEVICE}p1" /mnt/boot
 
-# XXX: dnf has a lot a weird (libX11?) dependencies, use microdnf instead?
-
 dnf -y \
 	--releasever=$RELEASE \
 	--installroot=/mnt \
@@ -125,9 +123,11 @@ curl -L "$ONE_CONTEXT_RPM_URL" > "/mnt$ONE_CONTEXT_RPM_PATH"
 run_root dnf -y install "$ONE_CONTEXT_RPM_PATH"
 run_root rm "$ONE_CONTEXT_RPM_PATH"
 
+# Install resize2fs, which is required to resize the root file-system.
+run_root dnf -y install e2fsprogs
+
 # Initalize base services.
 run_root systemd-machine-id-setup
-run_root systemctl enable systemd-networkd.service
 
 run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime
 run_root systemctl enable systemd-timesyncd.service
@@ -141,7 +141,8 @@ run_root dnf -y install kernel grub2
 cat > /mnt/etc/dracut.conf.d/virtio-blk.conf <<EOF
 add_drivers="virtio-blk"
 EOF
-run_root dracut --force
+kernel_version=$(ls /mnt/boot | grep "vmlinuz.*.$ARCH" | cut -d- -f2-)
+run_root dracut --force --kver $kernel_version
 
 # Configure grub2.
 run_root grub2-install --target=i386-pc "${NBD_DEVICE}"

pg_repair

@@ -0,0 +1,14 @@
+#!/bin/bash -e
+
+pglist_arr=( $(ceph health detail | grep pg | grep active | awk '{print $2}' ))
+
+echo ${pglist_arr[*]}
+
+for ((i=0; i<${#pglist_arr[@]}; i++)) do
+   if [ 1 -eq $(ceph pg repair ${pglist_arr[$i]} | grep repair | grep instructing | wc -l) ]; then
+      echo repair script error
+      break
+   fi
+   echo ${pglist_arr[$i]} repair script done
+   sleep 10
+done

vpn-statistics.sh

@@ -9,3 +9,10 @@ done
 
 # countries with counter
 ( for ip in $(wg | grep endpoint | sed -e 's/endpoint: //' -e 's/\(.*\):[0-9]*/\1/' -e 's/\[//' -e 's/\]//'); do curl -s ipinfo.io/$ip | grep -e country ; done  ) | sort | uniq -c | sort -g
+
+# Get number of configured VPNs
+configured_vpns=$(wg show  | grep ^peer | wc -l)
+active_vpns=$(wg show  | grep endpoint | wc -l)
+
+echo "Configured VPNs: ${configured_vpns}"
+echo "Active VPNs: ${active_vpns}"