Merge branch 'master' of code.ungleich.ch:ungleich-public/ungleich-tools

This commit is contained in:
Nico Schottelius 2020-05-21 17:09:49 +02:00
commit e0ac893ae6
9 changed files with 618 additions and 1 deletions

View file

@ -0,0 +1,16 @@
#!/bin/sh
rm -f /etc/apt/sources.list.d/ceph.list
cat > /etc/apt/sources.list <<EOF
deb http://pkgmaster.devuan.org/merged beowulf main contrib non-free
deb http://pkgmaster.devuan.org/merged beowulf-updates main contrib non-free
deb http://pkgmaster.devuan.org/merged beowulf-security main contrib non-free
EOF
echo deb http://ftp.debian.org/debian buster-backports main > /etc/apt/sources.list.d/backports.list
apt update
apt dist-upgrade -y
apt install -t buster-backports -y ceph

View file

@ -0,0 +1,154 @@
#!/bin/sh
# This script generates Debian images for OpenNebula.
#
# Test image locally (without network) with:
# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2
set -e
set -x
# XXX: Handle command-line arguments?
ARCH=amd64
IMAGE_PATH=arch-$(date --iso-8601).img.qcow2
IMAGE_SIZE=10G
NBD_DEVICE=/dev/nbd0
# TODO: find the package definition and built ourself, publish in some RPM repository.
ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb"
ONE_CONTEXT_DEB_PATH=/root/one-context.deb
cleanup() {
# The order here is important.
umount /mnt/dev/pts 2>/dev/null || true
umount /mnt/dev/shm 2>/dev/null || true
umount /mnt/dev 2>/dev/null || true
umount /mnt/proc 2>/dev/null || true
umount /mnt/run 2>/dev/null || true
umount /mnt/sys 2>/dev/null || true
umount /mnt/boot 2>/dev/null || true
umount /mnt 2>/dev/null || true
qemu-nbd --disconnect "$NBD_DEVICE" || true
}
run_root() {
chroot /mnt /usr/bin/env \
PATH=/sbin:/usr/sbin:/bin:/usr/bin \
sh -c "$*"
}
if [ "$(whoami)" != 'root' ]; then
echo "This script must be run as root." >&2
exit 1
fi
if [ $(lsb_release --short --id) != "Arch" ]; then
echo "WARNING: this script has been designed to run on an Ubuntu system." >&2
echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2
sleep 5
fi
# Create base QCOW2 image.
qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE"
modprobe nbd max_part=16
qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH"
# Wait for qemu-nbd to settle.
sleep 1
# Don't forget to cleanup, even if the script crash.
trap cleanup EXIT
# Create partition table, format partitions.
sfdisk --no-reread "$NBD_DEVICE" <<EOF
1M,500M,L,*
,,L
EOF
mkfs.ext4 "${NBD_DEVICE}p1"
mkfs.ext4 "${NBD_DEVICE}p2"
# Mount partitions, install base OS.
mount "${NBD_DEVICE}p2" /mnt
mkdir /mnt/boot
mount "${NBD_DEVICE}p1" /mnt/boot
# Install base system.
pacstrap /mnt nbase base-devel openssh
mount --bind /dev /mnt/dev
mount --bind /dev/pts /mnt/dev/pts
mount --bind /dev/shm /mnt/dev/shm
mount --bind /proc /mnt/proc
mount --bind /run /mnt/run
mount --bind /sys /mnt/sys
# Guest networking is to be handled by the one-context package.
# See https://github.com/OpenNebula/addon-context-linux for details.
# Required to resolve package mirror in chroot.
cp /etc/resolv.conf /mnt/etc/resolv.conf
# Initialize /etc/hosts.
cat > /mnt/etc/hosts << EOF
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
EOF
# Configure package sources and update package index.
cat > /mnt/etc/pacman.d/mirrorlist << EOF
##
## Arch Linux repository mirrorlist
## Generated on 2020-05-12
##
## Switzerland
Server = http://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch
Server = https://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch
Server = http://mirror.init7.net/archlinux/$repo/os/$arch
Server = https://mirror.init7.net/archlinux/$repo/os/$arch
Server = http://mirror.puzzle.ch/archlinux/$repo/os/$arch
Server = https://mirror.puzzle.ch/archlinux/$repo/os/$arch
Server = https://mirror.ungleich.ch/mirror/packages/archlinux/$repo/os/$arch
EOF
run_root pacman --sync --refresh --upgrade
# Initalize base services.
run_root systemd-machine-id-setup
run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime
run_root systemctl enable systemd-timesyncd.service
# Install kernel and generate initramfs.
run_root pacman --sync linux mkinitcpio
run_rot mkinitcpio -P
# Install and configure bootloader.
run_root pacman --sync grub
run_root grub-install --target=i386-pc "${NBD_DEVICE}"
run_root grub-mkconfig -o /boot/grub/grub.cfg
# Install en configure SSH daemon.
run_root pacman --sync openssh-server
# Install haveged due to lack of entropy in ONE environment.
run_root pacman --sync haveged
run_root systemctl enable haveged.service
# Generate fstab file.
boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1")
root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2")
cat >>/mnt/etc/fstab <<EOF
UUID=$boot_uuid /boot ext4 rw,relatime,data=ordered 0 2
UUID=$root_uuid / ext4 rw,relatime,data=ordered 0 1
EOF
# Reset systemd's environment.
run_root rm -f /etc/machine-id
run_root touch /etc/machine-id
rm -f /var/lib/systemd/random-seed
# Make sure everything is written to disk before exiting.
sync

View file

@ -0,0 +1,168 @@
#!/bin/sh
# This script generates Debian images for OpenNebula.
#
# Test image locally (without network) with:
# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2
set -e
set -x
# XXX: Handle command-line arguments?
RELEASE=buster # 10.X
ARCH=amd64
IMAGE_PATH=debian-$RELEASE-$(date --iso-8601).img.qcow2
IMAGE_SIZE=10G
NBD_DEVICE=/dev/nbd0
# TODO: find the package definition and built ourself, publish in some RPM repository.
ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb"
ONE_CONTEXT_DEB_PATH=/root/one-context.deb
cleanup() {
# The order here is important.
umount /mnt/dev/pts 2>/dev/null || true
umount /mnt/dev/shm 2>/dev/null || true
umount /mnt/dev 2>/dev/null || true
umount /mnt/proc 2>/dev/null || true
umount /mnt/run 2>/dev/null || true
umount /mnt/sys 2>/dev/null || true
umount /mnt/boot 2>/dev/null || true
umount /mnt 2>/dev/null || true
qemu-nbd --disconnect "$NBD_DEVICE" || true
}
run_root() {
chroot /mnt /usr/bin/env \
PATH=/sbin:/usr/sbin:/bin:/usr/bin \
sh -c "$*"
}
if [ "$(whoami)" != 'root' ]; then
echo "This script must be run as root." >&2
exit 1
fi
if [ $(lsb_release --short --id) != "Debian" ]; then
echo "WARNING: this script has been designed to run on an Debian system." >&2
echo "WARNING: Not running Debian. Giving you 5 seconds to abort." >&2
sleep 5
fi
# Create base QCOW2 image.
qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE"
modprobe nbd max_part=16
qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH"
# Wait for qemu-nbd to settle.
sleep 1
# Don't forget to cleanup, even if the script crash.
trap cleanup EXIT
# Create partition table, format partitions.
sfdisk --no-reread "$NBD_DEVICE" <<EOF
1M,500M,L,*
,,L
EOF
mkfs.ext4 "${NBD_DEVICE}p1"
mkfs.ext4 "${NBD_DEVICE}p2"
# Mount partitions, install base OS.
mount "${NBD_DEVICE}p2" /mnt
mkdir /mnt/boot
mount "${NBD_DEVICE}p1" /mnt/boot
debootstrap \
--arch=$ARCH $RELEASE \
/mnt http://ftp.ch.debian.org/debian
mount --bind /dev /mnt/dev
mount --bind /dev/pts /mnt/dev/pts
mount --bind /dev/shm /mnt/dev/shm
mount --bind /proc /mnt/proc
mount --bind /run /mnt/run
mount --bind /sys /mnt/sys
# Guest networking is to be handled by the one-context package.
# See https://github.com/OpenNebula/addon-context-linux for details.
# Required to resolve package mirror in chroot.
cp /etc/resolv.conf /mnt/etc/resolv.conf
# Initialize /etc/hosts.
cat > /mnt/etc/hosts << EOF
127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4
::1 localhost localhost.localdomain localhost6 localhost6.localdomain6
EOF
# Configure package sources and update package index.
cat >/mnt/etc/apt/sources.list <<EOF
# Stable
deb http://ftp.ch.debian.org/debian $RELEASE main contrib non-free
deb-src http://ftp.ch.debian.org/debian $RELEASE main contrib non-free
# Security updates
deb http://ftp.ch.debian.org/debian $RELEASE-updates main contrib non-free
deb-src http://ftp.ch.debian.org/debian $RELEASE-updates main contrib non-free
# Backports
#deb http://ftp.ch.debian.org/debian $RELEASE-backports main
#deb-src http://ftp.ch.debian.org/debian $RELEASE-backports main
EOF
run_root apt-get update
# Install (magic?) one-context DEB and hope things works as expected.
curl -L "$ONE_CONTEXT_DEB_URL" > "/mnt$ONE_CONTEXT_DEB_PATH"
run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH"
run_root rm "$ONE_CONTEXT_DEB_PATH"
# Manually install legacy network scripts used by one-context.
run_root apt-get -y install ifupdown
# Initalize base services.
run_root systemd-machine-id-setup
run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime
run_root systemctl enable systemd-timesyncd.service
# Install kernel and bootloader. Do not autoconfigure grub.
run_root 'echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections'
run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-image-amd64 grub-pc
# Configure grub.
run_root grub-install --target=i386-pc "${NBD_DEVICE}"
run_root grub-mkconfig -o /boot/grub/grub.cfg
# Install en configure SSH daemon.
run_root apt-get -y install openssh-server
# Install haveged due to lack of entropy in ONE environment.
run_root apt-get -y install haveged
run_root systemctl enable haveged.service
# Generate locales.
run_root sed -i 's/^# *\(en_GB.UTF-8\)/\1/' /etc/locale.gen
run_root locale-gen
# Generate fstab file.
boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1")
root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2")
cat >>/mnt/etc/fstab <<EOF
UUID=$boot_uuid /boot ext4 rw,relatime,data=ordered 0 2
UUID=$root_uuid / ext4 rw,relatime,data=ordered 0 1
EOF
# Reset systemd's environment.
run_root rm -f /etc/machine-id
run_root touch /etc/machine-id
rm -f /var/lib/systemd/random-seed
# Remove temporary files and reclaim freed disk space.
run_root apt-get clean
# Make sure everything is written to disk before exiting.
sync

View file

@ -18,7 +18,7 @@ set -e
set -x
# XXX: Handle command-line arguments?
RELEASE=31
RELEASE=32
ARCH=x86_64
IMAGE_PATH=fedora-$RELEASE-$(date +%+F).img.qcow2
IMAGE_SIZE=10G
@ -132,6 +132,10 @@ run_root systemd-machine-id-setup
run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime
run_root systemctl enable systemd-timesyncd.service
# Install haveged due to lack of entropy in ONE environment.
run_root dnf -y install haveged
run_root systemctl enable haveged.service
# Install kernel and bootloader.
# Note: linux-firmware is not required our environment and takes almost 200M
# uncompressed but is a direct dependency of kernel-core...

View file

@ -0,0 +1,243 @@
#!/bin/sh
#
# Copyright 2020 -- Evilham <contact@evilham.com>
# This is BSD licensed as it's based on BSD-licensed code
#
# We could have used e.g. something like:
# - https://git.sr.ht/~sircmpwn/builds.sr.ht/tree/master/images/freebsd/genimg
#
# But we actually do want to compile the kernel, so that the IPv6-only images
# are different and don't support INET.
# Explode if something goes wrong
set -e
# What are we building?
# These are the only configuration options.
# They default to current environment.
# RELEASE: should be 'CURRENT' for current or 'X.Y' Defaults to 'CURRENT'.
# ARCH: probably amd64 for DCL
# VMFORMATS: defaults to qcow2, can also be raw. See man mkimg.
# OPENNEBULA_CONTEXT_VERSION: For DCL's OpenNebula that'd be 5.10.0 (default)
# OPENNEBULA_CONTEXT_REVISION: Defaults to 1.
RELEASE=${RELEASE:-CURRENT}
if [ "${RELEASE}" == "CURRENT" ]; then
SRCBRANCH="master"
else
SRCBRANCH="releng/${RELEASE}"
fi
ARCH=${ARCH:-amd64}
VMFORMATS=${VMFORMATS:-qcow2}
OPENNEBULA_CONTEXT_VERSION=${OPENNEBULA_CONTEXT_VERSION:-5.10.0}
OPENNEBULA_CONTEXT_REVISION=${OPENNEBULA_CONTEXT_REVISION:-1}
# Didn't see a need to make these configurable.
CHROOTDIR="/scratch"
SRCDIR="${CHROOTDIR}/usr/src"
OUR_DIR="$(realpath $(dirname "${0}"))"
OUR_SRCCONF="${SRCDIR}/release/src.conf"
OUR_RELEASE_CONF="${SRCDIR}/release/release.conf"
# Shorthand for the package file name.
OPENNEBULA_CONTEXT="one-context-${OPENNEBULA_CONTEXT_VERSION}_${OPENNEBULA_CONTEXT_REVISION}.txz"
setup_sources() {
# Let's use git, we might need to install it
if ! which git 2>&1 > /dev/null; then
pkg install -y git
fi
if [ ! -d "$(dirname ${SRCDIR})" ]; then
mkdir -p "$(dirname ${SRCDIR})"
fi
# Checkout needed branch
if [ ! -d "${SRCDIR}" ]; then
git clone "https://github.com/freebsd/freebsd" \
--branch "${SRCBRANCH}" "${SRCDIR}"
else
GIT_CMD="git -C ${SRCDIR}"
${GIT_CMD} clean -df
${GIT_CMD} reset --hard
${GIT_CMD} fetch
${GIT_CMD} checkout "${SRCBRANCH}"
${GIT_CMD} pull
fi
# Add settings for IPv6-only kernel
cat > "${SRCDIR}/sys/${ARCH}/conf/GENERIC-IPV6ONLY" << EOF
include GENERIC
ident GENERIC-IPV6ONLY
makeoptions MKMODULESENV+="WITHOUT_INET_SUPPORT="
nooptions INET
nodevice gre
EOF
# Fix vmimage.subr to install custom package and fix other things
cat >> "${SRCDIR}/release/tools/vmimage.subr" << EOF
vm_extra_install_ports() {
# Make sure we install the opennbula context package
cp "/${OPENNEBULA_CONTEXT}" "\${DESTDIR}/tmp/${OPENNEBULA_CONTEXT}"
chroot \${DESTDIR} \${EMULATOR} env ASSUME_ALWAYS_YES=yes \\
/usr/sbin/pkg add '/tmp/${OPENNEBULA_CONTEXT}'
# Now make sure the system has better defaults
cat >> "\${DESTDIR}/etc/rc.conf" << eof
# Update to latest patch on first boot
firstboot_freebsd_update_enable="YES"
# Enable OpenNebula's service.
one_context_enable="YES"
# Enable SSH for customers
sshd_enable="YES"
# Clear tmp on boot
clear_tmp_enable="YES"
# Disable sendmail by default
sendmail_enable="NONE"
# Disable crash dumps
dumpdev="NO"
eof
# Enable root access with SSH key.
# It is user's responsibility to further secure their system.
sed -i '' -E \
's/(^#[ ]*|^)PermitRootLogin .*/PermitRootLogin without-password/' \
"\${DESTDIR}/etc/ssh/sshd_config"
}
EOF
# Skip building iso images
rm "${SRCDIR}/release/${ARCH}/mkisoimages.sh"
# This is a hack to not build the memstick
cat > "${SRCDIR}/release/${ARCH}/make-memstick.sh" <<EOF
# Create an empty file, else checksums fail
touch "\${2}" || true
EOF
}
setup_our_env() {
# Required by META_MODE to build faster next time
# This saves a lot of time when e.g. compiling GENERIC and GENERIC-IPV6ONLY
if ! kldstat | grep -q filemon; then
kldload filemon
fi
}
gen_releaseconf() {
cat << EOF
#!/bin/sh
#
# Based off FreeBSD's release/release.conf.sample
#
# This redefines the prototype defined in release.sh.
# At this stage, the build chroot exists.
buildenv_setup() {
# Ensure META_MODE is on
echo "WITH_META_MODE=yes" > \${CHROOTDIR}/etc/src-env.conf
}
## Set the directory within which the release will be built.
CHROOTDIR="${CHROOTDIR}"
## Set to override the default target architecture and kernel
TARGET="${ARCH}"
TARGET_ARCH="${ARCH}"
KERNEL="${KERNEL_CONFIG}"
## Set to specify a custom make.conf and/or src.conf
SRC_CONF="${OUR_SRCCONF}"
# Since these are VMs, users should add other components if they want to.
NODOC=YES
NOPORTS=YES
NOSRC=YES
# We manage sources manually
SRC_UPDATE_SKIP=YES
## Set to pass additional flags to make(1) for the build chroot setup, such
## as TARGET/TARGET_ARCH.
# This was necessary for "cross-compiling"
CHROOT_MAKEENV="MK_LLVM_TARGET_X86=yes"
WITH_VMIMAGES=YES
# VM image size, see man 1 truncate
VMSIZE="10G"
# List of disk image formats, see man mkgimg.
VMFORMATS="${VMFORMATS}"
# These variables have to be exported because they are needed in subprocesses.
export NOSWAP=YES
# Custom ports
# - firstboot-freebsd-update helps us not have to create an image for each
# patch level. We still will have to do it for each minor version update.
# - bash is apparently needed for one-context
export VM_EXTRA_PACKAGES="firstboot-freebsd-update bash"
EOF
}
_do_run_release() {
. "${SRCDIR}/release/release.sh"
}
run_release() {
_do_run_release -c "${OUR_RELEASE_CONF}"
}
build_image() {
# Generate configuration
echo "${2}" > "${OUR_SRCCONF}"
KERNEL_CONFIG="${1}"
gen_releaseconf > "${OUR_RELEASE_CONF}"
# Be paranoid about files and stuff
sync
# Continue with the release script
run_release
# Be paranoid about files and stuff
sync
mv "${CHROOTDIR}/R/vmimages" "${OUR_DIR}/FreeBSD-${RELEASE}-${1}"
# Be paranoid about files and stuff
sync
}
our_main() {
case "$1" in
--dualstack)
BUILD_DUALSTACK=yes
;;
--ipv6only)
BUILD_IPV6ONLY=yes
;;
*)
cat << EOF
Run with --dualstack or --ipv6only depending on the image you want.
EOF
exit 1
;;
esac
setup_sources
setup_our_env
# Fetch OpenNebula's context package
fetch "https://github.com/OpenNebula/addon-context-linux/releases/download/v${OPENNEBULA_CONTEXT_VERSION}/${OPENNEBULA_CONTEXT}" \
-o "${CHROOTDIR}/${OPENNEBULA_CONTEXT}"
# Do run
if [ -n "${BUILD_DUALSTACK}" ]; then
build_image "GENERIC"
fi
if [ -n "${BUILD_IPV6ONLY}" ]; then
build_image "GENERIC-IPV6ONLY" "$(cat << EOF
WITHOUT_INET=yes
WITHOUT_INET_SUPPORT=yes
EOF
)"
fi
cat << EOF
*************** DONE ***************
You will find the images under "${OUR_DIR}".
************************************
EOF
}
our_main "${@}"

View file

@ -0,0 +1,32 @@
#!/bin/sh
#
# Copyright 2020 -- Evilham <contact@evilham.com>
# This is BSD licensed as it's based on BSD-licensed code
#
#
# This builds all needed FreeBSD images for ungleich's Data Center Light
# When there are new releases, they should be updated here and the script
# should run.
# 11.4 is scheduled end of June 2020
# 12.2 is scheduled end of October 2020
#
SUPPORTED_RELEASES="11.3 12.1"
# This should run in a DCL VM with an OK amount of cores (4/8 minimum),
# 4G RAM, and storage of roughly 20G + 5G * #resulting_images.
#
# This is because there is the base system, a 'pristine chroot', and during the
# build there can be 2 copies of the resulting system written to the system.
# Since there are 4 combinations of images:
# {STABLE,RELEASE} x {dualstack, IPv6ONLY}
#
# That means we'll need to assign about 40G storage to be on the safe side.
date=$(date -I)
for release in ${SUPPORTED_RELEASES}; do
for build in dualstack ipv6only; do
env RELEASE=${release} sh freebsd-build-opennebula-image-generic.sh --${build} \
| tee "freebsd-${release}-${build}-${date}.log"
done
done