Merge branch 'master' into 'master'

[OpenNebula-images] Add scripts to generate FreeBSD images.

See merge request ungleich-public/ungleich-tools!5

opennebula-images/FreeBSD-build-opennebula-image-generic.sh

@@ -0,0 +1,243 @@
+#!/bin/sh
+#
+# Copyright 2020 -- Evilham <contact@evilham.com>
+# This is BSD licensed as it's based on BSD-licensed code
+#
+# We could have used e.g. something like:
+# - https://git.sr.ht/~sircmpwn/builds.sr.ht/tree/master/images/freebsd/genimg
+#
+# But we actually do want to compile the kernel, so that the IPv6-only images
+# are different and don't support INET.
+
+# Explode if something goes wrong
+set -e
+
+# What are we building?
+# These are the only configuration options.
+# They default to current environment.
+# RELEASE: should be 'CURRENT' for current or 'X.Y' Defaults to 'CURRENT'.
+# ARCH: probably amd64 for DCL
+# VMFORMATS: defaults to qcow2, can also be raw. See man mkimg.
+# OPENNEBULA_CONTEXT_VERSION: For DCL's OpenNebula that'd be 5.10.0 (default)
+# OPENNEBULA_CONTEXT_REVISION: Defaults to 1.
+RELEASE=${RELEASE:-CURRENT}
+if [ "${RELEASE}" == "CURRENT" ]; then
+        SRCBRANCH="master"
+else
+        SRCBRANCH="releng/${RELEASE}"
+fi
+ARCH=${ARCH:-amd64}
+VMFORMATS=${VMFORMATS:-qcow2}
+OPENNEBULA_CONTEXT_VERSION=${OPENNEBULA_CONTEXT_VERSION:-5.10.0}
+OPENNEBULA_CONTEXT_REVISION=${OPENNEBULA_CONTEXT_REVISION:-1}
+
+# Didn't see a need to make these configurable.
+CHROOTDIR="/scratch"
+SRCDIR="${CHROOTDIR}/usr/src"
+OUR_DIR="$(realpath $(dirname "${0}"))"
+OUR_SRCCONF="${SRCDIR}/release/src.conf"
+OUR_RELEASE_CONF="${SRCDIR}/release/release.conf"
+# Shorthand for the package file name.
+OPENNEBULA_CONTEXT="one-context-${OPENNEBULA_CONTEXT_VERSION}_${OPENNEBULA_CONTEXT_REVISION}.txz"
+
+setup_sources() {
+        # Let's use git, we might need to install it
+        if ! which git 2>&1 > /dev/null; then
+                pkg install -y git
+        fi
+
+        if [ ! -d "$(dirname ${SRCDIR})" ]; then
+                mkdir -p "$(dirname ${SRCDIR})"
+        fi
+
+        # Checkout needed branch
+        if [ ! -d "${SRCDIR}" ]; then
+                git clone "https://github.com/freebsd/freebsd" \
+                        --branch "${SRCBRANCH}" "${SRCDIR}"
+        else
+                GIT_CMD="git -C ${SRCDIR}"
+                ${GIT_CMD} clean -df
+                ${GIT_CMD} reset --hard
+                ${GIT_CMD} fetch
+                ${GIT_CMD} checkout "${SRCBRANCH}"
+                ${GIT_CMD} pull
+        fi
+
+        # Add settings for IPv6-only kernel
+        cat > "${SRCDIR}/sys/${ARCH}/conf/GENERIC-IPV6ONLY" << EOF
+include GENERIC
+ident GENERIC-IPV6ONLY
+makeoptions MKMODULESENV+="WITHOUT_INET_SUPPORT="
+nooptions INET
+nodevice gre
+EOF
+        # Fix vmimage.subr to install custom package and fix other things
+        cat >> "${SRCDIR}/release/tools/vmimage.subr" << EOF
+vm_extra_install_ports() {
+        # Make sure we install the opennbula context package
+        cp "/${OPENNEBULA_CONTEXT}" "\${DESTDIR}/tmp/${OPENNEBULA_CONTEXT}"
+        chroot \${DESTDIR} \${EMULATOR} env ASSUME_ALWAYS_YES=yes \\
+                /usr/sbin/pkg add '/tmp/${OPENNEBULA_CONTEXT}'
+
+        # Now make sure the system has better defaults
+        cat >> "\${DESTDIR}/etc/rc.conf" << eof
+# Update to latest patch on first boot
+firstboot_freebsd_update_enable="YES"
+# Enable OpenNebula's service.
+one_context_enable="YES"
+# Enable SSH for customers
+sshd_enable="YES"
+# Clear tmp on boot
+clear_tmp_enable="YES"
+# Disable sendmail by default
+sendmail_enable="NONE"
+# Disable crash dumps
+dumpdev="NO"
+eof
+        # Enable root access with SSH key.
+        # It is user's responsibility to further secure their system.
+        sed -i '' -E \
+            's/(^#[ ]*|^)PermitRootLogin .*/PermitRootLogin without-password/' \
+            "\${DESTDIR}/etc/ssh/sshd_config"
+}
+EOF
+        # Skip building iso images
+        rm "${SRCDIR}/release/${ARCH}/mkisoimages.sh"
+        # This is a hack to not build the memstick
+        cat > "${SRCDIR}/release/${ARCH}/make-memstick.sh" <<EOF
+# Create an empty file, else checksums fail
+touch "\${2}" || true
+EOF
+}
+
+setup_our_env() {
+        # Required by META_MODE to build faster next time
+        # This saves a lot of time when e.g. compiling GENERIC and GENERIC-IPV6ONLY
+        if ! kldstat | grep -q filemon; then
+                kldload filemon
+        fi
+}
+
+gen_releaseconf() {
+        cat << EOF
+#!/bin/sh
+#
+# Based off FreeBSD's release/release.conf.sample
+#
+
+# This redefines the prototype defined in release.sh.
+# At this stage, the build chroot exists.
+buildenv_setup() {
+  # Ensure META_MODE is on
+  echo "WITH_META_MODE=yes" > \${CHROOTDIR}/etc/src-env.conf
+}
+
+## Set the directory within which the release will be built.
+CHROOTDIR="${CHROOTDIR}"
+
+## Set to override the default target architecture and kernel
+TARGET="${ARCH}"
+TARGET_ARCH="${ARCH}"
+KERNEL="${KERNEL_CONFIG}"
+
+## Set to specify a custom make.conf and/or src.conf
+SRC_CONF="${OUR_SRCCONF}"
+
+# Since these are VMs, users should add other components if they want to.
+NODOC=YES
+NOPORTS=YES
+NOSRC=YES
+
+# We manage sources manually
+SRC_UPDATE_SKIP=YES
+
+## Set to pass additional flags to make(1) for the build chroot setup, such
+## as TARGET/TARGET_ARCH.
+# This was necessary for "cross-compiling"
+CHROOT_MAKEENV="MK_LLVM_TARGET_X86=yes"
+
+WITH_VMIMAGES=YES
+
+# VM image size, see man 1 truncate
+VMSIZE="10G"
+
+# List of disk image formats, see man mkgimg.
+VMFORMATS="${VMFORMATS}"
+
+# These variables have to be exported because they are needed in subprocesses.
+export NOSWAP=YES
+# Custom ports
+# - firstboot-freebsd-update helps us not have to create an image for each
+#   patch level. We still will have to do it for each minor version update.
+# - bash is apparently needed for one-context
+export VM_EXTRA_PACKAGES="firstboot-freebsd-update bash"
+EOF
+}
+
+_do_run_release() {
+        . "${SRCDIR}/release/release.sh"
+}
+run_release() {
+        _do_run_release -c "${OUR_RELEASE_CONF}"
+}
+
+
+build_image() {
+        # Generate configuration
+        echo "${2}" > "${OUR_SRCCONF}"
+        KERNEL_CONFIG="${1}"
+        gen_releaseconf > "${OUR_RELEASE_CONF}"
+        # Be paranoid about files and stuff
+        sync
+        # Continue with the release script
+        run_release
+        # Be paranoid about files and stuff
+        sync
+
+        mv "${CHROOTDIR}/R/vmimages" "${OUR_DIR}/FreeBSD-${RELEASE}-${1}"
+
+        # Be paranoid about files and stuff
+        sync
+}
+
+our_main() {
+        case "$1" in
+                --dualstack)
+                        BUILD_DUALSTACK=yes
+                        ;;
+                --ipv6only)
+                        BUILD_IPV6ONLY=yes
+                        ;;
+                *)
+                        cat << EOF
+Run with --dualstack or --ipv6only depending on the image you want.
+EOF
+                        exit 1
+                        ;;
+        esac
+        setup_sources
+        setup_our_env
+        # Fetch OpenNebula's context package
+        fetch "https://github.com/OpenNebula/addon-context-linux/releases/download/v${OPENNEBULA_CONTEXT_VERSION}/${OPENNEBULA_CONTEXT}" \
+                -o "${CHROOTDIR}/${OPENNEBULA_CONTEXT}"
+        # Do run
+        if [ -n "${BUILD_DUALSTACK}" ]; then
+                build_image "GENERIC"
+        fi
+        if [ -n "${BUILD_IPV6ONLY}" ]; then
+                build_image "GENERIC-IPV6ONLY" "$(cat << EOF
+WITHOUT_INET=yes
+WITHOUT_INET_SUPPORT=yes
+EOF
+)"
+        fi
+
+        cat << EOF
+
+*************** DONE ***************
+You will find the images under "${OUR_DIR}".
+************************************
+EOF
+}
+
+our_main "${@}"

opennebula-images/FreeBSD-build-opennebula-image.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+#
+# Copyright 2020 -- Evilham <contact@evilham.com>
+# This is BSD licensed as it's based on BSD-licensed code
+#
+#
+# This builds all needed FreeBSD images for ungleich's Data Center Light
+# When there are new releases, they should be updated here and the script
+# should run.
+# 11.4 is scheduled end of June 2020
+# 12.2 is scheduled end of October 2020
+#
+
+SUPPORTED_RELEASES="11.3 12.1"
+
+# This should run in a DCL VM with an OK amount of cores (4/8 minimum),
+# 4G RAM, and storage of roughly 20G + 5G * #resulting_images.
+#
+# This is because there is the base system, a 'pristine chroot', and during the
+# build there can be 2 copies of the resulting system written to the system.
+# Since there are 4 combinations of images:
+#   {STABLE,RELEASE} x {dualstack, IPv6ONLY}
+#
+# That means we'll need to assign about 40G storage to be on the safe side.
+
+for release in ${SUPPORTED_RELEASES}; do
+    for build in dualstack ipv6only; do
+        env RELEASE=${release} sh FreeBSD-build-opennebula-image-generic.sh --${build} \
+            | tee "FreeBSD-${release}-${build}.log"
+    done
+done