diff --git a/ceph-upgrade-server-to-nautilus.sh b/ceph-upgrade-server-to-nautilus.sh new file mode 100755 index 0000000..058e40d --- /dev/null +++ b/ceph-upgrade-server-to-nautilus.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +rm -f /etc/apt/sources.list.d/ceph.list + +cat > /etc/apt/sources.list < /etc/apt/sources.list.d/backports.list +apt update +apt dist-upgrade -y +apt install -t buster-backports -y ceph diff --git a/alpine-build-opennebula-image.sh b/opennebula-images/alpine-build-opennebula-image.sh similarity index 100% rename from alpine-build-opennebula-image.sh rename to opennebula-images/alpine-build-opennebula-image.sh diff --git a/opennebula-images/arch-build-opennebula-image.sh b/opennebula-images/arch-build-opennebula-image.sh new file mode 100755 index 0000000..00c454f --- /dev/null +++ b/opennebula-images/arch-build-opennebula-image.sh @@ -0,0 +1,154 @@ +#!/bin/sh + +# This script generates Debian images for OpenNebula. +# +# Test image locally (without network) with: +# qemu-system-x86_64 -enable-kvm -m 1G -drive file=$IMAGE,format=qcow2 + +set -e +set -x + +# XXX: Handle command-line arguments? +ARCH=amd64 +IMAGE_PATH=arch-$(date --iso-8601).img.qcow2 +IMAGE_SIZE=10G +NBD_DEVICE=/dev/nbd0 + +# TODO: find the package definition and built ourself, publish in some RPM repository. +ONE_CONTEXT_DEB_URL="https://github.com/OpenNebula/addon-context-linux/releases/download/v5.10.0/one-context_5.10.0-1.deb" +ONE_CONTEXT_DEB_PATH=/root/one-context.deb + +cleanup() { + # The order here is important. + umount /mnt/dev/pts 2>/dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Arch" ]; then + echo "WARNING: this script has been designed to run on an Ubuntu system." >&2 + echo "WARNING: Not running Ubuntu. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat > /mnt/etc/pacman.d/mirrorlist << EOF +## +## Arch Linux repository mirrorlist +## Generated on 2020-05-12 +## + +## Switzerland +Server = http://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch +Server = https://pkg.adfinis-sygroup.ch/archlinux/$repo/os/$arch +Server = http://mirror.init7.net/archlinux/$repo/os/$arch +Server = https://mirror.init7.net/archlinux/$repo/os/$arch +Server = http://mirror.puzzle.ch/archlinux/$repo/os/$arch +Server = https://mirror.puzzle.ch/archlinux/$repo/os/$arch +Server = https://mirror.ungleich.ch/mirror/packages/archlinux/$repo/os/$arch +EOF +run_root pacman --sync --refresh --upgrade + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and generate initramfs. +run_root pacman --sync linux mkinitcpio +run_rot mkinitcpio -P + +# Install and configure bootloader. +run_root pacman --sync grub +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root pacman --sync openssh-server + +# Install haveged due to lack of entropy in ONE environment. +run_root pacman --sync haveged +run_root systemctl enable haveged.service + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab </dev/null || true + umount /mnt/dev/shm 2>/dev/null || true + umount /mnt/dev 2>/dev/null || true + umount /mnt/proc 2>/dev/null || true + umount /mnt/run 2>/dev/null || true + umount /mnt/sys 2>/dev/null || true + umount /mnt/boot 2>/dev/null || true + umount /mnt 2>/dev/null || true + qemu-nbd --disconnect "$NBD_DEVICE" || true +} + +run_root() { + chroot /mnt /usr/bin/env \ + PATH=/sbin:/usr/sbin:/bin:/usr/bin \ + sh -c "$*" +} + +if [ "$(whoami)" != 'root' ]; then + echo "This script must be run as root." >&2 + exit 1 +fi + +if [ $(lsb_release --short --id) != "Debian" ]; then + echo "WARNING: this script has been designed to run on an Debian system." >&2 + echo "WARNING: Not running Debian. Giving you 5 seconds to abort." >&2 + sleep 5 +fi + +# Create base QCOW2 image. +qemu-img create -f qcow2 "$IMAGE_PATH" "$IMAGE_SIZE" +modprobe nbd max_part=16 +qemu-nbd --connect="$NBD_DEVICE" "$IMAGE_PATH" + +# Wait for qemu-nbd to settle. +sleep 1 + +# Don't forget to cleanup, even if the script crash. +trap cleanup EXIT + +# Create partition table, format partitions. +sfdisk --no-reread "$NBD_DEVICE" < /mnt/etc/hosts << EOF +127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4 +::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 + +EOF + +# Configure package sources and update package index. +cat >/mnt/etc/apt/sources.list < "/mnt$ONE_CONTEXT_DEB_PATH" +run_root apt-get -y install "$ONE_CONTEXT_DEB_PATH" +run_root rm "$ONE_CONTEXT_DEB_PATH" + +# Manually install legacy network scripts used by one-context. +run_root apt-get -y install ifupdown + +# Initalize base services. +run_root systemd-machine-id-setup + +run_root ln -sf /usr/share/zoneinfo/UTC /etc/localtime +run_root systemctl enable systemd-timesyncd.service + +# Install kernel and bootloader. Do not autoconfigure grub. +run_root 'echo "grub-pc grub-pc/install_devices_empty boolean true" | debconf-set-selections' +run_root DEBIAN_FRONTEND=noninteractive apt-get -y install locales linux-image-amd64 grub-pc + +# Configure grub. +run_root grub-install --target=i386-pc "${NBD_DEVICE}" +run_root grub-mkconfig -o /boot/grub/grub.cfg + +# Install en configure SSH daemon. +run_root apt-get -y install openssh-server + +# Install haveged due to lack of entropy in ONE environment. +run_root apt-get -y install haveged +run_root systemctl enable haveged.service + +# Generate locales. +run_root sed -i 's/^# *\(en_GB.UTF-8\)/\1/' /etc/locale.gen +run_root locale-gen + +# Generate fstab file. +boot_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p1") +root_uuid=$(blkid --match-tag UUID --output value "${NBD_DEVICE}p2") +cat >>/mnt/etc/fstab < +# This is BSD licensed as it's based on BSD-licensed code +# +# We could have used e.g. something like: +# - https://git.sr.ht/~sircmpwn/builds.sr.ht/tree/master/images/freebsd/genimg +# +# But we actually do want to compile the kernel, so that the IPv6-only images +# are different and don't support INET. + +# Explode if something goes wrong +set -e + +# What are we building? +# These are the only configuration options. +# They default to current environment. +# RELEASE: should be 'CURRENT' for current or 'X.Y' Defaults to 'CURRENT'. +# ARCH: probably amd64 for DCL +# VMFORMATS: defaults to qcow2, can also be raw. See man mkimg. +# OPENNEBULA_CONTEXT_VERSION: For DCL's OpenNebula that'd be 5.10.0 (default) +# OPENNEBULA_CONTEXT_REVISION: Defaults to 1. +RELEASE=${RELEASE:-CURRENT} +if [ "${RELEASE}" == "CURRENT" ]; then + SRCBRANCH="master" +else + SRCBRANCH="releng/${RELEASE}" +fi +ARCH=${ARCH:-amd64} +VMFORMATS=${VMFORMATS:-qcow2} +OPENNEBULA_CONTEXT_VERSION=${OPENNEBULA_CONTEXT_VERSION:-5.10.0} +OPENNEBULA_CONTEXT_REVISION=${OPENNEBULA_CONTEXT_REVISION:-1} + +# Didn't see a need to make these configurable. +CHROOTDIR="/scratch" +SRCDIR="${CHROOTDIR}/usr/src" +OUR_DIR="$(realpath $(dirname "${0}"))" +OUR_SRCCONF="${SRCDIR}/release/src.conf" +OUR_RELEASE_CONF="${SRCDIR}/release/release.conf" +# Shorthand for the package file name. +OPENNEBULA_CONTEXT="one-context-${OPENNEBULA_CONTEXT_VERSION}_${OPENNEBULA_CONTEXT_REVISION}.txz" + +setup_sources() { + # Let's use git, we might need to install it + if ! which git 2>&1 > /dev/null; then + pkg install -y git + fi + + if [ ! -d "$(dirname ${SRCDIR})" ]; then + mkdir -p "$(dirname ${SRCDIR})" + fi + + # Checkout needed branch + if [ ! -d "${SRCDIR}" ]; then + git clone "https://github.com/freebsd/freebsd" \ + --branch "${SRCBRANCH}" "${SRCDIR}" + else + GIT_CMD="git -C ${SRCDIR}" + ${GIT_CMD} clean -df + ${GIT_CMD} reset --hard + ${GIT_CMD} fetch + ${GIT_CMD} checkout "${SRCBRANCH}" + ${GIT_CMD} pull + fi + + # Add settings for IPv6-only kernel + cat > "${SRCDIR}/sys/${ARCH}/conf/GENERIC-IPV6ONLY" << EOF +include GENERIC +ident GENERIC-IPV6ONLY +makeoptions MKMODULESENV+="WITHOUT_INET_SUPPORT=" +nooptions INET +nodevice gre +EOF + # Fix vmimage.subr to install custom package and fix other things + cat >> "${SRCDIR}/release/tools/vmimage.subr" << EOF +vm_extra_install_ports() { + # Make sure we install the opennbula context package + cp "/${OPENNEBULA_CONTEXT}" "\${DESTDIR}/tmp/${OPENNEBULA_CONTEXT}" + chroot \${DESTDIR} \${EMULATOR} env ASSUME_ALWAYS_YES=yes \\ + /usr/sbin/pkg add '/tmp/${OPENNEBULA_CONTEXT}' + + # Now make sure the system has better defaults + cat >> "\${DESTDIR}/etc/rc.conf" << eof +# Update to latest patch on first boot +firstboot_freebsd_update_enable="YES" +# Enable OpenNebula's service. +one_context_enable="YES" +# Enable SSH for customers +sshd_enable="YES" +# Clear tmp on boot +clear_tmp_enable="YES" +# Disable sendmail by default +sendmail_enable="NONE" +# Disable crash dumps +dumpdev="NO" +eof + # Enable root access with SSH key. + # It is user's responsibility to further secure their system. + sed -i '' -E \ + 's/(^#[ ]*|^)PermitRootLogin .*/PermitRootLogin without-password/' \ + "\${DESTDIR}/etc/ssh/sshd_config" +} +EOF + # Skip building iso images + rm "${SRCDIR}/release/${ARCH}/mkisoimages.sh" + # This is a hack to not build the memstick + cat > "${SRCDIR}/release/${ARCH}/make-memstick.sh" < \${CHROOTDIR}/etc/src-env.conf +} + +## Set the directory within which the release will be built. +CHROOTDIR="${CHROOTDIR}" + +## Set to override the default target architecture and kernel +TARGET="${ARCH}" +TARGET_ARCH="${ARCH}" +KERNEL="${KERNEL_CONFIG}" + +## Set to specify a custom make.conf and/or src.conf +SRC_CONF="${OUR_SRCCONF}" + +# Since these are VMs, users should add other components if they want to. +NODOC=YES +NOPORTS=YES +NOSRC=YES + +# We manage sources manually +SRC_UPDATE_SKIP=YES + +## Set to pass additional flags to make(1) for the build chroot setup, such +## as TARGET/TARGET_ARCH. +# This was necessary for "cross-compiling" +CHROOT_MAKEENV="MK_LLVM_TARGET_X86=yes" + +WITH_VMIMAGES=YES + +# VM image size, see man 1 truncate +VMSIZE="10G" + +# List of disk image formats, see man mkgimg. +VMFORMATS="${VMFORMATS}" + +# These variables have to be exported because they are needed in subprocesses. +export NOSWAP=YES +# Custom ports +# - firstboot-freebsd-update helps us not have to create an image for each +# patch level. We still will have to do it for each minor version update. +# - bash is apparently needed for one-context +export VM_EXTRA_PACKAGES="firstboot-freebsd-update bash" +EOF +} + +_do_run_release() { + . "${SRCDIR}/release/release.sh" +} +run_release() { + _do_run_release -c "${OUR_RELEASE_CONF}" +} + + +build_image() { + # Generate configuration + echo "${2}" > "${OUR_SRCCONF}" + KERNEL_CONFIG="${1}" + gen_releaseconf > "${OUR_RELEASE_CONF}" + # Be paranoid about files and stuff + sync + # Continue with the release script + run_release + # Be paranoid about files and stuff + sync + + mv "${CHROOTDIR}/R/vmimages" "${OUR_DIR}/FreeBSD-${RELEASE}-${1}" + + # Be paranoid about files and stuff + sync +} + +our_main() { + case "$1" in + --dualstack) + BUILD_DUALSTACK=yes + ;; + --ipv6only) + BUILD_IPV6ONLY=yes + ;; + *) + cat << EOF +Run with --dualstack or --ipv6only depending on the image you want. +EOF + exit 1 + ;; + esac + setup_sources + setup_our_env + # Fetch OpenNebula's context package + fetch "https://github.com/OpenNebula/addon-context-linux/releases/download/v${OPENNEBULA_CONTEXT_VERSION}/${OPENNEBULA_CONTEXT}" \ + -o "${CHROOTDIR}/${OPENNEBULA_CONTEXT}" + # Do run + if [ -n "${BUILD_DUALSTACK}" ]; then + build_image "GENERIC" + fi + if [ -n "${BUILD_IPV6ONLY}" ]; then + build_image "GENERIC-IPV6ONLY" "$(cat << EOF +WITHOUT_INET=yes +WITHOUT_INET_SUPPORT=yes +EOF +)" + fi + + cat << EOF + +*************** DONE *************** +You will find the images under "${OUR_DIR}". +************************************ +EOF +} + +our_main "${@}" diff --git a/opennebula-images/freebsd-build-opennebula-image.sh b/opennebula-images/freebsd-build-opennebula-image.sh new file mode 100755 index 0000000..266f639 --- /dev/null +++ b/opennebula-images/freebsd-build-opennebula-image.sh @@ -0,0 +1,32 @@ +#!/bin/sh +# +# Copyright 2020 -- Evilham +# This is BSD licensed as it's based on BSD-licensed code +# +# +# This builds all needed FreeBSD images for ungleich's Data Center Light +# When there are new releases, they should be updated here and the script +# should run. +# 11.4 is scheduled end of June 2020 +# 12.2 is scheduled end of October 2020 +# + +SUPPORTED_RELEASES="11.3 12.1" + +# This should run in a DCL VM with an OK amount of cores (4/8 minimum), +# 4G RAM, and storage of roughly 20G + 5G * #resulting_images. +# +# This is because there is the base system, a 'pristine chroot', and during the +# build there can be 2 copies of the resulting system written to the system. +# Since there are 4 combinations of images: +# {STABLE,RELEASE} x {dualstack, IPv6ONLY} +# +# That means we'll need to assign about 40G storage to be on the safe side. + +date=$(date -I) +for release in ${SUPPORTED_RELEASES}; do + for build in dualstack ipv6only; do + env RELEASE=${release} sh freebsd-build-opennebula-image-generic.sh --${build} \ + | tee "freebsd-${release}-${build}-${date}.log" + done +done diff --git a/ubuntu-build-opennebula-image.sh b/opennebula-images/ubuntu-build-opennebula-image.sh old mode 100644 new mode 100755 similarity index 100% rename from ubuntu-build-opennebula-image.sh rename to opennebula-images/ubuntu-build-opennebula-image.sh