Commit 799ec723 authored by Darko Poljak's avatar Darko Poljak

Implement preos

parent eab540bd
......@@ -5,10 +5,11 @@ import logging
import collections
import functools
import cdist.configuration
import cdist.preos
# set of beta sub-commands
BETA_COMMANDS = set(('install', 'inventory', ))
BETA_COMMANDS = set(('install', 'inventory', 'preos', ))
# set of beta arguments for sub-commands
BETA_ARGS = {
'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )),
......@@ -422,6 +423,9 @@ def get_parsers():
parser['inventory'].set_defaults(
func=cdist.inventory.Inventory.commandline)
# PreOs
parser['preos'] = parser['sub'].add_parser('preos', add_help=False)
# Shell
parser['shell'] = parser['sub'].add_parser(
'shell', parents=[parser['loglevel']])
......
import os
import os.path
import sys
import inspect
import argparse
import cdist
import logging
_PREOS_CALL = "commandline"
_PREOS_NAME = "_preos_name"
_PREOS_MARKER = "_cdist_preos"
_PLUGINS_DIR = "preos"
_PLUGINS_PATH = [os.path.join(os.path.dirname(__file__), _PLUGINS_DIR), ]
cdist_home = cdist.home_dir()
if cdist_home:
cdist_home_preos = os.path.join(cdist_home, "preos")
if os.path.isdir(cdist_home_preos):
_PLUGINS_PATH.append(cdist_home_preos)
sys.path.extend(_PLUGINS_PATH)
log = logging.getLogger("PreOS")
def preos_plugin(obj):
"""It is preos if _PREOS_MARKER is True and has _PREOS_CALL."""
if hasattr(obj, _PREOS_MARKER):
is_preos = getattr(obj, _PREOS_MARKER)
else:
is_preos = False
if is_preos and hasattr(obj, _PREOS_CALL):
yield obj
def scan_preos_dir_plugins(dir):
for fname in os.listdir(dir):
if os.path.isfile(os.path.join(dir, fname)):
fname = os.path.splitext(fname)[0]
module_name = fname
try:
module = __import__(module_name)
yield from preos_plugin(module)
clsmembers = inspect.getmembers(module, inspect.isclass)
for cm in clsmembers:
c = cm[1]
yield from preos_plugin(c)
except ImportError as e:
log.warning("Cannot import '{}': {}".format(module_name, e))
def find_preos_plugins():
for dir in _PLUGINS_PATH:
yield from scan_preos_dir_plugins(dir)
def find_preoses():
preoses = {}
for preos in find_preos_plugins():
if hasattr(preos, _PREOS_NAME):
preos_name = getattr(preos, _PREOS_NAME)
else:
preos_name = preos.__name__.lower()
preoses[preos_name] = preos
return preoses
def check_root():
if os.geteuid() != 0:
raise cdist.Error("Must be run with root privileges")
class PreOS(object):
preoses = None
@classmethod
def commandline(cls, argv):
if not cls.preoses:
cls.preoses = find_preoses()
parser = argparse.ArgumentParser(
description="Create PreOS", prog="cdist preos")
parser.add_argument('preos', help='PreOS to create, one of: {}'.format(
set(cls.preoses)))
args = parser.parse_args(argv[1:2])
preos_name = args.preos
if preos_name in cls.preoses:
preos = cls.preoses[preos_name]
func = getattr(preos, _PREOS_CALL)
if inspect.ismodule(preos):
func_args = [preos, argv[2:], ]
else:
func_args = [argv[2:], ]
log.info("Running preos : {}".format(preos_name))
func(*func_args)
else:
log.error("Unknown preos: {}, available preoses: {}".format(
preos_name, set(cls.preoses.keys())))
from debootstrap.debootstrap import Debian, Ubuntu, Devuan
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# 2016 Darko Poljak (darko.poljak at ungleich.ch)
#
# This file is part of cdist.
#
# cdist is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# cdist is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with cdist. If not, see <http://www.gnu.org/licenses/>.
#
#
import cdist
import cdist.config
import cdist.core
import cdist.preos
import argparse
import cdist.argparse
import logging
import os
import subprocess
class Debian(object):
_preos_name = 'debian'
_cdist_preos = True
_files_dir = os.path.join(os.path.dirname(__file__), "files")
@classmethod
def default_args(cls):
default_remote_exec = os.path.join(cls._files_dir, "remote-exec.sh")
default_remote_copy = os.path.join(cls._files_dir, "remote-copy.sh")
default_init_manifest = os.path.join(
cls._files_dir, "init-manifest-{}".format(cls._preos_name))
defargs = argparse.Namespace()
defargs.arch = 'amd64'
defargs.bootstrap = False
defargs.configure = False
defargs.cdist_params = '-v'
defargs.rm_bootstrap_dir = False
defargs.suite = 'stable'
defargs.remote_exec = default_remote_exec
defargs.remote_copy = default_remote_copy
defargs.manifest = default_init_manifest
return defargs
@classmethod
def get_parser(cls):
defargs = cls.default_args()
cdist_parser = cdist.argparse.get_parsers()
parser = argparse.ArgumentParser(
prog='cdist preos {}'.format(cls._preos_name),
parents=[cdist_parser['loglevel'], cdist_parser['beta']])
parser.add_argument('target_dir', nargs=1,
help=("target directory where PreOS will be "
"bootstrapped"))
parser.add_argument(
'-a', '--arch',
help="target debootstrap architecture, by default '{}'".format(
defargs.arch), dest='arch', default=defargs.arch)
parser.add_argument(
'-B', '--bootstrap',
help='do bootstrap step',
dest='bootstrap', action='store_true', default=defargs.bootstrap)
parser.add_argument(
'-C', '--configure',
help='do configure step',
dest='configure', action='store_true', default=defargs.configure)
parser.add_argument(
'-c', '--cdist-params',
help=("parameters that will be passed to cdist config, by default"
" '{}' is used".format(defargs.cdist_params)),
dest='cdist_params', default=defargs.cdist_params)
parser.add_argument(
'-D', '--drive-boot',
help='create bootable PreOS on specified drive',
dest='drive')
parser.add_argument(
'-e', '--remote-exec',
help=("remote exec that cdist config will use, by default "
"internal script is used"),
dest='remote_exec', default=defargs.remote_exec)
parser.add_argument(
'-i', '--init-manifest',
help=("init manifest that cdist config will use, by default "
"internal init manifest is used"),
dest='manifest', default=defargs.manifest)
parser.add_argument(
'-k', '--keyfile', action="append",
help=("ssh key files that will be added to cdist config; "
"'__ssh_authorized_keys root ...' type is appended to "
"initial manifest"),
dest='keyfile')
parser.add_argument(
'-m', '--mirror',
help='use specified mirror for debootstrap',
dest='mirror')
parser.add_argument(
'-P', '--root-password',
help='Set specified password for root, generated by default',
dest='root_password')
parser.add_argument('-p', '--pxe-boot-dir', help='PXE boot directory',
dest='pxe_boot_dir')
parser.add_argument(
'-r', '--rm-bootstrap-dir',
help='remove target directory after finishing',
dest='rm_bootstrap_dir', action='store_true',
default=defargs.rm_bootstrap_dir)
parser.add_argument(
'-S', '--script',
help='use specified script for debootstrap',
dest='script')
parser.add_argument('-s', '--suite',
help="suite used for debootstrap, "
"by default '{}'".format(defargs.suite),
dest='suite', default=defargs.suite)
parser.add_argument(
'-y', '--remote-copy',
help=("remote copy that cdist config will use, by default "
"internal script is used"),
dest='remote_copy', default=defargs.remote_copy)
parser.epilog = cdist.argparse.EPILOG
return parser
@classmethod
def update_env(cls, env):
pass
@classmethod
def commandline(cls, argv):
log = logging.getLogger(cls.__name__)
parser = cls.get_parser()
cdist.argparse.add_beta_command(cls._preos_name)
args = parser.parse_args(argv)
if args.script and not args.mirror:
raise cdist.Error("script option cannot be used without "
"mirror option")
args.command = cls._preos_name
cdist.argparse.check_beta(vars(args))
cdist.preos.check_root()
args.target_dir = os.path.realpath(args.target_dir[0])
args.os = cls._preos_name
args.remote_exec = os.path.realpath(args.remote_exec)
args.remote_copy = os.path.realpath(args.remote_copy)
args.manifest = os.path.realpath(args.manifest)
if args.keyfile:
new_keyfile = [os.path.realpath(x) for x in args.keyfile]
args.keyfile = new_keyfile
if args.pxe_boot_dir:
args.pxe_boot_dir = os.path.realpath(args.pxe_boot_dir)
cdist.argparse.handle_loglevel(args)
log.debug("preos: {}, args: {}".format(cls._preos_name, args))
try:
env = vars(args)
new_env = {}
for key in env:
if key == 'verbose':
if env[key] >= 3:
new_env['debug'] = "yes"
elif env[key] == 2:
new_env['verbose'] = "yes"
elif not env[key]:
new_env[key] = ''
elif isinstance(env[key], bool) and env[key]:
new_env[key] = "yes"
elif isinstance(env[key], list):
val = env[key]
new_env[key + "_cnt"] = str(len(val))
for i, v in enumerate(val):
new_env[key + "_" + str(i)] = v
else:
new_env[key] = str(env[key])
env = new_env
env.update(os.environ)
cls.update_env(env)
log.debug("preos: {} env: {}".format(cls._preos_name, env))
cmd = os.path.join(cls._files_dir, "code")
info_msg = ["Running preos: {}, suite: {}, arch: {}".format(
cls._preos_name, args.suite, args.arch), ]
if args.mirror:
info_msg.append("mirror: {}".format(args.mirror))
if args.script:
info_msg.append("script: {}".format(args.script))
if args.bootstrap:
info_msg.append("bootstrapping")
if args.configure:
info_msg.append("configuring")
if args.pxe_boot_dir:
info_msg.append("creating PXE")
if args.drive:
info_msg.append("creating bootable drive")
log.info(info_msg)
log.debug("cmd={}".format(cmd))
subprocess.check_call(cmd, env=env, shell=True)
except subprocess.CalledProcessError as e:
log.error("preos {} failed: {}".format(cls._preos_name, e))
class Ubuntu(Debian):
_preos_name = "ubuntu"
@classmethod
def default_args(cls):
defargs = super().default_args()
defargs.suite = 'xenial'
return defargs
class Devuan(Debian):
_preos_name = "devuan"
@classmethod
def default_args(cls):
defargs = super().default_args()
defargs.suite = 'jessie'
return defargs
@classmethod
def update_env(cls, env):
env['DEBOOTSTRAP_DIR'] = os.path.join(cls._files_dir,
'devuan-debootstrap')
#!/bin/sh
##
## 2016 Darko Poljak (darko.poljak at ungleich.ch)
##
## This file is part of cdist.
##
## cdist is free software: you can redistribute it and/or modify
## it under the terms of the GNU General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## cdist is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU General Public License for more details.
##
## You should have received a copy of the GNU General Public License
## along with cdist. If not, see <http://www.gnu.org/licenses/>.
set -e
if [ "${debug}" ]
then
set -x
cdist_params="${cdist_params} -d"
fi
bootstrap_dir="${target_dir}"
case "${os}" in
ubuntu|debian|devuan)
# nothing, those are valid values
;;
*)
echo "ERROR: invalid os value: ${os}" >&2
exit 1
;;
esac
check_bootstrap_dir() {
if [ ! -e "$1" ]
then
echo "ERROR: bootstrap directory $1 does not exist" >&2
exit 1
fi
}
# bootstrap
if [ "${bootstrap}" ]
then
if [ "${DEBOOTSTRAP_DIR}" ]
then
debootstrap_cmd="${DEBOOTSTRAP_DIR}/debootstrap"
else
command -v debootstrap 2>&1 > /dev/null || {
echo "ERROR: debootstrap not found" >&2
exit 1
}
debootstrap_cmd="debootstrap"
fi
# If PreOS on drive then do not check for directory emptiness.
# Partition can at least contain 'lost+found' directory.
if [ ! "${drive}" ]
then
if [ -e "${bootstrap_dir}" ]
then
dir_content=$(ls -A "${bootstrap_dir}" | wc -l)
else
dir_content=0
fi
if [ "${dir_content}" -ne 0 ]
then
echo "ERROR: "${bootstrap_dir}" not empty " >&2
exit 1
fi
fi
if [ "${verbose}" -o "${debug}" ]
then
echo "bootstrapping..."
fi
mkdir -p "${bootstrap_dir}"
"${debootstrap_cmd}" --include=openssh-server --arch=${arch} ${suite} ${bootstrap_dir} \
${mirror} ${script}
if [ "${verbose}" -o "${debug}" ]
then
echo "bootstrap finished"
fi
fi
chroot_mount() {
mount -t proc none "${bootstrap_dir}/proc" || true
mount -t sysfs none "${bootstrap_dir}/sys" || true
mount -o bind /dev "${bootstrap_dir}/dev" || true
mount -t devpts none "${bootstrap_dir}/dev/pts" || true
}
chroot_umount() {
umount "${bootstrap_dir}/dev/pts" || true
umount "${bootstrap_dir}/dev" || true
umount "${bootstrap_dir}/sys" || true
umount "${bootstrap_dir}/proc" || true
}
TRAPFUNC="umount \"${bootstrap_dir}/dev/pts\" || true; \
umount \"${bootstrap_dir}/dev\" || true; \
umount \"${bootstrap_dir}/sys\" || true; \
umount \"${bootstrap_dir}/proc\" || true;"
# config
if [ "${configure}" ]
then
if [ ! -f "${manifest}" ]
then
echo "ERROR: ${manifest} does not exist" >&2
exit 1
fi
if [ ! -f "${remote_exec}" ]
then
echo "ERROR: ${remote_exec} does not exist" >&2
exit 1
fi
if [ ! -f "${remote_copy}" ]
then
echo "ERROR: ${remote_copy} does not exist" >&2
exit 1
fi
if [ "${keyfile_cnt}" -a "${keyfile_cnt}" -gt 0 ]
then
i="$((keyfile_cnt - 1))"
keyfiles=""
while [ "${i}" -ge 0 ]
do
kf_var="keyfile_${i}"
eval kf='$'"${kf_var}"
if [ ! -f "${kf}" ]
then
echo "ERROR: ${kf} does not exist" >&2
exit 1
fi
key=$(cat "${kf}")
keyfiles="${keyfiles} --key '${key}'"
i=$((i - 1))
done
ssh_auth_keys_line="__ssh_authorized_keys root ${keyfiles}\n"
else
ssh_auth_keys_line=""
fi
check_bootstrap_dir "${bootstrap_dir}"
if [ "${verbose}" -o "${debug}" ]
then
echo "configuring..."
fi
trap "${TRAPFUNC}" 0 1 2 3 15
chroot_mount
chroot "${bootstrap_dir}" /usr/bin/apt-get update
if [ "${drive}" ]
then
grub_manifest_line="__package grub-pc --state present\n"
grub_kern_params_line="__line linux_kernel_params \
--file /etc/default/grub \
--line 'GRUB_CMDLINE_LINUX_DEFAULT=\"quiet splash net.ifnames=0\"'\n"
else
grub_manifest_line=""
grub_kern_params_line=""
fi
grub_lines="${grub_manifest_line}${grub_kern_params_line}"
printf "${ssh_auth_keys_line}${grub_lines}" \
| cat "${manifest}" - |\
cdist config \
${cdist_params} -i - \
--remote-exec "${remote_exec}" \
--remote-copy "${remote_copy}" \
"${bootstrap_dir}"
# __hostname with systmed uses hostnamectl which needs dbus running
# set hostname explicitly here instead
printf "preos\n" > "${bootstrap_dir}/etc/hostname"
chroot "${bootstrap_dir}" /usr/bin/apt-get autoclean
chroot "${bootstrap_dir}" /usr/bin/apt-get clean
chroot "${bootstrap_dir}" /usr/bin/apt-get autoremove
chroot_umount
trap - 0 1 2 3 15
if [ "${verbose}" -o "${debug}" ]
then
echo "configuring finished"
fi
fi
if [ "${pxe_boot_dir}" ]
then
check_bootstrap_dir "${bootstrap_dir}"
if [ "${verbose}" -o "${debug}" ]
then
echo "creating pxe..."
fi
mkdir -p "${pxe_boot_dir}"
cp "${bootstrap_dir}"/boot/vmlinuz-* "${pxe_boot_dir}/kernel"
cd "${bootstrap_dir}"
find . -print0 | cpio --null -o --format=newc | gzip -9 > "${pxe_boot_dir}/initramfs"
mkdir -p "${pxe_boot_dir}/pxelinux.cfg"
cat <<EOPXEF > "${pxe_boot_dir}/pxelinux.cfg/default"
DEFAULT preos
LABEL preos
KERNEL kernel
APPEND utf8 load_ramdisk=1 root=/dev/ram nofb initrd=initramfs console=ttyS1,115200 net.ifnames=0
EOPXEF
cp "${bootstrap_dir}/usr/lib/PXELINUX/pxelinux.0" "${pxe_boot_dir}/pxelinux.0"
cp "${bootstrap_dir}/usr/lib/syslinux/modules/bios/ldlinux.c32" \
"${pxe_boot_dir}/ldlinux.c32"
# network boot need all files world readable
chmod -R 644 "${pxe_boot_dir}"/*
if [ "${verbose}" -o "${debug}" ]
then
echo "pxe creation finished"
fi
fi
if [ "${drive}" ]
then
trap "${TRAPFUNC}" 0 1 2 3 15
chroot_mount
chroot "${bootstrap_dir}" grub-install ${drive}
chroot "${bootstrap_dir}" /bin/sh -c "GRUB_DISABLE_OS_PROBER=true update-grub"
# set root password
if [ ! "${root_password}" ]
then
if ! which strings >/dev/null 2>&1
then
printf "strings is missing\n" >&2
exit 1
fi
root_password="$(head -n 1000 /dev/urandom | strings | \
grep -o '[[:alnum:]]' | head -n 30 | tr -d '\n')"
printf "Generated root password (without quotes):'${root_password}'\n"
fi
chroot "${bootstrap_dir}" /bin/sh -c "echo \"root:${root_password}\" | \
chpasswd"
# /etc/securetty must not be world writeable.
chmod 644 "${bootstrap_dir}"/etc/securetty
chroot_umount
trap - 0 1 2 3 15
fi
if [ "${rm_bootstrap_dir}" ]
then
if [ "${verbose}" -o "${debug}" ]
then
echo "removing bootstrap dir..."
fi
rm -r -f "${bootstrap_dir}"
if [ "${verbose}" -o "${debug}" ]
then
echo "removing bootstrap dir finished"
fi
fi
# avoid dpkg-dev dependency; fish out the version with sed
VERSION := $(shell sed 's/.*(\(.*\)).*/\1/; q' debian/changelog)
all:
clean:
DSDIR=$(DESTDIR)/usr/share/debootstrap
install:
mkdir -p $(DSDIR)/scripts
mkdir -p $(DESTDIR)/usr/sbin
cp -a scripts/* $(DSDIR)/scripts/
install -o root -g root -m 0644 functions $(DSDIR)/
sed 's/@VERSION@/$(VERSION)/g' debootstrap >$(DESTDIR)/usr/sbin/debootstrap
chown root:root $(DESTDIR)/usr/sbin/debootstrap
chmod 0755 $(DESTDIR)/usr/sbin/debootstrap
README for debootstrap
======================
See the manpage for (some) documentation.
Running debootstrap from source
-------------------------------
You can run debootstrap from its source tree without installing it. This
can be useful if you want a quick way to make a Debian chroot on another