From 321b39ee893c012341c988b151f1f186332a72de Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 28 Nov 2013 14:11:23 +0100 Subject: [PATCH 01/19] use new docs/ (from 2012 or so) Signed-off-by: Nico Schottelius --- {doc => docs}/dev/logs/2012-05-24.preos | 0 docs/dev/logs/2013-11-28.preos | 2 ++ 2 files changed, 2 insertions(+) rename {doc => docs}/dev/logs/2012-05-24.preos (100%) create mode 100644 docs/dev/logs/2013-11-28.preos diff --git a/doc/dev/logs/2012-05-24.preos b/docs/dev/logs/2012-05-24.preos similarity index 100% rename from doc/dev/logs/2012-05-24.preos rename to docs/dev/logs/2012-05-24.preos diff --git a/docs/dev/logs/2013-11-28.preos b/docs/dev/logs/2013-11-28.preos new file mode 100644 index 00000000..aa34f377 --- /dev/null +++ b/docs/dev/logs/2013-11-28.preos @@ -0,0 +1,2 @@ +- debootstrap for the moment +- From 65cab0d0e8adf4fac88ee0f3e909131273d4d45f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 28 Nov 2013 14:18:24 +0100 Subject: [PATCH 02/19] add "preos" subcommand to generate preos Signed-off-by: Nico Schottelius --- cdist/preos.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/cdist | 10 +++++++++ 2 files changed, 71 insertions(+) create mode 100644 cdist/preos.py diff --git a/cdist/preos.py b/cdist/preos.py new file mode 100644 index 00000000..29181057 --- /dev/null +++ b/cdist/preos.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import os +import subprocess + +# initialise cdist +import cdist.exec.local + +import cdist.config + +log = logging.getLogger(__name__) + +class PreOS(object): + + def __init__(self, target_dir, arch="amd64"): + + self.target_dir = target_dir + self.arch = arch + + self.command = "debootstrap" + self.suite = "wheezy" + self.options = [ "--include=openssh-server", + "--arch=%s" % self.arch ] + + def run(self): + cmd = [ self.command ] + cmd.extend(self.options) + cmd.append(self.suite) + cmd.append(self.target_dir) + + log.debug("Bootstrap: %s" % cmd) + + subprocess.call(cmd) + + + @classmethod + def commandline(cls, args): + print(args) + self = cls(target_dir=args.target_dir[0], + arch=args.arch) + self.run() diff --git a/scripts/cdist b/scripts/cdist index 39449666..f4d4ce93 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -26,6 +26,7 @@ def commandline(): import cdist.banner import cdist.config + import cdist.preos import cdist.shell # Construct parser others can reuse @@ -83,6 +84,15 @@ def commandline(): default=cdist.REMOTE_EXEC) parser['config'].set_defaults(func=cdist.config.Config.commandline) + # PreOS + parser['preos'] = parser['sub'].add_parser('preos', + parents=[parser['loglevel']]) + parser['preos'].add_argument('-a', '--arch', + help='Select architecture for preos', default="amd64") + parser['preos'].add_argument('target_dir', nargs=1, + help='Select target directory') + parser['preos'].set_defaults(func=cdist.preos.PreOS.commandline) + # Shell parser['shell'] = parser['sub'].add_parser('shell', parents=[parser['loglevel']]) From adab98799485adb826fff7b8ded384668c1df606 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 29 Nov 2013 16:02:00 +0100 Subject: [PATCH 03/19] minor updates for preos Signed-off-by: Nico Schottelius --- cdist/preos.py | 4 +++- docs/dev/logs/2013-11-28.preos | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 29181057..77d6d6dc 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -42,7 +42,7 @@ class PreOS(object): self.options = [ "--include=openssh-server", "--arch=%s" % self.arch ] - def run(self): + def bootstrap(self): cmd = [ self.command ] cmd.extend(self.options) cmd.append(self.suite) @@ -52,6 +52,8 @@ class PreOS(object): subprocess.call(cmd) + def run(self): + self.bootstrap() @classmethod def commandline(cls, args): diff --git a/docs/dev/logs/2013-11-28.preos b/docs/dev/logs/2013-11-28.preos index aa34f377..f8561135 100644 --- a/docs/dev/logs/2013-11-28.preos +++ b/docs/dev/logs/2013-11-28.preos @@ -1,2 +1,2 @@ - debootstrap for the moment -- +- add triggers: https://github.com/telmich/cdist/issues/214 From a6e2cf853e55e7123c9923b7e86ce31b0cc618fd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Dec 2013 22:33:13 +0100 Subject: [PATCH 04/19] mkdiropt needs to be unquoted => empty if not existing Otherwise this happens: root@lilly ~ # cat /var/lib/cdist/object/__directory/vm/.cdist/code-remote rm -f "/vm" mkdir "" "/vm" which results into mkdir: cannot create directory `': No such file or directory Signed-off-by: Nico Schottelius --- cdist/conf/type/__directory/gencode-remote | 2 +- docs/changelog | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index 23fa4ed3..800fc6e4 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -75,7 +75,7 @@ case "$state_should" in set_attributes=1 cat << DONE rm -f "$destination" -mkdir "$mkdiropt" "$destination" +mkdir $mkdiropt "$destination" DONE fi diff --git a/docs/changelog b/docs/changelog index 67eaca1a..b9056fa6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ Changelog * Type __file: Only remove file when state is absent (Steven Armstrong) * Type __link: Only remove link when state is absent (Steven Armstrong) * Type __directory: Only remove directory when state is absent (Steven Armstrong) + * Type __directory: Fix newly introduced quoting issue * Core: Fix backtrace when cache cannot be deleted 2.3.6: 2013-11-25 From 8af1add2a6059bf6a6c525f15a712a8a25464fa6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2014 00:04:46 +0100 Subject: [PATCH 05/19] preos: seperate parameters, create remote_exec, remote_copy and manifest on the fly Signed-off-by: Nico Schottelius --- cdist/preos.py | 98 +++++++++++++++++++++++++++++++--- docs/dev/logs/2014-01-09.preos | 20 +++++++ scripts/cdist | 9 ++++ 3 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 docs/dev/logs/2014-01-09.preos diff --git a/cdist/preos.py b/cdist/preos.py index 77d6d6dc..2f53791f 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -22,16 +22,26 @@ import logging import os import subprocess +import stat +import tempfile -# initialise cdist -import cdist.exec.local import cdist.config +import cdist.exec.local +import cdist.exec.remote log = logging.getLogger(__name__) +class PreOSExistsError(cdist.Error): + def __init__(self, path): + self.path = path + + def __str__(self): + return 'Path %s already exists' % self.path + + class PreOS(object): - + def __init__(self, target_dir, arch="amd64"): self.target_dir = target_dir @@ -42,7 +52,53 @@ class PreOS(object): self.options = [ "--include=openssh-server", "--arch=%s" % self.arch ] + self._init_helper() + + def _init_helper(self): + self.helper = {} + self.helper["manifest"] = """ +for pkg in linux-image-amd64 openssh-server; do + __package $pkg --state present +done +""" + self.helper["remote_exec"] = """#!/bin/sh +# echo $@ +# set -x +chroot="$1"; shift + +script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX") +trap cleanup INT TERM EXIT +cleanup() { + [ $__cdist_debug ] || rm "$script" +} + +echo "#!/bin/sh -l" > "$script" +echo "$@" >> "$script" +chmod +x "$script" + +relative_script="${script#$chroot}" + +# run in chroot +chroot "$chroot" "$relative_script" +""" + + self.helper["remote_copy"] = """#!/bin/sh + echo $@ + set -x +src=$1; shift +dst=$1; shift +real_dst=$(echo $dst | sed 's,:,,') +cp -L "$src" "$real_dst" +""" + + @property + def exists(self): + return os.path.exists(self.target_dir) + def bootstrap(self): + if self.exists: + raise PreOSExistsError(self.target_dir) + cmd = [ self.command ] cmd.extend(self.options) cmd.append(self.suite) @@ -52,12 +108,40 @@ class PreOS(object): subprocess.call(cmd) - def run(self): - self.bootstrap() + def create_helper_files(self, base_dir): + for key, val in self.helper.items(): + filename = os.path.join(base_dir, key) + with open(filename, "w") as fd: + fd.write(val) + os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) + + def config(self): + handle, path = tempfile.mkstemp(prefix='cdist.stdin.') + with tempfile.TemporaryDirectory() as tempdir: + host = self.target_dir + + self.create_helper_files(tempdir) + + local = cdist.exec.local.Local( + target_host=host, + initial_manifest=os.path.join(tempdir, "manifest") + ) + + remote = cdist.exec.remote.Remote( + target_host=host, + remote_exec=os.path.join(tempdir, "remote_exec"), + remote_copy=os.path.join(tempdir, "remote_copy"), + ) + + config = cdist.config.Config(local, remote) + config.run() @classmethod def commandline(cls, args): - print(args) self = cls(target_dir=args.target_dir[0], arch=args.arch) - self.run() + + if args.bootstrap: + self.bootstrap() + if args.config: + self.config() diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos new file mode 100644 index 00000000..b802825d --- /dev/null +++ b/docs/dev/logs/2014-01-09.preos @@ -0,0 +1,20 @@ +- debootstrap + x setup arch + + - include trigger + - replace with cdist config later? + + - get kernel + - create initramfs + - later: + - configure chroot using cdist + + + packages: + linux-image-amd64 + + - temporary manifest + + - bugs with sudo + [22:50:04] bento:~# ln -s ~nico/.cdist/ ~ + diff --git a/scripts/cdist b/scripts/cdist index f4d4ce93..872d00ff 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -89,6 +89,15 @@ def commandline(): parents=[parser['loglevel']]) parser['preos'].add_argument('-a', '--arch', help='Select architecture for preos', default="amd64") + parser['preos'].add_argument('-b', '--bootstrap', + help='Bootstrap directory with OS', action="store_true") + parser['preos'].add_argument('-c', '--configure', + help='Configure previously bootstrapped directory', action="store_true", + dest="config") + parser['preos'].add_argument('-i', '--initramfs', + help='Create Linux initramfs', action="store_true") + parser['preos'].add_argument('-k', '--kernel', + help='Create Linux kernel', action="store_true") parser['preos'].add_argument('target_dir', nargs=1, help='Select target directory') parser['preos'].set_defaults(func=cdist.preos.PreOS.commandline) From b535e848ada9720d01861292c8df79b1cbe51677 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2014 00:38:29 +0100 Subject: [PATCH 06/19] run apt-get update after deboostrap Signed-off-by: Nico Schottelius --- cdist/preos.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index 2f53791f..98fd7f35 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -108,6 +108,8 @@ cp -L "$src" "$real_dst" subprocess.call(cmd) + cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get update" ] + def create_helper_files(self, base_dir): for key, val in self.helper.items(): filename = os.path.join(base_dir, key) From 4fb55b8d92065690d383c21e6015f550c70122bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2014 10:46:09 +0100 Subject: [PATCH 07/19] various updates for preos Signed-off-by: Nico Schottelius --- cdist/preos.py | 51 +++++++++++++++++--- docs/dev/logs/2014-01-09.preos | 85 +++++++++++++++++++++++++++++----- scripts/cdist | 21 ++++++--- 3 files changed, 133 insertions(+), 24 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 98fd7f35..c24dbbe3 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2013-2014 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -39,6 +39,9 @@ class PreOSExistsError(cdist.Error): def __str__(self): return 'Path %s already exists' % self.path +class PreOSBootstrapError(cdist.Error): + pass + class PreOS(object): @@ -52,14 +55,39 @@ class PreOS(object): self.options = [ "--include=openssh-server", "--arch=%s" % self.arch ] + self.pxelinux = "/usr/lib/syslinux/pxelinux.0" + self.pxelinux-cfg = """ +DEFAULT linux +LABEL linux +KERNEL linux +INITRD initramfs +APPEND ro root=/dev/sda1 initrd=initrd.img + self._init_helper() def _init_helper(self): self.helper = {} self.helper["manifest"] = """ -for pkg in linux-image-amd64 openssh-server; do +for pkg in \ + file \ + linux-image-amd64 + openssh-server + syslinux \ + gdisk util-linux \ + btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do __package $pkg --state present done + +__file /etc/network/interfaces --source - --mode 0644 << eof +# The loopback network interface +auto lo +iface lo inet loopback + +# The primary network interface +auto eth0 +allow-hotplug eth0 +iface eth0 init dhcp +eof """ self.helper["remote_exec"] = """#!/bin/sh # echo $@ @@ -78,13 +106,16 @@ chmod +x "$script" relative_script="${script#$chroot}" +# ensure PATH is setup +export PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin + # run in chroot chroot "$chroot" "$relative_script" """ self.helper["remote_copy"] = """#!/bin/sh - echo $@ - set -x +# echo $@ +# set -x src=$1; shift dst=$1; shift real_dst=$(echo $dst | sed 's,:,,') @@ -106,9 +137,14 @@ cp -L "$src" "$real_dst" log.debug("Bootstrap: %s" % cmd) - subprocess.call(cmd) +# try: + subprocess.check_call(cmd) +# except subprocess.CalledProcessError: +# raise - cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get update" ] + # Required to run this - otherwise apt-get install fails + cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get", "update" ] + subprocess.check_call(cmd) def create_helper_files(self, base_dir): for key, val in self.helper.items(): @@ -117,6 +153,9 @@ cp -L "$src" "$real_dst" fd.write(val) os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) + def create_pxe(self, base_dir): + pass + def config(self): handle, path = tempfile.mkstemp(prefix='cdist.stdin.') with tempfile.TemporaryDirectory() as tempdir: diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos index b802825d..c0b840ee 100644 --- a/docs/dev/logs/2014-01-09.preos +++ b/docs/dev/logs/2014-01-09.preos @@ -1,20 +1,83 @@ - debootstrap x setup arch + x allow cdist to configure debootstrapped directory using cdist + x include sshd + x configure network (eth0, dhcp) + x various mkfs variants + - various fdisk tools - - include trigger - - replace with cdist config later? + - add option for different initial manifest + - allow -, stdin usage - - get kernel - - create initramfs - - later: - - configure chroot using cdist + - add option for additional manifest + - allow -, stdin usage + - trigger + - can be handled in the manifest of the user - packages: - linux-image-amd64 + - remove /var/cache/apt/archives/* ? - - temporary manifest + - fix linux-image name (amd64) - - bugs with sudo - [22:50:04] bento:~# ln -s ~nico/.cdist/ ~ + - blog! + - self configuring + + - pxe + /pxe/ + - pxelinux.0 + - linux + - initramfs + - pxelinux.cfg/ + - default + + - iso + + - add unit tests + +-------------------------------------------------------------------------------- + +[1:16] bento:~% sudo cdist preos -vc ~nico/preos-tests/preos03 +INFO: cdist: version 3.0.0-38-gea286c6 +INFO: /home/users/nico/preos-tests/preos03: Running global explorers +INFO: /home/users/nico/preos-tests/preos03: Running initial manifest /tmp/tmpxbquwe/manifest +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __file/etc/network/interfaces +INFO: /home/users/nico/preos-tests/preos03: Generating code for __file/etc/network/interfaces +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/xfsprogs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/reiser4progs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/jfsutils +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/e2fsprogs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/btrfs-tools +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/file +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/syslinux +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/openssh-server +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/linux-image-amd64 +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/linux-image-amd64 +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/linux-image-amd64 +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/openssh-server +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/openssh-server +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/syslinux +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/syslinux +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/file +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/file +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/btrfs-tools +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/btrfs-tools +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/e2fsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/e2fsprogs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/jfsutils +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/jfsutils +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/reiser4progs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/reiser4progs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/xfsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/xfsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/xfsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/reiser4progs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/jfsutils +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/e2fsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/btrfs-tools +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/file +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/syslinux +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/openssh-server +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/linux-image-amd64 +INFO: /home/users/nico/preos-tests/preos03: Finished successful run in 2.546635866165161 seconds +[1:16] bento:~% diff --git a/scripts/cdist b/scripts/cdist index 872d00ff..c1e1cd94 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -89,15 +89,22 @@ def commandline(): parents=[parser['loglevel']]) parser['preos'].add_argument('-a', '--arch', help='Select architecture for preos', default="amd64") + parser['preos'].add_argument('-A', '--additional-manifest', + help='Add stuff to configuration manifest', default="amd64") parser['preos'].add_argument('-b', '--bootstrap', - help='Bootstrap directory with OS', action="store_true") + help='Bootstrap directory with PreOS', action="store_true") parser['preos'].add_argument('-c', '--configure', - help='Configure previously bootstrapped directory', action="store_true", - dest="config") - parser['preos'].add_argument('-i', '--initramfs', - help='Create Linux initramfs', action="store_true") - parser['preos'].add_argument('-k', '--kernel', - help='Create Linux kernel', action="store_true") + help='Configure previously bootstrapped directory', + action="store_true", dest="config") + parser['preos'].add_argument('-i', '--initial-manifest', + help='Initial manifest for configuration (added to built in)') + parser['preos'].add_argument('-r', '--replace-manifest', + help='Instead of appending to the built in manifest, replace the internal manifest', + action="store_true") + parser['preos'].add_argument('-I', '--iso-boot', + help='Create ISO for booting in given location') + parser['preos'].add_argument('-p', '--pxe-boot', + help='Create PXE files for booting in given location') parser['preos'].add_argument('target_dir', nargs=1, help='Select target directory') parser['preos'].set_defaults(func=cdist.preos.PreOS.commandline) From 995e33afc9f62959fabc1888ec02952d6a68242a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2014 17:21:42 +0100 Subject: [PATCH 08/19] add command line handling for pxe generating Signed-off-by: Nico Schottelius --- cdist/preos.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index c24dbbe3..78b88f94 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -56,12 +56,13 @@ class PreOS(object): "--arch=%s" % self.arch ] self.pxelinux = "/usr/lib/syslinux/pxelinux.0" - self.pxelinux-cfg = """ + self.pxelinux_cfg = """ DEFAULT linux LABEL linux KERNEL linux INITRD initramfs APPEND ro root=/dev/sda1 initrd=initrd.img +""" self._init_helper() @@ -153,9 +154,29 @@ cp -L "$src" "$real_dst" fd.write(val) os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) - def create_pxe(self, base_dir): + def create_kernel(self): + cmd=[ "cp", '"$(ls boot/vmlinuz-* | tail -n1)"' ] + cmd.append + pass + def create_initramfs(self): + base_cmd="find . -print0 | sudo cpio --null -ov --format=newc | gzip -9" + + pass + + def create_iso(self, out_dir): + self.out_dir = out_dir + + raise cdist.Error("Generating ISO is not yet supported") + + def create_pxe(self, out_dir): + self.out_dir = out_dir + + self.create_kernel() + self.create_initramfs() + self.create_pxeconfig() + def config(self): handle, path = tempfile.mkstemp(prefix='cdist.stdin.') with tempfile.TemporaryDirectory() as tempdir: @@ -186,3 +207,7 @@ cp -L "$src" "$real_dst" self.bootstrap() if args.config: self.config() + if args.pxe_boot: + self.create_pxe(args.pxe_boot) + if args.iso_boot: + self.create_iso(args.pxe_boot) From c585e4876e46306afd56b9490e36e00d379b3fbd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 21:05:14 +0100 Subject: [PATCH 09/19] create kernel, pxeconfig and pxelinux.0 Signed-off-by: Nico Schottelius --- cdist/preos.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 78b88f94..1fb73ad7 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -20,9 +20,11 @@ # import logging +import glob import os import subprocess import stat +import shutil import tempfile @@ -57,11 +59,10 @@ class PreOS(object): self.pxelinux = "/usr/lib/syslinux/pxelinux.0" self.pxelinux_cfg = """ -DEFAULT linux -LABEL linux -KERNEL linux +DEFAULT preos +LABEL preos +KERNEL kernel INITRD initramfs -APPEND ro root=/dev/sda1 initrd=initrd.img """ self._init_helper() @@ -155,13 +156,29 @@ cp -L "$src" "$real_dst" os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) def create_kernel(self): - cmd=[ "cp", '"$(ls boot/vmlinuz-* | tail -n1)"' ] - cmd.append + dst = os.path.join(self.out_dir, "kernel") + srcglob = glob.glob("%s/boot/vmlinuz-*" % self.target_dir) + src = srcglob[0] - pass + shutil.copyfile(src, dst, follow_symlinks=True) + + def create_pxelinux(self): + dst = os.path.join(self.out_dir, "pxelinux.0") + src = "%s/usr/lib/syslinux/pxelinux.0" % self.target_dir + + shutil.copyfile(src, dst, follow_symlinks=True) + + def create_pxeconfig(self): + configdir = os.path.join(self.out_dir, "pxelinux.cfg") + configfile = os.path.join(configdir, "default") + if not os.path.isdir(configdir): + os.mkdir(configdir) + + with open(configfile, "w") as fd: + fd.write(self.pxelinux_cfg) def create_initramfs(self): - base_cmd="find . -print0 | sudo cpio --null -ov --format=newc | gzip -9" + base_cmd="find . -print0 | cpio --null -ov --format=newc | gzip -9" pass @@ -174,8 +191,9 @@ cp -L "$src" "$real_dst" self.out_dir = out_dir self.create_kernel() - self.create_initramfs() +# self.create_initramfs() self.create_pxeconfig() + self.create_pxelinux() def config(self): handle, path = tempfile.mkstemp(prefix='cdist.stdin.') @@ -210,4 +228,4 @@ cp -L "$src" "$real_dst" if args.pxe_boot: self.create_pxe(args.pxe_boot) if args.iso_boot: - self.create_iso(args.pxe_boot) + self.create_iso(args.iso_boot) From 0d78ab313ffcb81a0daf9b650f982cd3e1d1b90b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 21:14:04 +0100 Subject: [PATCH 10/19] create initramfs Signed-off-by: Nico Schottelius --- cdist/preos.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 1fb73ad7..6d473ddf 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -160,17 +160,20 @@ cp -L "$src" "$real_dst" srcglob = glob.glob("%s/boot/vmlinuz-*" % self.target_dir) src = srcglob[0] + log.info("Creating kernel ...") shutil.copyfile(src, dst, follow_symlinks=True) def create_pxelinux(self): dst = os.path.join(self.out_dir, "pxelinux.0") src = "%s/usr/lib/syslinux/pxelinux.0" % self.target_dir + log.info("Creating pxelinux.0 ...") shutil.copyfile(src, dst, follow_symlinks=True) def create_pxeconfig(self): configdir = os.path.join(self.out_dir, "pxelinux.cfg") configfile = os.path.join(configdir, "default") + log.info("Creating pxe configuration ...") if not os.path.isdir(configdir): os.mkdir(configdir) @@ -178,9 +181,11 @@ cp -L "$src" "$real_dst" fd.write(self.pxelinux_cfg) def create_initramfs(self): - base_cmd="find . -print0 | cpio --null -ov --format=newc | gzip -9" + out_file = os.path.join(self.out_dir, "initramfs") + cmd="cd {target_dir}; find . -print0 | cpio --null -o --format=newc | gzip -9 > {out_file}".format(target_dir = self.target_dir, out_file = out_file) - pass + log.info("Creating initramfs ...") + subprocess.check_call(cmd, shell=True) def create_iso(self, out_dir): self.out_dir = out_dir @@ -191,7 +196,7 @@ cp -L "$src" "$real_dst" self.out_dir = out_dir self.create_kernel() -# self.create_initramfs() + self.create_initramfs() self.create_pxeconfig() self.create_pxelinux() From 11ba4640b4c7d2e0018fcaafa0fab66f147a169f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 22:34:44 +0100 Subject: [PATCH 11/19] disable unsupported iso - create /init - include support for another initial manifest Signed-off-by: Nico Schottelius --- cdist/preos.py | 89 +++++++++++++++++++++++----------- docs/dev/logs/2014-01-09.preos | 29 +++++++++-- scripts/cdist | 6 +-- 3 files changed, 87 insertions(+), 37 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 6d473ddf..cbb9f23c 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -24,16 +24,42 @@ import glob import os import subprocess import stat +import sys import shutil import tempfile - import cdist.config import cdist.exec.local import cdist.exec.remote log = logging.getLogger(__name__) +DEFAULT_MANIFEST = """ +for pkg in \ + file \ + linux-image-amd64 \ + openssh-server \ + syslinux \ + gdisk util-linux \ + btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do + __package $pkg --state present +done + +# initramfs requires /init +__link /init --source /sbin/init --type symbolic + +__file /etc/network/interfaces --source - --mode 0644 << eof +# The loopback network interface +auto lo +iface lo inet loopback + +# The primary network interface +auto eth0 +allow-hotplug eth0 +iface eth0 init dhcp +eof +""" + class PreOSExistsError(cdist.Error): def __init__(self, path): self.path = path @@ -65,32 +91,9 @@ KERNEL kernel INITRD initramfs """ - self._init_helper() - def _init_helper(self): self.helper = {} - self.helper["manifest"] = """ -for pkg in \ - file \ - linux-image-amd64 - openssh-server - syslinux \ - gdisk util-linux \ - btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do - __package $pkg --state present -done - -__file /etc/network/interfaces --source - --mode 0644 << eof -# The loopback network interface -auto lo -iface lo inet loopback - -# The primary network interface -auto eth0 -allow-hotplug eth0 -iface eth0 init dhcp -eof -""" + self.helper["manifest"] = self.initial_manifest self.helper["remote_exec"] = """#!/bin/sh # echo $@ # set -x @@ -200,7 +203,25 @@ cp -L "$src" "$real_dst" self.create_pxeconfig() self.create_pxelinux() + + def setup_initial_manifest(self, user_initial_manifest, replace_manifest): + if user_initial_manifest: + if user_initial_manifest == '-': + user_initial_manifest_content = sys.stdin.read() + else: + with open(user_initial_manifest, "r") as fd: + user_initial_manifest_content = fd.read() + else: + user_initial_manifest_content = "" + + if replace_manifest: + self.initial_manifest = user_initial_manifest_content + else: + self.initial_manifest = "{default}\n# User supplied manifest\n{user}".format(default=DEFAULT_MANIFEST, user=user_initial_manifest_content) + def config(self): + self._init_helper() + handle, path = tempfile.mkstemp(prefix='cdist.stdin.') with tempfile.TemporaryDirectory() as tempdir: host = self.target_dir @@ -226,11 +247,21 @@ cp -L "$src" "$real_dst" self = cls(target_dir=args.target_dir[0], arch=args.arch) + # read initial manifest first - it may come from stdin + if args.config: + self.setup_initial_manifest(args.initial_manifest, args.replace_manifest) + + # Bootstrap: creates base directory if args.bootstrap: self.bootstrap() + + # Configure the OS if args.config: self.config() - if args.pxe_boot: - self.create_pxe(args.pxe_boot) - if args.iso_boot: - self.create_iso(args.iso_boot) + + # Output pxe files + if args.pxe_boot_dir: + self.create_pxe(args.pxe_boot_dir) + + #if args.iso_boot_dir: + # self.create_iso(args.iso_boot) diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos index c0b840ee..0f0b0384 100644 --- a/docs/dev/logs/2014-01-09.preos +++ b/docs/dev/logs/2014-01-09.preos @@ -8,21 +8,25 @@ - add option for different initial manifest - allow -, stdin usage + - allow to replace current manifest (later) - - add option for additional manifest - - allow -, stdin usage - - - trigger + x trigger - can be handled in the manifest of the user - remove /var/cache/apt/archives/* ? + - later, optimisation level + + + - bug: cdist config als root! - fix linux-image name (amd64) + - ln -s /sbin/init /init + - blog! - self configuring - - pxe + x pxe /pxe/ - pxelinux.0 - linux @@ -31,6 +35,9 @@ - default - iso + - later + - usb stick (+efi version) + - later - add unit tests @@ -81,3 +88,15 @@ INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/linux- INFO: /home/users/nico/preos-tests/preos03: Finished successful run in 2.546635866165161 seconds [1:16] bento:~% +-------------------------------------------------------------------------------- +[21:14] bento:vm-tests% qemu-system-x86_64 -m 2G -boot order=cn -drive file=testhd1,if=virtio -net nic -net user,tftp=$(pwd -P)/tftp,bootfile=/pxelinux.0 + +-------------------------------------------------------------------------------- +[21:16] bento:preos-tests% sudo cdist preos -vp /home/users/nico/vm-tests/tftp /home/users/nico/preos-tests/preos03/ +INFO: cdist: version 3.0.0-42-g0d78ab3 +INFO: cdist.preos: Creating kernel ... +INFO: cdist.preos: Creating initramfs ... +760780 blocks +INFO: cdist.preos: Creating pxe configuration ... +INFO: cdist.preos: Creating pxelinux.0 ... + diff --git a/scripts/cdist b/scripts/cdist index c1e1cd94..ed9d2eda 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -101,9 +101,9 @@ def commandline(): parser['preos'].add_argument('-r', '--replace-manifest', help='Instead of appending to the built in manifest, replace the internal manifest', action="store_true") - parser['preos'].add_argument('-I', '--iso-boot', - help='Create ISO for booting in given location') - parser['preos'].add_argument('-p', '--pxe-boot', +# parser['preos'].add_argument('-I', '--iso-boot-dir', +# help='Create ISO for booting in given location') + parser['preos'].add_argument('-p', '--pxe-boot-dir', help='Create PXE files for booting in given location') parser['preos'].add_argument('target_dir', nargs=1, help='Select target directory') From e7ad8f929804e02337c4c09b4045ade2859d2b85 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 22:45:38 +0100 Subject: [PATCH 12/19] inet not init Signed-off-by: Nico Schottelius --- cdist/preos.py | 2 +- docs/dev/logs/2014-01-09.preos | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index cbb9f23c..8b4a9a09 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -56,7 +56,7 @@ iface lo inet loopback # The primary network interface auto eth0 allow-hotplug eth0 -iface eth0 init dhcp +iface eth0 inet dhcp eof """ diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos index 0f0b0384..8d5a59b6 100644 --- a/docs/dev/logs/2014-01-09.preos +++ b/docs/dev/logs/2014-01-09.preos @@ -41,6 +41,10 @@ - add unit tests +- testing with qemu + [22:43] bento:vm-tests% qemu-system-x86_64 -m 2G -boot order=cn -drive file=testhd1,if=virtio -net nic -net user,tftp=$(pwd -P)/tftp,bootfile=/pxelinux.0,hostfwd=tcp::7777-:22 -enable-kvm + + -------------------------------------------------------------------------------- [1:16] bento:~% sudo cdist preos -vc ~nico/preos-tests/preos03 From 07545f4f7f9ddc806037a61babccefa6d3fa83a5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 22:47:34 +0100 Subject: [PATCH 13/19] update preos notes Signed-off-by: Nico Schottelius --- docs/dev/logs/2014-01-09.preos | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos index 8d5a59b6..1a3f2ddc 100644 --- a/docs/dev/logs/2014-01-09.preos +++ b/docs/dev/logs/2014-01-09.preos @@ -6,9 +6,9 @@ x various mkfs variants - various fdisk tools - - add option for different initial manifest - - allow -, stdin usage - - allow to replace current manifest (later) + x add option for different initial manifest + x allow -, stdin usage + x allow to replace current manifest (later) x trigger - can be handled in the manifest of the user @@ -44,6 +44,9 @@ - testing with qemu [22:43] bento:vm-tests% qemu-system-x86_64 -m 2G -boot order=cn -drive file=testhd1,if=virtio -net nic -net user,tftp=$(pwd -P)/tftp,bootfile=/pxelinux.0,hostfwd=tcp::7777-:22 -enable-kvm +- create preos + [22:43] bento:preos-tests% echo __panter_root_ssh_keys | sudo cdist preos -vp /home/users/nico/vm-tests/tftp -c /home/users/nico/preos-tests/preos03/ -i - + -------------------------------------------------------------------------------- From 3daa74e81dc225a9004ba276b20e25f1ad3d80be Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 22:48:47 +0100 Subject: [PATCH 14/19] fix 'stdin: is not a tty' problem (thanks, steven) Signed-off-by: Nico Schottelius --- cdist/preos.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index 8b4a9a09..433cf871 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -58,6 +58,10 @@ auto eth0 allow-hotplug eth0 iface eth0 inet dhcp eof + +# Steven found this out - coyping it 1:1 +# fix the bloody 'stdin: is not a tty' problem +__line /root/.profile --line 'mesg n' --state absent """ class PreOSExistsError(cdist.Error): From c3f79277b226ef5c97360566c868660e1ed1fbad Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 08:50:41 +0100 Subject: [PATCH 15/19] add some more packages for preos - fixes #267 Signed-off-by: Nico Schottelius --- cdist/preos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 433cf871..f573566d 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -39,8 +39,8 @@ for pkg in \ file \ linux-image-amd64 \ openssh-server \ - syslinux \ - gdisk util-linux \ + syslinux grub 2 \ + gdisk util-linux lvm2 mdadm \ btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do __package $pkg --state present done From 79cfdf578d1d93241648262b64c2243ffed49559 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 09:01:04 +0100 Subject: [PATCH 16/19] remove obsolete '--additional-manifest' parameter Signed-off-by: Nico Schottelius --- scripts/cdist | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/cdist b/scripts/cdist index ed9d2eda..7b7569ee 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -89,8 +89,6 @@ def commandline(): parents=[parser['loglevel']]) parser['preos'].add_argument('-a', '--arch', help='Select architecture for preos', default="amd64") - parser['preos'].add_argument('-A', '--additional-manifest', - help='Add stuff to configuration manifest', default="amd64") parser['preos'].add_argument('-b', '--bootstrap', help='Bootstrap directory with PreOS', action="store_true") parser['preos'].add_argument('-c', '--configure', From 55f26cbe25d8c87279a8221a9ef611fc2c90b8f1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 09:06:02 +0100 Subject: [PATCH 17/19] - ' ' Signed-off-by: Nico Schottelius --- cdist/preos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index f573566d..b76d2178 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -39,7 +39,7 @@ for pkg in \ file \ linux-image-amd64 \ openssh-server \ - syslinux grub 2 \ + syslinux grub2 \ gdisk util-linux lvm2 mdadm \ btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do __package $pkg --state present From b125c0a4f273acaf33a082f8179b69e3b765b7d0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 09:21:50 +0100 Subject: [PATCH 18/19] create output directory, if it does not exist Signed-off-by: Nico Schottelius --- cdist/preos.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index b76d2178..f3d34c21 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -194,14 +194,21 @@ cp -L "$src" "$real_dst" log.info("Creating initramfs ...") subprocess.check_call(cmd, shell=True) + def ensure_out_dir_exists(self): + os.makedirs(self.out_dir, exist_ok=True) + + def create_iso(self, out_dir): self.out_dir = out_dir + self.ensure_out_dir_exists() + raise cdist.Error("Generating ISO is not yet supported") def create_pxe(self, out_dir): self.out_dir = out_dir + self.ensure_out_dir_exists() self.create_kernel() self.create_initramfs() self.create_pxeconfig() From 4cfedb1787abe7af68aa3cb51bbfc84714c26f4b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 09:42:53 +0100 Subject: [PATCH 19/19] +curl Signed-off-by: Nico Schottelius --- cdist/preos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index f3d34c21..347b0cba 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -38,7 +38,7 @@ DEFAULT_MANIFEST = """ for pkg in \ file \ linux-image-amd64 \ - openssh-server \ + openssh-server curl \ syslinux grub2 \ gdisk util-linux lvm2 mdadm \ btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do