merge preos and install

Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>

cdist/preos.py

@@ -0,0 +1,278 @@
+# -*- coding: utf-8 -*-
+#
+# 2013-2014 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+import logging
+import glob
+import os
+import subprocess
+import stat
+import sys
+import shutil
+import tempfile
+
+import cdist.config
+import cdist.exec.local
+import cdist.exec.remote
+
+log = logging.getLogger(__name__)
+
+DEFAULT_MANIFEST = """
+for pkg in \
+    file \
+    linux-image-amd64 \
+    openssh-server curl \
+    syslinux grub2 \
+    gdisk util-linux lvm2 mdadm \
+    btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do
+    __package $pkg --state present
+done
+
+# initramfs requires /init
+__link /init --source /sbin/init --type symbolic
+
+__file /etc/network/interfaces --source - --mode 0644 << eof
+# The loopback network interface
+auto lo
+iface lo inet loopback
+
+# The primary network interface
+auto eth0
+allow-hotplug eth0
+iface eth0 inet dhcp
+eof
+
+# Steven found this out - coyping it 1:1
+# fix the bloody 'stdin: is not a tty' problem
+__line /root/.profile --line 'mesg n' --state absent
+"""
+
+class PreOSExistsError(cdist.Error):
+    def __init__(self, path):
+        self.path = path
+
+    def __str__(self):
+        return 'Path %s already exists' % self.path
+
+class PreOSBootstrapError(cdist.Error):
+    pass
+
+
+class PreOS(object):
+
+    def __init__(self, target_dir, arch="amd64"):
+
+        self.target_dir = target_dir
+        self.arch = arch
+
+        self.command = "debootstrap"
+        self.suite  = "wheezy"
+        self.options = [ "--include=openssh-server",
+            "--arch=%s" % self.arch ]
+
+        self.pxelinux = "/usr/lib/syslinux/pxelinux.0"
+        self.pxelinux_cfg = """
+DEFAULT preos
+LABEL preos
+KERNEL kernel
+INITRD initramfs
+"""
+
+    def _init_helper(self):
+        self.helper = {}
+        self.helper["manifest"]  = self.initial_manifest
+        self.helper["remote_exec"]  = """#!/bin/sh
+#        echo $@
+#        set -x
+chroot="$1"; shift
+
+script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX")
+trap cleanup INT TERM EXIT
+cleanup() {
+   [ $__cdist_debug ] || rm "$script"
+}
+
+echo "#!/bin/sh -l" > "$script"
+echo "$@" >> "$script"
+chmod +x "$script"
+
+relative_script="${script#$chroot}"
+
+# ensure PATH is setup
+export PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin
+
+# run in chroot
+chroot "$chroot" "$relative_script"
+"""
+
+        self.helper["remote_copy"]  = """#!/bin/sh
+#        echo $@
+#        set -x
+src=$1; shift
+dst=$1; shift
+real_dst=$(echo $dst | sed 's,:,,')
+cp -L "$src" "$real_dst"
+"""
+
+    @property
+    def exists(self):
+        return os.path.exists(self.target_dir)
+
+    def bootstrap(self):
+        if self.exists:
+            raise PreOSExistsError(self.target_dir)
+
+        cmd = [ self.command ]
+        cmd.extend(self.options)
+        cmd.append(self.suite)
+        cmd.append(self.target_dir)
+
+        log.debug("Bootstrap: %s" % cmd)
+
+#        try:
+        subprocess.check_call(cmd)
+#        except subprocess.CalledProcessError:
+#            raise 
+
+        # Required to run this - otherwise apt-get install fails
+        cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get", "update" ]
+        subprocess.check_call(cmd)
+
+    def create_helper_files(self, base_dir):
+        for key, val in self.helper.items():
+            filename = os.path.join(base_dir, key)
+            with open(filename, "w") as fd:
+                fd.write(val)
+            os.chmod(filename, stat.S_IRUSR |  stat.S_IXUSR)
+
+    def create_kernel(self):
+        dst = os.path.join(self.out_dir, "kernel")
+        srcglob = glob.glob("%s/boot/vmlinuz-*" % self.target_dir)
+        src = srcglob[0]
+
+        log.info("Creating kernel  ...")
+        shutil.copyfile(src, dst, follow_symlinks=True)
+
+    def create_pxelinux(self):
+        dst = os.path.join(self.out_dir, "pxelinux.0")
+        src = "%s/usr/lib/syslinux/pxelinux.0" % self.target_dir
+
+        log.info("Creating pxelinux.0  ...")
+        shutil.copyfile(src, dst, follow_symlinks=True)
+
+    def create_pxeconfig(self):
+        configdir = os.path.join(self.out_dir, "pxelinux.cfg")
+        configfile = os.path.join(configdir, "default")
+        log.info("Creating pxe configuration ...")
+        if not os.path.isdir(configdir):
+            os.mkdir(configdir)
+
+        with open(configfile, "w") as fd:
+            fd.write(self.pxelinux_cfg)
+
+    def create_initramfs(self):
+        out_file = os.path.join(self.out_dir, "initramfs")
+        cmd="cd {target_dir}; find . -print0 | cpio --null -o --format=newc | gzip -9 > {out_file}".format(target_dir = self.target_dir, out_file = out_file)
+
+        log.info("Creating initramfs ...")
+        subprocess.check_call(cmd, shell=True)
+
+    def ensure_out_dir_exists(self):
+        os.makedirs(self.out_dir, exist_ok=True)
+
+
+    def create_iso(self, out_dir):
+        self.out_dir = out_dir
+
+        self.ensure_out_dir_exists()
+
+        raise cdist.Error("Generating ISO is not yet supported")
+
+    def create_pxe(self, out_dir):
+        self.out_dir = out_dir
+
+        self.ensure_out_dir_exists()
+        self.create_kernel()
+        self.create_initramfs()
+        self.create_pxeconfig()
+        self.create_pxelinux()
+
+
+    def setup_initial_manifest(self, user_initial_manifest, replace_manifest):
+        if user_initial_manifest:
+            if user_initial_manifest == '-':
+                user_initial_manifest_content = sys.stdin.read()
+            else:
+                with open(user_initial_manifest, "r") as fd:
+                    user_initial_manifest_content = fd.read()
+        else:
+            user_initial_manifest_content = ""
+
+        if replace_manifest:
+            self.initial_manifest = user_initial_manifest_content
+        else:
+            self.initial_manifest = "{default}\n# User supplied manifest\n{user}".format(default=DEFAULT_MANIFEST, user=user_initial_manifest_content)
+
+    def config(self):
+        self._init_helper()
+
+        handle, path = tempfile.mkstemp(prefix='cdist.stdin.')
+        with tempfile.TemporaryDirectory() as tempdir:
+            host = self.target_dir
+
+            self.create_helper_files(tempdir)
+
+            local = cdist.exec.local.Local(
+                target_host=host,
+                initial_manifest=os.path.join(tempdir, "manifest")
+            )
+
+            remote = cdist.exec.remote.Remote(
+                target_host=host,
+                remote_exec=os.path.join(tempdir, "remote_exec"),
+                remote_copy=os.path.join(tempdir, "remote_copy"),
+            )
+
+            config = cdist.config.Config(local, remote)
+            config.run()
+
+    @classmethod
+    def commandline(cls, args):
+        self = cls(target_dir=args.target_dir[0],
+            arch=args.arch)
+
+        # read initial manifest first - it may come from stdin
+        if args.config:
+            self.setup_initial_manifest(args.initial_manifest, args.replace_manifest)
+
+        # Bootstrap: creates base directory
+        if args.bootstrap:
+            self.bootstrap()
+
+        # Configure the OS
+        if args.config:
+            self.config()
+
+        # Output pxe files
+        if args.pxe_boot_dir:
+            self.create_pxe(args.pxe_boot_dir)
+
+        #if args.iso_boot_dir:
+        #    self.create_iso(args.iso_boot)

docs/dev/logs/2013-11-28.preos

@@ -0,0 +1,2 @@
+- debootstrap for the moment
+- add triggers: https://github.com/telmich/cdist/issues/214

docs/dev/logs/2014-01-09.preos

@@ -0,0 +1,109 @@
+- debootstrap
+    x setup arch
+    x allow cdist to configure debootstrapped directory using cdist
+        x include sshd
+        x configure network (eth0, dhcp)
+        x various mkfs variants
+        - various fdisk tools
+
+    x add option for different initial manifest
+        x allow -, stdin usage
+        x allow to replace current manifest (later)
+
+    x trigger
+        - can be handled in the manifest of the user
+
+    - remove /var/cache/apt/archives/* ?
+        - later, optimisation level
+
+
+    - bug: cdist config als root!
+
+    - fix linux-image name (amd64)
+
+    - ln -s /sbin/init /init
+
+    - blog!
+        - self configuring
+
+    x pxe
+        /pxe/
+            - pxelinux.0
+            - linux
+            - initramfs
+            - pxelinux.cfg/
+                - default
+
+    - iso
+        - later
+    - usb stick (+efi version)
+        - later
+
+    - add unit tests
+
+- testing with qemu
+    [22:43] bento:vm-tests% qemu-system-x86_64 -m 2G -boot order=cn -drive file=testhd1,if=virtio -net nic -net user,tftp=$(pwd -P)/tftp,bootfile=/pxelinux.0,hostfwd=tcp::7777-:22 -enable-kvm
+
+- create preos
+    [22:43] bento:preos-tests% echo __panter_root_ssh_keys | sudo cdist preos -vp /home/users/nico/vm-tests/tftp -c /home/users/nico/preos-tests/preos03/ -i -
+
+
+--------------------------------------------------------------------------------
+
+[1:16] bento:~% sudo cdist preos -vc ~nico/preos-tests/preos03
+INFO: cdist: version 3.0.0-38-gea286c6
+INFO: /home/users/nico/preos-tests/preos03: Running global explorers
+INFO: /home/users/nico/preos-tests/preos03: Running initial manifest /tmp/tmpxbquwe/manifest
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __file/etc/network/interfaces
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __file/etc/network/interfaces
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/xfsprogs
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/reiser4progs
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/jfsutils
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/e2fsprogs
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/btrfs-tools
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/file
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/syslinux
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/openssh-server
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/linux-image-amd64
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/linux-image-amd64
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/linux-image-amd64
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/openssh-server
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/openssh-server
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/syslinux
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/syslinux
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/file
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/file
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/btrfs-tools
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/btrfs-tools
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/e2fsprogs
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/e2fsprogs
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/jfsutils
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/jfsutils
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/reiser4progs
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/reiser4progs
+INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/xfsprogs
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/xfsprogs
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/xfsprogs
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/reiser4progs
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/jfsutils
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/e2fsprogs
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/btrfs-tools
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/file
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/syslinux
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/openssh-server
+INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/linux-image-amd64
+INFO: /home/users/nico/preos-tests/preos03: Finished successful run in 2.546635866165161 seconds
+[1:16] bento:~% 
+
+--------------------------------------------------------------------------------
+[21:14] bento:vm-tests% qemu-system-x86_64 -m 2G -boot order=cn -drive file=testhd1,if=virtio -net nic -net user,tftp=$(pwd -P)/tftp,bootfile=/pxelinux.0                            
+
+--------------------------------------------------------------------------------
+[21:16] bento:preos-tests% sudo cdist preos -vp /home/users/nico/vm-tests/tftp /home/users/nico/preos-tests/preos03/                 
+INFO: cdist: version 3.0.0-42-g0d78ab3
+INFO: cdist.preos: Creating kernel  ...
+INFO: cdist.preos: Creating initramfs ...
+760780 blocks
+INFO: cdist.preos: Creating pxe configuration ...
+INFO: cdist.preos: Creating pxelinux.0  ...
+

scripts/cdist

@@ -27,6 +27,7 @@ def commandline():
     import cdist.banner
     import cdist.config
     import cdist.install
+    import cdist.preos
     import cdist.shell
 
     # Construct parser others can reuse
@@ -86,6 +87,29 @@ def commandline():
          default=cdist.REMOTE_EXEC)
     parser['config'].set_defaults(func=cdist.config.Config.commandline)
 
+    # PreOS
+    parser['preos'] = parser['sub'].add_parser('preos', 
+        parents=[parser['loglevel']])
+    parser['preos'].add_argument('-a', '--arch',
+         help='Select architecture for preos', default="amd64")
+    parser['preos'].add_argument('-b', '--bootstrap',
+         help='Bootstrap directory with PreOS',  action="store_true")
+    parser['preos'].add_argument('-c', '--configure',
+         help='Configure previously bootstrapped directory', 
+         action="store_true", dest="config")
+    parser['preos'].add_argument('-i', '--initial-manifest',
+         help='Initial manifest for configuration (added to built in)')
+    parser['preos'].add_argument('-r', '--replace-manifest',
+         help='Instead of appending to the built in manifest, replace the internal manifest', 
+         action="store_true")
+#    parser['preos'].add_argument('-I', '--iso-boot-dir',
+#         help='Create ISO for booting in given location')
+    parser['preos'].add_argument('-p', '--pxe-boot-dir',
+         help='Create PXE files for booting in given location')
+    parser['preos'].add_argument('target_dir', nargs=1,
+        help='Select target directory')
+    parser['preos'].set_defaults(func=cdist.preos.PreOS.commandline)
+
     # Shell
     parser['shell'] = parser['sub'].add_parser('shell', 
         parents=[parser['loglevel']])