From 321b39ee893c012341c988b151f1f186332a72de Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 28 Nov 2013 14:11:23 +0100 Subject: [PATCH 001/197] use new docs/ (from 2012 or so) Signed-off-by: Nico Schottelius --- {doc => docs}/dev/logs/2012-05-24.preos | 0 docs/dev/logs/2013-11-28.preos | 2 ++ 2 files changed, 2 insertions(+) rename {doc => docs}/dev/logs/2012-05-24.preos (100%) create mode 100644 docs/dev/logs/2013-11-28.preos diff --git a/doc/dev/logs/2012-05-24.preos b/docs/dev/logs/2012-05-24.preos similarity index 100% rename from doc/dev/logs/2012-05-24.preos rename to docs/dev/logs/2012-05-24.preos diff --git a/docs/dev/logs/2013-11-28.preos b/docs/dev/logs/2013-11-28.preos new file mode 100644 index 00000000..aa34f377 --- /dev/null +++ b/docs/dev/logs/2013-11-28.preos @@ -0,0 +1,2 @@ +- debootstrap for the moment +- From 65cab0d0e8adf4fac88ee0f3e909131273d4d45f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 28 Nov 2013 14:18:24 +0100 Subject: [PATCH 002/197] add "preos" subcommand to generate preos Signed-off-by: Nico Schottelius --- cdist/preos.py | 61 ++++++++++++++++++++++++++++++++++++++++++++++++++ scripts/cdist | 10 +++++++++ 2 files changed, 71 insertions(+) create mode 100644 cdist/preos.py diff --git a/cdist/preos.py b/cdist/preos.py new file mode 100644 index 00000000..29181057 --- /dev/null +++ b/cdist/preos.py @@ -0,0 +1,61 @@ +# -*- coding: utf-8 -*- +# +# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import logging +import os +import subprocess + +# initialise cdist +import cdist.exec.local + +import cdist.config + +log = logging.getLogger(__name__) + +class PreOS(object): + + def __init__(self, target_dir, arch="amd64"): + + self.target_dir = target_dir + self.arch = arch + + self.command = "debootstrap" + self.suite = "wheezy" + self.options = [ "--include=openssh-server", + "--arch=%s" % self.arch ] + + def run(self): + cmd = [ self.command ] + cmd.extend(self.options) + cmd.append(self.suite) + cmd.append(self.target_dir) + + log.debug("Bootstrap: %s" % cmd) + + subprocess.call(cmd) + + + @classmethod + def commandline(cls, args): + print(args) + self = cls(target_dir=args.target_dir[0], + arch=args.arch) + self.run() diff --git a/scripts/cdist b/scripts/cdist index 39449666..f4d4ce93 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -26,6 +26,7 @@ def commandline(): import cdist.banner import cdist.config + import cdist.preos import cdist.shell # Construct parser others can reuse @@ -83,6 +84,15 @@ def commandline(): default=cdist.REMOTE_EXEC) parser['config'].set_defaults(func=cdist.config.Config.commandline) + # PreOS + parser['preos'] = parser['sub'].add_parser('preos', + parents=[parser['loglevel']]) + parser['preos'].add_argument('-a', '--arch', + help='Select architecture for preos', default="amd64") + parser['preos'].add_argument('target_dir', nargs=1, + help='Select target directory') + parser['preos'].set_defaults(func=cdist.preos.PreOS.commandline) + # Shell parser['shell'] = parser['sub'].add_parser('shell', parents=[parser['loglevel']]) From adab98799485adb826fff7b8ded384668c1df606 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 29 Nov 2013 16:02:00 +0100 Subject: [PATCH 003/197] minor updates for preos Signed-off-by: Nico Schottelius --- cdist/preos.py | 4 +++- docs/dev/logs/2013-11-28.preos | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 29181057..77d6d6dc 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -42,7 +42,7 @@ class PreOS(object): self.options = [ "--include=openssh-server", "--arch=%s" % self.arch ] - def run(self): + def bootstrap(self): cmd = [ self.command ] cmd.extend(self.options) cmd.append(self.suite) @@ -52,6 +52,8 @@ class PreOS(object): subprocess.call(cmd) + def run(self): + self.bootstrap() @classmethod def commandline(cls, args): diff --git a/docs/dev/logs/2013-11-28.preos b/docs/dev/logs/2013-11-28.preos index aa34f377..f8561135 100644 --- a/docs/dev/logs/2013-11-28.preos +++ b/docs/dev/logs/2013-11-28.preos @@ -1,2 +1,2 @@ - debootstrap for the moment -- +- add triggers: https://github.com/telmich/cdist/issues/214 From a6e2cf853e55e7123c9923b7e86ce31b0cc618fd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Dec 2013 22:33:13 +0100 Subject: [PATCH 004/197] mkdiropt needs to be unquoted => empty if not existing Otherwise this happens: root@lilly ~ # cat /var/lib/cdist/object/__directory/vm/.cdist/code-remote rm -f "/vm" mkdir "" "/vm" which results into mkdir: cannot create directory `': No such file or directory Signed-off-by: Nico Schottelius --- cdist/conf/type/__directory/gencode-remote | 2 +- docs/changelog | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index 23fa4ed3..800fc6e4 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -75,7 +75,7 @@ case "$state_should" in set_attributes=1 cat << DONE rm -f "$destination" -mkdir "$mkdiropt" "$destination" +mkdir $mkdiropt "$destination" DONE fi diff --git a/docs/changelog b/docs/changelog index 67eaca1a..b9056fa6 100644 --- a/docs/changelog +++ b/docs/changelog @@ -9,6 +9,7 @@ Changelog * Type __file: Only remove file when state is absent (Steven Armstrong) * Type __link: Only remove link when state is absent (Steven Armstrong) * Type __directory: Only remove directory when state is absent (Steven Armstrong) + * Type __directory: Fix newly introduced quoting issue * Core: Fix backtrace when cache cannot be deleted 2.3.6: 2013-11-25 From 8af1add2a6059bf6a6c525f15a712a8a25464fa6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2014 00:04:46 +0100 Subject: [PATCH 005/197] preos: seperate parameters, create remote_exec, remote_copy and manifest on the fly Signed-off-by: Nico Schottelius --- cdist/preos.py | 98 +++++++++++++++++++++++++++++++--- docs/dev/logs/2014-01-09.preos | 20 +++++++ scripts/cdist | 9 ++++ 3 files changed, 120 insertions(+), 7 deletions(-) create mode 100644 docs/dev/logs/2014-01-09.preos diff --git a/cdist/preos.py b/cdist/preos.py index 77d6d6dc..2f53791f 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -22,16 +22,26 @@ import logging import os import subprocess +import stat +import tempfile -# initialise cdist -import cdist.exec.local import cdist.config +import cdist.exec.local +import cdist.exec.remote log = logging.getLogger(__name__) +class PreOSExistsError(cdist.Error): + def __init__(self, path): + self.path = path + + def __str__(self): + return 'Path %s already exists' % self.path + + class PreOS(object): - + def __init__(self, target_dir, arch="amd64"): self.target_dir = target_dir @@ -42,7 +52,53 @@ class PreOS(object): self.options = [ "--include=openssh-server", "--arch=%s" % self.arch ] + self._init_helper() + + def _init_helper(self): + self.helper = {} + self.helper["manifest"] = """ +for pkg in linux-image-amd64 openssh-server; do + __package $pkg --state present +done +""" + self.helper["remote_exec"] = """#!/bin/sh +# echo $@ +# set -x +chroot="$1"; shift + +script=$(mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX") +trap cleanup INT TERM EXIT +cleanup() { + [ $__cdist_debug ] || rm "$script" +} + +echo "#!/bin/sh -l" > "$script" +echo "$@" >> "$script" +chmod +x "$script" + +relative_script="${script#$chroot}" + +# run in chroot +chroot "$chroot" "$relative_script" +""" + + self.helper["remote_copy"] = """#!/bin/sh + echo $@ + set -x +src=$1; shift +dst=$1; shift +real_dst=$(echo $dst | sed 's,:,,') +cp -L "$src" "$real_dst" +""" + + @property + def exists(self): + return os.path.exists(self.target_dir) + def bootstrap(self): + if self.exists: + raise PreOSExistsError(self.target_dir) + cmd = [ self.command ] cmd.extend(self.options) cmd.append(self.suite) @@ -52,12 +108,40 @@ class PreOS(object): subprocess.call(cmd) - def run(self): - self.bootstrap() + def create_helper_files(self, base_dir): + for key, val in self.helper.items(): + filename = os.path.join(base_dir, key) + with open(filename, "w") as fd: + fd.write(val) + os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) + + def config(self): + handle, path = tempfile.mkstemp(prefix='cdist.stdin.') + with tempfile.TemporaryDirectory() as tempdir: + host = self.target_dir + + self.create_helper_files(tempdir) + + local = cdist.exec.local.Local( + target_host=host, + initial_manifest=os.path.join(tempdir, "manifest") + ) + + remote = cdist.exec.remote.Remote( + target_host=host, + remote_exec=os.path.join(tempdir, "remote_exec"), + remote_copy=os.path.join(tempdir, "remote_copy"), + ) + + config = cdist.config.Config(local, remote) + config.run() @classmethod def commandline(cls, args): - print(args) self = cls(target_dir=args.target_dir[0], arch=args.arch) - self.run() + + if args.bootstrap: + self.bootstrap() + if args.config: + self.config() diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos new file mode 100644 index 00000000..b802825d --- /dev/null +++ b/docs/dev/logs/2014-01-09.preos @@ -0,0 +1,20 @@ +- debootstrap + x setup arch + + - include trigger + - replace with cdist config later? + + - get kernel + - create initramfs + - later: + - configure chroot using cdist + + + packages: + linux-image-amd64 + + - temporary manifest + + - bugs with sudo + [22:50:04] bento:~# ln -s ~nico/.cdist/ ~ + diff --git a/scripts/cdist b/scripts/cdist index f4d4ce93..872d00ff 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -89,6 +89,15 @@ def commandline(): parents=[parser['loglevel']]) parser['preos'].add_argument('-a', '--arch', help='Select architecture for preos', default="amd64") + parser['preos'].add_argument('-b', '--bootstrap', + help='Bootstrap directory with OS', action="store_true") + parser['preos'].add_argument('-c', '--configure', + help='Configure previously bootstrapped directory', action="store_true", + dest="config") + parser['preos'].add_argument('-i', '--initramfs', + help='Create Linux initramfs', action="store_true") + parser['preos'].add_argument('-k', '--kernel', + help='Create Linux kernel', action="store_true") parser['preos'].add_argument('target_dir', nargs=1, help='Select target directory') parser['preos'].set_defaults(func=cdist.preos.PreOS.commandline) From b535e848ada9720d01861292c8df79b1cbe51677 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2014 00:38:29 +0100 Subject: [PATCH 006/197] run apt-get update after deboostrap Signed-off-by: Nico Schottelius --- cdist/preos.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index 2f53791f..98fd7f35 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -108,6 +108,8 @@ cp -L "$src" "$real_dst" subprocess.call(cmd) + cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get update" ] + def create_helper_files(self, base_dir): for key, val in self.helper.items(): filename = os.path.join(base_dir, key) From 4fb55b8d92065690d383c21e6015f550c70122bc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2014 10:46:09 +0100 Subject: [PATCH 007/197] various updates for preos Signed-off-by: Nico Schottelius --- cdist/preos.py | 51 +++++++++++++++++--- docs/dev/logs/2014-01-09.preos | 85 +++++++++++++++++++++++++++++----- scripts/cdist | 21 ++++++--- 3 files changed, 133 insertions(+), 24 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 98fd7f35..c24dbbe3 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2013-2014 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -39,6 +39,9 @@ class PreOSExistsError(cdist.Error): def __str__(self): return 'Path %s already exists' % self.path +class PreOSBootstrapError(cdist.Error): + pass + class PreOS(object): @@ -52,14 +55,39 @@ class PreOS(object): self.options = [ "--include=openssh-server", "--arch=%s" % self.arch ] + self.pxelinux = "/usr/lib/syslinux/pxelinux.0" + self.pxelinux-cfg = """ +DEFAULT linux +LABEL linux +KERNEL linux +INITRD initramfs +APPEND ro root=/dev/sda1 initrd=initrd.img + self._init_helper() def _init_helper(self): self.helper = {} self.helper["manifest"] = """ -for pkg in linux-image-amd64 openssh-server; do +for pkg in \ + file \ + linux-image-amd64 + openssh-server + syslinux \ + gdisk util-linux \ + btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do __package $pkg --state present done + +__file /etc/network/interfaces --source - --mode 0644 << eof +# The loopback network interface +auto lo +iface lo inet loopback + +# The primary network interface +auto eth0 +allow-hotplug eth0 +iface eth0 init dhcp +eof """ self.helper["remote_exec"] = """#!/bin/sh # echo $@ @@ -78,13 +106,16 @@ chmod +x "$script" relative_script="${script#$chroot}" +# ensure PATH is setup +export PATH=$PATH:/bin:/usr/bin:/sbin:/usr/sbin + # run in chroot chroot "$chroot" "$relative_script" """ self.helper["remote_copy"] = """#!/bin/sh - echo $@ - set -x +# echo $@ +# set -x src=$1; shift dst=$1; shift real_dst=$(echo $dst | sed 's,:,,') @@ -106,9 +137,14 @@ cp -L "$src" "$real_dst" log.debug("Bootstrap: %s" % cmd) - subprocess.call(cmd) +# try: + subprocess.check_call(cmd) +# except subprocess.CalledProcessError: +# raise - cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get update" ] + # Required to run this - otherwise apt-get install fails + cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get", "update" ] + subprocess.check_call(cmd) def create_helper_files(self, base_dir): for key, val in self.helper.items(): @@ -117,6 +153,9 @@ cp -L "$src" "$real_dst" fd.write(val) os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) + def create_pxe(self, base_dir): + pass + def config(self): handle, path = tempfile.mkstemp(prefix='cdist.stdin.') with tempfile.TemporaryDirectory() as tempdir: diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos index b802825d..c0b840ee 100644 --- a/docs/dev/logs/2014-01-09.preos +++ b/docs/dev/logs/2014-01-09.preos @@ -1,20 +1,83 @@ - debootstrap x setup arch + x allow cdist to configure debootstrapped directory using cdist + x include sshd + x configure network (eth0, dhcp) + x various mkfs variants + - various fdisk tools - - include trigger - - replace with cdist config later? + - add option for different initial manifest + - allow -, stdin usage - - get kernel - - create initramfs - - later: - - configure chroot using cdist + - add option for additional manifest + - allow -, stdin usage + - trigger + - can be handled in the manifest of the user - packages: - linux-image-amd64 + - remove /var/cache/apt/archives/* ? - - temporary manifest + - fix linux-image name (amd64) - - bugs with sudo - [22:50:04] bento:~# ln -s ~nico/.cdist/ ~ + - blog! + - self configuring + + - pxe + /pxe/ + - pxelinux.0 + - linux + - initramfs + - pxelinux.cfg/ + - default + + - iso + + - add unit tests + +-------------------------------------------------------------------------------- + +[1:16] bento:~% sudo cdist preos -vc ~nico/preos-tests/preos03 +INFO: cdist: version 3.0.0-38-gea286c6 +INFO: /home/users/nico/preos-tests/preos03: Running global explorers +INFO: /home/users/nico/preos-tests/preos03: Running initial manifest /tmp/tmpxbquwe/manifest +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __file/etc/network/interfaces +INFO: /home/users/nico/preos-tests/preos03: Generating code for __file/etc/network/interfaces +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/xfsprogs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/reiser4progs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/jfsutils +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/e2fsprogs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/btrfs-tools +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/file +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/syslinux +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/openssh-server +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package/linux-image-amd64 +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/linux-image-amd64 +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/linux-image-amd64 +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/openssh-server +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/openssh-server +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/syslinux +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/syslinux +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/file +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/file +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/btrfs-tools +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/btrfs-tools +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/e2fsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/e2fsprogs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/jfsutils +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/jfsutils +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/reiser4progs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/reiser4progs +INFO: /home/users/nico/preos-tests/preos03: Running manifest and explorers for __package_apt/xfsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package_apt/xfsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/xfsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/reiser4progs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/jfsutils +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/e2fsprogs +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/btrfs-tools +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/file +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/syslinux +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/openssh-server +INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/linux-image-amd64 +INFO: /home/users/nico/preos-tests/preos03: Finished successful run in 2.546635866165161 seconds +[1:16] bento:~% diff --git a/scripts/cdist b/scripts/cdist index 872d00ff..c1e1cd94 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -89,15 +89,22 @@ def commandline(): parents=[parser['loglevel']]) parser['preos'].add_argument('-a', '--arch', help='Select architecture for preos', default="amd64") + parser['preos'].add_argument('-A', '--additional-manifest', + help='Add stuff to configuration manifest', default="amd64") parser['preos'].add_argument('-b', '--bootstrap', - help='Bootstrap directory with OS', action="store_true") + help='Bootstrap directory with PreOS', action="store_true") parser['preos'].add_argument('-c', '--configure', - help='Configure previously bootstrapped directory', action="store_true", - dest="config") - parser['preos'].add_argument('-i', '--initramfs', - help='Create Linux initramfs', action="store_true") - parser['preos'].add_argument('-k', '--kernel', - help='Create Linux kernel', action="store_true") + help='Configure previously bootstrapped directory', + action="store_true", dest="config") + parser['preos'].add_argument('-i', '--initial-manifest', + help='Initial manifest for configuration (added to built in)') + parser['preos'].add_argument('-r', '--replace-manifest', + help='Instead of appending to the built in manifest, replace the internal manifest', + action="store_true") + parser['preos'].add_argument('-I', '--iso-boot', + help='Create ISO for booting in given location') + parser['preos'].add_argument('-p', '--pxe-boot', + help='Create PXE files for booting in given location') parser['preos'].add_argument('target_dir', nargs=1, help='Select target directory') parser['preos'].set_defaults(func=cdist.preos.PreOS.commandline) From 995e33afc9f62959fabc1888ec02952d6a68242a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 10 Jan 2014 17:21:42 +0100 Subject: [PATCH 008/197] add command line handling for pxe generating Signed-off-by: Nico Schottelius --- cdist/preos.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index c24dbbe3..78b88f94 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -56,12 +56,13 @@ class PreOS(object): "--arch=%s" % self.arch ] self.pxelinux = "/usr/lib/syslinux/pxelinux.0" - self.pxelinux-cfg = """ + self.pxelinux_cfg = """ DEFAULT linux LABEL linux KERNEL linux INITRD initramfs APPEND ro root=/dev/sda1 initrd=initrd.img +""" self._init_helper() @@ -153,9 +154,29 @@ cp -L "$src" "$real_dst" fd.write(val) os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) - def create_pxe(self, base_dir): + def create_kernel(self): + cmd=[ "cp", '"$(ls boot/vmlinuz-* | tail -n1)"' ] + cmd.append + pass + def create_initramfs(self): + base_cmd="find . -print0 | sudo cpio --null -ov --format=newc | gzip -9" + + pass + + def create_iso(self, out_dir): + self.out_dir = out_dir + + raise cdist.Error("Generating ISO is not yet supported") + + def create_pxe(self, out_dir): + self.out_dir = out_dir + + self.create_kernel() + self.create_initramfs() + self.create_pxeconfig() + def config(self): handle, path = tempfile.mkstemp(prefix='cdist.stdin.') with tempfile.TemporaryDirectory() as tempdir: @@ -186,3 +207,7 @@ cp -L "$src" "$real_dst" self.bootstrap() if args.config: self.config() + if args.pxe_boot: + self.create_pxe(args.pxe_boot) + if args.iso_boot: + self.create_iso(args.pxe_boot) From c585e4876e46306afd56b9490e36e00d379b3fbd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 21:05:14 +0100 Subject: [PATCH 009/197] create kernel, pxeconfig and pxelinux.0 Signed-off-by: Nico Schottelius --- cdist/preos.py | 38 ++++++++++++++++++++++++++++---------- 1 file changed, 28 insertions(+), 10 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 78b88f94..1fb73ad7 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -20,9 +20,11 @@ # import logging +import glob import os import subprocess import stat +import shutil import tempfile @@ -57,11 +59,10 @@ class PreOS(object): self.pxelinux = "/usr/lib/syslinux/pxelinux.0" self.pxelinux_cfg = """ -DEFAULT linux -LABEL linux -KERNEL linux +DEFAULT preos +LABEL preos +KERNEL kernel INITRD initramfs -APPEND ro root=/dev/sda1 initrd=initrd.img """ self._init_helper() @@ -155,13 +156,29 @@ cp -L "$src" "$real_dst" os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) def create_kernel(self): - cmd=[ "cp", '"$(ls boot/vmlinuz-* | tail -n1)"' ] - cmd.append + dst = os.path.join(self.out_dir, "kernel") + srcglob = glob.glob("%s/boot/vmlinuz-*" % self.target_dir) + src = srcglob[0] - pass + shutil.copyfile(src, dst, follow_symlinks=True) + + def create_pxelinux(self): + dst = os.path.join(self.out_dir, "pxelinux.0") + src = "%s/usr/lib/syslinux/pxelinux.0" % self.target_dir + + shutil.copyfile(src, dst, follow_symlinks=True) + + def create_pxeconfig(self): + configdir = os.path.join(self.out_dir, "pxelinux.cfg") + configfile = os.path.join(configdir, "default") + if not os.path.isdir(configdir): + os.mkdir(configdir) + + with open(configfile, "w") as fd: + fd.write(self.pxelinux_cfg) def create_initramfs(self): - base_cmd="find . -print0 | sudo cpio --null -ov --format=newc | gzip -9" + base_cmd="find . -print0 | cpio --null -ov --format=newc | gzip -9" pass @@ -174,8 +191,9 @@ cp -L "$src" "$real_dst" self.out_dir = out_dir self.create_kernel() - self.create_initramfs() +# self.create_initramfs() self.create_pxeconfig() + self.create_pxelinux() def config(self): handle, path = tempfile.mkstemp(prefix='cdist.stdin.') @@ -210,4 +228,4 @@ cp -L "$src" "$real_dst" if args.pxe_boot: self.create_pxe(args.pxe_boot) if args.iso_boot: - self.create_iso(args.pxe_boot) + self.create_iso(args.iso_boot) From 0d78ab313ffcb81a0daf9b650f982cd3e1d1b90b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 21:14:04 +0100 Subject: [PATCH 010/197] create initramfs Signed-off-by: Nico Schottelius --- cdist/preos.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 1fb73ad7..6d473ddf 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -160,17 +160,20 @@ cp -L "$src" "$real_dst" srcglob = glob.glob("%s/boot/vmlinuz-*" % self.target_dir) src = srcglob[0] + log.info("Creating kernel ...") shutil.copyfile(src, dst, follow_symlinks=True) def create_pxelinux(self): dst = os.path.join(self.out_dir, "pxelinux.0") src = "%s/usr/lib/syslinux/pxelinux.0" % self.target_dir + log.info("Creating pxelinux.0 ...") shutil.copyfile(src, dst, follow_symlinks=True) def create_pxeconfig(self): configdir = os.path.join(self.out_dir, "pxelinux.cfg") configfile = os.path.join(configdir, "default") + log.info("Creating pxe configuration ...") if not os.path.isdir(configdir): os.mkdir(configdir) @@ -178,9 +181,11 @@ cp -L "$src" "$real_dst" fd.write(self.pxelinux_cfg) def create_initramfs(self): - base_cmd="find . -print0 | cpio --null -ov --format=newc | gzip -9" + out_file = os.path.join(self.out_dir, "initramfs") + cmd="cd {target_dir}; find . -print0 | cpio --null -o --format=newc | gzip -9 > {out_file}".format(target_dir = self.target_dir, out_file = out_file) - pass + log.info("Creating initramfs ...") + subprocess.check_call(cmd, shell=True) def create_iso(self, out_dir): self.out_dir = out_dir @@ -191,7 +196,7 @@ cp -L "$src" "$real_dst" self.out_dir = out_dir self.create_kernel() -# self.create_initramfs() + self.create_initramfs() self.create_pxeconfig() self.create_pxelinux() From 11ba4640b4c7d2e0018fcaafa0fab66f147a169f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 22:34:44 +0100 Subject: [PATCH 011/197] disable unsupported iso - create /init - include support for another initial manifest Signed-off-by: Nico Schottelius --- cdist/preos.py | 89 +++++++++++++++++++++++----------- docs/dev/logs/2014-01-09.preos | 29 +++++++++-- scripts/cdist | 6 +-- 3 files changed, 87 insertions(+), 37 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 6d473ddf..cbb9f23c 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -24,16 +24,42 @@ import glob import os import subprocess import stat +import sys import shutil import tempfile - import cdist.config import cdist.exec.local import cdist.exec.remote log = logging.getLogger(__name__) +DEFAULT_MANIFEST = """ +for pkg in \ + file \ + linux-image-amd64 \ + openssh-server \ + syslinux \ + gdisk util-linux \ + btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do + __package $pkg --state present +done + +# initramfs requires /init +__link /init --source /sbin/init --type symbolic + +__file /etc/network/interfaces --source - --mode 0644 << eof +# The loopback network interface +auto lo +iface lo inet loopback + +# The primary network interface +auto eth0 +allow-hotplug eth0 +iface eth0 init dhcp +eof +""" + class PreOSExistsError(cdist.Error): def __init__(self, path): self.path = path @@ -65,32 +91,9 @@ KERNEL kernel INITRD initramfs """ - self._init_helper() - def _init_helper(self): self.helper = {} - self.helper["manifest"] = """ -for pkg in \ - file \ - linux-image-amd64 - openssh-server - syslinux \ - gdisk util-linux \ - btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do - __package $pkg --state present -done - -__file /etc/network/interfaces --source - --mode 0644 << eof -# The loopback network interface -auto lo -iface lo inet loopback - -# The primary network interface -auto eth0 -allow-hotplug eth0 -iface eth0 init dhcp -eof -""" + self.helper["manifest"] = self.initial_manifest self.helper["remote_exec"] = """#!/bin/sh # echo $@ # set -x @@ -200,7 +203,25 @@ cp -L "$src" "$real_dst" self.create_pxeconfig() self.create_pxelinux() + + def setup_initial_manifest(self, user_initial_manifest, replace_manifest): + if user_initial_manifest: + if user_initial_manifest == '-': + user_initial_manifest_content = sys.stdin.read() + else: + with open(user_initial_manifest, "r") as fd: + user_initial_manifest_content = fd.read() + else: + user_initial_manifest_content = "" + + if replace_manifest: + self.initial_manifest = user_initial_manifest_content + else: + self.initial_manifest = "{default}\n# User supplied manifest\n{user}".format(default=DEFAULT_MANIFEST, user=user_initial_manifest_content) + def config(self): + self._init_helper() + handle, path = tempfile.mkstemp(prefix='cdist.stdin.') with tempfile.TemporaryDirectory() as tempdir: host = self.target_dir @@ -226,11 +247,21 @@ cp -L "$src" "$real_dst" self = cls(target_dir=args.target_dir[0], arch=args.arch) + # read initial manifest first - it may come from stdin + if args.config: + self.setup_initial_manifest(args.initial_manifest, args.replace_manifest) + + # Bootstrap: creates base directory if args.bootstrap: self.bootstrap() + + # Configure the OS if args.config: self.config() - if args.pxe_boot: - self.create_pxe(args.pxe_boot) - if args.iso_boot: - self.create_iso(args.iso_boot) + + # Output pxe files + if args.pxe_boot_dir: + self.create_pxe(args.pxe_boot_dir) + + #if args.iso_boot_dir: + # self.create_iso(args.iso_boot) diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos index c0b840ee..0f0b0384 100644 --- a/docs/dev/logs/2014-01-09.preos +++ b/docs/dev/logs/2014-01-09.preos @@ -8,21 +8,25 @@ - add option for different initial manifest - allow -, stdin usage + - allow to replace current manifest (later) - - add option for additional manifest - - allow -, stdin usage - - - trigger + x trigger - can be handled in the manifest of the user - remove /var/cache/apt/archives/* ? + - later, optimisation level + + + - bug: cdist config als root! - fix linux-image name (amd64) + - ln -s /sbin/init /init + - blog! - self configuring - - pxe + x pxe /pxe/ - pxelinux.0 - linux @@ -31,6 +35,9 @@ - default - iso + - later + - usb stick (+efi version) + - later - add unit tests @@ -81,3 +88,15 @@ INFO: /home/users/nico/preos-tests/preos03: Generating code for __package/linux- INFO: /home/users/nico/preos-tests/preos03: Finished successful run in 2.546635866165161 seconds [1:16] bento:~% +-------------------------------------------------------------------------------- +[21:14] bento:vm-tests% qemu-system-x86_64 -m 2G -boot order=cn -drive file=testhd1,if=virtio -net nic -net user,tftp=$(pwd -P)/tftp,bootfile=/pxelinux.0 + +-------------------------------------------------------------------------------- +[21:16] bento:preos-tests% sudo cdist preos -vp /home/users/nico/vm-tests/tftp /home/users/nico/preos-tests/preos03/ +INFO: cdist: version 3.0.0-42-g0d78ab3 +INFO: cdist.preos: Creating kernel ... +INFO: cdist.preos: Creating initramfs ... +760780 blocks +INFO: cdist.preos: Creating pxe configuration ... +INFO: cdist.preos: Creating pxelinux.0 ... + diff --git a/scripts/cdist b/scripts/cdist index c1e1cd94..ed9d2eda 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -101,9 +101,9 @@ def commandline(): parser['preos'].add_argument('-r', '--replace-manifest', help='Instead of appending to the built in manifest, replace the internal manifest', action="store_true") - parser['preos'].add_argument('-I', '--iso-boot', - help='Create ISO for booting in given location') - parser['preos'].add_argument('-p', '--pxe-boot', +# parser['preos'].add_argument('-I', '--iso-boot-dir', +# help='Create ISO for booting in given location') + parser['preos'].add_argument('-p', '--pxe-boot-dir', help='Create PXE files for booting in given location') parser['preos'].add_argument('target_dir', nargs=1, help='Select target directory') From e7ad8f929804e02337c4c09b4045ade2859d2b85 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 22:45:38 +0100 Subject: [PATCH 012/197] inet not init Signed-off-by: Nico Schottelius --- cdist/preos.py | 2 +- docs/dev/logs/2014-01-09.preos | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index cbb9f23c..8b4a9a09 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -56,7 +56,7 @@ iface lo inet loopback # The primary network interface auto eth0 allow-hotplug eth0 -iface eth0 init dhcp +iface eth0 inet dhcp eof """ diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos index 0f0b0384..8d5a59b6 100644 --- a/docs/dev/logs/2014-01-09.preos +++ b/docs/dev/logs/2014-01-09.preos @@ -41,6 +41,10 @@ - add unit tests +- testing with qemu + [22:43] bento:vm-tests% qemu-system-x86_64 -m 2G -boot order=cn -drive file=testhd1,if=virtio -net nic -net user,tftp=$(pwd -P)/tftp,bootfile=/pxelinux.0,hostfwd=tcp::7777-:22 -enable-kvm + + -------------------------------------------------------------------------------- [1:16] bento:~% sudo cdist preos -vc ~nico/preos-tests/preos03 From 07545f4f7f9ddc806037a61babccefa6d3fa83a5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 22:47:34 +0100 Subject: [PATCH 013/197] update preos notes Signed-off-by: Nico Schottelius --- docs/dev/logs/2014-01-09.preos | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/dev/logs/2014-01-09.preos b/docs/dev/logs/2014-01-09.preos index 8d5a59b6..1a3f2ddc 100644 --- a/docs/dev/logs/2014-01-09.preos +++ b/docs/dev/logs/2014-01-09.preos @@ -6,9 +6,9 @@ x various mkfs variants - various fdisk tools - - add option for different initial manifest - - allow -, stdin usage - - allow to replace current manifest (later) + x add option for different initial manifest + x allow -, stdin usage + x allow to replace current manifest (later) x trigger - can be handled in the manifest of the user @@ -44,6 +44,9 @@ - testing with qemu [22:43] bento:vm-tests% qemu-system-x86_64 -m 2G -boot order=cn -drive file=testhd1,if=virtio -net nic -net user,tftp=$(pwd -P)/tftp,bootfile=/pxelinux.0,hostfwd=tcp::7777-:22 -enable-kvm +- create preos + [22:43] bento:preos-tests% echo __panter_root_ssh_keys | sudo cdist preos -vp /home/users/nico/vm-tests/tftp -c /home/users/nico/preos-tests/preos03/ -i - + -------------------------------------------------------------------------------- From 3daa74e81dc225a9004ba276b20e25f1ad3d80be Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 11 Jan 2014 22:48:47 +0100 Subject: [PATCH 014/197] fix 'stdin: is not a tty' problem (thanks, steven) Signed-off-by: Nico Schottelius --- cdist/preos.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index 8b4a9a09..433cf871 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -58,6 +58,10 @@ auto eth0 allow-hotplug eth0 iface eth0 inet dhcp eof + +# Steven found this out - coyping it 1:1 +# fix the bloody 'stdin: is not a tty' problem +__line /root/.profile --line 'mesg n' --state absent """ class PreOSExistsError(cdist.Error): From 54815e2b29ae2642f17c600a3c7afa1e24262d7d Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 29 Aug 2013 22:32:26 +0200 Subject: [PATCH 015/197] implement cdist install Signed-off-by: Steven Armstrong --- cdist/config.py | 1 - cdist/core/cdist_type.py | 5 ++ cdist/emulator.py | 2 +- cdist/install.py | 37 ++++++++++ cdist/test/cdist_type/__init__.py | 10 +++ docs/dev/todo/steven | 118 ------------------------------ scripts/cdist | 5 ++ 7 files changed, 58 insertions(+), 120 deletions(-) create mode 100644 cdist/install.py delete mode 100644 docs/dev/todo/steven diff --git a/cdist/config.py b/cdist/config.py index 3f8a7fc6..59cf339a 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -158,7 +158,6 @@ class Config(object): self.local.save_cache() self.log.info("Finished successful run in %s seconds", time.time() - start_time) - def object_list(self): """Short name for object list retrieval""" for cdist_object in core.CdistObject.list_objects(self.local.object_path, diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 46e126f9..ff1ebaec 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -101,6 +101,11 @@ class CdistType(object): """Check whether a type is a singleton.""" return os.path.isfile(os.path.join(self.absolute_path, "singleton")) + @property + def is_install(self): + """Check whether a type is used for installation (if not: for configuration)""" + return os.path.isfile(os.path.join(self.absolute_path, "install")) + @property def explorers(self): """Return a list of available explorers""" diff --git a/cdist/emulator.py b/cdist/emulator.py index b70ef956..b8f7b10b 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# 2012-2013 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # diff --git a/cdist/install.py b/cdist/install.py new file mode 100644 index 00000000..4530029a --- /dev/null +++ b/cdist/install.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2013 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import cdist.config +import cdist.core + + +class Install(cdist.config.Config): + def object_list(self): + """Short name for object list retrieval. + In install mode, we only care about install objects. + """ + for cdist_object in cdist.core.CdistObject.list_objects(self.local.object_path, + self.local.type_path): + if cdist_object.cdist_type.is_install: + yield cdist_object + else: + self.log.debug("Running in install mode, ignoring non install object: {0}".format(cdist_object)) diff --git a/cdist/test/cdist_type/__init__.py b/cdist/test/cdist_type/__init__.py index 79f824d3..8cc1f2e4 100644 --- a/cdist/test/cdist_type/__init__.py +++ b/cdist/test/cdist_type/__init__.py @@ -106,6 +106,16 @@ class TypeTestCase(test.CdistTestCase): cdist_type = core.CdistType(base_path, '__not_singleton') self.assertFalse(cdist_type.is_singleton) + def test_install_is_install(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__install') + self.assertTrue(cdist_type.is_install) + + def test_not_install_is_install(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__not_install') + self.assertFalse(cdist_type.is_install) + def test_with_explorers(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__with_explorers') diff --git a/docs/dev/todo/steven b/docs/dev/todo/steven deleted file mode 100644 index 2aacaa42..00000000 --- a/docs/dev/todo/steven +++ /dev/null @@ -1,118 +0,0 @@ -autorequire: - - objects defined in type manifests should be automatically prerequisites of the current object - - __foo/some-id - __other other-id --state present - => require="__other/other-id" __foo/some-id - - -metaparameters: - - steal the metaparameters from puppet: - - # I have to be there before the other one - __directory /etc/ssh \ - --before __file/etc/ssh/sshd_config - - # the other one has to be there before me - __file /etc/ssh/sshd_config \ - --after __directory/etc/ssh - - # if I change, tell the other one about it - __file /etc/ssh/sshd_config \ - --notify __init_script/etc/rc.d/sshd - - # whenever the other one changes, I want to know - __init_script /etc/rc.d/sshd \ - --subscribe __file/etc/ssh/sshd_config - - - how does a type react to a received 'event'? - - maybe something like: - __some_type/ - manifest - ... - gencode-refresh - ... - - gencode-refresh -> code-refresh -> ssh $target sh -e code-refresh - - - - -logging: - - logging from type emulator without clobbering stdout - maybe implement logging server as described here [1] - [1] http://docs.python.org/py3k/howto/logging-cookbook.html#configuration-server-example - - - use different logger to limit output to current area of interest, - e.g. - explorer.$target_host: explorer related messages for the run for $target_host - manifest.$target_host: manifest related messages for the run for $target_host - ... - then one could filter e.g. on explorer.* - - - more granular debug output, - [2] http://blog.ooz.ie/2011/03/python-logging-extending-standard.html - - - -tests: - - __init__(): - - sets up env: __target_host - - run_initial_manifest(): - - parameter is actually used (from __init__) - - ensure changing the manifest actually runs a different manifest - -> give ConfigInstall Constructor different manifest - -> different manifest is executed. - - test all submitted (from core to type manifest) variables: - - ENVIRONMENT - - they are set - - they contain the correct values - - run_type_manifest(): - - test all submitted (from core to type manifest) variables: - - ENVIRONMENT - - they are set - - they contain the correct values - - same tests as for test_initial_manifest_*? - - run_manifest(): - - test all submitted variables: - - ENVIRONMENT - - including __debug, if debug - - they are set - - they contain the correct values - - does $require work? - - check that exception raised, if manifest is not existent - - object_run(): - - ensure no object is run twice - - ensure requirements are taken into account? - - and order of run is adjusted - - check (from extern?) that all needed variables are setup - - ensure no code-{local, remote} is created, - if gencode is not producing code - - ensure THAT code-{local, remote} contains what gencode created - - abort if gencode-* fails - - abort if code-* fails - - abort == raise(FooException) - - gencode-*: ensure ENVIRONMENT is setup correctly - - run_type_explorer() - - ensure ALL type explores have been run - - ensure output is saved to correct path - - ensure a type with {0,1,2} explorers works ? - - none, one, multiple - - ensure ENVIRONMENT is setup correctly - - fails if ANY of the given explorer fails - - run_global_explorers(): - - ensure ALL type explores have been run - - ensure output is saved to correct path - - ensure a type with {0,1,2} explorers works ? - - none, one, multiple - - ensure ENVIRONMENT is setup correctly - - fails if ANY of the given explorer fails - -Code fixes needed: - - - shutil, os.mkdir, etc. everywhere: catch/reraise exceptions correctly diff --git a/scripts/cdist b/scripts/cdist index 39449666..31bd7373 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -26,6 +26,7 @@ def commandline(): import cdist.banner import cdist.config + import cdist.install import cdist.shell # Construct parser others can reuse @@ -90,6 +91,10 @@ def commandline(): help='Select shell to use, defaults to current shell') parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) + # Install + parser['install'] = parser['sub'].add_parser('install', + parents=[parser['loglevel'], parser['config']]) + parser['install'].set_defaults(func=cdist.install.Install.commandline) for p in parser: parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/" From 02476073aac96ce59d25e71ace119489616d4551 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 29 Aug 2013 22:33:43 +0200 Subject: [PATCH 016/197] add install types Signed-off-by: Steven Armstrong --- .../__install_bootloader_grub/gencode-remote | 73 ++++++++ .../type/__install_bootloader_grub/install | 0 .../type/__install_bootloader_grub/man.text | 47 +++++ .../type/__install_bootloader_grub/manifest | 25 +++ .../parameter/optional | 2 + .../__install_chroot_mount/gencode-remote | 1 + .../conf/type/__install_chroot_mount/install | 0 .../conf/type/__install_chroot_mount/man.text | 1 + .../__install_chroot_umount/gencode-remote | 1 + .../conf/type/__install_chroot_umount/install | 0 .../type/__install_chroot_umount/man.text | 1 + .../type/__install_config/files/remote/copy | 48 +++++ .../type/__install_config/files/remote/exec | 73 ++++++++ .../conf/type/__install_config/gencode-local | 50 +++++ cdist/conf/type/__install_config/install | 0 cdist/conf/type/__install_config/man.text | 47 +++++ cdist/conf/type/__install_config/manifest | 23 +++ .../type/__install_config/parameter/optional | 1 + cdist/conf/type/__install_config/singleton | 0 cdist/conf/type/__install_file/explorer | 1 + cdist/conf/type/__install_file/gencode-local | 1 + cdist/conf/type/__install_file/gencode-remote | 1 + cdist/conf/type/__install_file/install | 0 cdist/conf/type/__install_file/man.text | 1 + cdist/conf/type/__install_file/parameter | 1 + cdist/conf/type/__install_fstab/install | 0 cdist/conf/type/__install_fstab/man.text | 48 +++++ cdist/conf/type/__install_fstab/manifest | 29 +++ .../type/__install_fstab/parameter/optional | 1 + cdist/conf/type/__install_fstab/singleton | 0 .../files/fstab.header | 1 + .../__install_generate_fstab/gencode-local | 59 ++++++ .../type/__install_generate_fstab/install | 0 .../type/__install_generate_fstab/man.text | 52 ++++++ .../parameter/boolean | 1 + .../parameter/required | 1 + .../type/__install_generate_fstab/singleton | 0 cdist/conf/type/__install_mkfs/gencode-remote | 71 +++++++ cdist/conf/type/__install_mkfs/install | 0 cdist/conf/type/__install_mkfs/man.text | 57 ++++++ cdist/conf/type/__install_mkfs/manifest | 31 ++++ .../type/__install_mkfs/parameter/optional | 3 + .../type/__install_mkfs/parameter/required | 1 + .../conf/type/__install_mount/gencode-remote | 78 ++++++++ cdist/conf/type/__install_mount/install | 0 cdist/conf/type/__install_mount/man.text | 61 ++++++ cdist/conf/type/__install_mount/manifest | 29 +++ .../type/__install_mount/parameter/optional | 3 + .../type/__install_mount/parameter/required | 1 + .../type/__install_partition_msdos/install | 0 .../type/__install_partition_msdos/man.text | 62 +++++++ .../type/__install_partition_msdos/manifest | 41 ++++ .../parameter/optional | 3 + .../parameter/required | 1 + .../explorer/partitions | 3 + .../files/lib.sh | 68 +++++++ .../gencode-remote | 175 ++++++++++++++++++ .../__install_partition_msdos_apply/install | 0 .../__install_partition_msdos_apply/man.text | 42 +++++ .../__install_partition_msdos_apply/singleton | 0 .../conf/type/__install_reboot/gencode-remote | 23 +++ cdist/conf/type/__install_reboot/install | 0 cdist/conf/type/__install_reboot/man.text | 43 +++++ cdist/conf/type/__install_reboot/manifest | 23 +++ cdist/conf/type/__install_reboot/singleton | 0 .../type/__install_reset_disk/gencode-remote | 65 +++++++ cdist/conf/type/__install_reset_disk/install | 0 cdist/conf/type/__install_reset_disk/man.text | 43 +++++ .../conf/type/__install_stage/gencode-remote | 65 +++++++ cdist/conf/type/__install_stage/install | 0 cdist/conf/type/__install_stage/man.text | 58 ++++++ cdist/conf/type/__install_stage/manifest | 33 ++++ .../type/__install_stage/parameter/optional | 2 + .../type/__install_stage/parameter/required | 1 + cdist/conf/type/__install_stage/singleton | 0 .../conf/type/__install_umount/gencode-remote | 25 +++ cdist/conf/type/__install_umount/install | 0 cdist/conf/type/__install_umount/man.text | 43 +++++ cdist/conf/type/__install_umount/manifest | 23 +++ 79 files changed, 1767 insertions(+) create mode 100755 cdist/conf/type/__install_bootloader_grub/gencode-remote create mode 100644 cdist/conf/type/__install_bootloader_grub/install create mode 100644 cdist/conf/type/__install_bootloader_grub/man.text create mode 100755 cdist/conf/type/__install_bootloader_grub/manifest create mode 100644 cdist/conf/type/__install_bootloader_grub/parameter/optional create mode 120000 cdist/conf/type/__install_chroot_mount/gencode-remote create mode 100644 cdist/conf/type/__install_chroot_mount/install create mode 120000 cdist/conf/type/__install_chroot_mount/man.text create mode 120000 cdist/conf/type/__install_chroot_umount/gencode-remote create mode 100644 cdist/conf/type/__install_chroot_umount/install create mode 120000 cdist/conf/type/__install_chroot_umount/man.text create mode 100755 cdist/conf/type/__install_config/files/remote/copy create mode 100755 cdist/conf/type/__install_config/files/remote/exec create mode 100755 cdist/conf/type/__install_config/gencode-local create mode 100644 cdist/conf/type/__install_config/install create mode 100644 cdist/conf/type/__install_config/man.text create mode 100755 cdist/conf/type/__install_config/manifest create mode 100644 cdist/conf/type/__install_config/parameter/optional create mode 100644 cdist/conf/type/__install_config/singleton create mode 120000 cdist/conf/type/__install_file/explorer create mode 120000 cdist/conf/type/__install_file/gencode-local create mode 120000 cdist/conf/type/__install_file/gencode-remote create mode 100644 cdist/conf/type/__install_file/install create mode 120000 cdist/conf/type/__install_file/man.text create mode 120000 cdist/conf/type/__install_file/parameter create mode 100644 cdist/conf/type/__install_fstab/install create mode 100644 cdist/conf/type/__install_fstab/man.text create mode 100755 cdist/conf/type/__install_fstab/manifest create mode 100644 cdist/conf/type/__install_fstab/parameter/optional create mode 100644 cdist/conf/type/__install_fstab/singleton create mode 100644 cdist/conf/type/__install_generate_fstab/files/fstab.header create mode 100755 cdist/conf/type/__install_generate_fstab/gencode-local create mode 100644 cdist/conf/type/__install_generate_fstab/install create mode 100644 cdist/conf/type/__install_generate_fstab/man.text create mode 100644 cdist/conf/type/__install_generate_fstab/parameter/boolean create mode 100644 cdist/conf/type/__install_generate_fstab/parameter/required create mode 100644 cdist/conf/type/__install_generate_fstab/singleton create mode 100755 cdist/conf/type/__install_mkfs/gencode-remote create mode 100644 cdist/conf/type/__install_mkfs/install create mode 100644 cdist/conf/type/__install_mkfs/man.text create mode 100755 cdist/conf/type/__install_mkfs/manifest create mode 100644 cdist/conf/type/__install_mkfs/parameter/optional create mode 100644 cdist/conf/type/__install_mkfs/parameter/required create mode 100755 cdist/conf/type/__install_mount/gencode-remote create mode 100644 cdist/conf/type/__install_mount/install create mode 100644 cdist/conf/type/__install_mount/man.text create mode 100755 cdist/conf/type/__install_mount/manifest create mode 100644 cdist/conf/type/__install_mount/parameter/optional create mode 100644 cdist/conf/type/__install_mount/parameter/required create mode 100644 cdist/conf/type/__install_partition_msdos/install create mode 100644 cdist/conf/type/__install_partition_msdos/man.text create mode 100755 cdist/conf/type/__install_partition_msdos/manifest create mode 100644 cdist/conf/type/__install_partition_msdos/parameter/optional create mode 100644 cdist/conf/type/__install_partition_msdos/parameter/required create mode 100755 cdist/conf/type/__install_partition_msdos_apply/explorer/partitions create mode 100644 cdist/conf/type/__install_partition_msdos_apply/files/lib.sh create mode 100755 cdist/conf/type/__install_partition_msdos_apply/gencode-remote create mode 100644 cdist/conf/type/__install_partition_msdos_apply/install create mode 100644 cdist/conf/type/__install_partition_msdos_apply/man.text create mode 100644 cdist/conf/type/__install_partition_msdos_apply/singleton create mode 100755 cdist/conf/type/__install_reboot/gencode-remote create mode 100644 cdist/conf/type/__install_reboot/install create mode 100644 cdist/conf/type/__install_reboot/man.text create mode 100755 cdist/conf/type/__install_reboot/manifest create mode 100644 cdist/conf/type/__install_reboot/singleton create mode 100755 cdist/conf/type/__install_reset_disk/gencode-remote create mode 100644 cdist/conf/type/__install_reset_disk/install create mode 100644 cdist/conf/type/__install_reset_disk/man.text create mode 100755 cdist/conf/type/__install_stage/gencode-remote create mode 100644 cdist/conf/type/__install_stage/install create mode 100644 cdist/conf/type/__install_stage/man.text create mode 100755 cdist/conf/type/__install_stage/manifest create mode 100644 cdist/conf/type/__install_stage/parameter/optional create mode 100644 cdist/conf/type/__install_stage/parameter/required create mode 100644 cdist/conf/type/__install_stage/singleton create mode 100755 cdist/conf/type/__install_umount/gencode-remote create mode 100644 cdist/conf/type/__install_umount/install create mode 100644 cdist/conf/type/__install_umount/man.text create mode 100755 cdist/conf/type/__install_umount/manifest diff --git a/cdist/conf/type/__install_bootloader_grub/gencode-remote b/cdist/conf/type/__install_bootloader_grub/gencode-remote new file mode 100755 index 00000000..ed57331a --- /dev/null +++ b/cdist/conf/type/__install_bootloader_grub/gencode-remote @@ -0,0 +1,73 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +device="$(cat "$__object/parameter/device")" +chroot="$(cat "$__object/parameter/chroot")" + + +cat << DONE +os=\$( +if grep -q ^DISTRIB_ID=Ubuntu ${chroot}/etc/lsb-release 2>/dev/null; then + echo ubuntu + exit 0 +fi + +if [ -f ${chroot}/etc/arch-release ]; then + echo archlinux + exit 0 +fi + +if [ -f ${chroot}/etc/debian_version ]; then + echo debian + exit 0 +fi +) + +# Ensure /tmp exists +[ -d "${chroot}/tmp" ] || mkdir -m 1777 "${chroot}/tmp" +# Generate script to run in chroot +script=\$(mktemp "${chroot}/tmp/__install_bootloader_grub.XXXXXXXXXX") +# Link file descriptor #6 with stdout +exec 6>&1 +# Link stdout with \$script +exec > \$script + +echo "#!/bin/sh -l" +echo "grub-install $device" +case \$os in + archlinux) + # bugfix/workarround: rebuild initramfs + # FIXME: doesn't belong here + echo "mkinitcpio -p linux" + echo "grub-mkconfig -o /boot/grub/grub.cfg" + ;; + ubuntu|debian) echo "update-grub" ;; +esac + +# Restore stdout and close file descriptor #6. +exec 1>&6 6>&- + +# Make script executable +chmod +x "\$script" + +# Run script in chroot +relative_script="\${script#$chroot}" +chroot "$chroot" "\$relative_script" +DONE diff --git a/cdist/conf/type/__install_bootloader_grub/install b/cdist/conf/type/__install_bootloader_grub/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_bootloader_grub/man.text b/cdist/conf/type/__install_bootloader_grub/man.text new file mode 100644 index 00000000..858e6a67 --- /dev/null +++ b/cdist/conf/type/__install_bootloader_grub/man.text @@ -0,0 +1,47 @@ +cdist-type__install_bootloader_grub(7) +====================================== +Steven Armstrong + + +NAME +---- +cdist-type__install_bootloader_grub - install grub2 bootloader on given disk + + +DESCRIPTION +----------- +This cdist type allows you to install grub2 bootloader on given disk. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +device:: + The device to install grub to. Defaults to object_id + +chroot:: + where to chroot before running grub-install. Defaults to /target. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_bootloader_grub /dev/sda +__install_bootloader_grub /dev/sda --chroot /mnt/foobar +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_bootloader_grub/manifest b/cdist/conf/type/__install_bootloader_grub/manifest new file mode 100755 index 00000000..4c7c4955 --- /dev/null +++ b/cdist/conf/type/__install_bootloader_grub/manifest @@ -0,0 +1,25 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +device="$(cat "$__object/parameter/device" 2>/dev/null \ + || echo "/$__object_id" | tee "$__object/parameter/device")" +chroot="$(cat "$__object/parameter/chroot" 2>/dev/null \ + || echo "/target" | tee "$__object/parameter/chroot")" diff --git a/cdist/conf/type/__install_bootloader_grub/parameter/optional b/cdist/conf/type/__install_bootloader_grub/parameter/optional new file mode 100644 index 00000000..0bd1ce46 --- /dev/null +++ b/cdist/conf/type/__install_bootloader_grub/parameter/optional @@ -0,0 +1,2 @@ +device +chroot diff --git a/cdist/conf/type/__install_chroot_mount/gencode-remote b/cdist/conf/type/__install_chroot_mount/gencode-remote new file mode 120000 index 00000000..b1a5485e --- /dev/null +++ b/cdist/conf/type/__install_chroot_mount/gencode-remote @@ -0,0 +1 @@ +../__chroot_mount/gencode-remote \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_mount/install b/cdist/conf/type/__install_chroot_mount/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_chroot_mount/man.text b/cdist/conf/type/__install_chroot_mount/man.text new file mode 120000 index 00000000..e131fceb --- /dev/null +++ b/cdist/conf/type/__install_chroot_mount/man.text @@ -0,0 +1 @@ +../__chroot_mount/man.text \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_umount/gencode-remote b/cdist/conf/type/__install_chroot_umount/gencode-remote new file mode 120000 index 00000000..f2bd2681 --- /dev/null +++ b/cdist/conf/type/__install_chroot_umount/gencode-remote @@ -0,0 +1 @@ +../__chroot_umount/gencode-remote \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_umount/install b/cdist/conf/type/__install_chroot_umount/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_chroot_umount/man.text b/cdist/conf/type/__install_chroot_umount/man.text new file mode 120000 index 00000000..f615c734 --- /dev/null +++ b/cdist/conf/type/__install_chroot_umount/man.text @@ -0,0 +1 @@ +../__chroot_umount/man.text \ No newline at end of file diff --git a/cdist/conf/type/__install_config/files/remote/copy b/cdist/conf/type/__install_config/files/remote/copy new file mode 100755 index 00000000..5b6f555c --- /dev/null +++ b/cdist/conf/type/__install_config/files/remote/copy @@ -0,0 +1,48 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# __remote_copy script to run cdist against a chroot on a remote host via ssh. +# +# Usage: +# __remote_copy="/path/to/this/script /path/to/your/chroot" cdist config target-id +# + +log() { + echo "$@" | logger -t "__install_config copy" + : +} + +chroot="$1"; shift +target_host="$__target_host" + +scp="scp -o User=root -q" + +# postfix target_host with chroot location +code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")" + +log "target_host: $target_host" +log "chroot: $chroot" +log "@: $@" +log "code: $code" + +# copy files into chroot +$scp $code + +log "-----" diff --git a/cdist/conf/type/__install_config/files/remote/exec b/cdist/conf/type/__install_config/files/remote/exec new file mode 100755 index 00000000..4822bcf3 --- /dev/null +++ b/cdist/conf/type/__install_config/files/remote/exec @@ -0,0 +1,73 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# __remote_exec script to run cdist against a chroot on a remote host via ssh. +# +# Usage: +# __remote_exec="/path/to/this/script /path/to/your/chroot" cdist config target-id +# + +log() { + echo "$@" | logger -t "__install_config exec" + : +} + +chroot="$1"; shift +target_host="$__target_host" +# In exec mode the first argument is the __target_host which we already got from env. Get rid of it. +shift + +ssh="ssh -o User=root -q $target_host" +scp="scp -o User=root -q" + +local_script=$(mktemp "/tmp/chroot-${0##*/}.XXXXXXXXXX") +remote_script=$($ssh mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX") +relative_script="${remote_script#$chroot}" +trap cleanup INT TERM EXIT +cleanup() { + [ $__cdist_debug ] || { + rm "$local_script" + $ssh "rm $remote_script"; + } +} + +log "chroot: $chroot" +log "target_host: $target_host" +log "local_script: $local_script" +log "remote_script: $remote_script" +log "relative_script: $relative_script" +log "@: $@" +cat > "$local_script" << DONE +#!/bin/sh -l +# FIXME: fix the dependency bug, then test if the below is required or not +#if [ -f /etc/environment ]; then +# . /etc/environment +#fi +$@ +DONE + +# Upload script to target +$scp $local_script $target_host:$remote_script +$ssh "chmod +x $remote_script" + +# run in chroot +$ssh "chroot $chroot $relative_script" + +log "-----" diff --git a/cdist/conf/type/__install_config/gencode-local b/cdist/conf/type/__install_config/gencode-local new file mode 100755 index 00000000..da87a4ac --- /dev/null +++ b/cdist/conf/type/__install_config/gencode-local @@ -0,0 +1,50 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +chroot="$(cat "$__object/parameter/chroot")" +remote_exec="$__type/files/remote/exec" +remote_copy="$__type/files/remote/copy" + +cdist_args="-v" +[ "$__debug" = "yes" ] && cdist_args="$cdist_args -d" + +cat << DONE +echo "__apt_noautostart --state present" \ + | cdist $cdist_args \ + config \ + --initial-manifest - \ + --remote-exec="$remote_exec $chroot" \ + --remote-copy="$remote_copy $chroot" \ + $__target_host + +cdist $cdist_args \ + config \ + --remote-exec="$remote_exec $chroot" \ + --remote-copy="$remote_copy $chroot" \ + $__target_host + +echo "__apt_noautostart --state absent" \ + | cdist $cdist_args \ + config \ + --initial-manifest - \ + --remote-exec="$remote_exec $chroot" \ + --remote-copy="$remote_copy $chroot" \ + $__target_host +DONE diff --git a/cdist/conf/type/__install_config/install b/cdist/conf/type/__install_config/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_config/man.text b/cdist/conf/type/__install_config/man.text new file mode 100644 index 00000000..def0439b --- /dev/null +++ b/cdist/conf/type/__install_config/man.text @@ -0,0 +1,47 @@ +cdist-type__install_config(7) +============================= +Steven Armstrong + + +NAME +---- +cdist-type__install_config - run cdist config as part of the installation + + +DESCRIPTION +----------- +This cdist type allows you to run cdist config as part of the installation. +It does this by using a custom __remote_{copy,exec} prefix which runs +cdist config against the /target chroot on the remote host. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +chroot:: + where to chroot before running grub-install. Defaults to /target. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_config + +__install_config --chroot /mnt/somewhere +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_config/manifest b/cdist/conf/type/__install_config/manifest new file mode 100755 index 00000000..f26297b4 --- /dev/null +++ b/cdist/conf/type/__install_config/manifest @@ -0,0 +1,23 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +chroot="$(cat "$__object/parameter/chroot" 2>/dev/null \ + || echo "/target" | tee "$__object/parameter/chroot")" diff --git a/cdist/conf/type/__install_config/parameter/optional b/cdist/conf/type/__install_config/parameter/optional new file mode 100644 index 00000000..fa32393d --- /dev/null +++ b/cdist/conf/type/__install_config/parameter/optional @@ -0,0 +1 @@ +chroot diff --git a/cdist/conf/type/__install_config/singleton b/cdist/conf/type/__install_config/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_file/explorer b/cdist/conf/type/__install_file/explorer new file mode 120000 index 00000000..8479ee44 --- /dev/null +++ b/cdist/conf/type/__install_file/explorer @@ -0,0 +1 @@ +../__file/explorer \ No newline at end of file diff --git a/cdist/conf/type/__install_file/gencode-local b/cdist/conf/type/__install_file/gencode-local new file mode 120000 index 00000000..9ce4e805 --- /dev/null +++ b/cdist/conf/type/__install_file/gencode-local @@ -0,0 +1 @@ +../__file/gencode-local \ No newline at end of file diff --git a/cdist/conf/type/__install_file/gencode-remote b/cdist/conf/type/__install_file/gencode-remote new file mode 120000 index 00000000..f390bba4 --- /dev/null +++ b/cdist/conf/type/__install_file/gencode-remote @@ -0,0 +1 @@ +../__file/gencode-remote \ No newline at end of file diff --git a/cdist/conf/type/__install_file/install b/cdist/conf/type/__install_file/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_file/man.text b/cdist/conf/type/__install_file/man.text new file mode 120000 index 00000000..ba483161 --- /dev/null +++ b/cdist/conf/type/__install_file/man.text @@ -0,0 +1 @@ +../__file/man.text \ No newline at end of file diff --git a/cdist/conf/type/__install_file/parameter b/cdist/conf/type/__install_file/parameter new file mode 120000 index 00000000..e5099e86 --- /dev/null +++ b/cdist/conf/type/__install_file/parameter @@ -0,0 +1 @@ +../__file/parameter \ No newline at end of file diff --git a/cdist/conf/type/__install_fstab/install b/cdist/conf/type/__install_fstab/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_fstab/man.text b/cdist/conf/type/__install_fstab/man.text new file mode 100644 index 00000000..7c509427 --- /dev/null +++ b/cdist/conf/type/__install_fstab/man.text @@ -0,0 +1,48 @@ +cdist-type__install_fstab(7) +============================ +Steven Armstrong + + +NAME +---- +cdist-type__install_fstab - generate /etc/fstab during installation + + +DESCRIPTION +----------- +Uses __install_generate_fstab to generate a /etc/fstab file and uploads it +to the target machine at ${prefix}/etc/fstab. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +prefix:: + The prefix under which to generate the /etc/fstab file. + Defaults to /target. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_fstab +__install_fstab --prefix /mnt/target +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__install_mount(7) +- cdist-type__install_generate_fstab(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_fstab/manifest b/cdist/conf/type/__install_fstab/manifest new file mode 100755 index 00000000..74af53c0 --- /dev/null +++ b/cdist/conf/type/__install_fstab/manifest @@ -0,0 +1,29 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +prefix="$(cat "$__object/parameter/prefix" 2>/dev/null || echo "/target")" + +[ -d "$__object/files" ] || mkdir "$__object/files" +__install_generate_fstab --uuid --destination "$__object/files/fstab" +require="__install_generate_fstab" \ + __install_file "${prefix}/etc/fstab" --source "$__object/files/fstab" \ + --mode 644 \ + --owner root \ + --group root diff --git a/cdist/conf/type/__install_fstab/parameter/optional b/cdist/conf/type/__install_fstab/parameter/optional new file mode 100644 index 00000000..f73f3093 --- /dev/null +++ b/cdist/conf/type/__install_fstab/parameter/optional @@ -0,0 +1 @@ +file diff --git a/cdist/conf/type/__install_fstab/singleton b/cdist/conf/type/__install_fstab/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_generate_fstab/files/fstab.header b/cdist/conf/type/__install_generate_fstab/files/fstab.header new file mode 100644 index 00000000..7653cc78 --- /dev/null +++ b/cdist/conf/type/__install_generate_fstab/files/fstab.header @@ -0,0 +1 @@ +# Generated by cdist __install_generate_fstab diff --git a/cdist/conf/type/__install_generate_fstab/gencode-local b/cdist/conf/type/__install_generate_fstab/gencode-local new file mode 100755 index 00000000..d10e5b92 --- /dev/null +++ b/cdist/conf/type/__install_generate_fstab/gencode-local @@ -0,0 +1,59 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="$(cat "$__object/parameter/destination")" +cat "$__type/files/fstab.header" > "$destination" + +mkdir "$__object/files" +# get current UUID's from target_host +$__remote_exec $__target_host blkid > "$__object/files/blkid" + +for object in $(find "$__global/object/__install_mount" -path "*.cdist"); do + device="$(cat "$object/parameter/device")" + dir="$(cat "$object/parameter/dir")" + prefix="$(cat "$object/parameter/prefix")" + type="$(cat "$object/parameter/type")" + if [ -f "$object/parameter/options" ]; then + options="$(cat "$object/parameter/options")" + else + options="defaults" + fi + dump=0 + case "$type" in + swap) + pass=0 + dir="$type" + ;; + tmpfs) + pass=0 + ;; + *) + pass=1 + ;; + esac + if [ -f "$__object/parameter/uuid" ]; then + uuid="$(grep -w $device "$__object/files/blkid" | awk '{print $2}')" + if [ -n "$uuid" ]; then + echo "# $dir was on $device during installation" >> "$destination" + device="$uuid" + fi + fi + echo "$device $dir $type $options $dump $pass" >> "$destination" +done diff --git a/cdist/conf/type/__install_generate_fstab/install b/cdist/conf/type/__install_generate_fstab/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_generate_fstab/man.text b/cdist/conf/type/__install_generate_fstab/man.text new file mode 100644 index 00000000..d7d747a0 --- /dev/null +++ b/cdist/conf/type/__install_generate_fstab/man.text @@ -0,0 +1,52 @@ +cdist-type__install_generate_fstab(7) +===================================== +Steven Armstrong + + +NAME +---- +cdist-type__install_generate_fstab - generate /etc/fstab during installation + + +DESCRIPTION +----------- +Generates a /etc/fstab file from information retreived from +__install_mount definitions. + + +REQUIRED PARAMETERS +------------------- +destination:: + The path where to store the generated fstab file. + Note that this is a path on the server, where cdist is running, not the target host. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------- +uuid:: + use UUID instead of device in fstab + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_generate_fstab --destination /path/where/you/want/fstab +__install_generate_fstab --uuid --destination /path/where/you/want/fstab +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_generate_fstab/parameter/boolean b/cdist/conf/type/__install_generate_fstab/parameter/boolean new file mode 100644 index 00000000..43ab6159 --- /dev/null +++ b/cdist/conf/type/__install_generate_fstab/parameter/boolean @@ -0,0 +1 @@ +uuid diff --git a/cdist/conf/type/__install_generate_fstab/parameter/required b/cdist/conf/type/__install_generate_fstab/parameter/required new file mode 100644 index 00000000..ac459b09 --- /dev/null +++ b/cdist/conf/type/__install_generate_fstab/parameter/required @@ -0,0 +1 @@ +destination diff --git a/cdist/conf/type/__install_generate_fstab/singleton b/cdist/conf/type/__install_generate_fstab/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_mkfs/gencode-remote b/cdist/conf/type/__install_mkfs/gencode-remote new file mode 100755 index 00000000..6a71b8ed --- /dev/null +++ b/cdist/conf/type/__install_mkfs/gencode-remote @@ -0,0 +1,71 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +device="$(cat "$__object/parameter/device")" +type="$(cat "$__object/parameter/type")" + +# show command output in debug mode +cat << DONE +__debug=$__cdist_debug +if [ "\$__debug" != "yes" ]; then + # Link file descriptor #6 with stdout + exec 6>&1 + # redirect all output to /dev/null + exec > /dev/null +fi +DONE + +case "$type" in + swap) + echo "mkswap $device" + ;; + xfs) + command="mkfs.xfs -f -q" + if [ -f "$__object/parameter/options" ]; then + options="$(cat "$__object/parameter/options")" + command="$command $options" + fi + command="$command $device" + if [ -f "$__object/parameter/blocks" ]; then + blocks="$(cat "$__object/parameter/blocks")" + command="$command $blocks" + fi + echo "$command" + ;; + *) + command="mkfs -t $type -q" + if [ -f "$__object/parameter/options" ]; then + options="$(cat "$__object/parameter/options")" + command="$command $options" + fi + command="$command $device" + if [ -f "$__object/parameter/blocks" ]; then + blocks="$(cat "$__object/parameter/blocks")" + command="$command $blocks" + fi + echo "$command" +esac + +cat << DONE +if [ "\$__debug" != "yes" ]; then + # Restore stdout and close file descriptor #6. + exec 1>&6 6>&- +fi +DONE diff --git a/cdist/conf/type/__install_mkfs/install b/cdist/conf/type/__install_mkfs/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_mkfs/man.text b/cdist/conf/type/__install_mkfs/man.text new file mode 100644 index 00000000..3a9a325d --- /dev/null +++ b/cdist/conf/type/__install_mkfs/man.text @@ -0,0 +1,57 @@ +cdist-type__install_mkfs(7) +=========================== +Steven Armstrong + + +NAME +---- +cdist-type__install_mkfs - build a linux file system + + +DESCRIPTION +----------- +This cdist type is a wrapper for the mkfs command. + + +REQUIRED PARAMETERS +------------------- +type:: + The filesystem type to use. Same as used with mkfs -t. + + +OPTIONAL PARAMETERS +------------------- +device:: + defaults to object_id + +options:: + file system-specific options to be passed to the mkfs command + +blocks:: + the number of blocks to be used for the file system + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# reiserfs /dev/sda5 +__install_mkfs /dev/sda5 --type reiserfs +# same thing with explicit device +__install_mkfs whatever --device /dev/sda5 --type reiserfs + +# jfs with journal on /dev/sda2 +__install_mkfs /dev/sda1 --type jfs --options "-j /dev/sda2" +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- mkfs(8) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_mkfs/manifest b/cdist/conf/type/__install_mkfs/manifest new file mode 100755 index 00000000..e9d275a4 --- /dev/null +++ b/cdist/conf/type/__install_mkfs/manifest @@ -0,0 +1,31 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +if [ -f "$__object/parameter/device" ]; then + device="(cat "$__object/parameter/device")" +else + device="/$__object_id" + echo "$device" > "$__object/parameter/device" +fi + +type="(cat "$__object/parameter/type")" + +options="(cat "$__object/parameter/options")" diff --git a/cdist/conf/type/__install_mkfs/parameter/optional b/cdist/conf/type/__install_mkfs/parameter/optional new file mode 100644 index 00000000..86aeae30 --- /dev/null +++ b/cdist/conf/type/__install_mkfs/parameter/optional @@ -0,0 +1,3 @@ +device +options +blocks diff --git a/cdist/conf/type/__install_mkfs/parameter/required b/cdist/conf/type/__install_mkfs/parameter/required new file mode 100644 index 00000000..aa80e646 --- /dev/null +++ b/cdist/conf/type/__install_mkfs/parameter/required @@ -0,0 +1 @@ +type diff --git a/cdist/conf/type/__install_mount/gencode-remote b/cdist/conf/type/__install_mount/gencode-remote new file mode 100755 index 00000000..0ab5c069 --- /dev/null +++ b/cdist/conf/type/__install_mount/gencode-remote @@ -0,0 +1,78 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# show command output in debug mode +cat << DONE +__debug=$__cdist_debug +if [ "\$__debug" != "yes" ]; then + # Link file descriptor #6 with stdout + exec 6>&1 + # redirect all output to /dev/null + exec > /dev/null +fi +DONE + + +get_type_from_mkfs() { + _device="$1" + for mkfs_object in $(find "$__global/object/__install_mkfs" -path "*.cdist"); do + mkfs_device="$(cat "$mkfs_object/parameter/device")" + if [ "$_device" = "$mkfs_device" ]; then + cat "$mkfs_object/parameter/type" + break + fi + done + unset _device + unset mkfs_device + unset mkfs_object +} + +device="$(cat "$__object/parameter/device")" +dir="$(cat "$__object/parameter/dir")" +prefix="$(cat "$__object/parameter/prefix")" +if [ -f "$__object/parameter/type" ]; then + type="$(cat "$__object/parameter/type")" +else + type="$(get_type_from_mkfs "$device")" + # store for later use by others + echo "$type" > "$__object/parameter/type" +fi +[ -n "$type" ] || die "Can't determine type for $__object" +if [ "$type" = "swap" ]; then + echo "swapon \"$device\"" +else + if [ -f "$__object/parameter/options" ]; then + options="$(cat "$__object/parameter/options")" + else + options="" + fi + [ -n "$options" ] && options="-o $options" + mount_point="${prefix}${dir}" + + echo "[ -d \"$mount_point\" ] || mkdir -p \"$mount_point\"" + echo "mount -t \"$type\" $options \"$device\" \"$mount_point\"" +fi + +cat << DONE +if [ "\$__debug" != "yes" ]; then + # Restore stdout and close file descriptor #6. + exec 1>&6 6>&- +fi +DONE diff --git a/cdist/conf/type/__install_mount/install b/cdist/conf/type/__install_mount/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_mount/man.text b/cdist/conf/type/__install_mount/man.text new file mode 100644 index 00000000..b55cb83e --- /dev/null +++ b/cdist/conf/type/__install_mount/man.text @@ -0,0 +1,61 @@ +cdist-type__install_mount(7) +============================ +Steven Armstrong + + +NAME +---- +cdist-type__install_mount - mount filesystems in the installer + + +DESCRIPTION +----------- +Mounts filesystems in the installer. Collects data to generate /etc/fstab. + + +REQUIRED PARAMETERS +------------------- +device:: + the device to mount + + +OPTIONAL PARAMETERS +------------------- +dir:: + where to mount device. Defaults to object_id. + +options:: + mount options passed to mount(8) and used in /etc/fstab + +type:: + filesystem type passed to mount(8) and used in /etc/fstab. + If type is swap, 'dir' is ignored. + Defaults to the filesystem used in __install_mkfs for the same 'device'. + +prefix:: + the prefix to prepend to 'dir' when mounting in the installer. + Defaults to /target. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_mount slash --dir / --device /dev/sda5 --options noatime +require="__install_mount/slash" __install_mount /boot --device /dev/sda1 +__install_mount swap --device /dev/sda2 --type swap +require="__install_mount/slash" __install_mount /tmp --device tmpfs --type tmpfs +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__install_mount_apply(7) +- cdist-type__install_mkfs(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_mount/manifest b/cdist/conf/type/__install_mount/manifest new file mode 100755 index 00000000..5afae7fc --- /dev/null +++ b/cdist/conf/type/__install_mount/manifest @@ -0,0 +1,29 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +if [ ! -f "$__object/parameter/dir" ]; then + dir="/$__object_id" + echo "$dir" > "$__object/parameter/dir" +fi +if [ ! -f "$__object/parameter/prefix" ]; then + prefix="/target" + echo "$prefix" > "$__object/parameter/prefix" +fi diff --git a/cdist/conf/type/__install_mount/parameter/optional b/cdist/conf/type/__install_mount/parameter/optional new file mode 100644 index 00000000..08b6ad04 --- /dev/null +++ b/cdist/conf/type/__install_mount/parameter/optional @@ -0,0 +1,3 @@ +dir +type +options diff --git a/cdist/conf/type/__install_mount/parameter/required b/cdist/conf/type/__install_mount/parameter/required new file mode 100644 index 00000000..f89ee6a8 --- /dev/null +++ b/cdist/conf/type/__install_mount/parameter/required @@ -0,0 +1 @@ +device diff --git a/cdist/conf/type/__install_partition_msdos/install b/cdist/conf/type/__install_partition_msdos/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_partition_msdos/man.text b/cdist/conf/type/__install_partition_msdos/man.text new file mode 100644 index 00000000..8d403b67 --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos/man.text @@ -0,0 +1,62 @@ +cdist-type__install_partition_msdos(7) +============================== +Steven Armstrong + + +NAME +---- +cdist-type__install_partition_msdos - creates msdos partitions + + +DESCRIPTION +----------- +This cdist type allows you to create msdos paritions. + + +REQUIRED PARAMETERS +------------------- +type:: + the partition type used in fdisk (such as 82 or 83) or "extended" + + +OPTIONAL PARAMETERS +------------------- +partition:: + defaults to object_id +bootable:: + mark partition as bootable, true or false, defaults to false +size:: + the size of the partition (such as 32M or 15G, whole numbers + only), '+' for remaining space, or 'n%' for percentage of remaining + (these should only be used after all specific partition sizes are + specified). Defaults to +. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# 128MB, linux, bootable +__install_partition_msdos /dev/sda1 --type 83 --size 128M --bootable true +# 512MB, swap +__install_partition_msdos /dev/sda2 --type 82 --size 512M +# 100GB, extended +__install_partition_msdos /dev/sda3 --type extended --size 100G +# 10GB, linux +__install_partition_msdos /dev/sda5 --type 83 --size 10G +# 50% of the free space of the extended partition, linux +__install_partition_msdos /dev/sda6 --type 83 --size 50% +# rest of the extended partition, linux +__install_partition_msdos /dev/sda7 --type 83 --size + +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_partition_msdos/manifest b/cdist/conf/type/__install_partition_msdos/manifest new file mode 100755 index 00000000..e55d3f24 --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos/manifest @@ -0,0 +1,41 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +if [ -f "$__object/parameter/partition" ]; then + partition="(cat "$__object/parameter/partition")" +else + partition="/$__object_id" + echo "$partition" > "$__object/parameter/partition" +fi +device="$(echo "$partition" | sed 's/[0-9]//g')" +echo "$device" > "$__object/parameter/device" +minor="$(echo "$partition" | sed 's/[^0-9]//g')" +echo "$minor" > "$__object/parameter/minor" + +if [ ! -f "$__object/parameter/bootable" ]; then + echo "false" > "$__object/parameter/bootable" +fi +if [ ! -f "$__object/parameter/size" ]; then + echo "+" > "$__object/parameter/size" +fi + +# pull in the type that actually does something with the above parameters +require="$__object_name" __install_partition_msdos_apply diff --git a/cdist/conf/type/__install_partition_msdos/parameter/optional b/cdist/conf/type/__install_partition_msdos/parameter/optional new file mode 100644 index 00000000..b2b0a4c2 --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos/parameter/optional @@ -0,0 +1,3 @@ +partition +bootable +size diff --git a/cdist/conf/type/__install_partition_msdos/parameter/required b/cdist/conf/type/__install_partition_msdos/parameter/required new file mode 100644 index 00000000..aa80e646 --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos/parameter/required @@ -0,0 +1 @@ +type diff --git a/cdist/conf/type/__install_partition_msdos_apply/explorer/partitions b/cdist/conf/type/__install_partition_msdos_apply/explorer/partitions new file mode 100755 index 00000000..6be61af4 --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos_apply/explorer/partitions @@ -0,0 +1,3 @@ +#!/bin/sh + +cat /proc/partitions diff --git a/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh b/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh new file mode 100644 index 00000000..cddc575d --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh @@ -0,0 +1,68 @@ +die() { + echo "[__install_partition_msdos_apply] $@" >&2 + exit 1 +} +debug() { + #echo "[__install_partition_msdos_apply] $@" >&2 + : +} + +fdisk_command() { + local device="$1" + local cmd="$2" + + debug fdisk_command "running fdisk command '${cmd}' on device ${device}" + printf "${cmd}\nw\n" | fdisk -c -u "$device" + ret=$? + # give disk some time + sleep 1 + return $ret +} + +create_disklabel() { + local device=$1 + + debug create_disklabel "creating new msdos disklabel" + fdisk_command ${device} "o" + return $? +} + +toggle_bootable() { + local device="$1" + local minor="$2" + fdisk_command ${device} "a\n${minor}\n" + return $? +} + +create_partition() { + local device="$1" + local minor="$2" + local size="$3" + local type="$4" + local primary_count="$5" + + if [ "$type" = "extended" -o "$type" = "5" ]; then + # Extended partition + primary_extended="e\n" + first_minor="${minor}\n" + [ "${minor}" = "4" ] && first_minor="" + type_minor="${minor}\n" + [ "${minor}" = "1" ] && type_minor="" + type="5" + elif [ "${minor}" -lt "5" ]; then + primary_extended="p\n" + first_minor="${minor}\n" + [ "${minor}" = "4" ] && first_minor="" + type_minor="${minor}\n" + [ "${minor}" = "1" ] && type_minor="" + else + # Logical partitions + first_minor="${minor}\n" + type_minor="${minor}\n" + primary_extended="l\n" + [ "$primary_count" -gt "3" ] && primary_extended="" + fi + [ -n "${size}" ] && size="+${size}M" + fdisk_command ${device} "n\n${primary_extended}${first_minor}\n${size}\nt\n${type_minor}${type}\n" + return $? +} diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote new file mode 100755 index 00000000..c683673d --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -0,0 +1,175 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +die() { + echo "[__install_partition_msdos_apply] $@" >&2 + exit 1 +} +debug() { + #echo "[__install_partition_msdos_apply] $@" >&2 + : +} + +# Convert a size specifier 1G 100M or 50% into the corresponding numeric MB. +size_to_mb() { + local size=$1 + local available_size="$2" + + local number_suffix="$(echo ${size} | sed -e 's:\.[0-9]\+::' -e 's:\([0-9]\+\)\([KkMmGg%]\)[Bb]\?:\1|\2:')" + local number="$(echo ${number_suffix} | cut -d '|' -f1)" + local suffix="$(echo ${number_suffix} | cut -d '|' -f2)" + + case "$suffix" in + K|k) + size="$(( $number / 1024 ))" + ;; + M|m) + size="$number" + ;; + G|g) + size="$(( $number * 1024 ))" + ;; + %) + size="$(( $available_size * $number / 100 ))" + ;; + *) + size="-1" + esac + echo "$size" +} + +get_objects() { + objects_file=$(mktemp) + for object in $(find "$__global/object/__install_partition_msdos" -path "*.cdist"); do + object_device="$(cat "$object/parameter/device")" + object_minor="$(cat "$object/parameter/minor")" + echo "$object_device $object_minor $object" >> $objects_file + done + sort -k 1,2 $objects_file | cut -d' ' -f 3 + rm $objects_file + unset objects_file + unset object + unset object_device + unset object_minor +} + +# include function library for use on target +cat << DONE +__debug=$__cdist_debug +if [ "\$__debug" != "yes" ]; then + # Link file descriptor #6 with stdout + exec 6>&1 + # redirect all output to /dev/null + exec > /dev/null +fi +DONE +cat "$__type/files/lib.sh" + +partitions="$__object/explorer/partitions" +objects=$(get_objects) +current_device="" +available_device_size= +available_extended_size= +available_size= +primary_count=0 +for object in $objects; do + device="$(cat "$object/parameter/device")" + if [ "$current_device" != "$device" ]; then + echo "create_disklabel \"$device\" || die 'Failed to create disklabel for $device'" + current_device="$device" + device_name=$(echo ${device} | sed -e 's:^/dev/::;s:/:\\/:g') + available_device_size=$(( $(awk "/${device_name}\$/ { print \$3; }" "$partitions") / 1024)) + # make sure we don't go past the end of the drive + available_device_size=$((available_device_size - 2)) + available_extended_size=0 + primary_count=0 + debug "----- $device" + debug "current_device=$current_device" + debug "available_device_size=$available_device_size" + fi + + type="$(cat "$object/parameter/type")" + partition="$(cat "$object/parameter/partition")" + minor="$(cat "$object/parameter/minor")" + + bootable="$(cat "$object/parameter/bootable")" + size="$(cat "$object/parameter/size")" + + + if [ "${minor}" -lt "5" ]; then + # Primary partitions + primary_count=$(( $primary_count + 1 )) + available_size=$available_device_size + else + # Logical partitions + available_size=$available_extended_size + fi + + if [ "$size" = "+" ]; then + # use rest of device + partition_size="" + available_size=0 + else + partition_size=$(size_to_mb "$size" "$available_size") + available_size="$(( $available_size - $partition_size ))" + fi + + if [ "${minor}" -lt "5" ]; then + # Primary partitions + available_device_size=$available_size + if [ "$type" = "extended" -o "$type" = "5" ]; then + # Extended partition + available_extended_size=$partition_size + fi + else + # Logical paritions + available_extended_size=$available_size + fi + + [ "$partition_size" = "-1" ] && die "could not translate size '$size' to a usable value" + debug "----- $partition" + debug "primary_count=$primary_count" + debug "current_device=$current_device" + debug "device=$device" + debug "type=$type" + debug "partition=$partition" + debug "minor=$minor" + debug "bootable=$bootable" + debug "size=$size" + debug "partition_size=$partition_size" + debug "available_size=$available_size" + debug "available_device_size=$available_device_size" + debug "available_extended_size=$available_extended_size" + debug "----------" + + echo "create_partition '$device' '$minor' '$partition_size' '$type' '$primary_count' \ + || die 'Failed to create partition: $partition'" + + if [ "$bootable" = "true" ]; then + echo "toggle_bootable '$device' '$minor' || die 'Failed to toogle bootable flag for partition: $partition'" + fi +done + +cat << DONE +if [ "\$__debug" != "yes" ]; then + # Restore stdout and close file descriptor #6. + exec 1>&6 6>&- +fi +DONE diff --git a/cdist/conf/type/__install_partition_msdos_apply/install b/cdist/conf/type/__install_partition_msdos_apply/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_partition_msdos_apply/man.text b/cdist/conf/type/__install_partition_msdos_apply/man.text new file mode 100644 index 00000000..5399afb7 --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos_apply/man.text @@ -0,0 +1,42 @@ +cdist-type__install_partition_msdos_apply(7) +============================================ +Steven Armstrong + + +NAME +---- +cdist-type__install_partition_msdos_apply - Apply dos partition settings + + +DESCRIPTION +----------- +Create the partitions defined with __install_partition_msdos + + +REQUIRED PARAMETERS +------------------- +None + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_partition_msdos_apply +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__install_partition_msdos_apply(7) + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_partition_msdos_apply/singleton b/cdist/conf/type/__install_partition_msdos_apply/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_reboot/gencode-remote b/cdist/conf/type/__install_reboot/gencode-remote new file mode 100755 index 00000000..4358347d --- /dev/null +++ b/cdist/conf/type/__install_reboot/gencode-remote @@ -0,0 +1,23 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +options="$(cat "$__object/parameter/options")" + +echo "reboot $options" diff --git a/cdist/conf/type/__install_reboot/install b/cdist/conf/type/__install_reboot/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_reboot/man.text b/cdist/conf/type/__install_reboot/man.text new file mode 100644 index 00000000..91aec19a --- /dev/null +++ b/cdist/conf/type/__install_reboot/man.text @@ -0,0 +1,43 @@ +cdist-type__install_reboot(7) +============================= +Steven Armstrong + + +NAME +---- +cdist-type__install_reboot - run reboot + + +DESCRIPTION +----------- +This cdist type allows you to reboot a machine. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +options:: + options to pass to the reboot command. e.g. -f + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_reboot +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_reboot/manifest b/cdist/conf/type/__install_reboot/manifest new file mode 100755 index 00000000..fab80a1e --- /dev/null +++ b/cdist/conf/type/__install_reboot/manifest @@ -0,0 +1,23 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +options="$(cat "$__object/parameter/options" 2>/dev/null \ + || echo "" | tee "$__object/parameter/options")" diff --git a/cdist/conf/type/__install_reboot/singleton b/cdist/conf/type/__install_reboot/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_reset_disk/gencode-remote b/cdist/conf/type/__install_reset_disk/gencode-remote new file mode 100755 index 00000000..e9278a7e --- /dev/null +++ b/cdist/conf/type/__install_reset_disk/gencode-remote @@ -0,0 +1,65 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +disk="/$__object_id" +disk_name="${disk##*/}" + +cat << DONE +# stop mdadm raids if any +if [ -r /proc/mdstat ]; then + md_name="\$(awk "/$disk_name/ {print \$1}" /proc/mdstat)" + if [ -n "\$md_name" ]; then + if command -v mdadm >/dev/null; then + mdadm --stop "/dev/\$md_name" + else + echo "WARNING: mdadm command not found" >&2 + echo "WARNING: could not stop active mdadm raid for disk $disk" >&2 + fi + fi +fi + +# stop lvm's if any +if find /sys/class/block/$disk_name*/holders/ -mindepth 1 | grep -q holders/dm; then + if command -v vgchange >/dev/null; then + vgchange -a n + else + echo "WARNING: vgchange command not found" >&2 + fi +fi + +# clean disks from any legacy signatures +if command -v wipefs >/dev/null; then + wipefs -a "$disk" || true +fi +if command -v mdadm >/dev/null; then + mdadm --zero-superblock --force "$disk" || true +else + echo "WARNING: mdadm command not found" >&2 +fi +if command -v pvremove >/dev/null; then + pvremove --force --force --yes "$disk" || true +else + echo "WARNING: pvremove command not found" >&2 +fi + +# erase partition table +dd if=/dev/zero of=$disk bs=512 count=1 +printf 'w\n' | fdisk -u -c $disk || true +DONE diff --git a/cdist/conf/type/__install_reset_disk/install b/cdist/conf/type/__install_reset_disk/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_reset_disk/man.text b/cdist/conf/type/__install_reset_disk/man.text new file mode 100644 index 00000000..542d68ba --- /dev/null +++ b/cdist/conf/type/__install_reset_disk/man.text @@ -0,0 +1,43 @@ +cdist-type__install_reset_disk(7) +================================= +Steven Armstrong + + +NAME +---- +cdist-type__install_reset_disk - reset a disk + + +DESCRIPTION +----------- +Remove partition table. +Remove all lvm labels. +Remove mdadm superblock. + + +REQUIRED PARAMETERS +------------------- +None. + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_reset_disk /dev/sdb +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_stage/gencode-remote b/cdist/conf/type/__install_stage/gencode-remote new file mode 100755 index 00000000..17eeda0d --- /dev/null +++ b/cdist/conf/type/__install_stage/gencode-remote @@ -0,0 +1,65 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# show command output in debug mode +cat << DONE +__debug=$__cdist_debug +if [ "\$__debug" != "yes" ]; then + # Link file descriptor #6 with stdout + exec 6>&1 + # redirect all output to /dev/null + exec > /dev/null +fi +DONE + +uri="$(cat "$__object/parameter/uri")" +target="$(cat "$__object/parameter/target")" +post_install="$(cat "$__object/parameter/post_install" 2>/dev/null || true)" + + +[ "$__debug" = "yes" ] && curl="curl" || curl="curl -s" +[ "$__debug" = "yes" ] && tar="tar -xvzp" || tar="tar -xzp" + +echo "$curl '$uri' | $tar -C '$target'" +if [ -n "$post_install" ]; then + post_install_script="$(cat "$__object/parameter/post_install_script")" + cat << DONE +[ -d "${target}/proc" ] || mkdir "${target}/proc" +mount -t proc none "${target}/proc" +[ -d "${target}/sys" ] || mkdir "${target}/sys" +mount -t sysfs none "${target}/sys" +[ -d "${target}/dev" ] || mkdir "${target}/dev" +mount --rbind /dev "${target}/dev" +[ -d "${target}/tmp" ] || mkdir -m 1777 "${target}/tmp" +mount -t tmpfs none "${target}/tmp" +cp "$post_install_script" "${target}/tmp/post_install" +chmod +x "${target}/tmp/post_install" +cp /etc/resolv.conf "${target}/etc/" +chroot "$target" /tmp/post_install +umount -l "${target}/tmp" "${target}/dev" "${target}/sys" "${target}/proc" +DONE +fi + +cat << DONE +if [ "\$__debug" != "yes" ]; then + # Restore stdout and close file descriptor #6. + exec 1>&6 6>&- +fi +DONE diff --git a/cdist/conf/type/__install_stage/install b/cdist/conf/type/__install_stage/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_stage/man.text b/cdist/conf/type/__install_stage/man.text new file mode 100644 index 00000000..0e657fdc --- /dev/null +++ b/cdist/conf/type/__install_stage/man.text @@ -0,0 +1,58 @@ +cdist-type__install_stage(7) +============================ +Steven Armstrong + + +NAME +---- +cdist-type__install_stage - download and unpack a stage file + + +DESCRIPTION +----------- +Downloads a operating system stage using curl and unpacks it to /target +using tar. The stage tarball is expected to be gzip compressed. + + +REQUIRED PARAMETERS +------------------- +uri:: + The uri from which to fetch the tarball. + Can be anything understood by curl, e.g: + http://path/to/stage.tgz + tftp:///path/to/stage.tgz + file:///local/path/stage.tgz + + +OPTIONAL PARAMETERS +------------------- +target:: + where to unpack the tarball to. Defaults to /target. + +post_install:: + path to an optional local script. The script is uploaded to the target and + executed inside (chroot) the target after the stage has been unpacked. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_stage --uri tftp:///path/to/stage.tgz +__install_stage --uri http://path/to/stage.tgz --target /mnt/foobar +__install_stage --uri file:///path/to/stage.tgz --target /target +__install_stage --uri file:///path/to/stage.tgz \ + --target /target \ + --post_install /path/to/file/on/server +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_stage/manifest b/cdist/conf/type/__install_stage/manifest new file mode 100755 index 00000000..ab5f4d79 --- /dev/null +++ b/cdist/conf/type/__install_stage/manifest @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +uri="$(cat "$__object/parameter/uri" 2>/dev/null \ + || echo "$__object_id" | tee "$__object/parameter/uri")" +target="$(cat "$__object/parameter/target" 2>/dev/null \ + || echo "/target" | tee "$__object/parameter/target")" + +if [ -f "$__object/parameter/post_install" ]; then + post_install="$(cat "$__object/parameter/post_install")" + post_install_script="/tmp/post_install" + __install_file $post_install_script --source $post_install + echo "$post_install_script" > "$__object/parameter/post_install_script" +fi + diff --git a/cdist/conf/type/__install_stage/parameter/optional b/cdist/conf/type/__install_stage/parameter/optional new file mode 100644 index 00000000..8e1a11b5 --- /dev/null +++ b/cdist/conf/type/__install_stage/parameter/optional @@ -0,0 +1,2 @@ +target +post_install diff --git a/cdist/conf/type/__install_stage/parameter/required b/cdist/conf/type/__install_stage/parameter/required new file mode 100644 index 00000000..c7954952 --- /dev/null +++ b/cdist/conf/type/__install_stage/parameter/required @@ -0,0 +1 @@ +uri diff --git a/cdist/conf/type/__install_stage/singleton b/cdist/conf/type/__install_stage/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_umount/gencode-remote b/cdist/conf/type/__install_umount/gencode-remote new file mode 100755 index 00000000..c275fe5d --- /dev/null +++ b/cdist/conf/type/__install_umount/gencode-remote @@ -0,0 +1,25 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +target="$(cat "$__object/parameter/target")" + +echo "swapoff -a" +echo "umount -l ${target}/* || true" +echo "umount -l ${target}" diff --git a/cdist/conf/type/__install_umount/install b/cdist/conf/type/__install_umount/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_umount/man.text b/cdist/conf/type/__install_umount/man.text new file mode 100644 index 00000000..8d9d1f55 --- /dev/null +++ b/cdist/conf/type/__install_umount/man.text @@ -0,0 +1,43 @@ +cdist-type__install_umount(7) +============================= +Steven Armstrong + + +NAME +---- +cdist-type__install_umount - umount target directory + + +DESCRIPTION +----------- +This cdist type allows you to recursively umount the given target directory. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +target:: + the mount point to umount. Defaults to object_id + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_umount /target +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_umount/manifest b/cdist/conf/type/__install_umount/manifest new file mode 100755 index 00000000..c547e167 --- /dev/null +++ b/cdist/conf/type/__install_umount/manifest @@ -0,0 +1,23 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# set defaults +target="$(cat "$__object/parameter/target" 2>/dev/null \ + || echo "/target" | tee "$__object/parameter/target")" From f5aad522cc7328243c8338f29602838d789b094d Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 29 Aug 2013 23:28:15 +0200 Subject: [PATCH 017/197] dont add help to 2 parsers Signed-off-by: Steven Armstrong --- scripts/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cdist b/scripts/cdist index 31bd7373..dfdef691 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -92,7 +92,7 @@ def commandline(): parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) # Install - parser['install'] = parser['sub'].add_parser('install', + parser['install'] = parser['sub'].add_parser('install', add_help=False, parents=[parser['loglevel'], parser['config']]) parser['install'].set_defaults(func=cdist.install.Install.commandline) From 82612bc312eb8c367fdac4d46f88306beee87429 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 29 Aug 2013 23:29:54 +0200 Subject: [PATCH 018/197] config already inherits from loglevel parser Signed-off-by: Steven Armstrong --- scripts/cdist | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/cdist b/scripts/cdist index dfdef691..24a955ae 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -93,7 +93,7 @@ def commandline(): # Install parser['install'] = parser['sub'].add_parser('install', add_help=False, - parents=[parser['loglevel'], parser['config']]) + parents=[parser['config']]) parser['install'].set_defaults(func=cdist.install.Install.commandline) for p in parser: From 4ace4348a7386d3750eaf6ed67873a48f11f4fc4 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 30 Aug 2013 12:56:12 +0200 Subject: [PATCH 019/197] filter out install objects when running config Signed-off-by: Steven Armstrong --- cdist/config.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cdist/config.py b/cdist/config.py index 59cf339a..988695ed 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -162,7 +162,10 @@ class Config(object): """Short name for object list retrieval""" for cdist_object in core.CdistObject.list_objects(self.local.object_path, self.local.type_path): - yield cdist_object + if cdist_object.cdist_type.is_install: + self.log.debug("Running in config mode, ignoring install object: {0}".format(cdist_object)) + else: + yield cdist_object def iterate_once(self): """ From a9109c94a480ef74937ac6680aa86f2682a17b4c Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Fri, 6 Sep 2013 12:22:57 +0200 Subject: [PATCH 020/197] add missing types from private repo Signed-off-by: Steven Armstrong --- cdist/conf/type/__chroot_mount/gencode-remote | 36 ++++++++++++++++ cdist/conf/type/__chroot_mount/man.text | 42 +++++++++++++++++++ .../conf/type/__chroot_umount/gencode-remote | 33 +++++++++++++++ cdist/conf/type/__chroot_umount/man.text | 42 +++++++++++++++++++ 4 files changed, 153 insertions(+) create mode 100755 cdist/conf/type/__chroot_mount/gencode-remote create mode 100644 cdist/conf/type/__chroot_mount/man.text create mode 100755 cdist/conf/type/__chroot_umount/gencode-remote create mode 100644 cdist/conf/type/__chroot_umount/man.text diff --git a/cdist/conf/type/__chroot_mount/gencode-remote b/cdist/conf/type/__chroot_mount/gencode-remote new file mode 100755 index 00000000..ec0b83ae --- /dev/null +++ b/cdist/conf/type/__chroot_mount/gencode-remote @@ -0,0 +1,36 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +chroot="/$__object_id" + +cat << DONE +# Prepare chroot +[ -d "${chroot}/proc" ] || mkdir "${chroot}/proc" +mount -t proc none "${chroot}/proc" +[ -d "${chroot}/sys" ] || mkdir "${chroot}/sys" +mount -t sysfs none "${chroot}/sys" +[ -d "${chroot}/dev" ] || mkdir "${chroot}/dev" +mount --rbind /dev "${chroot}/dev" +[ -d "${chroot}/tmp" ] || mkdir -m 1777 "${chroot}/tmp" +mount -t tmpfs none "${chroot}/tmp" +if [ ! -f "${chroot}/etc/resolv.conf" ]; then + cp /etc/resolv.conf "${chroot}/etc/" +fi +DONE diff --git a/cdist/conf/type/__chroot_mount/man.text b/cdist/conf/type/__chroot_mount/man.text new file mode 100644 index 00000000..adce80d9 --- /dev/null +++ b/cdist/conf/type/__chroot_mount/man.text @@ -0,0 +1,42 @@ +cdist-type__install_chroot_mount(7) +=================================== +Steven Armstrong + + +NAME +---- +cdist-type__install_chroot_mount - mount a chroot + + +DESCRIPTION +----------- +Mount and prepare a chroot for running commands within it. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_chroot_mount /path/to/chroot +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__chroot_umount/gencode-remote b/cdist/conf/type/__chroot_umount/gencode-remote new file mode 100755 index 00000000..aad9ac76 --- /dev/null +++ b/cdist/conf/type/__chroot_umount/gencode-remote @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +chroot="/$__object_id" + +cat << DONE +umount -l "${chroot}/tmp" +umount -l "${chroot}/dev" +umount -l "${chroot}/sys" +umount -l "${chroot}/proc" +rm -f "${chroot}/etc/resolv.conf" +# ensure /etc/resolvconf/resolv.conf.d/tail is not linked to \ +# e.g. /etc/resolvconf/resolv.conf.d/original +rm -f "${chroot}/etc/resolvconf/resolv.conf.d/tail" +touch "${chroot}/etc/resolvconf/resolv.conf.d/tail" +DONE diff --git a/cdist/conf/type/__chroot_umount/man.text b/cdist/conf/type/__chroot_umount/man.text new file mode 100644 index 00000000..a5ca1ef0 --- /dev/null +++ b/cdist/conf/type/__chroot_umount/man.text @@ -0,0 +1,42 @@ +cdist-type__install_chroot_umount(7) +==================================== +Steven Armstrong + + +NAME +---- +cdist-type__install_chroot_umount - unmount a chroot mounted by __chroot_mount + + +DESCRIPTION +----------- +Undo what __chroot_mount did. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__install_chroot_umount /path/to/chroot +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). From a035b52a0dfd88b0383e22edcfc9f0c42d8129b2 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Sat, 14 Sep 2013 21:55:51 +0200 Subject: [PATCH 021/197] better mounting of virtual filesystems in chroot Signed-off-by: Steven Armstrong --- cdist/conf/type/__chroot_mount/gencode-remote | 20 +++++++++++++++---- .../conf/type/__chroot_umount/gencode-remote | 11 ++++++---- 2 files changed, 23 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__chroot_mount/gencode-remote b/cdist/conf/type/__chroot_mount/gencode-remote index ec0b83ae..6d855f41 100755 --- a/cdist/conf/type/__chroot_mount/gencode-remote +++ b/cdist/conf/type/__chroot_mount/gencode-remote @@ -23,13 +23,25 @@ chroot="/$__object_id" cat << DONE # Prepare chroot [ -d "${chroot}/proc" ] || mkdir "${chroot}/proc" -mount -t proc none "${chroot}/proc" +mountpoint -q "${chroot}/proc" \ + || mount -t proc -o nosuid,noexec,nodev proc "${chroot}/proc" + [ -d "${chroot}/sys" ] || mkdir "${chroot}/sys" -mount -t sysfs none "${chroot}/sys" +mountpoint -q "${chroot}/sys" \ + || mount -t sysfs -o nosuid,noexec,nodev sys "${chroot}/sys" + [ -d "${chroot}/dev" ] || mkdir "${chroot}/dev" -mount --rbind /dev "${chroot}/dev" +mountpoint -q "${chroot}/dev" \ + || mount -t devtmpfs -o mode=0755,nosuid udev "${chroot}/dev" + +[ -d "${chroot}/dev/pts" ] || mkdir "${chroot}/dev/pts" +mountpoint -q "${chroot}/dev/pts" \ + || mount -t devpts -o mode=0620,gid=5,nosuid,noexec devpts "${chroot}/dev/pts" + [ -d "${chroot}/tmp" ] || mkdir -m 1777 "${chroot}/tmp" -mount -t tmpfs none "${chroot}/tmp" +mountpoint -q "${chroot}/tmp" \ + || mount -t tmpfs -o mode=1777,strictatime,nodev,nosuid tmpfs "${chroot}/tmp" + if [ ! -f "${chroot}/etc/resolv.conf" ]; then cp /etc/resolv.conf "${chroot}/etc/" fi diff --git a/cdist/conf/type/__chroot_umount/gencode-remote b/cdist/conf/type/__chroot_umount/gencode-remote index aad9ac76..caf2c40c 100755 --- a/cdist/conf/type/__chroot_umount/gencode-remote +++ b/cdist/conf/type/__chroot_umount/gencode-remote @@ -22,12 +22,15 @@ chroot="/$__object_id" cat << DONE umount -l "${chroot}/tmp" +umount -l "${chroot}/dev/pts" umount -l "${chroot}/dev" umount -l "${chroot}/sys" umount -l "${chroot}/proc" rm -f "${chroot}/etc/resolv.conf" -# ensure /etc/resolvconf/resolv.conf.d/tail is not linked to \ -# e.g. /etc/resolvconf/resolv.conf.d/original -rm -f "${chroot}/etc/resolvconf/resolv.conf.d/tail" -touch "${chroot}/etc/resolvconf/resolv.conf.d/tail" +if [ -d "${chroot}/etc/resolvconf/resolv.conf.d" ]; then + # ensure /etc/resolvconf/resolv.conf.d/tail is not linked to \ + # e.g. /etc/resolvconf/resolv.conf.d/original + rm -f "${chroot}/etc/resolvconf/resolv.conf.d/tail" + touch "${chroot}/etc/resolvconf/resolv.conf.d/tail" +fi DONE From f67cdc8afa0f898986ddd628dccbf0255f0b9a85 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 17 Sep 2013 21:35:00 +0200 Subject: [PATCH 022/197] cleanup, remove unused/useless post_install parameter Signed-off-by: Steven Armstrong --- .../conf/type/__install_stage/gencode-remote | 24 ++------------ cdist/conf/type/__install_stage/man.text | 7 ---- cdist/conf/type/__install_stage/manifest | 33 ------------------- .../__install_stage/parameter/default/target | 1 + .../type/__install_stage/parameter/optional | 1 - 5 files changed, 4 insertions(+), 62 deletions(-) delete mode 100755 cdist/conf/type/__install_stage/manifest create mode 100644 cdist/conf/type/__install_stage/parameter/default/target diff --git a/cdist/conf/type/__install_stage/gencode-remote b/cdist/conf/type/__install_stage/gencode-remote index 17eeda0d..12b9f1ed 100755 --- a/cdist/conf/type/__install_stage/gencode-remote +++ b/cdist/conf/type/__install_stage/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -29,33 +29,15 @@ if [ "\$__debug" != "yes" ]; then fi DONE -uri="$(cat "$__object/parameter/uri")" +uri="$(cat "$__object/parameter/uri" 2>/dev/null \ + || echo "$__object_id")" target="$(cat "$__object/parameter/target")" -post_install="$(cat "$__object/parameter/post_install" 2>/dev/null || true)" [ "$__debug" = "yes" ] && curl="curl" || curl="curl -s" [ "$__debug" = "yes" ] && tar="tar -xvzp" || tar="tar -xzp" echo "$curl '$uri' | $tar -C '$target'" -if [ -n "$post_install" ]; then - post_install_script="$(cat "$__object/parameter/post_install_script")" - cat << DONE -[ -d "${target}/proc" ] || mkdir "${target}/proc" -mount -t proc none "${target}/proc" -[ -d "${target}/sys" ] || mkdir "${target}/sys" -mount -t sysfs none "${target}/sys" -[ -d "${target}/dev" ] || mkdir "${target}/dev" -mount --rbind /dev "${target}/dev" -[ -d "${target}/tmp" ] || mkdir -m 1777 "${target}/tmp" -mount -t tmpfs none "${target}/tmp" -cp "$post_install_script" "${target}/tmp/post_install" -chmod +x "${target}/tmp/post_install" -cp /etc/resolv.conf "${target}/etc/" -chroot "$target" /tmp/post_install -umount -l "${target}/tmp" "${target}/dev" "${target}/sys" "${target}/proc" -DONE -fi cat << DONE if [ "\$__debug" != "yes" ]; then diff --git a/cdist/conf/type/__install_stage/man.text b/cdist/conf/type/__install_stage/man.text index 0e657fdc..7abc77e8 100644 --- a/cdist/conf/type/__install_stage/man.text +++ b/cdist/conf/type/__install_stage/man.text @@ -29,10 +29,6 @@ OPTIONAL PARAMETERS target:: where to unpack the tarball to. Defaults to /target. -post_install:: - path to an optional local script. The script is uploaded to the target and - executed inside (chroot) the target after the stage has been unpacked. - EXAMPLES -------- @@ -41,9 +37,6 @@ EXAMPLES __install_stage --uri tftp:///path/to/stage.tgz __install_stage --uri http://path/to/stage.tgz --target /mnt/foobar __install_stage --uri file:///path/to/stage.tgz --target /target -__install_stage --uri file:///path/to/stage.tgz \ - --target /target \ - --post_install /path/to/file/on/server -------------------------------------------------------------------------------- diff --git a/cdist/conf/type/__install_stage/manifest b/cdist/conf/type/__install_stage/manifest deleted file mode 100755 index ab5f4d79..00000000 --- a/cdist/conf/type/__install_stage/manifest +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -# -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# - -# set defaults -uri="$(cat "$__object/parameter/uri" 2>/dev/null \ - || echo "$__object_id" | tee "$__object/parameter/uri")" -target="$(cat "$__object/parameter/target" 2>/dev/null \ - || echo "/target" | tee "$__object/parameter/target")" - -if [ -f "$__object/parameter/post_install" ]; then - post_install="$(cat "$__object/parameter/post_install")" - post_install_script="/tmp/post_install" - __install_file $post_install_script --source $post_install - echo "$post_install_script" > "$__object/parameter/post_install_script" -fi - diff --git a/cdist/conf/type/__install_stage/parameter/default/target b/cdist/conf/type/__install_stage/parameter/default/target new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/cdist/conf/type/__install_stage/parameter/default/target @@ -0,0 +1 @@ +/target diff --git a/cdist/conf/type/__install_stage/parameter/optional b/cdist/conf/type/__install_stage/parameter/optional index 8e1a11b5..eb5a316c 100644 --- a/cdist/conf/type/__install_stage/parameter/optional +++ b/cdist/conf/type/__install_stage/parameter/optional @@ -1,2 +1 @@ target -post_install From bfae291cf797da1bb9ce653398aea826bc2f3535 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 17 Sep 2013 22:41:10 +0200 Subject: [PATCH 023/197] remove pseudo debug output redirection Signed-off-by: Steven Armstrong --- cdist/conf/type/__install_mkfs/gencode-remote | 20 +----------------- .../conf/type/__install_mount/gencode-remote | 21 +------------------ .../gencode-remote | 18 +--------------- .../conf/type/__install_stage/gencode-remote | 18 ---------------- 4 files changed, 3 insertions(+), 74 deletions(-) diff --git a/cdist/conf/type/__install_mkfs/gencode-remote b/cdist/conf/type/__install_mkfs/gencode-remote index 6a71b8ed..2fe680e5 100755 --- a/cdist/conf/type/__install_mkfs/gencode-remote +++ b/cdist/conf/type/__install_mkfs/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -21,17 +21,6 @@ device="$(cat "$__object/parameter/device")" type="$(cat "$__object/parameter/type")" -# show command output in debug mode -cat << DONE -__debug=$__cdist_debug -if [ "\$__debug" != "yes" ]; then - # Link file descriptor #6 with stdout - exec 6>&1 - # redirect all output to /dev/null - exec > /dev/null -fi -DONE - case "$type" in swap) echo "mkswap $device" @@ -62,10 +51,3 @@ case "$type" in fi echo "$command" esac - -cat << DONE -if [ "\$__debug" != "yes" ]; then - # Restore stdout and close file descriptor #6. - exec 1>&6 6>&- -fi -DONE diff --git a/cdist/conf/type/__install_mount/gencode-remote b/cdist/conf/type/__install_mount/gencode-remote index 0ab5c069..3a35c139 100755 --- a/cdist/conf/type/__install_mount/gencode-remote +++ b/cdist/conf/type/__install_mount/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -18,18 +18,6 @@ # along with cdist. If not, see . # -# show command output in debug mode -cat << DONE -__debug=$__cdist_debug -if [ "\$__debug" != "yes" ]; then - # Link file descriptor #6 with stdout - exec 6>&1 - # redirect all output to /dev/null - exec > /dev/null -fi -DONE - - get_type_from_mkfs() { _device="$1" for mkfs_object in $(find "$__global/object/__install_mkfs" -path "*.cdist"); do @@ -69,10 +57,3 @@ else echo "[ -d \"$mount_point\" ] || mkdir -p \"$mount_point\"" echo "mount -t \"$type\" $options \"$device\" \"$mount_point\"" fi - -cat << DONE -if [ "\$__debug" != "yes" ]; then - # Restore stdout and close file descriptor #6. - exec 1>&6 6>&- -fi -DONE diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index c683673d..a1547296 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -71,15 +71,6 @@ get_objects() { } # include function library for use on target -cat << DONE -__debug=$__cdist_debug -if [ "\$__debug" != "yes" ]; then - # Link file descriptor #6 with stdout - exec 6>&1 - # redirect all output to /dev/null - exec > /dev/null -fi -DONE cat "$__type/files/lib.sh" partitions="$__object/explorer/partitions" @@ -166,10 +157,3 @@ for object in $objects; do echo "toggle_bootable '$device' '$minor' || die 'Failed to toogle bootable flag for partition: $partition'" fi done - -cat << DONE -if [ "\$__debug" != "yes" ]; then - # Restore stdout and close file descriptor #6. - exec 1>&6 6>&- -fi -DONE diff --git a/cdist/conf/type/__install_stage/gencode-remote b/cdist/conf/type/__install_stage/gencode-remote index 12b9f1ed..bbc27679 100755 --- a/cdist/conf/type/__install_stage/gencode-remote +++ b/cdist/conf/type/__install_stage/gencode-remote @@ -18,17 +18,6 @@ # along with cdist. If not, see . # -# show command output in debug mode -cat << DONE -__debug=$__cdist_debug -if [ "\$__debug" != "yes" ]; then - # Link file descriptor #6 with stdout - exec 6>&1 - # redirect all output to /dev/null - exec > /dev/null -fi -DONE - uri="$(cat "$__object/parameter/uri" 2>/dev/null \ || echo "$__object_id")" target="$(cat "$__object/parameter/target")" @@ -38,10 +27,3 @@ target="$(cat "$__object/parameter/target")" [ "$__debug" = "yes" ] && tar="tar -xvzp" || tar="tar -xzp" echo "$curl '$uri' | $tar -C '$target'" - -cat << DONE -if [ "\$__debug" != "yes" ]; then - # Restore stdout and close file descriptor #6. - exec 1>&6 6>&- -fi -DONE From 2f70a0d70e5c4aae07bf93d61752e34df4f2c2eb Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 17 Sep 2013 23:33:30 +0200 Subject: [PATCH 024/197] need a way to set remote.base_path from the command line Signed-off-by: Steven Armstrong --- cdist/config.py | 3 ++- cdist/exec/remote.py | 7 ++----- scripts/cdist | 2 ++ 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 988695ed..dcaac740 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -124,7 +124,8 @@ class Config(object): remote = cdist.exec.remote.Remote( target_host=host, remote_exec=args.remote_exec, - remote_copy=args.remote_copy) + remote_copy=args.remote_copy, + base_path=args.remote_out_path) c = cls(local, remote, dry_run=args.dry_run) c.run() diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 7c807092..3ffda12f 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -48,15 +48,12 @@ class Remote(object): target_host, remote_exec, remote_copy, - base_path=None): + base_path="/var/lib/cdist"): self.target_host = target_host self._exec = remote_exec self._copy = remote_copy - if base_path: - self.base_path = base_path - else: - self.base_path = "/var/lib/cdist" + self.base_path = base_path self.conf_path = os.path.join(self.base_path, "conf") self.object_path = os.path.join(self.base_path, "object") diff --git a/scripts/cdist b/scripts/cdist index 24a955ae..7bd58e58 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -68,6 +68,8 @@ def commandline(): help='Do not execute code', action='store_true') parser['config'].add_argument('-o', '--out-dir', help='Directory to save cdist output in', dest="out_path") + parser['config'].add_argument('-r', '--remote-out-dir', + help='Directory to save cdist output in on the target host', dest="remote_out_path") parser['config'].add_argument('-p', '--parallel', help='Operate on multiple hosts in parallel', action='store_true', dest='parallel') From fc988a5c228e0f778aa04840b18c8f2ea8bdddd7 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 17 Sep 2013 23:48:45 +0200 Subject: [PATCH 025/197] oh my, never mind Signed-off-by: Steven Armstrong --- cdist/exec/remote.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 3ffda12f..7c807092 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -48,12 +48,15 @@ class Remote(object): target_host, remote_exec, remote_copy, - base_path="/var/lib/cdist"): + base_path=None): self.target_host = target_host self._exec = remote_exec self._copy = remote_copy - self.base_path = base_path + if base_path: + self.base_path = base_path + else: + self.base_path = "/var/lib/cdist" self.conf_path = os.path.join(self.base_path, "conf") self.object_path = os.path.join(self.base_path, "object") From a2318983970e580492f1940d39dd7e3ab3548316 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 15 Oct 2013 22:29:58 +0200 Subject: [PATCH 026/197] get rid of unnecessary tmp files Signed-off-by: Steven Armstrong --- .../type/__install_config/files/remote/exec | 37 +++---------------- 1 file changed, 6 insertions(+), 31 deletions(-) diff --git a/cdist/conf/type/__install_config/files/remote/exec b/cdist/conf/type/__install_config/files/remote/exec index 4822bcf3..58e6b162 100755 --- a/cdist/conf/type/__install_config/files/remote/exec +++ b/cdist/conf/type/__install_config/files/remote/exec @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -35,39 +35,14 @@ target_host="$__target_host" shift ssh="ssh -o User=root -q $target_host" -scp="scp -o User=root -q" +code="$ssh chroot $chroot sh -c '$@'" -local_script=$(mktemp "/tmp/chroot-${0##*/}.XXXXXXXXXX") -remote_script=$($ssh mktemp "${chroot}/tmp/chroot-${0##*/}.XXXXXXXXXX") -relative_script="${remote_script#$chroot}" -trap cleanup INT TERM EXIT -cleanup() { - [ $__cdist_debug ] || { - rm "$local_script" - $ssh "rm $remote_script"; - } -} - -log "chroot: $chroot" log "target_host: $target_host" -log "local_script: $local_script" -log "remote_script: $remote_script" -log "relative_script: $relative_script" +log "chroot: $chroot" log "@: $@" -cat > "$local_script" << DONE -#!/bin/sh -l -# FIXME: fix the dependency bug, then test if the below is required or not -#if [ -f /etc/environment ]; then -# . /etc/environment -#fi -$@ -DONE +log "code: $code" -# Upload script to target -$scp $local_script $target_host:$remote_script -$ssh "chmod +x $remote_script" - -# run in chroot -$ssh "chroot $chroot $relative_script" +# Run the code +$code log "-----" From f9cac131c9c4e125b85cc6b152839bb97b461383 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 10 Dec 2013 11:13:25 +0100 Subject: [PATCH 027/197] add parameter to run curl in insecure mode: thanks Thorsten! Signed-off-by: Steven Armstrong --- cdist/conf/type/__install_stage/gencode-remote | 5 ++++- cdist/conf/type/__install_stage/man.text | 9 ++++++++- cdist/conf/type/__install_stage/parameter/boolean | 1 + 3 files changed, 13 insertions(+), 2 deletions(-) create mode 100644 cdist/conf/type/__install_stage/parameter/boolean diff --git a/cdist/conf/type/__install_stage/gencode-remote b/cdist/conf/type/__install_stage/gencode-remote index bbc27679..3b83ea61 100755 --- a/cdist/conf/type/__install_stage/gencode-remote +++ b/cdist/conf/type/__install_stage/gencode-remote @@ -22,8 +22,11 @@ uri="$(cat "$__object/parameter/uri" 2>/dev/null \ || echo "$__object_id")" target="$(cat "$__object/parameter/target")" - [ "$__debug" = "yes" ] && curl="curl" || curl="curl -s" [ "$__debug" = "yes" ] && tar="tar -xvzp" || tar="tar -xzp" +if [ -f "$__object/parameter/insecure" ] ; then + curl="$curl -k" +fi + echo "$curl '$uri' | $tar -C '$target'" diff --git a/cdist/conf/type/__install_stage/man.text b/cdist/conf/type/__install_stage/man.text index 7abc77e8..289c8621 100644 --- a/cdist/conf/type/__install_stage/man.text +++ b/cdist/conf/type/__install_stage/man.text @@ -30,6 +30,12 @@ target:: where to unpack the tarball to. Defaults to /target. +BOOLEAN PARAMETERS +------------------ +insecure:: + run curl in insecure mode so it does not check the servers ssl certificate + + EXAMPLES -------- @@ -37,6 +43,7 @@ EXAMPLES __install_stage --uri tftp:///path/to/stage.tgz __install_stage --uri http://path/to/stage.tgz --target /mnt/foobar __install_stage --uri file:///path/to/stage.tgz --target /target +__install_stage --uri https://path/to/stage.tgz --target /mnt/foobar --insecure -------------------------------------------------------------------------------- @@ -47,5 +54,5 @@ SEE ALSO COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is +Copyright \(C) 2011 - 2013 Steven Armstrong. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_stage/parameter/boolean b/cdist/conf/type/__install_stage/parameter/boolean new file mode 100644 index 00000000..e86bf3fc --- /dev/null +++ b/cdist/conf/type/__install_stage/parameter/boolean @@ -0,0 +1 @@ +insecure From be8df7999be8c45206004e024eb8621ab32e0b52 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Tue, 17 Dec 2013 14:56:20 +0100 Subject: [PATCH 028/197] first stop lvm, then mdadm Signed-off-by: Steven Armstrong --- .../type/__install_reset_disk/gencode-remote | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/cdist/conf/type/__install_reset_disk/gencode-remote b/cdist/conf/type/__install_reset_disk/gencode-remote index e9278a7e..e8e9cf8c 100755 --- a/cdist/conf/type/__install_reset_disk/gencode-remote +++ b/cdist/conf/type/__install_reset_disk/gencode-remote @@ -22,6 +22,15 @@ disk="/$__object_id" disk_name="${disk##*/}" cat << DONE +# stop lvm's if any +if find /sys/class/block/$disk_name*/holders/ -mindepth 1 | grep -q holders/dm; then + if command -v vgchange >/dev/null; then + vgchange -a n + else + echo "WARNING: vgchange command not found" >&2 + fi +fi + # stop mdadm raids if any if [ -r /proc/mdstat ]; then md_name="\$(awk "/$disk_name/ {print \$1}" /proc/mdstat)" @@ -35,28 +44,19 @@ if [ -r /proc/mdstat ]; then fi fi -# stop lvm's if any -if find /sys/class/block/$disk_name*/holders/ -mindepth 1 | grep -q holders/dm; then - if command -v vgchange >/dev/null; then - vgchange -a n - else - echo "WARNING: vgchange command not found" >&2 - fi -fi - -# clean disks from any legacy signatures -if command -v wipefs >/dev/null; then - wipefs -a "$disk" || true +if command -v pvremove >/dev/null; then + pvremove --force --force --yes "$disk" || true +else + echo "WARNING: pvremove command not found" >&2 fi if command -v mdadm >/dev/null; then mdadm --zero-superblock --force "$disk" || true else echo "WARNING: mdadm command not found" >&2 fi -if command -v pvremove >/dev/null; then - pvremove --force --force --yes "$disk" || true -else - echo "WARNING: pvremove command not found" >&2 +# clean disks from any legacy signatures +if command -v wipefs >/dev/null; then + wipefs -a "$disk" || true fi # erase partition table From 4ca13d59a66e49a90148166f8b623772615a7a5f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 17 Jan 2014 10:40:42 +0100 Subject: [PATCH 029/197] comment out __apt_noautostart for the moment Signed-off-by: Nico Schottelius --- .../conf/type/__install_config/gencode-local | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/cdist/conf/type/__install_config/gencode-local b/cdist/conf/type/__install_config/gencode-local index da87a4ac..674dec25 100755 --- a/cdist/conf/type/__install_config/gencode-local +++ b/cdist/conf/type/__install_config/gencode-local @@ -26,13 +26,13 @@ cdist_args="-v" [ "$__debug" = "yes" ] && cdist_args="$cdist_args -d" cat << DONE -echo "__apt_noautostart --state present" \ - | cdist $cdist_args \ - config \ - --initial-manifest - \ - --remote-exec="$remote_exec $chroot" \ - --remote-copy="$remote_copy $chroot" \ - $__target_host +#echo "__apt_noautostart --state present" \ +# | cdist $cdist_args \ +# config \ +# --initial-manifest - \ +# --remote-exec="$remote_exec $chroot" \ +# --remote-copy="$remote_copy $chroot" \ +# $__target_host cdist $cdist_args \ config \ @@ -40,11 +40,11 @@ cdist $cdist_args \ --remote-copy="$remote_copy $chroot" \ $__target_host -echo "__apt_noautostart --state absent" \ - | cdist $cdist_args \ - config \ - --initial-manifest - \ - --remote-exec="$remote_exec $chroot" \ - --remote-copy="$remote_copy $chroot" \ - $__target_host +#echo "__apt_noautostart --state absent" \ +# | cdist $cdist_args \ +# config \ +# --initial-manifest - \ +# --remote-exec="$remote_exec $chroot" \ +# --remote-copy="$remote_copy $chroot" \ +# $__target_host DONE From c3f79277b226ef5c97360566c868660e1ed1fbad Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 08:50:41 +0100 Subject: [PATCH 030/197] add some more packages for preos - fixes #267 Signed-off-by: Nico Schottelius --- cdist/preos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 433cf871..f573566d 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -39,8 +39,8 @@ for pkg in \ file \ linux-image-amd64 \ openssh-server \ - syslinux \ - gdisk util-linux \ + syslinux grub 2 \ + gdisk util-linux lvm2 mdadm \ btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do __package $pkg --state present done From 79cfdf578d1d93241648262b64c2243ffed49559 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 09:01:04 +0100 Subject: [PATCH 031/197] remove obsolete '--additional-manifest' parameter Signed-off-by: Nico Schottelius --- scripts/cdist | 2 -- 1 file changed, 2 deletions(-) diff --git a/scripts/cdist b/scripts/cdist index ed9d2eda..7b7569ee 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -89,8 +89,6 @@ def commandline(): parents=[parser['loglevel']]) parser['preos'].add_argument('-a', '--arch', help='Select architecture for preos', default="amd64") - parser['preos'].add_argument('-A', '--additional-manifest', - help='Add stuff to configuration manifest', default="amd64") parser['preos'].add_argument('-b', '--bootstrap', help='Bootstrap directory with PreOS', action="store_true") parser['preos'].add_argument('-c', '--configure', From 55f26cbe25d8c87279a8221a9ef611fc2c90b8f1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 09:06:02 +0100 Subject: [PATCH 032/197] - ' ' Signed-off-by: Nico Schottelius --- cdist/preos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index f573566d..b76d2178 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -39,7 +39,7 @@ for pkg in \ file \ linux-image-amd64 \ openssh-server \ - syslinux grub 2 \ + syslinux grub2 \ gdisk util-linux lvm2 mdadm \ btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do __package $pkg --state present From b125c0a4f273acaf33a082f8179b69e3b765b7d0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 09:21:50 +0100 Subject: [PATCH 033/197] create output directory, if it does not exist Signed-off-by: Nico Schottelius --- cdist/preos.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index b76d2178..f3d34c21 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -194,14 +194,21 @@ cp -L "$src" "$real_dst" log.info("Creating initramfs ...") subprocess.check_call(cmd, shell=True) + def ensure_out_dir_exists(self): + os.makedirs(self.out_dir, exist_ok=True) + + def create_iso(self, out_dir): self.out_dir = out_dir + self.ensure_out_dir_exists() + raise cdist.Error("Generating ISO is not yet supported") def create_pxe(self, out_dir): self.out_dir = out_dir + self.ensure_out_dir_exists() self.create_kernel() self.create_initramfs() self.create_pxeconfig() From 4cfedb1787abe7af68aa3cb51bbfc84714c26f4b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 09:42:53 +0100 Subject: [PATCH 034/197] +curl Signed-off-by: Nico Schottelius --- cdist/preos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index f3d34c21..347b0cba 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -38,7 +38,7 @@ DEFAULT_MANIFEST = """ for pkg in \ file \ linux-image-amd64 \ - openssh-server \ + openssh-server curl \ syslinux grub2 \ gdisk util-linux lvm2 mdadm \ btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do From e463f84333204605d948f81de7ada5683bb0ce64 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 10:04:42 +0100 Subject: [PATCH 035/197] add changelog for 4.x cdist series Signed-off-by: Nico Schottelius --- cdist/conf/explorer/disks | 2 ++ docs/changelog.4 | 10 ++++++++++ 2 files changed, 12 insertions(+) create mode 100644 cdist/conf/explorer/disks create mode 100644 docs/changelog.4 diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks new file mode 100644 index 00000000..52fef81e --- /dev/null +++ b/cdist/conf/explorer/disks @@ -0,0 +1,2 @@ +cd /dev +echo sd? hd? vd? diff --git a/docs/changelog.4 b/docs/changelog.4 new file mode 100644 index 00000000..79478c86 --- /dev/null +++ b/docs/changelog.4 @@ -0,0 +1,10 @@ +Changelog +--------- + + * Changes are always commented with their author in (braces) + * Exception: No braces means author == Nico Schottelius + +4.0.0pre1: + * Core: Integrate initial install support + * Core: Integrate initial preos support + From b4644c9c2eae1fdabb2b5a33f69ae92a774c1bc0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 10:08:46 +0100 Subject: [PATCH 036/197] add readme / warning for 4.x series Signed-off-by: Nico Schottelius --- README.4 | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 README.4 diff --git a/README.4 b/README.4 new file mode 100644 index 00000000..04258873 --- /dev/null +++ b/README.4 @@ -0,0 +1,24 @@ +This branch contains experimental features for cdist 4.x: + + - install support + - preos support + +They are not yet stable: + + - use them at your own risk + - all __install types may change at any time (syntax, parameter, etc.) + - explorers for install may be broken + + - core code is based on the master branch, but + contains changes for install and preos feature + + +In short: + + _ _ _ + _ _ ___ ___ __ _| |_ _ _ ___ _ _ _ __ _____ ___ __ _ __(_)___| | __ +| | | / __|/ _ \ / _` | __| | | | |/ _ \| | | | '__| / _ \ \ /\ / / '_ \ | '__| / __| |/ / +| |_| \__ \ __/ | (_| | |_ | |_| | (_) | |_| | | | (_) \ V V /| | | | | | | \__ \ < + \__,_|___/\___| \__,_|\__| \__, |\___/ \__,_|_| \___/ \_/\_/ |_| |_| |_| |_|___/_|\_\ + |___/ + From 3ca911dbc641ed22596c18293d5f4c9271133eec Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 14:37:50 +0100 Subject: [PATCH 037/197] integrate install and preos support Signed-off-by: Nico Schottelius --- docs/changelog | 4 ++++ docs/changelog.4 | 10 ---------- 2 files changed, 4 insertions(+), 10 deletions(-) delete mode 100644 docs/changelog.4 diff --git a/docs/changelog b/docs/changelog index f43131a1..98260564 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,10 @@ Changelog * Changes are always commented with their author in (braces) * Exception: No braces means author == Nico Schottelius +4.0.0pre1: 2014-01-20 + * Core: Integrate initial install support + * Core: Integrate initial preos support + 3.0.3: * Core: Enhance error message when requirement is missing object id * Explorer hostname: Return host name by using uname -n diff --git a/docs/changelog.4 b/docs/changelog.4 deleted file mode 100644 index 79478c86..00000000 --- a/docs/changelog.4 +++ /dev/null @@ -1,10 +0,0 @@ -Changelog ---------- - - * Changes are always commented with their author in (braces) - * Exception: No braces means author == Nico Schottelius - -4.0.0pre1: - * Core: Integrate initial install support - * Core: Integrate initial preos support - From a3c5d32a5453b25057acd238377fe002f98f9e74 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 14:45:42 +0100 Subject: [PATCH 038/197] fix building manpage of __install_partition_msdos Signed-off-by: Nico Schottelius --- cdist/conf/type/__install_partition_msdos/man.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__install_partition_msdos/man.text b/cdist/conf/type/__install_partition_msdos/man.text index 8d403b67..82d81ac5 100644 --- a/cdist/conf/type/__install_partition_msdos/man.text +++ b/cdist/conf/type/__install_partition_msdos/man.text @@ -1,5 +1,5 @@ cdist-type__install_partition_msdos(7) -============================== +====================================== Steven Armstrong From 5a0a3971b0c2769bf3b1c0202b0d48be6db71bb9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 20 Jan 2014 14:48:05 +0100 Subject: [PATCH 039/197] do not change to the masterbranch... Signed-off-by: Nico Schottelius --- bin/build-helper | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/bin/build-helper b/bin/build-helper index 6210b907..bfd7d31c 100755 --- a/bin/build-helper +++ b/bin/build-helper @@ -247,8 +247,10 @@ eof "$0" release-git-tag # Also merge back the version branch - git checkout master - git merge "$target_branch" + if [ "$masterbranch" = yes ]; then + git checkout master + git merge "$target_branch" + fi # Publish git changes make pub From ca47ea00382a8f5da655540d6032c1338b51c8e5 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Mon, 3 Feb 2014 15:29:48 +0100 Subject: [PATCH 040/197] cleanup apt cache before packing initramfs Signed-off-by: Steven Armstrong --- cdist/preos.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index 347b0cba..dc400ba9 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -253,6 +253,12 @@ cp -L "$src" "$real_dst" config = cdist.config.Config(local, remote) config.run() + def cleanup(self): + # Remove cruft from chroot + for action in 'autoclean clean autoremove'.split(): + cmd = [ 'chroot', self.target_dir, '/usr/bin/apt-get', action] + subprocess.check_call(cmd) + @classmethod def commandline(cls, args): self = cls(target_dir=args.target_dir[0], @@ -270,6 +276,9 @@ cp -L "$src" "$real_dst" if args.config: self.config() + # Cleanup chroot + self.cleanup() + # Output pxe files if args.pxe_boot_dir: self.create_pxe(args.pxe_boot_dir) From ee5731fc960fa7849daa32f8ee4df130adc7aa66 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 6 Feb 2014 00:08:02 +0100 Subject: [PATCH 041/197] add __ccollect_source type Signed-off-by: Nico Schottelius --- .../type/__ccollect_source/explorer/cksum | 34 +++++++ .../conf/type/__ccollect_source/explorer/stat | 47 ++++++++++ .../conf/type/__ccollect_source/explorer/type | 33 +++++++ .../type/__ccollect_source/gencode-remote | 93 +++++++++++++++++++ cdist/conf/type/__ccollect_source/man.text | 64 +++++++++++++ cdist/conf/type/__ccollect_source/manifest | 53 +++++++++++ .../type/__ccollect_source/parameter/boolean | 1 + .../parameter/default/ccollectconf | 1 + .../__ccollect_source/parameter/default/state | 1 + .../type/__ccollect_source/parameter/optional | 2 + .../parameter/optional_multiple | 1 + .../type/__ccollect_source/parameter/required | 2 + 12 files changed, 332 insertions(+) create mode 100755 cdist/conf/type/__ccollect_source/explorer/cksum create mode 100755 cdist/conf/type/__ccollect_source/explorer/stat create mode 100755 cdist/conf/type/__ccollect_source/explorer/type create mode 100755 cdist/conf/type/__ccollect_source/gencode-remote create mode 100644 cdist/conf/type/__ccollect_source/man.text create mode 100755 cdist/conf/type/__ccollect_source/manifest create mode 100644 cdist/conf/type/__ccollect_source/parameter/boolean create mode 100644 cdist/conf/type/__ccollect_source/parameter/default/ccollectconf create mode 100644 cdist/conf/type/__ccollect_source/parameter/default/state create mode 100644 cdist/conf/type/__ccollect_source/parameter/optional create mode 100644 cdist/conf/type/__ccollect_source/parameter/optional_multiple create mode 100644 cdist/conf/type/__ccollect_source/parameter/required diff --git a/cdist/conf/type/__ccollect_source/explorer/cksum b/cdist/conf/type/__ccollect_source/explorer/cksum new file mode 100755 index 00000000..335e4e7a --- /dev/null +++ b/cdist/conf/type/__ccollect_source/explorer/cksum @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Retrieve the md5sum of a file to be created, if it is already existing. +# + +destination="/$__object_id" + +if [ -e "$destination" ]; then + if [ -f "$destination" ]; then + cksum < "$destination" + else + echo "NO REGULAR FILE" + fi +else + echo "NO FILE FOUND, NO CHECKSUM CALCULATED." +fi diff --git a/cdist/conf/type/__ccollect_source/explorer/stat b/cdist/conf/type/__ccollect_source/explorer/stat new file mode 100755 index 00000000..298221b7 --- /dev/null +++ b/cdist/conf/type/__ccollect_source/explorer/stat @@ -0,0 +1,47 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +# nothing to work with, nothing we could do +[ -e "$destination" ] || exit 0 + +os=$("$__explorer/os") +case "$os" in + "freebsd") + # FIXME: should be something like this based on man page, but can not test + stat -f "type: %ST +owner: %Du %Su +group: %Dg %Sg +mode: %Op %Sp +size: %Dz +links: %Dl +" "$destination" + ;; + *) + stat --printf="type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h +" "$destination" + ;; +esac diff --git a/cdist/conf/type/__ccollect_source/explorer/type b/cdist/conf/type/__ccollect_source/explorer/type new file mode 100755 index 00000000..e723047c --- /dev/null +++ b/cdist/conf/type/__ccollect_source/explorer/type @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2013 Steven Armstrong (steven-cdist armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" + +if [ ! -e "$destination" ]; then + echo none +elif [ -h "$destination" ]; then + echo symlink +elif [ -f "$destination" ]; then + echo file +elif [ -d "$destination" ]; then + echo directory +else + echo unknown +fi diff --git a/cdist/conf/type/__ccollect_source/gencode-remote b/cdist/conf/type/__ccollect_source/gencode-remote new file mode 100755 index 00000000..c41b5179 --- /dev/null +++ b/cdist/conf/type/__ccollect_source/gencode-remote @@ -0,0 +1,93 @@ +#!/bin/sh +# +# 2014 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +destination="/$__object_id" +state_should="$(cat "$__object/parameter/state")" +type="$(cat "$__object/explorer/type")" +stat_file="$__object/explorer/stat" + + +get_current_value() { + if [ -s "$stat_file" ]; then + _name="$1" + _value="$2" + case "$_value" in + [0-9]*) + _index=2 + ;; + *) + _index=3 + ;; + esac + awk '/'"$_name"':/ { print $'$_index' }' "$stat_file" + unset _name _value _index + fi +} + +set_group() { + echo chgrp \"$1\" \"$destination\" + echo chgrp $1 >> "$__messages_out" +} + +set_owner() { + echo chown \"$1\" \"$destination\" + echo chown $1 >> "$__messages_out" +} + +set_mode() { + echo chmod \"$1\" \"$destination\" + echo chmod $1 >> "$__messages_out" +} + +set_attributes= +case "$state_should" in + present|exists) + # Note: Mode - needs to happen last as a chown/chgrp can alter mode by + # clearing S_ISUID and S_ISGID bits (see chown(2)) + for attribute in group owner mode; do + if [ -f "$__object/parameter/$attribute" ]; then + value_should="$(cat "$__object/parameter/$attribute")" + + # change 0xxx format to xxx format => same as stat returns + if [ "$attribute" = mode ]; then + value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')" + fi + + value_is="$(get_current_value "$attribute" "$value_should")" + if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then + "set_$attribute" "$value_should" + fi + fi + done + + ;; + + absent) + if [ "$type" = "file" ]; then + echo rm -f \"$destination\" + echo remove >> "$__messages_out" + fi + ;; + + *) + echo "Unknown state: $state_should" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__ccollect_source/man.text b/cdist/conf/type/__ccollect_source/man.text new file mode 100644 index 00000000..32a7467e --- /dev/null +++ b/cdist/conf/type/__ccollect_source/man.text @@ -0,0 +1,64 @@ +cdist-type__ccollect_source(7) +============================== +Nico Schottelius + + +NAME +---- +cdist-type__ccollect_source - Manage ccollect sources + + +DESCRIPTION +----------- +This cdist type allows you to create or delete ccollect sources. + +REQUIRED PARAMETERS +------------------- +source:: + The source from which to backup +destination:: + The destination directory + + +OPTIONAL PARAMETERS +------------------- +state:: + 'present' or 'absent', defaults to 'present' +ccollectconf:: + The CCOLLECT_CONF directory. Defaults to /etc/ccollect. + + +OPTIONAL MULTIPLE PARAMETERS +---------------------------- +exclude:: + Paths to exclude of backup + +BOOLEAN PARAMETERS +------------------ +verbose:: + Whether to report backup verbosely + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__ccollect_source doc.ungleich.ch \ + --source doc.ungleich.ch:/ \ + --destination /backup/doc.ungleich.ch \ + --exclude '/proc/*' --exclude '/sys/*' \ + --verbose + +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- ccollect(1) +- http://www.nico.schottelius.org/software/ccollect/ + + +COPYING +------- +Copyright \(C) 2014 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__ccollect_source/manifest b/cdist/conf/type/__ccollect_source/manifest new file mode 100755 index 00000000..89c2ef2b --- /dev/null +++ b/cdist/conf/type/__ccollect_source/manifest @@ -0,0 +1,53 @@ +#!/bin/sh +# +# 2014 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +name="$__object_id" +state="$(cat "$__object/parameter/state")" +source="$(cat "$__object/parameter/source")" +destination="$(cat "$__object/parameter/destination")" +ccollectconf="$(cat "$__object/parameter/ccollectconf" | sed 's,/$,,')" + +sourcedir="$ccollectconf/sources" +basedir="$sourcedir/$name" + +destination_file="$basedir/destination" +source_file="$basedir/source" +exclude_file="$basedir/exclude" +verbose_file="$basedir/verbose" + +__directory "$basedir" --state "$state" + +export require="__directory$basedir" +echo "$destination" | __file "$destination_file" --source - --state "$state" +echo "$source" | __file "$source_file" --source - --state "$state" + +################################################################################ +# Booleans +if [ -f "$__object/parameter/verbose" ]; then + verbosestate="present" +else + verbosestate="absent" +fi +__file "$verbose_file" --state "$verbosestate" + +if [ -f "$__object/parameter/exclude" ]; then + __file "$exclude_file" --source - --state "$state" \ + < "$__object/parameter/exclude" +fi diff --git a/cdist/conf/type/__ccollect_source/parameter/boolean b/cdist/conf/type/__ccollect_source/parameter/boolean new file mode 100644 index 00000000..c00ee94a --- /dev/null +++ b/cdist/conf/type/__ccollect_source/parameter/boolean @@ -0,0 +1 @@ +verbose diff --git a/cdist/conf/type/__ccollect_source/parameter/default/ccollectconf b/cdist/conf/type/__ccollect_source/parameter/default/ccollectconf new file mode 100644 index 00000000..a9fda009 --- /dev/null +++ b/cdist/conf/type/__ccollect_source/parameter/default/ccollectconf @@ -0,0 +1 @@ +/etc/ccollect diff --git a/cdist/conf/type/__ccollect_source/parameter/default/state b/cdist/conf/type/__ccollect_source/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__ccollect_source/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__ccollect_source/parameter/optional b/cdist/conf/type/__ccollect_source/parameter/optional new file mode 100644 index 00000000..0249d11e --- /dev/null +++ b/cdist/conf/type/__ccollect_source/parameter/optional @@ -0,0 +1,2 @@ +ccollectconf +state diff --git a/cdist/conf/type/__ccollect_source/parameter/optional_multiple b/cdist/conf/type/__ccollect_source/parameter/optional_multiple new file mode 100644 index 00000000..9ba870ea --- /dev/null +++ b/cdist/conf/type/__ccollect_source/parameter/optional_multiple @@ -0,0 +1 @@ +exclude diff --git a/cdist/conf/type/__ccollect_source/parameter/required b/cdist/conf/type/__ccollect_source/parameter/required new file mode 100644 index 00000000..9239646e --- /dev/null +++ b/cdist/conf/type/__ccollect_source/parameter/required @@ -0,0 +1,2 @@ +source +destination From 4efe8553da4ccf4ec58197bb1422eefa9713e908 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 10 Feb 2014 21:40:11 +0100 Subject: [PATCH 042/197] run apt-get clean before creating preos Signed-off-by: Nico Schottelius --- cdist/preos.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index 347b0cba..ff157f5b 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -162,6 +162,10 @@ cp -L "$src" "$real_dst" fd.write(val) os.chmod(filename, stat.S_IRUSR | stat.S_IXUSR) + def remove_archives(self, base_dir): + cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get", "clean" ] + subprocess.check_call(cmd) + def create_kernel(self): dst = os.path.join(self.out_dir, "kernel") srcglob = glob.glob("%s/boot/vmlinuz-*" % self.target_dir) @@ -270,6 +274,9 @@ cp -L "$src" "$real_dst" if args.config: self.config() + # Cleanup archives before creating any image + self.remove_archives() + # Output pxe files if args.pxe_boot_dir: self.create_pxe(args.pxe_boot_dir) From ac23fa3e1018d80f52bf05686bc6a9e70b23eeeb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 14 Feb 2014 20:53:43 +0100 Subject: [PATCH 043/197] ++changes Signed-off-by: Nico Schottelius --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 00d2734e..4f6b1064 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,9 @@ Changelog * Changes are always commented with their author in (braces) * Exception: No braces means author == Nico Schottelius +4.0.0pre2: 2014-02-14 + * Core: Remove archives from generated preos (Steven Armstrong) + 4.0.0pre1: 2014-01-20 * Core: Integrate initial install support * Core: Integrate initial preos support From 9ad7e055024c067d6dbb35eab53b49b33a48344c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 31 Mar 2014 23:58:45 +0200 Subject: [PATCH 044/197] ++;; Signed-off-by: Nico Schottelius --- cdist/conf/type/__dog_vdi/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__dog_vdi/manifest b/cdist/conf/type/__dog_vdi/manifest index ab533c4b..be327a3a 100644 --- a/cdist/conf/type/__dog_vdi/manifest +++ b/cdist/conf/type/__dog_vdi/manifest @@ -26,6 +26,7 @@ case "$state_should" in echo "Size is required when state is present" >&2 exit 1 fi + ;; absent) : ;; From 8e060a1d838c38081eda0c8fb796cddaec20593d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 13 Jun 2014 13:51:03 +0200 Subject: [PATCH 045/197] release 4.0.0pre3 Signed-off-by: Nico Schottelius --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index 509394d9..a6e11e08 100644 --- a/docs/changelog +++ b/docs/changelog @@ -4,6 +4,9 @@ Changelog * Changes are always commented with their author in (braces) * Exception: No braces means author == Nico Schottelius +4.0.0pre3: 2014-06-13 + * Update to include changes from cdist 3.1.5 + 4.0.0pre2: 2014-02-14 * Core: Remove archives from generated preos (Steven Armstrong) From 3d82a0d25cda5f3cbbd8e9cb6ac67584f053ed69 Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Fri, 13 Jun 2014 14:56:24 +0200 Subject: [PATCH 046/197] Set hostname in preos --- cdist/preos.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/preos.py b/cdist/preos.py index dc400ba9..0aa7eb6f 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -62,6 +62,8 @@ eof # Steven found this out - coyping it 1:1 # fix the bloody 'stdin: is not a tty' problem __line /root/.profile --line 'mesg n' --state absent + +__hostname preos """ class PreOSExistsError(cdist.Error): From 1e582dceece14690034d609f8610269c6cfa37dc Mon Sep 17 00:00:00 2001 From: Daniel Heule Date: Wed, 23 Jul 2014 09:22:03 +0200 Subject: [PATCH 047/197] bugfix of raw_command with raw_command, you can set env variables in crontab, but cron don't accept env vars if the definition is like this: SHELL=/bin/bash # marker of something ... so we need to make sure that raw commands are not apended by a marker --- cdist/conf/type/__cron/explorer/entry | 7 ++++++- cdist/conf/type/__cron/gencode-remote | 14 +++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) mode change 100755 => 100644 cdist/conf/type/__cron/gencode-remote diff --git a/cdist/conf/type/__cron/explorer/entry b/cdist/conf/type/__cron/explorer/entry index c3bf02d2..2167e045 100644 --- a/cdist/conf/type/__cron/explorer/entry +++ b/cdist/conf/type/__cron/explorer/entry @@ -22,4 +22,9 @@ name="$__object_name" user="$(cat "$__object/parameter/user")" -crontab -u $user -l 2>/dev/null | grep "# $name\$" || true +if [ -f "$__object/parameter/raw_command" ]; then + command="$(cat "$__object/parameter/command")" + crontab -u $user -l 2>/dev/null | grep "^$command\$" || true +else + crontab -u $user -l 2>/dev/null | grep "# $name\$" || true +fi diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote old mode 100755 new mode 100644 index 77a63b9b..f5c1cc62 --- a/cdist/conf/type/__cron/gencode-remote +++ b/cdist/conf/type/__cron/gencode-remote @@ -26,7 +26,7 @@ command="$(cat "$__object/parameter/command")" if [ -f "$__object/parameter/raw" ]; then raw="$(cat "$__object/parameter/raw")" - entry="$raw $command" + entry="$raw $command # $name" elif [ -f "$__object/parameter/raw_command" ]; then entry="$command" else @@ -35,10 +35,9 @@ else day_of_month="$(cat "$__object/parameter/day_of_month" 2>/dev/null || echo "*")" month="$(cat "$__object/parameter/month" 2>/dev/null || echo "*")" day_of_week="$(cat "$__object/parameter/day_of_week" 2>/dev/null || echo "*")" - entry="$minute $hour $day_of_month $month $day_of_week $command" + entry="$minute $hour $day_of_month $month $day_of_week $command # $name" fi -entry="$entry # $name" mkdir "$__object/files" echo "$entry" > "$__object/files/entry" @@ -85,7 +84,12 @@ case "$state_should" in echo ") | crontab -u $user -" ;; absent) - echo "( crontab -u $user -l 2>/dev/null | grep -v -E \"$filter\" 2>/dev/null || true ) | \\" - echo "grep -v \"# $name\\$\" | crontab -u $user -" + if [ -f "$__object/parameter/raw_command" ]; then + echo "( crontab -u $user -l 2>/dev/null | grep -v -E \"$filter\" 2>/dev/null || true ) | \\" + echo "grep -v \"^$entry\\$\" | crontab -u $user -" + else + echo "( crontab -u $user -l 2>/dev/null | grep -v -E \"$filter\" 2>/dev/null || true ) | \\" + echo "grep -v \"# $name\\$\" | crontab -u $user -" + fi ;; esac From 82bcc83d4164c5e83b5ccde75d74637bceca0887 Mon Sep 17 00:00:00 2001 From: Tomas Pospisek Date: Mon, 18 Aug 2014 19:09:19 +0200 Subject: [PATCH 048/197] add new __package_dpkg type --- cdist/conf/type/__package_dpkg/gencode-local | 57 +++++++++++++++++++ cdist/conf/type/__package_dpkg/man.text | 47 +++++++++++++++ .../type/__package_dpkg/parameter/required | 1 + cdist/conf/type/__package_dpkg/singleton | 0 4 files changed, 105 insertions(+) create mode 100755 cdist/conf/type/__package_dpkg/gencode-local create mode 100644 cdist/conf/type/__package_dpkg/man.text create mode 100644 cdist/conf/type/__package_dpkg/parameter/required create mode 100644 cdist/conf/type/__package_dpkg/singleton diff --git a/cdist/conf/type/__package_dpkg/gencode-local b/cdist/conf/type/__package_dpkg/gencode-local new file mode 100755 index 00000000..9704c46c --- /dev/null +++ b/cdist/conf/type/__package_dpkg/gencode-local @@ -0,0 +1,57 @@ +#!/bin/sh +# +# 2013 Tomas Pospisek (tpo_deb sourcepole.ch) +# +# This file is based on cdist's __file/gencode-local and part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# This __package_dpkg type does not check whether a *.deb package is +# allready installed. It just copies the *.deb package over to the +# destination and installs it. We could use __package_apt to check +# whether a *.deb package is allready installed and only install it +# if we're given a --force argument or similar (would be clever not +# to conflict with dpkg's --force options). But currently we don't +# do any checks or --force'ing. +# + +local_package_path=$( cat "$__object/parameter/install" ) +package=$( basename "$local_package_path" ) + +if [ ! -f "$local_package_path" ]; then + echo "Package \"$local_package_path\" does not exist." >&2 + exit 1 +fi + +# upload package to temp directory +temp_dir="cdist.XXXXXXXXXX" +cat << DONE +destination_dir="\$($__remote_exec $__target_host "mktemp -d $temp_dir")" +DONE + +cat << DONE +$__remote_copy $local_package_path ${__target_host}:\$destination_dir +DONE + +# install package +echo "3" >&2 +cat << DONE +$__remote_exec $__target_host "dpkg -i \"\$destination_dir/$package\"" +DONE + +# clean up: remove tmp_dir and contents on remote host +cat << DONE +$__remote_exec $__target_host "rm -rf \"\$destination_dir\"" +DONE diff --git a/cdist/conf/type/__package_dpkg/man.text b/cdist/conf/type/__package_dpkg/man.text new file mode 100644 index 00000000..e6a67e79 --- /dev/null +++ b/cdist/conf/type/__package_dpkg/man.text @@ -0,0 +1,47 @@ +cdist-type__package_dpkg(7) +========================== +Tomas Pospisek + + +NAME +---- +cdist-type__package_dpkg - Manage packages with dpkg + + +DESCRIPTION +----------- +dpkg is usually used on Debian and variants (like Ubuntu) to +install packages. + + +REQUIRED PARAMETERS +------------------- +install:: + Specifies the local path to the *.deb package to be installed + + +OPTIONAL PARAMETERS +------------------- +None + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +# Install foo package +__package_dpkg --install /tmp/foo_0.1_all.deb +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__package(7) + + +COPYING +------- +Copyright \(C) 2013 Tomas Pospisek. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). +This type is based on __package_apt diff --git a/cdist/conf/type/__package_dpkg/parameter/required b/cdist/conf/type/__package_dpkg/parameter/required new file mode 100644 index 00000000..7c32f559 --- /dev/null +++ b/cdist/conf/type/__package_dpkg/parameter/required @@ -0,0 +1 @@ +install diff --git a/cdist/conf/type/__package_dpkg/singleton b/cdist/conf/type/__package_dpkg/singleton new file mode 100644 index 00000000..e69de29b From d8af4d3ad5e53b8aac3b571aac969f36a63d2d6b Mon Sep 17 00:00:00 2001 From: Tomas Pospisek Date: Mon, 18 Aug 2014 19:11:28 +0200 Subject: [PATCH 049/197] fix email address --- cdist/conf/type/__package_dpkg/man.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_dpkg/man.text b/cdist/conf/type/__package_dpkg/man.text index e6a67e79..6dc07c41 100644 --- a/cdist/conf/type/__package_dpkg/man.text +++ b/cdist/conf/type/__package_dpkg/man.text @@ -1,6 +1,6 @@ cdist-type__package_dpkg(7) ========================== -Tomas Pospisek +Tomas Pospisek NAME From 8b53003a16d1f80a6e3031a58f29b7bb60b264cd Mon Sep 17 00:00:00 2001 From: Tomas Pospisek Date: Tue, 9 Sep 2014 15:42:30 +0200 Subject: [PATCH 050/197] add __postgres_extension type --- .../type/__postgres_extension/gencode-remote | 39 ++++++++++++++ cdist/conf/type/__postgres_extension/man.text | 52 +++++++++++++++++++ .../parameter/default/state | 1 + .../__postgres_extension/parameter/optional | 1 + 4 files changed, 93 insertions(+) create mode 100755 cdist/conf/type/__postgres_extension/gencode-remote create mode 100644 cdist/conf/type/__postgres_extension/man.text create mode 100644 cdist/conf/type/__postgres_extension/parameter/default/state create mode 100644 cdist/conf/type/__postgres_extension/parameter/optional diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote new file mode 100755 index 00000000..3408df86 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/gencode-remote @@ -0,0 +1,39 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2013 Tomas Pospisek (tpo_deb at sourcepole.ch) +# +# This type was created by Tomas Pospisek based on the +#__postgres_role type by Steven Armstrong +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +dbname=$( echo "$__object_id" | cut -d":" -f1 ) +extension=$( echo "$__object_id" | cut -d":" -f2 ) + +state_should=$( cat "$__object/parameter/state" ) + +case "$state_should" in + present) + cmd="CREATE EXTENSION IF NOT EXISTS $extension" + echo "su - postgres -c 'psql -c \"$cmd\" \"$dbname\"'" + ;; + absent) + cmd="DROP EXTENSION IF EXISTS $extenstion" + echo "su - postgres -c 'psql -c \"$cmd\" \"$dbname\"'" + ;; +esac diff --git a/cdist/conf/type/__postgres_extension/man.text b/cdist/conf/type/__postgres_extension/man.text new file mode 100644 index 00000000..6d722d68 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/man.text @@ -0,0 +1,52 @@ +cdist-type__postgres_extension(7) +================================= +Tomas Pospisek + + +NAME +---- +cdist-type__postgres_extension - manage postgres extensions + + +DESCRIPTION +----------- +This cdist type allows you to create or drop postgres extensions. + +The object you need to pass to __postgres_extension consists of +the database name and the extension name joined by a colon in the +following form: + + dbname:extension + +f.ex. + + rails_test:unaccent + + +OPTIONAL PARAMETERS +------------------- +state:: + Either "present" or "absent", defaults to "present" + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__postgres_extension rails_test:unaccent +__postgres_extension --present rails_test:unaccent +__postgres_extension --absent rails_test:unaccent +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__postgres_database(7) +- http://www.postgresql.org/docs/current/static/sql-createextension.html + + +COPYING +------- +Copyright \(C) 2014 Tomas Pospisek. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__postgres_extension/parameter/default/state b/cdist/conf/type/__postgres_extension/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__postgres_extension/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__postgres_extension/parameter/optional b/cdist/conf/type/__postgres_extension/parameter/optional new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/cdist/conf/type/__postgres_extension/parameter/optional @@ -0,0 +1 @@ +state From 44941137d67d3a534b066bc4ddc93a466ca4e180 Mon Sep 17 00:00:00 2001 From: Tomas Pospisek Date: Wed, 10 Sep 2014 11:21:09 +0200 Subject: [PATCH 051/197] change implementation and API of __package_dpkg __package_dpkg wasn't working as intended - being a singleton meant that it could only install one package. Now we missuse /var/cache/apt/archives to copy our package into and `dpkg -i` from there --- .../{gencode-local => gencode-remote} | 29 +--------------- cdist/conf/type/__package_dpkg/man.text | 21 +++++------- cdist/conf/type/__package_dpkg/manifest | 34 +++++++++++++++++++ .../type/__package_dpkg/parameter/required | 2 +- cdist/conf/type/__package_dpkg/singleton | 0 5 files changed, 45 insertions(+), 41 deletions(-) rename cdist/conf/type/__package_dpkg/{gencode-local => gencode-remote} (62%) create mode 100644 cdist/conf/type/__package_dpkg/manifest delete mode 100644 cdist/conf/type/__package_dpkg/singleton diff --git a/cdist/conf/type/__package_dpkg/gencode-local b/cdist/conf/type/__package_dpkg/gencode-remote similarity index 62% rename from cdist/conf/type/__package_dpkg/gencode-local rename to cdist/conf/type/__package_dpkg/gencode-remote index 9704c46c..d4186e66 100755 --- a/cdist/conf/type/__package_dpkg/gencode-local +++ b/cdist/conf/type/__package_dpkg/gencode-remote @@ -27,31 +27,4 @@ # do any checks or --force'ing. # -local_package_path=$( cat "$__object/parameter/install" ) -package=$( basename "$local_package_path" ) - -if [ ! -f "$local_package_path" ]; then - echo "Package \"$local_package_path\" does not exist." >&2 - exit 1 -fi - -# upload package to temp directory -temp_dir="cdist.XXXXXXXXXX" -cat << DONE -destination_dir="\$($__remote_exec $__target_host "mktemp -d $temp_dir")" -DONE - -cat << DONE -$__remote_copy $local_package_path ${__target_host}:\$destination_dir -DONE - -# install package -echo "3" >&2 -cat << DONE -$__remote_exec $__target_host "dpkg -i \"\$destination_dir/$package\"" -DONE - -# clean up: remove tmp_dir and contents on remote host -cat << DONE -$__remote_exec $__target_host "rm -rf \"\$destination_dir\"" -DONE +echo "dpkg -i /var/cache/apt/archives/$__object_id" diff --git a/cdist/conf/type/__package_dpkg/man.text b/cdist/conf/type/__package_dpkg/man.text index 6dc07c41..ae98be99 100644 --- a/cdist/conf/type/__package_dpkg/man.text +++ b/cdist/conf/type/__package_dpkg/man.text @@ -10,27 +10,24 @@ cdist-type__package_dpkg - Manage packages with dpkg DESCRIPTION ----------- -dpkg is usually used on Debian and variants (like Ubuntu) to -install packages. +__package_dpkg is used on Debian and variants (like Ubuntu) to +install packages that are provided locally as *.deb files. + +The object given to __package_dpkg must be the name of the deb package. REQUIRED PARAMETERS ------------------- -install:: - Specifies the local path to the *.deb package to be installed - - -OPTIONAL PARAMETERS -------------------- -None - +source:: + path to the *.deb package EXAMPLES -------- -------------------------------------------------------------------------------- -# Install foo package -__package_dpkg --install /tmp/foo_0.1_all.deb +# Install foo and bar packages +__package_dpkg --source /tmp/foo_0.1_all.deb foo_0.1_all.deb +__package_dpkg --source $__type/files/bar_1.4.deb bar_1.4.deb -------------------------------------------------------------------------------- diff --git a/cdist/conf/type/__package_dpkg/manifest b/cdist/conf/type/__package_dpkg/manifest new file mode 100644 index 00000000..ff477c2d --- /dev/null +++ b/cdist/conf/type/__package_dpkg/manifest @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2013 Tomas Pospisek (tpo_deb sourcepole.ch) +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# This __package_dpkg type does not check whether a *.deb package is +# allready installed. It just copies the *.deb package over to the +# destination and installs it. We could use __package_apt to check +# whether a *.deb package is allready installed and only install it +# if we're given a --force argument or similar (would be clever not +# to conflict with dpkg's --force options). But currently we don't +# do any checks or --force'ing. + + +package_path=$( cat "$__object/parameter/source" ) +package=$( basename "$__object_id" ) + +__file "/var/cache/apt/archives/$package" \ + --source "$package_path" \ + --state present + diff --git a/cdist/conf/type/__package_dpkg/parameter/required b/cdist/conf/type/__package_dpkg/parameter/required index 7c32f559..5a18cd2f 100644 --- a/cdist/conf/type/__package_dpkg/parameter/required +++ b/cdist/conf/type/__package_dpkg/parameter/required @@ -1 +1 @@ -install +source diff --git a/cdist/conf/type/__package_dpkg/singleton b/cdist/conf/type/__package_dpkg/singleton deleted file mode 100644 index e69de29b..00000000 From 339167c23c0ccd68841f3f3b36490fce4743aaa6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 14 Sep 2014 13:51:13 +0200 Subject: [PATCH 052/197] catch some errors Signed-off-by: Nico Schottelius --- cdist/preos.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 0aa7eb6f..b61318ed 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -148,14 +148,17 @@ cp -L "$src" "$real_dst" log.debug("Bootstrap: %s" % cmd) -# try: - subprocess.check_call(cmd) -# except subprocess.CalledProcessError: -# raise + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + raise cdist.Error("Debootstrap failed (root priviliges required)") # Required to run this - otherwise apt-get install fails cmd = [ "chroot", self.target_dir, "/usr/bin/apt-get", "update" ] - subprocess.check_call(cmd) + try: + subprocess.check_call(cmd) + except subprocess.CalledProcessError: + raise cdist.Error("chrooted apt-get update failed (root priviliges required)") def create_helper_files(self, base_dir): for key, val in self.helper.items(): From 1b1b345263e72f9dfa4a4bd8b8a61892b45f04a3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 28 Mar 2015 03:20:28 +0900 Subject: [PATCH 053/197] add initial type for building preos Signed-off-by: Nico Schottelius --- cdist/conf/type/__cdist_preos/man.text | 38 ++++++++++++++ cdist/conf/type/__cdist_preos/manifest | 49 +++++++++++++++++++ .../__cdist_preos/parameter/default/branch | 1 + .../__cdist_preos/parameter/default/source | 1 + .../__cdist_preos/parameter/default/username | 1 + .../type/__cdist_preos/parameter/optional | 4 ++ 6 files changed, 94 insertions(+) create mode 100644 cdist/conf/type/__cdist_preos/man.text create mode 100755 cdist/conf/type/__cdist_preos/manifest create mode 100644 cdist/conf/type/__cdist_preos/parameter/default/branch create mode 100644 cdist/conf/type/__cdist_preos/parameter/default/source create mode 100644 cdist/conf/type/__cdist_preos/parameter/default/username create mode 100644 cdist/conf/type/__cdist_preos/parameter/optional diff --git a/cdist/conf/type/__cdist_preos/man.text b/cdist/conf/type/__cdist_preos/man.text new file mode 100644 index 00000000..19caa8e2 --- /dev/null +++ b/cdist/conf/type/__cdist_preos/man.text @@ -0,0 +1,38 @@ +cdist-type__cdist_preos(7) +========================== +Nico Schottelius + + +NAME +---- +cdist-type__cdist - Manage cdist installations + + +DESCRIPTION +----------- +This cdist type creates a directory containing an operating +suitable for installation using cdist. + +REQUIRED PARAMETERS +------------------- + +OPTIONAL PARAMETERS +------------------- +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__cdist_preos /tmp/random_name_for_packaging +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) +- cdist-type__cdist(7) + + +COPYING +------- +Copyright \(C) 2015 Nico Schottelius. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__cdist_preos/manifest b/cdist/conf/type/__cdist_preos/manifest new file mode 100755 index 00000000..8bb2175f --- /dev/null +++ b/cdist/conf/type/__cdist_preos/manifest @@ -0,0 +1,49 @@ +#!/bin/sh +# +# 2015 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +destination="/$__object_id" + +os=$(cat "$__global/explorer/os") + +case "$os" in + archlinux) + # any linux should work + : + ;; + *) + echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; +esac + +# Our root +__directory "$destination" \ + --mode 0755 + +for rootdir in boot bin etc lib; do + require="__directory/$destination" __directory "$destination/$rootdir" \ + --mode 0755 +done + + +require="__directory/$destination/etc" __cdistmarker \ + --destination "$destination/etc/cdist-configured" diff --git a/cdist/conf/type/__cdist_preos/parameter/default/branch b/cdist/conf/type/__cdist_preos/parameter/default/branch new file mode 100644 index 00000000..1f7391f9 --- /dev/null +++ b/cdist/conf/type/__cdist_preos/parameter/default/branch @@ -0,0 +1 @@ +master diff --git a/cdist/conf/type/__cdist_preos/parameter/default/source b/cdist/conf/type/__cdist_preos/parameter/default/source new file mode 100644 index 00000000..d669308f --- /dev/null +++ b/cdist/conf/type/__cdist_preos/parameter/default/source @@ -0,0 +1 @@ +git://github.com/telmich/cdist.git diff --git a/cdist/conf/type/__cdist_preos/parameter/default/username b/cdist/conf/type/__cdist_preos/parameter/default/username new file mode 100644 index 00000000..a585e141 --- /dev/null +++ b/cdist/conf/type/__cdist_preos/parameter/default/username @@ -0,0 +1 @@ +cdist diff --git a/cdist/conf/type/__cdist_preos/parameter/optional b/cdist/conf/type/__cdist_preos/parameter/optional new file mode 100644 index 00000000..a5f14343 --- /dev/null +++ b/cdist/conf/type/__cdist_preos/parameter/optional @@ -0,0 +1,4 @@ +branch +source +username +shell From 909c58de4e2908afaa2c6f9feaf8ea2810ddf628 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 2 May 2015 12:24:57 +0200 Subject: [PATCH 054/197] add hacking dir for new preos-setup Signed-off-by: Nico Schottelius --- hacking/recursive-ldd.sh | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 hacking/recursive-ldd.sh diff --git a/hacking/recursive-ldd.sh b/hacking/recursive-ldd.sh new file mode 100644 index 00000000..8093145d --- /dev/null +++ b/hacking/recursive-ldd.sh @@ -0,0 +1,31 @@ +#!/bin/sh +# Nico Schottelius +# Fri May 1 17:31:50 CEST 2015 + +# [18:09] wurzel:.cdist-ruag% ldd /usr/bin/ls | sed -e 's/=>//' -e 's/(.*//' | awk '{ if(NF == 2) { print $2 } else { print $1 } }' + +PATH=/bin:/sbin:/usr/bin:/usr/sbin + +bin_list="fdisk + +for command in command_list; + + + +exit 0 + + +bin=$1 + +list="" +new_list=$(objdump -p /usr/bin/ls | awk '$1 ~ /NEEDED/ { print $2 }') + +[18:16] wurzel:.cdist-ruag% ldconfig -p | grep 'libBrokenLocale.so.1$' | sed 's/.* => //' + + +for new_item in $new_list; do + + +done + +ldconfig -p | From a0aba11e772a9e8dbef645f91fb12b1a5844c0b0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 5 May 2015 13:53:19 +0200 Subject: [PATCH 055/197] +some tools / updates for preos Signed-off-by: Nico Schottelius --- hacking/README | 3 +++ hacking/bin_to_pkg.sh | 4 ++++ hacking/recursive-ldd.sh | 7 +++++-- 3 files changed, 12 insertions(+), 2 deletions(-) create mode 100644 hacking/README create mode 100755 hacking/bin_to_pkg.sh diff --git a/hacking/README b/hacking/README new file mode 100644 index 00000000..f675f106 --- /dev/null +++ b/hacking/README @@ -0,0 +1,3 @@ +- Target: + - get working iso + - have it configured and gathered by cdist? diff --git a/hacking/bin_to_pkg.sh b/hacking/bin_to_pkg.sh new file mode 100755 index 00000000..111b1fa9 --- /dev/null +++ b/hacking/bin_to_pkg.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +abspath=$(command -v "$1") +pacman -Qoq "$abspath" diff --git a/hacking/recursive-ldd.sh b/hacking/recursive-ldd.sh index 8093145d..d66d2b0e 100644 --- a/hacking/recursive-ldd.sh +++ b/hacking/recursive-ldd.sh @@ -6,10 +6,13 @@ PATH=/bin:/sbin:/usr/bin:/usr/sbin -bin_list="fdisk +pkg=" -for command in command_list; +bin_list="fdisk mount" +for bin in command_list; do + +done exit 0 From 56c7431467aa12f12494462277a604f8cc78e60c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 5 May 2015 13:53:47 +0200 Subject: [PATCH 056/197] __cdist_preos enhancements Signed-off-by: Nico Schottelius --- cdist/conf/type/__cdist_preos/manifest | 50 ++++++++++++++++++++------ 1 file changed, 40 insertions(+), 10 deletions(-) diff --git a/cdist/conf/type/__cdist_preos/manifest b/cdist/conf/type/__cdist_preos/manifest index 8bb2175f..78166b38 100755 --- a/cdist/conf/type/__cdist_preos/manifest +++ b/cdist/conf/type/__cdist_preos/manifest @@ -24,15 +24,16 @@ destination="/$__object_id" os=$(cat "$__global/explorer/os") case "$os" in - archlinux) - # any linux should work - : - ;; - *) - echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 - echo "Please contribute an implementation for it if you can." >&2 - exit 1 - ;; + archlinux) + kernel=/boot/vmlinuz-linux + initramfs=/boot/initramfs-linux-fallback.img + required_pkg="cdrkit syslinux" + ;; + *) + echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; esac # Our root @@ -44,6 +45,35 @@ for rootdir in boot bin etc lib; do --mode 0755 done - require="__directory/$destination/etc" __cdistmarker \ --destination "$destination/etc/cdist-configured" + +for pkg in $required_pkg; do + __package "$pkg" --state present +done + +# Create full dependency chain, because we don't know which file depends on which package +export CDIST_ORDER_DEPENDENCY=1 + +require="__directory/$destination/boot" __file "$destination/boot/linux" \ + --source "$kernel" --mode 0644 + +require="__directory/$destination/boot" __file "$destination/boot/initramfs" \ + --source "$initramfs" --mode 0644 + +require="__directory/$destination/boot" __file "$destination/boot/syslinux.cfg" \ + + + PROMPT 1 + TIMEOUT 50 + DEFAULT arch + + LABEL arch + LINUX ../vmlinuz-linux + APPEND root=/dev/sda2 rw + INITRD ../initramfs-linux.img + + LABEL archfallback + LINUX ../vmlinuz-linux + APPEND root=/dev/sda2 rw + INITRD ../initramfs-linux-fallback.img From 8c97ad3d9530423167ce0df768f1977925c9ffac Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 8 May 2015 08:22:22 +0200 Subject: [PATCH 057/197] generate list of files from packages Signed-off-by: Nico Schottelius --- hacking/archlinux/file_list_of_packages.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 hacking/archlinux/file_list_of_packages.sh diff --git a/hacking/archlinux/file_list_of_packages.sh b/hacking/archlinux/file_list_of_packages.sh new file mode 100644 index 00000000..608fdfbc --- /dev/null +++ b/hacking/archlinux/file_list_of_packages.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +pacman -Qlq "$@" From 9f3a8c0956ce97ecd73d8851ec863a4658863616 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 8 May 2015 13:42:13 +0200 Subject: [PATCH 058/197] +tools Signed-off-by: Nico Schottelius --- hacking/README | 9 +++++++++ hacking/recursive-ldd.sh | 2 +- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/hacking/README b/hacking/README index f675f106..ffd4f6a0 100644 --- a/hacking/README +++ b/hacking/README @@ -1,3 +1,12 @@ - Target: - get working iso - have it configured and gathered by cdist? + +- boot process via ...? + - systemd? + +- packaging via ... + - packages? + - binlist + - bootstrap of os + -> root permissions! diff --git a/hacking/recursive-ldd.sh b/hacking/recursive-ldd.sh index d66d2b0e..600df529 100644 --- a/hacking/recursive-ldd.sh +++ b/hacking/recursive-ldd.sh @@ -8,7 +8,7 @@ PATH=/bin:/sbin:/usr/bin:/usr/sbin pkg=" -bin_list="fdisk mount" +bin_list="bash fdisk mount syslinux umount rm mv" for bin in command_list; do From 75e3f3c90fd471a24aeb48023af93d2b2b16466c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 13 May 2015 08:13:54 +0200 Subject: [PATCH 059/197] write script to generate outdir from filelist Signed-off-by: Nico Schottelius --- hacking/copy_files_for_iso.sh | 23 +++++++++++++++++++++++ hacking/filelist_from_package.sh | 7 +++++++ hacking/filelist_to_dir.sh | 18 ++++++++++++++++++ hacking/qemu-test.sh | 12 ++++++++++++ hacking/recursive-ldd.sh | 0 5 files changed, 60 insertions(+) create mode 100755 hacking/copy_files_for_iso.sh create mode 100755 hacking/filelist_from_package.sh create mode 100755 hacking/filelist_to_dir.sh create mode 100755 hacking/qemu-test.sh mode change 100644 => 100755 hacking/recursive-ldd.sh diff --git a/hacking/copy_files_for_iso.sh b/hacking/copy_files_for_iso.sh new file mode 100755 index 00000000..0318c072 --- /dev/null +++ b/hacking/copy_files_for_iso.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +set -ex + +testdir=./iso-root-dir + +# Create base +rm -rf "$testdir" +mkdir "$testdir" + +# Copy binaries + +# Copy kernel +mkdir -p "$testdir/boot" +cp /boot/vmlinuz-linux "$testdir/boot/kernel" +cp /boot/initramfs-linux-fallback.img "$testdir/boot/initramfs" + +# Create iso +genisoimage -v -V "cdist preos v0.1" \ + -cache-inodes -J -l \ + -r -no-emul-boot \ + -boot-load-size 4 -b isolinux.bin -c boot.cat -o cdist-preos.iso iso + diff --git a/hacking/filelist_from_package.sh b/hacking/filelist_from_package.sh new file mode 100755 index 00000000..58ac48f4 --- /dev/null +++ b/hacking/filelist_from_package.sh @@ -0,0 +1,7 @@ +#!/bin/sh + +# Generate filelist excluding stuff that takes only space +for pkg in bash systemd util-linux; do + pacman -Qlq $pkg | grep -v \ + -e /usr/share/man/ +done diff --git a/hacking/filelist_to_dir.sh b/hacking/filelist_to_dir.sh new file mode 100755 index 00000000..3ce19b9f --- /dev/null +++ b/hacking/filelist_to_dir.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +if [ "$#" -ne 1 ]; then + echo "$0 outdir" + exit 1 +fi + +outdir=$1; shift + +mkdir -p "$outdir" + +while read file; do + if [ -d "$file" ]; then + mkdir -p "$outdir$file" + else + cp --preserve=mode,links "$file" "$outdir$file" + fi +done diff --git a/hacking/qemu-test.sh b/hacking/qemu-test.sh new file mode 100755 index 00000000..03695222 --- /dev/null +++ b/hacking/qemu-test.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +if [ "$#" -ne 1 ]; then + echo "$0 iso" + exit 1 +fi + +iso=$1; shift + +qemu-system-x86_64 -m 512 -boot order=cd \ + -drive=$iso,media=cdrom + diff --git a/hacking/recursive-ldd.sh b/hacking/recursive-ldd.sh old mode 100644 new mode 100755 From 1a52df0ddc06660eea4fff7e7f59759415a07401 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 13 May 2015 08:39:22 +0200 Subject: [PATCH 060/197] begin to write down how to create iso Signed-off-by: Nico Schottelius --- hacking/create_iso.sh | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 hacking/create_iso.sh diff --git a/hacking/create_iso.sh b/hacking/create_iso.sh new file mode 100644 index 00000000..23e8a0e3 --- /dev/null +++ b/hacking/create_iso.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +# FIXME: include os explorer to name preos + +indir=./iso + +version=0.3 +out=preos-${version}.iso + +genisoimage -r -V "cdist preos v0.2" \ + -cache-inodes -J -l \ + -no-emul-boot \ + -boot-load-size 4 -b isolinux.bin -c boot.cat \ + -o cdist-preos.iso $indir From 59d81ddd4b4ba330e920b8e7bb96de1ec1c587ae Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 13 May 2015 09:10:38 +0200 Subject: [PATCH 061/197] create iso works without checksum error Signed-off-by: Nico Schottelius --- hacking/create_iso.sh | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) mode change 100644 => 100755 hacking/create_iso.sh diff --git a/hacking/create_iso.sh b/hacking/create_iso.sh old mode 100644 new mode 100755 index 23e8a0e3..0902bc20 --- a/hacking/create_iso.sh +++ b/hacking/create_iso.sh @@ -2,13 +2,20 @@ # FIXME: include os explorer to name preos -indir=./iso +if [ "$#" -ne 2 ]; then + echo "$0 dir-in iso-out" + exit 1 +fi + +indir=$1; shift +iso=$1; shift version=0.3 + out=preos-${version}.iso -genisoimage -r -V "cdist preos v0.2" \ - -cache-inodes -J -l \ - -no-emul-boot \ - -boot-load-size 4 -b isolinux.bin -c boot.cat \ - -o cdist-preos.iso $indir + # -cache-inodes \ +genisoimage -r -J -l \ + -V "cdist PreOS $version" \ + -b boot/isolinux.bin -no-emul-boot -c boot.cat -boot-load-size 4 -boot-info-table \ + -o "$iso" "$indir" From a1f003bd9c6fa01906c5d36399564afa605f67ba Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 13 May 2015 09:24:55 +0200 Subject: [PATCH 062/197] get iso to boot into kernel + initramfs Signed-off-by: Nico Schottelius --- hacking/.gitignore | 2 ++ hacking/README | 6 ++++++ hacking/add_kernel_isolinux.sh | 29 +++++++++++++++++++++++++++++ hacking/all.sh | 10 ++++++++++ hacking/qemu-test.sh | 2 +- 5 files changed, 48 insertions(+), 1 deletion(-) create mode 100644 hacking/.gitignore create mode 100755 hacking/add_kernel_isolinux.sh create mode 100755 hacking/all.sh diff --git a/hacking/.gitignore b/hacking/.gitignore new file mode 100644 index 00000000..e3f3d036 --- /dev/null +++ b/hacking/.gitignore @@ -0,0 +1,2 @@ +iso/ +*.iso diff --git a/hacking/README b/hacking/README index ffd4f6a0..d5aa9423 100644 --- a/hacking/README +++ b/hacking/README @@ -10,3 +10,9 @@ - binlist - bootstrap of os -> root permissions! + +- uefi support + [9:15] wurzel:hacking% pacman -Ql syslinux | grep ldlin + syslinux /usr/lib/syslinux/bios/ldlinux.c32 + syslinux /usr/lib/syslinux/efi32/ldlinux.e32 + syslinux /usr/lib/syslinux/efi64/ldlinux.e64 diff --git a/hacking/add_kernel_isolinux.sh b/hacking/add_kernel_isolinux.sh new file mode 100755 index 00000000..ec7b610c --- /dev/null +++ b/hacking/add_kernel_isolinux.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# FIXME: Write cdist type / explorer that finds +# package for a file, distro independent + +if [ "$#" -ne 1 ]; then + echo "$0 dir-out" + exit 1 +fi + +dir=$1; shift +boot=$dir/boot + +mkdir -p "$boot" +cp /boot/vmlinuz-linux \ + /boot/initramfs-linux-fallback.img \ + /usr/lib/syslinux/bios/isolinux.bin \ + "$boot" + +cp /usr/lib/syslinux/bios/ldlinux.c32 \ + "$dir" + +cat > "$dir/isolinux.cfg" << eof +default preos +label preos +title cdist PreOS +linux /boot/vmlinuz-linux +initrd /boot/initramfs-linux-fallback.img +eof diff --git a/hacking/all.sh b/hacking/all.sh new file mode 100755 index 00000000..c1b4fb93 --- /dev/null +++ b/hacking/all.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +set -e + +dir=./iso +iso=preos.iso + +./filelist_from_package.sh | ./filelist_to_dir.sh "$dir" +./add_kernel_isolinux.sh "$dir" +./create_iso.sh "$dir" "$iso" diff --git a/hacking/qemu-test.sh b/hacking/qemu-test.sh index 03695222..02afc2e6 100755 --- a/hacking/qemu-test.sh +++ b/hacking/qemu-test.sh @@ -8,5 +8,5 @@ fi iso=$1; shift qemu-system-x86_64 -m 512 -boot order=cd \ - -drive=$iso,media=cdrom + -drive file=$iso,media=cdrom From f5edb02fbd8c3f1b00359bc5f04cb38178393c68 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 13 May 2015 09:30:24 +0200 Subject: [PATCH 063/197] library copy still missing Signed-off-by: Nico Schottelius --- hacking/all.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/hacking/all.sh b/hacking/all.sh index c1b4fb93..fe3d6d11 100755 --- a/hacking/all.sh +++ b/hacking/all.sh @@ -6,5 +6,6 @@ dir=./iso iso=preos.iso ./filelist_from_package.sh | ./filelist_to_dir.sh "$dir" +echo "MISSING: copy libraries" >&2 ./add_kernel_isolinux.sh "$dir" ./create_iso.sh "$dir" "$iso" From 648809ff444cd3716d3d8de6a78699dfa0c92ffd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 13 May 2015 10:25:49 +0200 Subject: [PATCH 064/197] begin ssh integration Signed-off-by: Nico Schottelius --- hacking/README | 10 ++++++++++ hacking/filelist_from_package.sh | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/hacking/README b/hacking/README index d5aa9423..50ba4616 100644 --- a/hacking/README +++ b/hacking/README @@ -1,7 +1,15 @@ +- next step + - rootfs fix + - get working to login + - have sshd enabled + +- everything into initramfs? + - Target: - get working iso - have it configured and gathered by cdist? + - boot process via ...? - systemd? @@ -11,6 +19,8 @@ - bootstrap of os -> root permissions! +- boot device + - uefi support [9:15] wurzel:hacking% pacman -Ql syslinux | grep ldlin syslinux /usr/lib/syslinux/bios/ldlinux.c32 diff --git a/hacking/filelist_from_package.sh b/hacking/filelist_from_package.sh index 58ac48f4..1bf24a7c 100755 --- a/hacking/filelist_from_package.sh +++ b/hacking/filelist_from_package.sh @@ -1,7 +1,7 @@ #!/bin/sh # Generate filelist excluding stuff that takes only space -for pkg in bash systemd util-linux; do +for pkg in bash systemd util-linux openssh; do pacman -Qlq $pkg | grep -v \ -e /usr/share/man/ done From b7ed5b7d124d980f3f192a6bca01432be3662f3d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 19 May 2015 10:02:18 +0200 Subject: [PATCH 065/197] add some scripts to try bootstraping using arch/debian methods Signed-off-by: Nico Schottelius --- hacking/arch_bootstrap.sh | 4 ++++ hacking/debian_bootstrap.sh | 3 +++ 2 files changed, 7 insertions(+) create mode 100644 hacking/arch_bootstrap.sh create mode 100644 hacking/debian_bootstrap.sh diff --git a/hacking/arch_bootstrap.sh b/hacking/arch_bootstrap.sh new file mode 100644 index 00000000..0472bf3c --- /dev/null +++ b/hacking/arch_bootstrap.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +fakeroot pacman -r $(pwd -P)/preos -Syu --noconfirm --cachedir $(pwd -P)/preos/var/cache/pacman base + diff --git a/hacking/debian_bootstrap.sh b/hacking/debian_bootstrap.sh new file mode 100644 index 00000000..75628116 --- /dev/null +++ b/hacking/debian_bootstrap.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +fakeroot debootstrap jessie ./preos-debootstrap/ From 6ff6604941918f3a36734fa47f817c519ffc2fc6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 19 May 2015 10:02:53 +0200 Subject: [PATCH 066/197] +logs Signed-off-by: Nico Schottelius --- .../2015-02-10.installation_from_usb_stick | 49 +++++++++++++++++++ docs/dev/logs/2015-03-28.preos-from-os | 32 ++++++++++++ 2 files changed, 81 insertions(+) create mode 100644 docs/dev/logs/2015-02-10.installation_from_usb_stick create mode 100644 docs/dev/logs/2015-03-28.preos-from-os diff --git a/docs/dev/logs/2015-02-10.installation_from_usb_stick b/docs/dev/logs/2015-02-10.installation_from_usb_stick new file mode 100644 index 00000000..b655bc18 --- /dev/null +++ b/docs/dev/logs/2015-02-10.installation_from_usb_stick @@ -0,0 +1,49 @@ +Objective: + + Create a bootable media that contains everything to install and configure a system. + +Ideas: + +* usb stick +** uefi vs. bios +** contains cdist config +** static ip (?) (if at all) +** hostname setup to localhost +** install and config support +* preos from existing OS? +** requires kernel +** requires initramfs (self build) +** missing tools: cdist preos --config hostname... +* testing with qemu +* syslinux/isolinux? + +Program: + +- get tools +- get kernel + - provide fallback on cdist page + - archlinux: /boot/vmlinuz-linux +- create initramfs? +- create bootable media + - iso + - uefi-usb + - bios-usb + +Tasks: + +- Setup test environment + - qemu launcher + /usr/bin/qemu-system-x86_64 -boot d -m 256 -cdrom '/home/users/nico/oeffentlich/rechner/projekte/cdist/cdist/cdist-preos.iso' +- Create bootable image +- Test image + +Log: + +mkdir iso +cp /boot/vmlinuz-linux iso/ +cp /usr/lib/syslinux/bios/isolinux.bin iso/ + +[22:36] freiheit:cdist% genisoimage -v -V "cdist preos v0.1" -cache-inodes -J -l -no-emul-boot -boot-load-size 4 -b isolinux.bin -c boot.cat -o cdist-preos.iso iso + +[22:38] freiheit:cdist% genisoimage -r -V "cdist preos v0.2" -cache-inodes -J -l -no-emul-boot -boot-load-size 4 -b isolinux.bin -c boot.cat -o cdist-preos.iso iso + diff --git a/docs/dev/logs/2015-03-28.preos-from-os b/docs/dev/logs/2015-03-28.preos-from-os new file mode 100644 index 00000000..93dc9e79 --- /dev/null +++ b/docs/dev/logs/2015-03-28.preos-from-os @@ -0,0 +1,32 @@ +- basics of config + - wrapping to config + - testbed for CaaS! +- allow to include .cdist +- generate + - pxe + - iso +- package... + - mkfs + - fdisk* + - kernel + +- types (?) + - iso? + - + +- based on Arch Linux + +- new types for iso? + +- change __cdistmarker to accept prefix + +- ISO / USB + genisoimage -r -V "cdist preos v0.2" -cache-inodes -J -l -no-emul-boot -boot-load-size 4 -b isolinux.bin -c boot.cat -o cdist-preos.iso iso + + - have a look at archiso? + + http://www.syslinux.org/wiki/index.php/Isohybrid + -> uefi + -> mbr + +- PXE From f51a444012e465847de01d15bc290a8f9f90c9a7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 19 May 2015 10:19:33 +0200 Subject: [PATCH 067/197] generate sorted / filtered list Signed-off-by: Nico Schottelius --- hacking/filelist_from_package.sh | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/hacking/filelist_from_package.sh b/hacking/filelist_from_package.sh index 1bf24a7c..b5dd8eb7 100755 --- a/hacking/filelist_from_package.sh +++ b/hacking/filelist_from_package.sh @@ -1,7 +1,19 @@ #!/bin/sh # Generate filelist excluding stuff that takes only space -for pkg in bash systemd util-linux openssh; do - pacman -Qlq $pkg | grep -v \ - -e /usr/share/man/ -done +( + for pkg in systemd openssh \ + bash bzip2 coreutils cryptsetup device-mapper dhcpcd \ + diffutils e2fsprogs file filesystem findutils gawk \ + gettext glibc grep gzip inetutils iproute2 \ + iputils jfsutils less licenses linux logrotate lvm2 \ + man-db man-pages mdadm nano pacman pciutils \ + pcmciautils perl procps-ng psmisc reiserfsprogs \ + s-nail sed shadow sysfsutils systemd-sysvcompat tar \ + texinfo usbutils util-linux vi which xfsprogs \ + ; do + pacman -Qlq $pkg | grep -v \ + -e /usr/share/man/ \ + -e /usr/share/doc/ + done +) | sort | uniq From 7ba6c0a44a44317993cb032cdf0caefbfa08d055 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 19 May 2015 11:06:54 +0200 Subject: [PATCH 068/197] can generate initramfs from busybox Signed-off-by: Nico Schottelius --- hacking/add_kernel_isolinux.sh | 29 --------- .../arch_bootstrap.sh | 0 .../debian_bootstrap.sh | 0 .../file_list_of_packages.sh | 0 .../filelist_from_package.sh | 0 .../filelist_to_dir.sh | 0 hacking/v3-busybox/add_kernel_isolinux.sh | 24 ++++++++ hacking/v3-busybox/all.sh | 8 +++ hacking/v3-busybox/generate.sh | 27 +++++++++ hacking/v3-busybox/init | 60 +++++++++++++++++++ 10 files changed, 119 insertions(+), 29 deletions(-) delete mode 100755 hacking/add_kernel_isolinux.sh rename hacking/{ => v1-debootstrap-pacstrap}/arch_bootstrap.sh (100%) rename hacking/{ => v1-debootstrap-pacstrap}/debian_bootstrap.sh (100%) rename hacking/{archlinux => v2-initramfs-from-os}/file_list_of_packages.sh (100%) rename hacking/{ => v2-initramfs-from-os}/filelist_from_package.sh (100%) rename hacking/{ => v2-initramfs-from-os}/filelist_to_dir.sh (100%) create mode 100755 hacking/v3-busybox/add_kernel_isolinux.sh create mode 100755 hacking/v3-busybox/all.sh create mode 100755 hacking/v3-busybox/generate.sh create mode 100644 hacking/v3-busybox/init diff --git a/hacking/add_kernel_isolinux.sh b/hacking/add_kernel_isolinux.sh deleted file mode 100755 index ec7b610c..00000000 --- a/hacking/add_kernel_isolinux.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh - -# FIXME: Write cdist type / explorer that finds -# package for a file, distro independent - -if [ "$#" -ne 1 ]; then - echo "$0 dir-out" - exit 1 -fi - -dir=$1; shift -boot=$dir/boot - -mkdir -p "$boot" -cp /boot/vmlinuz-linux \ - /boot/initramfs-linux-fallback.img \ - /usr/lib/syslinux/bios/isolinux.bin \ - "$boot" - -cp /usr/lib/syslinux/bios/ldlinux.c32 \ - "$dir" - -cat > "$dir/isolinux.cfg" << eof -default preos -label preos -title cdist PreOS -linux /boot/vmlinuz-linux -initrd /boot/initramfs-linux-fallback.img -eof diff --git a/hacking/arch_bootstrap.sh b/hacking/v1-debootstrap-pacstrap/arch_bootstrap.sh similarity index 100% rename from hacking/arch_bootstrap.sh rename to hacking/v1-debootstrap-pacstrap/arch_bootstrap.sh diff --git a/hacking/debian_bootstrap.sh b/hacking/v1-debootstrap-pacstrap/debian_bootstrap.sh similarity index 100% rename from hacking/debian_bootstrap.sh rename to hacking/v1-debootstrap-pacstrap/debian_bootstrap.sh diff --git a/hacking/archlinux/file_list_of_packages.sh b/hacking/v2-initramfs-from-os/file_list_of_packages.sh similarity index 100% rename from hacking/archlinux/file_list_of_packages.sh rename to hacking/v2-initramfs-from-os/file_list_of_packages.sh diff --git a/hacking/filelist_from_package.sh b/hacking/v2-initramfs-from-os/filelist_from_package.sh similarity index 100% rename from hacking/filelist_from_package.sh rename to hacking/v2-initramfs-from-os/filelist_from_package.sh diff --git a/hacking/filelist_to_dir.sh b/hacking/v2-initramfs-from-os/filelist_to_dir.sh similarity index 100% rename from hacking/filelist_to_dir.sh rename to hacking/v2-initramfs-from-os/filelist_to_dir.sh diff --git a/hacking/v3-busybox/add_kernel_isolinux.sh b/hacking/v3-busybox/add_kernel_isolinux.sh new file mode 100755 index 00000000..ac5d495b --- /dev/null +++ b/hacking/v3-busybox/add_kernel_isolinux.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +# FIXME: distro specific kernel location + +if [ "$#" -ne 1 ]; then + echo "$0 dir-out" + exit 1 +fi + +dir=$1; shift +boot=$dir/boot + +mkdir -p "$boot" +cp /boot/vmlinuz-linux "$boot/linux" +cp /usr/lib/syslinux/bios/isolinux.bin "$boot" +cp /usr/lib/syslinux/bios/ldlinux.c32 "$dir" + +cat > "$dir/isolinux.cfg" << eof +default preos +label preos +title cdist PreOS +linux /boot/linux +initrd /boot/initramfs +eof diff --git a/hacking/v3-busybox/all.sh b/hacking/v3-busybox/all.sh new file mode 100755 index 00000000..51eac9af --- /dev/null +++ b/hacking/v3-busybox/all.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +rm -rf preos +mkdir -p preos/boot + +./generate.sh > preos/boot/initramfs +./add_kernel_isolinux.sh preos + diff --git a/hacking/v3-busybox/generate.sh b/hacking/v3-busybox/generate.sh new file mode 100755 index 00000000..00227a57 --- /dev/null +++ b/hacking/v3-busybox/generate.sh @@ -0,0 +1,27 @@ +#!/bin/sh +set -ex + + +initramfs_dir=$(mktemp -d /tmp/cdist-preos.XXXXXXX) +# initramfs_dir=$1 + +for dir in bin sbin etc proc sys newroot; do + mkdir -p ${initramfs_dir}/$dir +done +touch ${initramfs_dir}/etc/mdev.conf + +cp init "${initramfs_dir}/init" +cp $(which busybox) "${initramfs_dir}/bin" +ln -fs busybox "${initramfs_dir}/bin/sh" + +cd "${initramfs_dir}" +find . | cpio -H newc -o | gzip + +exit 0 + +# TODO: +# - Kernel modules +# - ssh +# - various mkfs +# - libs + diff --git a/hacking/v3-busybox/init b/hacking/v3-busybox/init new file mode 100644 index 00000000..40507339 --- /dev/null +++ b/hacking/v3-busybox/init @@ -0,0 +1,60 @@ +#!/bin/sh + +#Mount things needed by this script +mount -t proc proc /proc +mount -t sysfs sysfs /sys + +#Disable kernel messages from popping onto the screen +echo 0 > /proc/sys/kernel/printk + +#Create all the symlinks to /bin/busybox +busybox --install -s + +#Create device nodes +mknod /dev/null c 1 3 +mknod /dev/tty c 5 0 +mdev -s + +#Function for parsing command line options with "=" in them +# get_opt("init=/sbin/init") will return "/sbin/init" +get_opt() { + echo "$@" | cut -d "=" -f 2 +} + +#Defaults +init="/sbin/init" +root="/dev/hda1" + +#Process command line options +for i in $(cat /proc/cmdline); do + case $i in + root\=*) + root=$(get_opt $i) + ;; + init\=*) + init=$(get_opt $i) + ;; + esac +done + + +exec sh + +# Skipping the rest + +#Mount the root device +mount "${root}" /newroot + +#Check if $init exists and is executable +if [[ -x "/newroot/${init}" ]] ; then + #Unmount all other mounts so that the ram used by + #the initramfs can be cleared after switch_root + umount /sys /proc + + #Switch to the new root and execute init + exec switch_root /newroot "${init}" +fi + +#This will only be run if the exec above failed +echo "Failed to switch_root, dropping to a shell" +exec sh From 7d7aa60e191a1bfa50ef45814c7e6f398c865edd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 19 May 2015 11:38:06 +0200 Subject: [PATCH 069/197] create directories required by busybox Signed-off-by: Nico Schottelius --- hacking/{ => v2-initramfs-from-os}/all.sh | 0 hacking/v3-busybox/{generate.sh => create_initramfs.sh} | 7 +++++-- hacking/{ => v3-busybox}/create_iso.sh | 0 hacking/{ => v3-busybox}/qemu-test.sh | 0 4 files changed, 5 insertions(+), 2 deletions(-) rename hacking/{ => v2-initramfs-from-os}/all.sh (100%) rename hacking/v3-busybox/{generate.sh => create_initramfs.sh} (72%) rename hacking/{ => v3-busybox}/create_iso.sh (100%) rename hacking/{ => v3-busybox}/qemu-test.sh (100%) diff --git a/hacking/all.sh b/hacking/v2-initramfs-from-os/all.sh similarity index 100% rename from hacking/all.sh rename to hacking/v2-initramfs-from-os/all.sh diff --git a/hacking/v3-busybox/generate.sh b/hacking/v3-busybox/create_initramfs.sh similarity index 72% rename from hacking/v3-busybox/generate.sh rename to hacking/v3-busybox/create_initramfs.sh index 00227a57..6d57b05c 100755 --- a/hacking/v3-busybox/generate.sh +++ b/hacking/v3-busybox/create_initramfs.sh @@ -5,14 +5,17 @@ set -ex initramfs_dir=$(mktemp -d /tmp/cdist-preos.XXXXXXX) # initramfs_dir=$1 -for dir in bin sbin etc proc sys newroot; do +for dir in bin sbin etc proc sys newroot usr/bin usr/sbin; do mkdir -p ${initramfs_dir}/$dir done touch ${initramfs_dir}/etc/mdev.conf cp init "${initramfs_dir}/init" cp $(which busybox) "${initramfs_dir}/bin" -ln -fs busybox "${initramfs_dir}/bin/sh" + +for link in sh mount; do + ln -fs busybox "${initramfs_dir}/bin/$link" +done cd "${initramfs_dir}" find . | cpio -H newc -o | gzip diff --git a/hacking/create_iso.sh b/hacking/v3-busybox/create_iso.sh similarity index 100% rename from hacking/create_iso.sh rename to hacking/v3-busybox/create_iso.sh diff --git a/hacking/qemu-test.sh b/hacking/v3-busybox/qemu-test.sh similarity index 100% rename from hacking/qemu-test.sh rename to hacking/v3-busybox/qemu-test.sh From 50bca7891f6496dcc732384d249ac386f8de085e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 19 May 2015 11:39:39 +0200 Subject: [PATCH 070/197] preos 0.4 Signed-off-by: Nico Schottelius --- hacking/v3-busybox/all.sh | 3 ++- hacking/v3-busybox/create_iso.sh | 2 +- hacking/v3-busybox/init | 5 +++-- 3 files changed, 6 insertions(+), 4 deletions(-) mode change 100644 => 100755 hacking/v3-busybox/init diff --git a/hacking/v3-busybox/all.sh b/hacking/v3-busybox/all.sh index 51eac9af..99b0c0ec 100755 --- a/hacking/v3-busybox/all.sh +++ b/hacking/v3-busybox/all.sh @@ -3,6 +3,7 @@ rm -rf preos mkdir -p preos/boot -./generate.sh > preos/boot/initramfs +./create_initramfs.sh > preos/boot/initramfs ./add_kernel_isolinux.sh preos +./create_iso.sh preos preos.iso diff --git a/hacking/v3-busybox/create_iso.sh b/hacking/v3-busybox/create_iso.sh index 0902bc20..c6b39be6 100755 --- a/hacking/v3-busybox/create_iso.sh +++ b/hacking/v3-busybox/create_iso.sh @@ -10,7 +10,7 @@ fi indir=$1; shift iso=$1; shift -version=0.3 +version=0.4 out=preos-${version}.iso diff --git a/hacking/v3-busybox/init b/hacking/v3-busybox/init old mode 100644 new mode 100755 index 40507339..a961526f --- a/hacking/v3-busybox/init +++ b/hacking/v3-busybox/init @@ -1,5 +1,8 @@ #!/bin/sh +#Create all the symlinks to /bin/busybox +/bin/busybox --install -s + #Mount things needed by this script mount -t proc proc /proc mount -t sysfs sysfs /sys @@ -7,8 +10,6 @@ mount -t sysfs sysfs /sys #Disable kernel messages from popping onto the screen echo 0 > /proc/sys/kernel/printk -#Create all the symlinks to /bin/busybox -busybox --install -s #Create device nodes mknod /dev/null c 1 3 From bb9d889d158e0082f02815b64f9ecd87afadbc0d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 19 May 2015 11:52:14 +0200 Subject: [PATCH 071/197] rename & cleanup Signed-off-by: Nico Schottelius --- hacking/.gitignore | 1 + hacking/README | 5 +++++ hacking/{ => v2-initramfs-from-os}/bin_to_pkg.sh | 0 .../{ => v2-initramfs-from-os}/copy_files_for_iso.sh | 0 .../v2-initramfs-from-os/filelist_from_package.sh | 8 +++++--- .../copy_bin_with_libs.sh} | 12 +++++++++--- hacking/v3-busybox/create_initramfs.sh | 2 ++ 7 files changed, 22 insertions(+), 6 deletions(-) rename hacking/{ => v2-initramfs-from-os}/bin_to_pkg.sh (100%) rename hacking/{ => v2-initramfs-from-os}/copy_files_for_iso.sh (100%) rename hacking/{recursive-ldd.sh => v3-busybox/copy_bin_with_libs.sh} (66%) diff --git a/hacking/.gitignore b/hacking/.gitignore index e3f3d036..375edb27 100644 --- a/hacking/.gitignore +++ b/hacking/.gitignore @@ -1,2 +1,3 @@ iso/ *.iso +preos/ diff --git a/hacking/README b/hacking/README index 50ba4616..937564d5 100644 --- a/hacking/README +++ b/hacking/README @@ -2,9 +2,14 @@ - rootfs fix - get working to login - have sshd enabled + - kernel -> initramfs? + http://jootamam.net/howto-initramfs-image.htm + - busybox!! - everything into initramfs? +- permission problem on various files below /etc + - Target: - get working iso - have it configured and gathered by cdist? diff --git a/hacking/bin_to_pkg.sh b/hacking/v2-initramfs-from-os/bin_to_pkg.sh similarity index 100% rename from hacking/bin_to_pkg.sh rename to hacking/v2-initramfs-from-os/bin_to_pkg.sh diff --git a/hacking/copy_files_for_iso.sh b/hacking/v2-initramfs-from-os/copy_files_for_iso.sh similarity index 100% rename from hacking/copy_files_for_iso.sh rename to hacking/v2-initramfs-from-os/copy_files_for_iso.sh diff --git a/hacking/v2-initramfs-from-os/filelist_from_package.sh b/hacking/v2-initramfs-from-os/filelist_from_package.sh index b5dd8eb7..5652ba66 100755 --- a/hacking/v2-initramfs-from-os/filelist_from_package.sh +++ b/hacking/v2-initramfs-from-os/filelist_from_package.sh @@ -8,12 +8,14 @@ gettext glibc grep gzip inetutils iproute2 \ iputils jfsutils less licenses linux logrotate lvm2 \ man-db man-pages mdadm nano pacman pciutils \ - pcmciautils perl procps-ng psmisc reiserfsprogs \ + pcmciautils procps-ng psmisc reiserfsprogs \ s-nail sed shadow sysfsutils systemd-sysvcompat tar \ - texinfo usbutils util-linux vi which xfsprogs \ + usbutils util-linux vi which xfsprogs \ ; do pacman -Qlq $pkg | grep -v \ -e /usr/share/man/ \ - -e /usr/share/doc/ + -e /usr/share/doc/ \ + -e /usr/include + done ) | sort | uniq diff --git a/hacking/recursive-ldd.sh b/hacking/v3-busybox/copy_bin_with_libs.sh similarity index 66% rename from hacking/recursive-ldd.sh rename to hacking/v3-busybox/copy_bin_with_libs.sh index 600df529..b3824a78 100755 --- a/hacking/recursive-ldd.sh +++ b/hacking/v3-busybox/copy_bin_with_libs.sh @@ -6,14 +6,20 @@ PATH=/bin:/sbin:/usr/bin:/usr/sbin -pkg=" +#bin_list="udevadm bash fdisk mount syslinux umount rm mv" +bin_list="udevadm fdisk" -bin_list="bash fdisk mount syslinux umount rm mv" +libs=$(mktemp /tmp/cdist-preos-libs.XXXXXXXXXXXXX) + +for bin in bin_list; do -for bin in command_list; do done +rm -f "$libs" + +# lfs +## ldd /bin/$f | sed "s/\t//" | cut -d " " -f1 >> $unsorted exit 0 diff --git a/hacking/v3-busybox/create_initramfs.sh b/hacking/v3-busybox/create_initramfs.sh index 6d57b05c..f87a7ef6 100755 --- a/hacking/v3-busybox/create_initramfs.sh +++ b/hacking/v3-busybox/create_initramfs.sh @@ -20,6 +20,8 @@ done cd "${initramfs_dir}" find . | cpio -H newc -o | gzip +rm -rf "${initramfs_dir}" + exit 0 # TODO: From 48096620267eb20b285c66f705e3d51a19cdcf30 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Tue, 19 May 2015 11:52:33 +0200 Subject: [PATCH 072/197] add v2 ideas Signed-off-by: Nico Schottelius --- .../add_kernel_isolinux.sh | 29 +++++++++++++++++++ hacking/v2-initramfs-from-os/packages_arch | 29 +++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100755 hacking/v2-initramfs-from-os/add_kernel_isolinux.sh create mode 100644 hacking/v2-initramfs-from-os/packages_arch diff --git a/hacking/v2-initramfs-from-os/add_kernel_isolinux.sh b/hacking/v2-initramfs-from-os/add_kernel_isolinux.sh new file mode 100755 index 00000000..ec7b610c --- /dev/null +++ b/hacking/v2-initramfs-from-os/add_kernel_isolinux.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +# FIXME: Write cdist type / explorer that finds +# package for a file, distro independent + +if [ "$#" -ne 1 ]; then + echo "$0 dir-out" + exit 1 +fi + +dir=$1; shift +boot=$dir/boot + +mkdir -p "$boot" +cp /boot/vmlinuz-linux \ + /boot/initramfs-linux-fallback.img \ + /usr/lib/syslinux/bios/isolinux.bin \ + "$boot" + +cp /usr/lib/syslinux/bios/ldlinux.c32 \ + "$dir" + +cat > "$dir/isolinux.cfg" << eof +default preos +label preos +title cdist PreOS +linux /boot/vmlinuz-linux +initrd /boot/initramfs-linux-fallback.img +eof diff --git a/hacking/v2-initramfs-from-os/packages_arch b/hacking/v2-initramfs-from-os/packages_arch new file mode 100644 index 00000000..ed879512 --- /dev/null +++ b/hacking/v2-initramfs-from-os/packages_arch @@ -0,0 +1,29 @@ +base syslinux + +[10:06] wurzel:hacking% sudo !! +sudo pacman -S base +[sudo] password for nico: +:: linux is in IgnorePkg/IgnoreGroup. Install anyway? [Y/n] y +:: There are 50 members in group base: +:: Repository core + 1) bash 2) bzip2 3) coreutils 4) cryptsetup 5) device-mapper 6) dhcpcd 7) diffutils 8) e2fsprogs 9) file + 10) filesystem 11) findutils 12) gawk 13) gcc-libs 14) gettext 15) glibc 16) grep 17) gzip 18) inetutils + 19) iproute2 20) iputils 21) jfsutils 22) less 23) licenses 24) linux 25) logrotate 26) lvm2 27) man-db + 28) man-pages 29) mdadm 30) nano 31) netctl 32) pacman 33) pciutils 34) pcmciautils 35) perl 36) procps-ng + 37) psmisc 38) reiserfsprogs 39) s-nail 40) sed 41) shadow 42) sysfsutils 43) systemd-sysvcompat 44) tar + 45) texinfo 46) usbutils 47) util-linux 48) vi 49) which 50) xfsprogs + +Enter a selection (default=all): + +:18,23s/ [0-9]*)//g + + bash bzip2 coreutils cryptsetup device-mapper dhcpcd diffutils e2fsprogs file + filesystem findutils gawk gcc-libs gettext glibc grep gzip inetutils + iproute2 iputils jfsutils less licenses linux logrotate lvm2 man-db + man-pages mdadm nano netctl pacman pciutils pcmciautils perl procps-ng + psmisc reiserfsprogs s-nail sed shadow sysfsutils systemd-sysvcompat tar + texinfo usbutils util-linux vi which xfsprogs + +6J + +bash bzip2 coreutils cryptsetup device-mapper dhcpcd diffutils e2fsprogs file filesystem findutils gawk gcc-libs gettext glibc grep gzip inetutils iproute2 iputils jfsutils less licenses linux logrotate lvm2 man-db man-pages mdadm nano netctl pacman pciutils pcmciautils perl procps-ng psmisc reiserfsprogs s-nail sed shadow sysfsutils systemd-sysvcompat tar texinfo usbutils util-linux vi which xfsprogs From 9dc7160903e3b118020dfad3e2082ca80147fdbf Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 20 May 2015 14:04:34 +0200 Subject: [PATCH 073/197] begin to import bin/libs Signed-off-by: Nico Schottelius --- hacking/v3-busybox/all.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hacking/v3-busybox/all.sh b/hacking/v3-busybox/all.sh index 99b0c0ec..65a3706b 100755 --- a/hacking/v3-busybox/all.sh +++ b/hacking/v3-busybox/all.sh @@ -5,5 +5,5 @@ mkdir -p preos/boot ./create_initramfs.sh > preos/boot/initramfs ./add_kernel_isolinux.sh preos +./copy_bin_with_libs.sh preos ./create_iso.sh preos preos.iso - From 85c825438ed25b39678a8a69c4bd3423b3709b35 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 28 May 2015 15:37:30 +0200 Subject: [PATCH 074/197] update manpage of __install_generate_fstab (typo) Signed-off-by: Nico Schottelius --- cdist/conf/type/__install_generate_fstab/man.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__install_generate_fstab/man.text b/cdist/conf/type/__install_generate_fstab/man.text index d7d747a0..d229f4df 100644 --- a/cdist/conf/type/__install_generate_fstab/man.text +++ b/cdist/conf/type/__install_generate_fstab/man.text @@ -10,7 +10,7 @@ cdist-type__install_generate_fstab - generate /etc/fstab during installation DESCRIPTION ----------- -Generates a /etc/fstab file from information retreived from +Generates a /etc/fstab file from information retrieved from __install_mount definitions. From 348867ff6a84958a2388799fa702396c16c2ae9f Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Wed, 6 Jul 2016 10:15:34 +0200 Subject: [PATCH 075/197] Correctly set hostname in preos --- cdist/preos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index b61318ed..8a17974e 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -63,7 +63,7 @@ eof # fix the bloody 'stdin: is not a tty' problem __line /root/.profile --line 'mesg n' --state absent -__hostname preos +__hostname --name preos """ class PreOSExistsError(cdist.Error): From 78d7d91e42dc7bcbc882eaf735fe48e3714c7f6d Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Wed, 6 Jul 2016 10:30:13 +0200 Subject: [PATCH 076/197] Install lsb-release in preos Several types use the lsb_release command to determine the distribution, if this command is missing unexpected things will happen. --- cdist/preos.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/preos.py b/cdist/preos.py index 8a17974e..df5be1fa 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -38,6 +38,7 @@ DEFAULT_MANIFEST = """ for pkg in \ file \ linux-image-amd64 \ + lsb-release \ openssh-server curl \ syslinux grub2 \ gdisk util-linux lvm2 mdadm \ From 0e114c37ac1756d916a393822a8744eba5d55eb0 Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Wed, 6 Jul 2016 10:47:28 +0200 Subject: [PATCH 077/197] Always use current stable release for preos --- cdist/preos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/preos.py b/cdist/preos.py index df5be1fa..8eced936 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -86,7 +86,7 @@ class PreOS(object): self.arch = arch self.command = "debootstrap" - self.suite = "wheezy" + self.suite = "stable" self.options = [ "--include=openssh-server", "--arch=%s" % self.arch ] From e79610f23c585017da411eea0a5d76b9876eacc4 Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Wed, 6 Jul 2016 11:22:15 +0200 Subject: [PATCH 078/197] Don't try to use hostnamectl when systemd isn't running --- cdist/conf/type/__hostname/explorer/has_hostnamectl | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__hostname/explorer/has_hostnamectl b/cdist/conf/type/__hostname/explorer/has_hostnamectl index 9040023d..7e8a57fd 100755 --- a/cdist/conf/type/__hostname/explorer/has_hostnamectl +++ b/cdist/conf/type/__hostname/explorer/has_hostnamectl @@ -18,7 +18,12 @@ # along with cdist. If not, see . # # -# Check whether system has hostnamectl +# Check whether system has hostnamectl and it's usable, i.e. don't +# try to use it when systemd isn't running yet. # -command -v hostnamectl || true +if command -v hostnamectl >/dev/null && hostnamectl status >/dev/null 2>&1; then + echo "yes" +else + true +fi From 16cb3a5ff11bae3f0f99e477bcfe202599a21119 Mon Sep 17 00:00:00 2001 From: Markus Koller Date: Wed, 6 Jul 2016 14:57:59 +0200 Subject: [PATCH 079/197] Update PXE setup for Debian 8 --- cdist/preos.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cdist/preos.py b/cdist/preos.py index 8eced936..c21dc00a 100644 --- a/cdist/preos.py +++ b/cdist/preos.py @@ -40,7 +40,7 @@ for pkg in \ linux-image-amd64 \ lsb-release \ openssh-server curl \ - syslinux grub2 \ + pxelinux syslinux-common grub2 \ gdisk util-linux lvm2 mdadm \ btrfs-tools e2fsprogs jfsutils reiser4progs xfsprogs; do __package $pkg --state present @@ -90,7 +90,7 @@ class PreOS(object): self.options = [ "--include=openssh-server", "--arch=%s" % self.arch ] - self.pxelinux = "/usr/lib/syslinux/pxelinux.0" + self.pxelinux = "/usr/lib/PXELINUX/pxelinux.0" self.pxelinux_cfg = """ DEFAULT preos LABEL preos @@ -178,11 +178,18 @@ cp -L "$src" "$real_dst" def create_pxelinux(self): dst = os.path.join(self.out_dir, "pxelinux.0") - src = "%s/usr/lib/syslinux/pxelinux.0" % self.target_dir + src = "%s/usr/lib/PXELINUX/pxelinux.0" % self.target_dir log.info("Creating pxelinux.0 ...") shutil.copyfile(src, dst, follow_symlinks=True) + def create_ldlinux(self): + dst = os.path.join(self.out_dir, "ldlinux.c32") + src = "%s/usr/lib/syslinux/modules/bios/ldlinux.c32" % self.target_dir + + log.info("Creating ldlinux.c32 ...") + shutil.copyfile(src, dst, follow_symlinks=True) + def create_pxeconfig(self): configdir = os.path.join(self.out_dir, "pxelinux.cfg") configfile = os.path.join(configdir, "default") @@ -219,6 +226,7 @@ cp -L "$src" "$real_dst" self.create_initramfs() self.create_pxeconfig() self.create_pxelinux() + self.create_ldlinux() def setup_initial_manifest(self, user_initial_manifest, replace_manifest): From dd1b95f5964528d3d8535c15bc8c4d3b7e5b87d2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 19 Aug 2016 12:27:35 +0200 Subject: [PATCH 080/197] Add target_host file with original host to cache since cache dir is hash. --- cdist/exec/local.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index c6e25be3..50f0ab65 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -254,6 +254,10 @@ class Local(object): "Cannot delete old cache %s: %s" % (destination, e)) shutil.move(self.base_path, destination) + # add target_host since cache dir is hash-ed target_host + host_cache_path = os.path.join(destination, "target_host") + with open(host_cache_path, 'w') as hostf: + print(self.target_host[0], file=hostf) def _create_messages(self): """Create empty messages""" From 7aa4b2d40ab176ceee1800fd6fe115fca6d5c93e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 19 Aug 2016 13:37:57 +0200 Subject: [PATCH 081/197] Support comments in hostfile, skip empty lines. --- cdist/config.py | 11 +++++++++-- docs/changelog | 1 + 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 31b41781..8d83a072 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -99,8 +99,15 @@ class Config(object): import fileinput try: for host in fileinput.input(files=(source)): - # remove leading and trailing whitespace - yield host.strip() + # remove comment if present + comment_index = host.find('#') + if comment_index >= 0: + host = host[:comment_index] + # remove leading and trailing whitespaces + host = host.strip() + # skip empty lines + if host: + yield host except (IOError, OSError) as e: raise cdist.Error("Error reading hosts from \'{}\'".format( source)) diff --git a/docs/changelog b/docs/changelog index db86de45..12510041 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,7 @@ Changelog --------- next: + * Core: Improve hostfile: support comments, skip empty lines (Darko Poljak) * Documentation: Add Parallelization chapter (Darko Poljak) * Core: Add -b, --enable-beta option for enabling beta functionalities (Darko Poljak) * Core: Add -j, --jobs option for parallel execution and add parallel support for global explorers (currently in beta) (Darko Poljak) From 72505e0f5f75878ae5127dc18ac14e292c877a9a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 19 Aug 2016 14:33:41 +0200 Subject: [PATCH 082/197] Add hostfile format to cdist man page. --- docs/src/man1/cdist.rst | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 47fe195c..eeba97f0 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -82,7 +82,7 @@ Configure one or more hosts. Read additional hosts to operate on from specified file or from stdin if '-' (each host on separate line). If no host or host file is specified then, by default, - read hosts from stdin. + read hosts from stdin. For the file format see below. .. option:: -i MANIFEST, --initial-manifest MANIFEST @@ -117,6 +117,15 @@ Configure one or more hosts. Command to use for remote execution (should behave like ssh) + +HOSTFILE FORMAT +~~~~~~~~~~~~~~~ +HOSTFILE contains hosts per line. +All characters after and including '#' until the end of line is a comment +and is stripped away. +Empty lines and comment lines (line that starts with '#') are skipped. + + SHELL ----- This command allows you to spawn a shell that enables access From 7f1e41f76904ee1d53646396e9f8e4253ca0b9be Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 19 Aug 2016 21:56:24 +0200 Subject: [PATCH 083/197] Move hostfile line processing to new method. --- cdist/config.py | 27 ++++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 8d83a072..2f28a22f 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -89,6 +89,25 @@ class Config(object): self.local.create_files_dirs() self.remote.create_files_dirs() + @staticmethod + def hostfile_process_line(line): + """Return host from read line or None if no host present.""" + if not line: + return None + # remove comment if present + comment_index = line.find('#') + if comment_index >= 0: + host = line[:comment_index] + else: + host = line + # remove leading and trailing whitespaces + host = host.strip() + # skip empty lines + if host: + return host + else: + return None + @staticmethod def hosts(source): """Yield hosts from source. @@ -99,13 +118,7 @@ class Config(object): import fileinput try: for host in fileinput.input(files=(source)): - # remove comment if present - comment_index = host.find('#') - if comment_index >= 0: - host = host[:comment_index] - # remove leading and trailing whitespaces - host = host.strip() - # skip empty lines + host = Config.hostfile_process_line(host) if host: yield host except (IOError, OSError) as e: From d9d739cd44ec005b3cf02bda54c44faf727f59d4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 20 Aug 2016 07:49:52 +0200 Subject: [PATCH 084/197] Update HOSTFILE FORMAT description. --- docs/src/man1/cdist.rst | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index eeba97f0..baeb0025 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -120,10 +120,15 @@ Configure one or more hosts. HOSTFILE FORMAT ~~~~~~~~~~~~~~~ -HOSTFILE contains hosts per line. -All characters after and including '#' until the end of line is a comment -and is stripped away. -Empty lines and comment lines (line that starts with '#') are skipped. +HOSTFILE contains hosts per line. +All characters after and including '#' until the end of line is a comment. +In a line, all leading and trailing whitespace characters are ignored. +Empty lines are ignored/skipped. + +Hostfile line is processed like the following. First, all comments are +removed. Then all leading and trailing whitespace characters are stripped. +If such a line results in empty line it is ignored/skipped. Otherwise, +host string is used. SHELL From b83e6993c1a4067c662fcaeb918de4acf6651e1d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 21 Aug 2016 17:10:24 +0200 Subject: [PATCH 085/197] Make comment better. --- cdist/exec/local.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 50f0ab65..93301063 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -254,7 +254,7 @@ class Local(object): "Cannot delete old cache %s: %s" % (destination, e)) shutil.move(self.base_path, destination) - # add target_host since cache dir is hash-ed target_host + # add target_host since cache dir can be hash-ed target_host host_cache_path = os.path.join(destination, "target_host") with open(host_cache_path, 'w') as hostf: print(self.target_host[0], file=hostf) From b5262c850ed6f918a1ed9154ff9a4ce915b30999 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 21 Aug 2016 21:48:21 +0200 Subject: [PATCH 086/197] Exit cleanly in case of non UTF-8 file. --- cdist/config.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index 2f28a22f..e20f1a7c 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -121,9 +121,10 @@ class Config(object): host = Config.hostfile_process_line(host) if host: yield host - except (IOError, OSError) as e: - raise cdist.Error("Error reading hosts from \'{}\'".format( - source)) + except (IOError, OSError, UnicodeError) as e: + raise cdist.Error( + "Error reading hosts from file \'{}\': {}".format( + source, e)) else: if source: for host in source: From 4121d14eabf4e4b3f8da65463e63884378da3386 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Aug 2016 08:00:23 +0200 Subject: [PATCH 087/197] Update changelog for target_host in cache dir. --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 518be192..61441d98 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Core: Add target_host file to cache since cache dir name can be hash (Darko Poljak) * Core: Improve hostfile: support comments, skip empty lines (Darko Poljak) 4.3.0: 2016-08-19 From b5a79fbc8f0c410608a4915429cb27f1b006270d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Aug 2016 08:11:49 +0200 Subject: [PATCH 088/197] Fix spelling (Dmitry Bogatov patch). --- cdist/conf/type/__filesystem/man.rst | 14 +++++++------- docs/changelog | 1 + 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__filesystem/man.rst b/cdist/conf/type/__filesystem/man.rst index d88e27c1..73f576d5 100644 --- a/cdist/conf/type/__filesystem/man.rst +++ b/cdist/conf/type/__filesystem/man.rst @@ -13,7 +13,7 @@ This cdist type allows you to create filesystems on devices. If the device is mounted on target, it refuses to do anything. If the device has a filesystem other then the specified and/or -the label is not correct, it only makes a new filesystem +the label is not correct, it only makes a new filesystem if you have specified --force option. @@ -30,9 +30,9 @@ device Blockdevice for filesystem, Defaults to object_id. On linux, it can be any lsblk accepted device notation. - | + | | For example: - | /dev/sdx + | /dev/sdx | or /dev/disk/by-xxxx/xxx | or /dev/mapper/xxxx @@ -46,15 +46,15 @@ mkfsoptions BOOLEAN PARAMETERS ------------------ force - Normaly, this type does nothing if a filesystem is found - on the target device. If you specify force, it's formated + Normally, this type does nothing if a filesystem is found + on the target device. If you specify force, it's formatted if the filesystem type or label differs from parameters. Warning: This option can easily lead into data loss! MESSAGES -------- filesystem on \: created - Filesytem was created on + Filesystem was created on EXAMPLES @@ -62,7 +62,7 @@ EXAMPLES .. code-block:: sh - # Ensures that device /dev/sdb is formated with xfs + # Ensures that device /dev/sdb is formatted with xfs __filesystem /dev/sdb --fstype xfs --label Testdisk1 # The same thing with btrfs and disk spezified by pci path to disk 1:0 on vmware __filesystem dev_sdb --fstype btrfs --device /dev/disk/by-path/pci-0000:0b:00.0-scsi-0:0:0:0 --label Testdisk2 diff --git a/docs/changelog b/docs/changelog index 61441d98..ec1c9f98 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Type __filesystem: Spelling fixes (Dmitry Bogatov) * Core: Add target_host file to cache since cache dir name can be hash (Darko Poljak) * Core: Improve hostfile: support comments, skip empty lines (Darko Poljak) From 3df61be7b8b85f981b17855e2fcc58daf1e95623 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Aug 2016 08:12:14 +0200 Subject: [PATCH 089/197] Update cdist support web page. --- docs/web/cdist/support.mdwn | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/web/cdist/support.mdwn b/docs/web/cdist/support.mdwn index 39388c5a..4c89ee79 100644 --- a/docs/web/cdist/support.mdwn +++ b/docs/web/cdist/support.mdwn @@ -8,7 +8,7 @@ You can join the development ***IRC channel*** ### Mailing list Bug reports, questions, patches, etc. should be send to the -[cdist mailing list](http://l.schottelius.org/mailman/listinfo/cdist). +[cdist mailing list](https://groups.google.com/forum/#!forum/cdist-configuration-management). ### Linkedin @@ -17,6 +17,9 @@ at [Linked in](http://www.linkedin.com/), you can join the [cdist group](http://www.linkedin.com/groups/cdist-configuration-management-3952797). +### Chat +Chat with us: [ungleich chat] (https://chat.ungleich.ch/channel/cdist). + ### Commercial support You can request commercial support for cdist from From 2d72c08e9b896822d5050912d4f5c733e0fdd540 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Aug 2016 09:24:48 +0200 Subject: [PATCH 090/197] Fix spelling. --- cdist/conf/type/__consul_check/man.rst | 2 +- cdist/conf/type/__filesystem/man.rst | 2 +- docs/changelog | 1 + docs/src/cdist-best-practice.rst | 2 +- docs/src/cdist-features.rst | 2 +- docs/src/cdist-hacker.rst | 2 +- docs/src/cdist-manifest.rst | 4 ++-- docs/src/cdist-type.rst | 2 +- docs/src/cdist-update.rst | 6 +++--- docs/src/man1/cdist.rst | 2 +- docs/web/cdist/features.mdwn | 2 +- docs/web/cdist/update.mdwn | 6 +++--- docs/web/cdist/update/2.0-to-2.1.mdwn | 4 ++-- 13 files changed, 19 insertions(+), 18 deletions(-) diff --git a/cdist/conf/type/__consul_check/man.rst b/cdist/conf/type/__consul_check/man.rst index dc7895de..9694c7af 100644 --- a/cdist/conf/type/__consul_check/man.rst +++ b/cdist/conf/type/__consul_check/man.rst @@ -11,7 +11,7 @@ DESCRIPTION Generate and deploy check definitions for a consul agent. See http://www.consul.io/docs/agent/checks.html for parameter documentation. -Use either script toghether with interval, or use ttl. +Use either script together with interval, or use ttl. REQUIRED PARAMETERS diff --git a/cdist/conf/type/__filesystem/man.rst b/cdist/conf/type/__filesystem/man.rst index 73f576d5..1c103ac9 100644 --- a/cdist/conf/type/__filesystem/man.rst +++ b/cdist/conf/type/__filesystem/man.rst @@ -37,7 +37,7 @@ device | or /dev/mapper/xxxx label - Label which sould apply on the filesystem. + Label which should be applied on the filesystem. mkfsoptions Additional options which are inserted to the mkfs.xxx call. diff --git a/docs/changelog b/docs/changelog index ec1c9f98..9d7bc427 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Documentation: Spelling fixes (Darko Poljak) * Type __filesystem: Spelling fixes (Dmitry Bogatov) * Core: Add target_host file to cache since cache dir name can be hash (Darko Poljak) * Core: Improve hostfile: support comments, skip empty lines (Darko Poljak) diff --git a/docs/src/cdist-best-practice.rst b/docs/src/cdist-best-practice.rst index 57ce5cc1..493f1506 100644 --- a/docs/src/cdist-best-practice.rst +++ b/docs/src/cdist-best-practice.rst @@ -58,7 +58,7 @@ you can clone it multiple times:: machine-b % git clone git://your-git-server/cdist -Seperating work by groups +Separating work by groups ------------------------- If you are working with different groups on one cdist-configuration, you can delegate to other manifests and have the groups edit only diff --git a/docs/src/cdist-features.rst b/docs/src/cdist-features.rst index 8a147741..7018d248 100644 --- a/docs/src/cdist-features.rst +++ b/docs/src/cdist-features.rst @@ -7,7 +7,7 @@ Simplicity There is only one type to extend cdist called **type** Design - + Type and core cleanly seperated + + Type and core cleanly separated + Sticks completly to the KISS (keep it simple and stupid) paradigma + Meaningful error messages - do not lose time debugging error messages + Consistency in behaviour, naming and documentation diff --git a/docs/src/cdist-hacker.rst b/docs/src/cdist-hacker.rst index 326d83ba..efc5da4b 100644 --- a/docs/src/cdist-hacker.rst +++ b/docs/src/cdist-hacker.rst @@ -56,7 +56,7 @@ or open a pull request at http://github.com/telmich/cdist. How to submit a new type ------------------------ -For detailled information about types, see `cdist type `_. +For detailed information about types, see `cdist type `_. Submitting a type works as described above, with the additional requirement that a corresponding manpage named man.text in asciidoc format with diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst index 5655c6c2..b29cf0d8 100644 --- a/docs/src/cdist-manifest.rst +++ b/docs/src/cdist-manifest.rst @@ -118,7 +118,7 @@ On line 4 you can see that the instantion of a type "\__link" object needs the object "__file/etc/cdist-configured" to be present, before it can proceed. This also means that the "\__link" command must make sure, that either -"\__file/etc/cdist-configured" allready is present, or, if it's not, it needs +"\__file/etc/cdist-configured" already is present, or, if it's not, it needs to be created. The task of cdist is to make sure, that the dependency will be resolved appropriately and thus "\__file/etc/cdist-configured" be created if necessary before "__link" proceeds (or to abort execution with an error). @@ -216,7 +216,7 @@ How to override objects: .. code-block:: sh - # for example in the inital manifest + # for example in the initial manifest # create user account foobar with some hash for password __user foobar --password 'some_fancy_hash' --home /home/foobarexample diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index c75d0a52..62694fd8 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -126,7 +126,7 @@ Example: (e.g. in cdist/conf/type/__nginx_vhost/manifest) # parameter with multiple values if [ -f "$__object/parameter/server_alias" ]; then for alias in $(cat "$__object/parameter/server_alias"); do - echo $alias > /some/where/usefull + echo $alias > /some/where/useful done fi diff --git a/docs/src/cdist-update.rst b/docs/src/cdist-update.rst index 0b445ba4..e810d6e9 100644 --- a/docs/src/cdist-update.rst +++ b/docs/src/cdist-update.rst @@ -85,7 +85,7 @@ Use `messaging `_ instead. Updating from 2.2 to 2.3 ~~~~~~~~~~~~~~~~~~~~~~~~ -No incompatiblities. +No incompatibilities. Updating from 2.1 to 2.2 ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -164,7 +164,7 @@ Updating from 1.5 to 1.6 Updating from 1.3 to 1.5 ~~~~~~~~~~~~~~~~~~~~~~~~ -No incompatiblities. +No incompatibilities. Updating from 1.2 to 1.3 ~~~~~~~~~~~~~~~~~~~~~~~~ @@ -174,7 +174,7 @@ Rename **gencode** of every type to **gencode-remote**. Updating from 1.1 to 1.2 ~~~~~~~~~~~~~~~~~~~~~~~~ -No incompatiblities. +No incompatibilities. Updating from 1.0 to 1.1 ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index baeb0025..52562e14 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -207,7 +207,7 @@ CDIST_LOCAL_SHELL Selects shell for local script execution, defaults to /bin/sh. CDIST_REMOTE_SHELL - Selects shell for remote scirpt execution, defaults to /bin/sh. + Selects shell for remote script execution, defaults to /bin/sh. CDIST_OVERRIDE Allow overwriting type parameters. diff --git a/docs/web/cdist/features.mdwn b/docs/web/cdist/features.mdwn index a97f2013..77c61382 100644 --- a/docs/web/cdist/features.mdwn +++ b/docs/web/cdist/features.mdwn @@ -3,7 +3,7 @@ But cdist ticks differently, here is the feature set that makes it unique: [[!table data=""" Keywords | Description Simplicity | There is only one type to extend cdist called ***type*** -Design | Type and core cleanly seperated +Design | Type and core cleanly separated Design | Sticks completly to the KISS (keep it simple and stupid) paradigma Design | Meaningful error messages - do not lose time debugging error messages Design | Consistency in behaviour, naming and documentation diff --git a/docs/web/cdist/update.mdwn b/docs/web/cdist/update.mdwn index 28f41da7..df4617bb 100644 --- a/docs/web/cdist/update.mdwn +++ b/docs/web/cdist/update.mdwn @@ -67,7 +67,7 @@ Use [messaging](/software/cdist/man/3.0.0/man7/cdist-messaging.html) instead. ### Updating from 2.2 to 2.3 -No incompatiblities. +No incompatibilities. ### Updating from 2.1 to 2.2 @@ -134,7 +134,7 @@ Have a look at the update guide for [[2.0 to 2.1|2.0-to-2.1]]. ### Updating from 1.3 to 1.5 -No incompatiblities. +No incompatibilities. ### Updating from 1.2 to 1.3 @@ -142,7 +142,7 @@ Rename **gencode** of every type to **gencode-remote**. ### Updating from 1.1 to 1.2 -No incompatiblities. +No incompatibilities. ### Updating from 1.0 to 1.1 diff --git a/docs/web/cdist/update/2.0-to-2.1.mdwn b/docs/web/cdist/update/2.0-to-2.1.mdwn index 1d0037ab..3b5f5dc4 100644 --- a/docs/web/cdist/update/2.0-to-2.1.mdwn +++ b/docs/web/cdist/update/2.0-to-2.1.mdwn @@ -10,7 +10,7 @@ explorers and manifest to custom directories. This document will guide you to a successful update. -## Preperation +## Preparation As for every software and system you use in production, you should first of all make a backup of your data. To prevent any breakage, it is @@ -35,7 +35,7 @@ Now try to merge upstream into the new branch. % git merge origin/2.1 Fix any conflicts that may have been occurred due to local changes -and then **git add** and *git commit** those changes. This should seldomly +and then **git add** and *git commit** those changes. This should seldom occur and if, it's mostly for people hacking on the cdist core. ## Move "conf" directory From 49506406fd6ff3ead0c69bebaf038a68b0f48153 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Aug 2016 09:27:12 +0200 Subject: [PATCH 091/197] Fix spelling. --- docs/changelog | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/changelog b/docs/changelog index 9d7bc427..1ab5e321 100644 --- a/docs/changelog +++ b/docs/changelog @@ -82,7 +82,7 @@ next: * Type __consul_agent: Use systemd for Debian 8 (Nico Schottelius) * Type __firewalld_rule: Ensure firewalld package is present (David Hürlimann) * Type __locale: Support CentOS (David Hürlimann) - * Type __staged_file: Fix comparision operator (Nico Schottelius) + * Type __staged_file: Fix comparison operator (Nico Schottelius) * Type __user_groups: Support old Linux versions (Daniel Heule) 3.1.12: 2015-03-19 @@ -269,7 +269,7 @@ next: 3.0.1: 2014-01-14 * Core: Copy only files, not directories (Steven Armstrong) * Core: Allow hostnames to start with / (Nico Schottelius) - * Type __line: Remove unecessary backslash escape (Nico Schottelius) + * Type __line: Remove unnecessary backslash escape (Nico Schottelius) * Type __directory: Add messaging support (Daniel Heule) * Type __directory: Do not generate code if mode is 0xxx (Daniel Heule) * Type __package: Fix typo in optional parameter ptype (Daniel Heule) From 94d37913d125ea3fba01f2eef69db61fa0e51766 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Aug 2016 10:36:40 +0200 Subject: [PATCH 092/197] Fix Makefile tabulation. --- docs/src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/Makefile b/docs/src/Makefile index 800a80d8..2ecf7a32 100644 --- a/docs/src/Makefile +++ b/docs/src/Makefile @@ -11,7 +11,7 @@ _BUILDDIR = _build # User-friendly check for sphinx-build ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) - $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) + $(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don\'t have Sphinx installed, grab it from http://sphinx-doc.org/) endif # Internal variables. From e7e7cfdce26c0c5badbe25237e90c65221fc1e62 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Aug 2016 18:49:14 +0200 Subject: [PATCH 093/197] Update changelog for release 4.3.1. --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 1ab5e321..ba0ce9c5 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.3.1: 2016-08-22 * Documentation: Spelling fixes (Darko Poljak) * Type __filesystem: Spelling fixes (Dmitry Bogatov) * Core: Add target_host file to cache since cache dir name can be hash (Darko Poljak) From bc0efa9c4e3c22a7a9e788a2a1a7ef3a4980334f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 22 Aug 2016 19:07:13 +0200 Subject: [PATCH 094/197] Remove extra whitespace. --- docs/web/cdist/support.mdwn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/web/cdist/support.mdwn b/docs/web/cdist/support.mdwn index 4c89ee79..4f92853b 100644 --- a/docs/web/cdist/support.mdwn +++ b/docs/web/cdist/support.mdwn @@ -18,7 +18,7 @@ you can join the [cdist group](http://www.linkedin.com/groups/cdist-configuration-management-3952797). ### Chat -Chat with us: [ungleich chat] (https://chat.ungleich.ch/channel/cdist). +Chat with us: [ungleich chat](https://chat.ungleich.ch/channel/cdist). ### Commercial support From cd8373fe506cec09944a95d515538f0111a2b349 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Tue, 23 Aug 2016 15:55:07 +0200 Subject: [PATCH 095/197] Hotfix: Changed source of consul 0.5.1 --- cdist/conf/type/__consul/files/versions/0.5.1/source | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__consul/files/versions/0.5.1/source b/cdist/conf/type/__consul/files/versions/0.5.1/source index f02a1103..a47eb69c 100644 --- a/cdist/conf/type/__consul/files/versions/0.5.1/source +++ b/cdist/conf/type/__consul/files/versions/0.5.1/source @@ -1 +1 @@ -https://dl.bintray.com/mitchellh/consul/0.5.1_linux_amd64.zip +https://releases.hashicorp.com/consul/0.5.1/consul_0.5.1_linux_amd64.zip From 428c06c8d39ab7777cc1bece34dd0ae23f924d2a Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Tue, 23 Aug 2016 16:40:13 +0200 Subject: [PATCH 096/197] Hotfix: Changed sources of all consul version + cksum files --- cdist/conf/type/__consul/files/versions/0.4.1/cksum | 2 +- cdist/conf/type/__consul/files/versions/0.4.1/source | 2 +- cdist/conf/type/__consul/files/versions/0.5.0/cksum | 2 +- cdist/conf/type/__consul/files/versions/0.5.0/source | 2 +- cdist/conf/type/__consul/files/versions/0.5.1/cksum | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__consul/files/versions/0.4.1/cksum b/cdist/conf/type/__consul/files/versions/0.4.1/cksum index edba1a68..3e114ece 100644 --- a/cdist/conf/type/__consul/files/versions/0.4.1/cksum +++ b/cdist/conf/type/__consul/files/versions/0.4.1/cksum @@ -1 +1 @@ -428915666 15738724 consul +2024850362 4073679 consul diff --git a/cdist/conf/type/__consul/files/versions/0.4.1/source b/cdist/conf/type/__consul/files/versions/0.4.1/source index b1e9908d..7fb949c8 100644 --- a/cdist/conf/type/__consul/files/versions/0.4.1/source +++ b/cdist/conf/type/__consul/files/versions/0.4.1/source @@ -1 +1 @@ -https://dl.bintray.com/mitchellh/consul/0.4.1_linux_amd64.zip +https://releases.hashicorp.com/consul/0.4.1/consul_0.4.1_linux_amd64.zip diff --git a/cdist/conf/type/__consul/files/versions/0.5.0/cksum b/cdist/conf/type/__consul/files/versions/0.5.0/cksum index fe9888ae..66bb482a 100644 --- a/cdist/conf/type/__consul/files/versions/0.5.0/cksum +++ b/cdist/conf/type/__consul/files/versions/0.5.0/cksum @@ -1 +1 @@ -131560372 17734417 consul +915590436 4669655 consul diff --git a/cdist/conf/type/__consul/files/versions/0.5.0/source b/cdist/conf/type/__consul/files/versions/0.5.0/source index 00a209a5..dc1c33c4 100644 --- a/cdist/conf/type/__consul/files/versions/0.5.0/source +++ b/cdist/conf/type/__consul/files/versions/0.5.0/source @@ -1 +1 @@ -https://dl.bintray.com/mitchellh/consul/0.5.0_linux_amd64.zip +https://releases.hashicorp.com/consul/0.5.0/consul_0.5.0_linux_amd64.zip diff --git a/cdist/conf/type/__consul/files/versions/0.5.1/cksum b/cdist/conf/type/__consul/files/versions/0.5.1/cksum index a176ed43..c1c59011 100644 --- a/cdist/conf/type/__consul/files/versions/0.5.1/cksum +++ b/cdist/conf/type/__consul/files/versions/0.5.1/cksum @@ -1 +1 @@ -2564582176 18232733 consul +190723740 4802625 consul From 232a9098518151c98356dc7a6d4d28b0c4cbc827 Mon Sep 17 00:00:00 2001 From: smwalter Date: Wed, 24 Aug 2016 17:34:18 +0900 Subject: [PATCH 097/197] change documentation from git://git.schottelius.org/cdist to git://github.com/ungleich/cdist because git.schottelius.org does not exist. --- cdist/conf/type/__cdist/man.rst | 2 +- docs/src/cdist-quickstart.rst | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__cdist/man.rst b/cdist/conf/type/__cdist/man.rst index f02f848a..72b11e78 100644 --- a/cdist/conf/type/__cdist/man.rst +++ b/cdist/conf/type/__cdist/man.rst @@ -47,7 +47,7 @@ EXAMPLES __cdist /home/cdist/cdist # Use alternative source - __cdist --source "git://git.schottelius.org/cdist" /home/cdist/cdist + __cdist --source "git://github.com/telmich/cdist" /home/cdist/cdist AUTHORS diff --git a/docs/src/cdist-quickstart.rst b/docs/src/cdist-quickstart.rst index 7c967691..0020568d 100644 --- a/docs/src/cdist-quickstart.rst +++ b/docs/src/cdist-quickstart.rst @@ -56,7 +56,7 @@ code into your shell to get started and configure localhost:: # Get cdist # Mirrors can be found on # http://www.nico.schottelius.org/software/cdist/install/#index2h4 - git clone git://git.schottelius.org/cdist + git clone git://github.com/ungleich/cdist # Create manifest (maps configuration to host(s) cd cdist From 72001b237e060d47ec63d95060d9a4a1ec128337 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 24 Aug 2016 13:46:47 +0200 Subject: [PATCH 098/197] Remove relict comment. --- cdist/emulator.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cdist/emulator.py b/cdist/emulator.py index 2c5e567b..b04ed130 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -166,8 +166,6 @@ class Emulator(object): self.parameters[key] = value if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env: - # make existing requirements a set, so we can compare it - # later with new requirements if self.cdist_object.parameters != self.parameters: errmsg = ("Object %s already exists with conflicting " "parameters:\n%s: %s\n%s: %s" % ( From 3b91443f813a154e3dc0e7afa523b1e6970607aa Mon Sep 17 00:00:00 2001 From: Andres Erbsen Date: Fri, 2 Sep 2016 11:39:22 -0400 Subject: [PATCH 099/197] __hostname: openbsd support --- cdist/conf/type/__hostname/gencode-remote | 4 ++-- cdist/conf/type/__hostname/manifest | 12 +++++++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index c1808de0..4eb08723 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -40,7 +40,7 @@ case "$os" in exit 0 fi ;; - scientific|centos) + scientific|centos|openbsd) if [ "$name_sysconfig" = "$name_should" -a "$name_running" = "$name_should" ]; then exit 0 fi @@ -64,7 +64,7 @@ else echo "hostname '$name_should'" echo "printf '%s\n' '$name_should' > /etc/hostname" ;; - centos) + centos|openbsd) echo "hostname '$name_should'" ;; suse) diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index 842075e0..823d2f7e 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -23,7 +23,14 @@ os=$(cat "$__global/explorer/os") if [ -f "$__object/parameter/name" ]; then name_should="$(cat "$__object/parameter/name")" else - name_should="$(echo "${__target_host%%.*}")" + case "$os" in + openbsd) + name_should="$(echo "${__target_host}")" + ;; + *) + name_should="$(echo "${__target_host%%.*}")" + ;; + esac fi @@ -45,6 +52,9 @@ case "$os" in --key HOSTNAME \ --value "$name_should" --exact_delimiter ;; + openbsd) + echo "$name_should" | __file /etc/myname --source - + ;; *) not_supported ;; From 1a4bec21bff07413a070503c1a215d4d91acb8b6 Mon Sep 17 00:00:00 2001 From: Andres Erbsen Date: Fri, 2 Sep 2016 11:21:05 -0400 Subject: [PATCH 100/197] __package: call __package_pkg_openbsd on openbsd --- cdist/conf/type/__package/manifest | 1 + 1 file changed, 1 insertion(+) diff --git a/cdist/conf/type/__package/manifest b/cdist/conf/type/__package/manifest index c745f85f..525691bb 100755 --- a/cdist/conf/type/__package/manifest +++ b/cdist/conf/type/__package/manifest @@ -43,6 +43,7 @@ else gentoo) type="emerge" ;; suse) type="zypper" ;; openwrt) type="opkg" ;; + openbsd) type="pkg_openbsd" ;; *) echo "Don't know how to manage packages on: $os" >&2 exit 1 From db32d0de3a62044ce23c036cca0d79ffe34a90a5 Mon Sep 17 00:00:00 2001 From: Andres Erbsen Date: Fri, 2 Sep 2016 11:13:37 -0400 Subject: [PATCH 101/197] __package_pkg_openbsd: support --version --- .../type/__package_pkg_openbsd/gencode-remote | 22 +++++++++++++++---- cdist/conf/type/__package_pkg_openbsd/man.rst | 3 +++ .../__package_pkg_openbsd/parameter/optional | 1 + 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/cdist/conf/type/__package_pkg_openbsd/gencode-remote b/cdist/conf/type/__package_pkg_openbsd/gencode-remote index dea7f711..5ba5f7ef 100755 --- a/cdist/conf/type/__package_pkg_openbsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_openbsd/gencode-remote @@ -29,6 +29,10 @@ os_version="$(cat "$__global/explorer/os_version")" machine="$(cat "$__global/explorer/machine")" +if [ -f "$__object/parameter/version" ]; then + version="$(cat "$__object/parameter/version")" +fi + if [ -f "$__object/parameter/flavor" ]; then flavor="$(cat "$__object/parameter/flavor")" fi @@ -42,6 +46,16 @@ else name="$__object_id" fi +if [ -n "$version" -a -n "$flavor" ]; then + pkgid="$name-$version-$flavor" +elif [ -n "$version" ]; then + pkgid="$name-$version" +elif [ -n "$flavor" ]; then + pkgid="$name--$flavor" +else + pkgid="$name" +fi + state_should="$(cat "$__object/parameter/state")" pkg_version="$(cat "$__object/explorer/pkg_version")" @@ -65,8 +79,8 @@ case "$state_should" in # use this because pkg_add doesn't properly handle errors cat << eof export PKG_PATH="$pkg_path" -status=\$(pkg_add "$pkgopts" "$name--$flavor" 2>&1) -pkg_info | grep "^${name}.*${flavor}" > /dev/null 2>&1 +status=\$(pkg_add "$pkgopts" "$pkgid" 2>&1) +pkg_info | grep "^${name}.*${version}.*${flavor}" > /dev/null 2>&1 # We didn't find the package in the list of 'installed packages', so it failed # This is necessary because pkg_add doesn't return properly @@ -83,8 +97,8 @@ eof absent) # use this because pkg_add doesn't properly handle errors cat << eof -status=\$(pkg_delete "$pkgopts" "$name--$flavor") -pkg_info | grep "^${name}.*${flavor}" > /dev/null 2>&1 +status=\$(pkg_delete "$pkgopts" "$pkgid") +pkg_info | grep "^${name}.*${version}.*${flavor}" > /dev/null 2>&1 # We found the package in the list of 'installed packages' # This would indicate that pkg_delete failed, send the output of pkg_delete diff --git a/cdist/conf/type/__package_pkg_openbsd/man.rst b/cdist/conf/type/__package_pkg_openbsd/man.rst index 0814029f..dcfd0719 100644 --- a/cdist/conf/type/__package_pkg_openbsd/man.rst +++ b/cdist/conf/type/__package_pkg_openbsd/man.rst @@ -24,6 +24,9 @@ name flavor If supplied, use to avoid ambiguity. +version + If supplied, use to avoid ambiguity. + state Either "present" or "absent", defaults to "present" diff --git a/cdist/conf/type/__package_pkg_openbsd/parameter/optional b/cdist/conf/type/__package_pkg_openbsd/parameter/optional index 43278d16..6a5f9277 100644 --- a/cdist/conf/type/__package_pkg_openbsd/parameter/optional +++ b/cdist/conf/type/__package_pkg_openbsd/parameter/optional @@ -1,4 +1,5 @@ name +version flavor state pkg_path From 1875bce52e9da62a4f0eb7bd3922c32a26dd7f86 Mon Sep 17 00:00:00 2001 From: Dmitry Bogatov Date: Thu, 1 Sep 2016 17:47:51 +0300 Subject: [PATCH 102/197] Add support for guixsd into os explorer --- cdist/conf/explorer/os | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 550192d4..094685ea 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -39,6 +39,11 @@ if [ -f /etc/cdist-preos ]; then exit 0 fi +if [ -d /gnu/store ]; then + echo guixsd + exit 0 +fi + ### Debian and derivatives if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then echo ubuntu From 493c8d61f400653ce235cbd27d717e51656a0e9b Mon Sep 17 00:00:00 2001 From: Andres Erbsen Date: Sun, 11 Sep 2016 22:23:06 -0400 Subject: [PATCH 103/197] __user_groups: refactor, support FreeBSD --- .../type/__user_groups/explorer/oldusermod | 7 ++-- cdist/conf/type/__user_groups/gencode-remote | 37 ++++++++++--------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/cdist/conf/type/__user_groups/explorer/oldusermod b/cdist/conf/type/__user_groups/explorer/oldusermod index bf43fcec..248d3922 100644 --- a/cdist/conf/type/__user_groups/explorer/oldusermod +++ b/cdist/conf/type/__user_groups/explorer/oldusermod @@ -22,7 +22,8 @@ os="$($__explorer/os)" if [ "$os" = "netbsd" ]; then echo netbsd - exit +elif [ "$os" = "freebsd" ]; then + echo freebsd +else + usermod --help | grep -q -- '-A group' && echo true || echo false fi - -usermod --help | grep -q -- '-A group' && echo true || echo false diff --git a/cdist/conf/type/__user_groups/gencode-remote b/cdist/conf/type/__user_groups/gencode-remote index 8b13f32c..6728228c 100755 --- a/cdist/conf/type/__user_groups/gencode-remote +++ b/cdist/conf/type/__user_groups/gencode-remote @@ -23,19 +23,6 @@ state_should="$(cat "$__object/parameter/state")" oldusermod="$(cat "$__object/explorer/oldusermod")" os=$(cat "$__global/explorer/os") -if [ "$os" = "netbsd" ]; then - # NetBSD does not have a command to remove a user from a group - oldusermod="true" - addparam="-G" - delparam=";;#" -elif [ "$oldusermod" = "true" ]; then - addparam="-A" - delparam="-R" -else - addparam="-a" - delparam="-d" -fi - mkdir "$__object/files" # file has to be sorted for comparison with `comm` sort "$__object/parameter/group" > "$__object/files/group.sorted" @@ -43,11 +30,9 @@ sort "$__object/parameter/group" > "$__object/files/group.sorted" case "$state_should" in present) changed_groups="$(comm -13 "$__object/explorer/group" "$__object/files/group.sorted")" - action="$addparam" ;; absent) changed_groups="$(comm -12 "$__object/explorer/group" "$__object/files/group.sorted")" - action="$delparam" ;; esac @@ -57,9 +42,25 @@ if [ -z "$changed_groups" ]; then fi for group in $changed_groups; do - if [ "$oldusermod" = "true" ]; then - echo "usermod $action \"$group\" \"$user\"" + if [ "$os" = "netbsd" ]; then + case "$state_should" in + present) echo "usermod -G \"$group\" \"$user\"" ;; + absent) echo 'NetBSD does not have a command to remove a user from a group' >&2 ; exit 1 ;; + esac + elif [ "$os" = "freebsd" ]; then + case "$state_should" in + present) echo "pw groupmod \"$group\" -m \"$user\"" ;; + absent) echo "pw groupmod \"$group\" -d \"$user\"" ;; + esac + elif [ "$oldusermod" = "true" ]; then + case "$state_should" in + present) echo "usermod -A \"$group\" \"$user\"" ;; + absent) echo "usermod -R \"$group\" \"$user\"" ;; + esac else - echo "gpasswd $action \"$user\" \"$group\"" + case "$state_should" in + present) echo "gpasswd -a \"$group\" \"$user\"" ;; + absent) echo "gpasswd -d \"$group\" \"$user\"" ;; + esac fi done From f7381e261ae12eedfd9746937145ce0e28e78d08 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 17 Sep 2016 09:48:15 +0200 Subject: [PATCH 104/197] Add new type __firewalld_start. --- .../type/__firewalld_start/gencode-remote | 84 +++++++++++++++++++ cdist/conf/type/__firewalld_start/man.rst | 53 ++++++++++++ cdist/conf/type/__firewalld_start/manifest | 23 +++++ .../parameter/default/bootstate | 1 + .../parameter/default/startstate | 1 + .../type/__firewalld_start/parameter/optional | 2 + cdist/conf/type/__firewalld_start/singleton | 0 7 files changed, 164 insertions(+) create mode 100644 cdist/conf/type/__firewalld_start/gencode-remote create mode 100644 cdist/conf/type/__firewalld_start/man.rst create mode 100644 cdist/conf/type/__firewalld_start/manifest create mode 100644 cdist/conf/type/__firewalld_start/parameter/default/bootstate create mode 100644 cdist/conf/type/__firewalld_start/parameter/default/startstate create mode 100644 cdist/conf/type/__firewalld_start/parameter/optional create mode 100644 cdist/conf/type/__firewalld_start/singleton diff --git a/cdist/conf/type/__firewalld_start/gencode-remote b/cdist/conf/type/__firewalld_start/gencode-remote new file mode 100644 index 00000000..7a3b6298 --- /dev/null +++ b/cdist/conf/type/__firewalld_start/gencode-remote @@ -0,0 +1,84 @@ +#!/bin/sh +# +# 2016 Darko Poljak(darko.poljak at ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +startstate="$(cat "$__object/parameter/startstate")" +init=$(cat "$__global/explorer/init") + +os=$(cat "$__global/explorer/os") +os_version=$(cat "$__global/explorer/os_version") +name="firewalld" + +case "${startstate}" in + present) + cmd="start" + ;; + absent) + cmd="stop" + ;; + *) + echo "Unknown startstate: ${startstate}" >&2 + exit 1 + ;; +esac + +if [ "$init" = 'systemd' ]; then + # this handles ALL linux distros with systemd + # e.g. archlinux, gentoo in some cases, new RHEL and SLES versions + echo "systemctl \"$cmd\" \"$name\"" +else + case "$os" in + debian) + case "$os_version" in + [1-7]*) + echo "service \"$name\" \"$cmd\"" + ;; + 8*) + echo "systemctl \"$cmd\" \"$name\"" + ;; + *) + echo "Unsupported version $os_version of $os" >&2 + exit 1 + ;; + esac + ;; + + gentoo) + echo service \"$name\" \"$cmd\" + ;; + + amazon|scientific|centos|fedora|owl|redhat|suse) + echo service \"$name\" \"$cmd\" + ;; + + openwrt) + echo "/etc/init.d/\"$name\" \"$cmd\"" + ;; + + ubuntu) + echo "service \"$name\" \"$cmd\"" + ;; + + *) + echo "Unsupported os: $os" >&2 + exit 1 + ;; + esac +fi diff --git a/cdist/conf/type/__firewalld_start/man.rst b/cdist/conf/type/__firewalld_start/man.rst new file mode 100644 index 00000000..03232b72 --- /dev/null +++ b/cdist/conf/type/__firewalld_start/man.rst @@ -0,0 +1,53 @@ +cdist-type__firewalld_start(7) +============================= + +NAME +---- +cdist-type__firewalld_start - start and enable firewalld + + +DESCRIPTION +----------- +This cdist type allows you to start and enable firewalld. + + +REQUIRED PARAMETERS +------------------- +None + +OPTIONAL PARAMETERS +------------------- +startstate + 'present' or 'absent', start/stop firewalld. Default is 'present'. +bootstate + 'present' or 'absent', enable/disable firewalld on boot. Default is 'present'. + + +EXAMPLES +-------- + +.. code-block:: sh + + # start and enable firewalld + __firewalld_start + + # only enable firewalld to start on boot + __firewalld_start --startstate present --bootstate absent + + +SEE ALSO +-------- +:strong:`firewalld`\ (8) + + +AUTHORS +------- +Darko Poljak + + +COPYING +------- +Copyright \(C) 2016 Darko Poljak. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__firewalld_start/manifest b/cdist/conf/type/__firewalld_start/manifest new file mode 100644 index 00000000..2c6a0219 --- /dev/null +++ b/cdist/conf/type/__firewalld_start/manifest @@ -0,0 +1,23 @@ +#!/bin/sh +# +# 2016 Darko Poljak (darko.poljak at ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +bootstate="$(cat "$__object/parameter/bootstate")" + +__package firewalld +require="__package/firewalld" __start_on_boot firewalld --state "${bootstate}" diff --git a/cdist/conf/type/__firewalld_start/parameter/default/bootstate b/cdist/conf/type/__firewalld_start/parameter/default/bootstate new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__firewalld_start/parameter/default/bootstate @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__firewalld_start/parameter/default/startstate b/cdist/conf/type/__firewalld_start/parameter/default/startstate new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__firewalld_start/parameter/default/startstate @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__firewalld_start/parameter/optional b/cdist/conf/type/__firewalld_start/parameter/optional new file mode 100644 index 00000000..934c7d0d --- /dev/null +++ b/cdist/conf/type/__firewalld_start/parameter/optional @@ -0,0 +1,2 @@ +bootstate +startstate diff --git a/cdist/conf/type/__firewalld_start/singleton b/cdist/conf/type/__firewalld_start/singleton new file mode 100644 index 00000000..e69de29b From f87d31f42375c13a059fb306baf111eedf48533b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 17 Sep 2016 17:33:53 +0200 Subject: [PATCH 105/197] Update changelog: new type __firewalld_start. --- docs/changelog | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/changelog b/docs/changelog index ba0ce9c5..dac0e205 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,9 @@ Changelog --------- +next: + * New type: __firewalld_start: start/stop firewalld and/or enable/disable start on boot (Darko Poljak) + 4.3.1: 2016-08-22 * Documentation: Spelling fixes (Darko Poljak) * Type __filesystem: Spelling fixes (Dmitry Bogatov) From 74652cec1327e12271675d712cb47ca49b0d0bbf Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 21 Sep 2016 19:22:24 +0200 Subject: [PATCH 106/197] Write more informative warning messages. --- cdist/config.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index e20f1a7c..b0131601 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -266,12 +266,10 @@ class Config(object): host_name = socket.gethostbyaddr(ip_addr)[0] log.debug("derived host_name for host \"{}\": {}".format( host, host_name)) - except socket.gaierror as e: - log.warn("host_name: {}".format(e)) - # in case of error provide empty value - host_name = '' - except socket.herror as e: - log.warn("host_name: {}".format(e)) + except (socket.gaierror, socket.herror) as e: + log.warn("Could not derive host_name for {}" + ", $host_name will be empty. Error is: {}".format( + host, e)) # in case of error provide empty value host_name = '' @@ -280,9 +278,12 @@ class Config(object): log.debug("derived host_fqdn for host \"{}\": {}".format( host, host_fqdn)) except socket.herror as e: - log.warn("host_fqdn: {}".format(e)) + log.warn("Could not derive host_fqdn for {}" + ", $host_fqdn will be empty. Error is: {}".format( + host, e)) # in case of error provide empty value host_fqdn = '' + target_host = (host, host_name, host_fqdn) local = cdist.exec.local.Local( From bee55935700b51b037d0216b5f034fe3d382c6ef Mon Sep 17 00:00:00 2001 From: Daniel Heule Date: Tue, 4 Oct 2016 11:29:48 +0200 Subject: [PATCH 107/197] use /etc/os-release instead of /etc/SuSE-release --- cdist/conf/explorer/os_version | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 58f750b0..380782cc 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -61,7 +61,11 @@ case "$($__explorer/os)" in cat /etc/slackware-version ;; suse) - cat /etc/SuSE-release + if [ -f /etc/os-release ]; then + cat /etc/os-release + else + cat /etc/SuSE-release + fi ;; ubuntu) lsb_release -sr From 8e967424de8bff0de7707043fb94b0ae80e4b48e Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Wed, 5 Oct 2016 23:22:36 +0200 Subject: [PATCH 108/197] consul syslog config option should be called enable_syslog instead Signed-off-by: Steven Armstrong --- cdist/conf/type/__consul_agent/man.rst | 2 +- cdist/conf/type/__consul_agent/manifest | 2 +- cdist/conf/type/__consul_agent/parameter/boolean | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__consul_agent/man.rst b/cdist/conf/type/__consul_agent/man.rst index 8285cb25..d75425d7 100644 --- a/cdist/conf/type/__consul_agent/man.rst +++ b/cdist/conf/type/__consul_agent/man.rst @@ -106,7 +106,7 @@ rejoin-after-leave server used to control if an agent is in server or client mode -syslog +enable-syslog enables logging to syslog verify-incoming diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index b4d1d75c..e98d770b 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -98,7 +98,7 @@ for param in $(ls "$__object/parameter/"); do key="$(echo "${param%-*}" | tr '-' '_')" printf ' ,"%s": "%s"\n' "$key" "$destination" ;; - disable-remote-exec|disable-update-check|leave-on-terminate|rejoin-after-leave|server|syslog|verify-incoming|verify-outgoing) + disable-remote-exec|disable-update-check|leave-on-terminate|rejoin-after-leave|server|enable-syslog|verify-incoming|verify-outgoing) # handle boolean parameters key="$(echo "$param" | tr '-' '_')" printf ' ,"%s": true\n' "$key" diff --git a/cdist/conf/type/__consul_agent/parameter/boolean b/cdist/conf/type/__consul_agent/parameter/boolean index 9efecf49..91f7f17e 100644 --- a/cdist/conf/type/__consul_agent/parameter/boolean +++ b/cdist/conf/type/__consul_agent/parameter/boolean @@ -3,6 +3,6 @@ disable-update-check leave-on-terminate rejoin-after-leave server -syslog +enable-syslog verify-incoming verify-outgoing From 88b436b4c17f594cbcd747b936961da0b674d570 Mon Sep 17 00:00:00 2001 From: Steven Armstrong Date: Thu, 6 Oct 2016 23:34:50 +0200 Subject: [PATCH 109/197] changelog++ Signed-off-by: Steven Armstrong --- docs/changelog | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/changelog b/docs/changelog index dac0e205..61fc93cf 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,6 +3,8 @@ Changelog next: * New type: __firewalld_start: start/stop firewalld and/or enable/disable start on boot (Darko Poljak) + * Bugfix __consul_agent: config option was misnamed 'syslog' instead of + 'enable_syslog' (Steven Armstrong) 4.3.1: 2016-08-22 * Documentation: Spelling fixes (Darko Poljak) From d49af95d3c3665ebb86465862449292bf04aa03e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 8 Oct 2016 11:40:32 +0200 Subject: [PATCH 110/197] Add warning message for faulty dependencies case. --- cdist/conf/type/__firewalld_start/man.rst | 2 +- cdist/emulator.py | 28 +++++++++- cdist/test/config/__init__.py | 16 ++++++ .../fixtures/manifest/init-deps-resolver | 8 +++ cdist/test/config/fixtures/type/__a/.keep | 0 cdist/test/config/fixtures/type/__b/.keep | 0 cdist/test/config/fixtures/type/__c/.keep | 0 cdist/test/config/fixtures/type/__d/.keep | 0 cdist/test/config/fixtures/type/__e/.keep | 0 cdist/test/config/fixtures/type/__f/.keep | 0 cdist/test/config/fixtures/type/__g/.keep | 0 cdist/test/config/fixtures/type/__g/manifest | 1 + cdist/test/config/fixtures/type/__h/.keep | 0 cdist/test/config/fixtures/type/__h/manifest | 3 ++ cdist/test/config/fixtures/type/__i/.keep | 0 cdist/test/config/fixtures/type/__j/.keep | 0 cdist/test/emulator/__init__.py | 54 +++++++++++++++++++ docs/src/man1/cdist.rst | 29 ++++++++++ 18 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 cdist/test/config/fixtures/manifest/init-deps-resolver create mode 100644 cdist/test/config/fixtures/type/__a/.keep create mode 100644 cdist/test/config/fixtures/type/__b/.keep create mode 100644 cdist/test/config/fixtures/type/__c/.keep create mode 100644 cdist/test/config/fixtures/type/__d/.keep create mode 100644 cdist/test/config/fixtures/type/__e/.keep create mode 100644 cdist/test/config/fixtures/type/__f/.keep create mode 100644 cdist/test/config/fixtures/type/__g/.keep create mode 100644 cdist/test/config/fixtures/type/__g/manifest create mode 100644 cdist/test/config/fixtures/type/__h/.keep create mode 100644 cdist/test/config/fixtures/type/__h/manifest create mode 100644 cdist/test/config/fixtures/type/__i/.keep create mode 100644 cdist/test/config/fixtures/type/__j/.keep diff --git a/cdist/conf/type/__firewalld_start/man.rst b/cdist/conf/type/__firewalld_start/man.rst index 03232b72..74199cd6 100644 --- a/cdist/conf/type/__firewalld_start/man.rst +++ b/cdist/conf/type/__firewalld_start/man.rst @@ -1,5 +1,5 @@ cdist-type__firewalld_start(7) -============================= +============================== NAME ---- diff --git a/cdist/emulator.py b/cdist/emulator.py index b04ed130..6744de8b 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -84,6 +84,10 @@ class Emulator(object): self.type_name = os.path.basename(argv[0]) self.cdist_type = core.CdistType(self.type_base_path, self.type_name) + # If set then object alreay exists and this var holds existing + # requirements. + self._existing_reqs = None + self.__init_log() def run(self): @@ -166,6 +170,9 @@ class Emulator(object): self.parameters[key] = value if self.cdist_object.exists and 'CDIST_OVERRIDE' not in self.env: + # Make existing requirements a set so that we can compare it + # later with new requirements. + self._existing_reqs = set(self.cdist_object.requirements) if self.cdist_object.parameters != self.parameters: errmsg = ("Object %s already exists with conflicting " "parameters:\n%s: %s\n%s: %s" % ( @@ -243,7 +250,7 @@ class Emulator(object): return cdist_object.name def record_requirements(self): - """record requirements""" + """Record requirements.""" # Inject the predecessor, but not if its an override # (this would leed to an circular dependency) @@ -267,6 +274,7 @@ class Emulator(object): # so do not set a requirement pass + reqs = set() if "require" in self.env: requirements = self.env['require'] self.log.debug("reqs = " + requirements) @@ -274,7 +282,23 @@ class Emulator(object): # Ignore empty fields - probably the only field anyway if len(requirement) == 0: continue - self.record_requirement(requirement) + object_name = self.record_requirement(requirement) + reqs.add(object_name) + if self._existing_reqs is not None: + # If object exists then compare existing and new requirements. + if self._existing_reqs != reqs: + warnmsg = ("Object {} already exists with requirements:\n" + "{}: {}\n" + "{}: {}\n" + "Dependency resolver could not handle dependencies " + "as expected.".format( + self.cdist_object.name, + " ".join(self.cdist_object.source), + self._existing_reqs, + self.object_source, + reqs + )) + self.log.warning(warnmsg) def record_auto_requirements(self): """An object shall automatically depend on all objects that it diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py index db753f41..af1aa38f 100644 --- a/cdist/test/config/__init__.py +++ b/cdist/test/config/__init__.py @@ -177,6 +177,22 @@ class ConfigRunTestCase(test.CdistTestCase): dryrun.run() # if we are here, dryrun works like expected + def test_desp_resolver(self): + """Test to show dependency resolver warning message.""" + local = cdist.exec.local.Local( + target_host=self.target_host, + base_root_path=self.host_base_path, + host_dir_name=self.hostdir, + exec_path=os.path.abspath(os.path.join( + my_dir, '../../../scripts/cdist')), + initial_manifest=os.path.join( + fixtures, 'manifest/init-deps-resolver'), + add_conf_dirs=[fixtures]) + + # dry_run is ok for dependency testing + config = cdist.config.Config(local, self.remote, dry_run=True) + config.run() + # Currently the resolving code will simply detect that this object does # not exist. It should probably check if the type is a singleton as well diff --git a/cdist/test/config/fixtures/manifest/init-deps-resolver b/cdist/test/config/fixtures/manifest/init-deps-resolver new file mode 100644 index 00000000..f67ab61c --- /dev/null +++ b/cdist/test/config/fixtures/manifest/init-deps-resolver @@ -0,0 +1,8 @@ +__a a +require="__e/e" __b b +require="__f/f" __c c +__e e +__f f +require="__c/c" __d d +__g g +__h h diff --git a/cdist/test/config/fixtures/type/__a/.keep b/cdist/test/config/fixtures/type/__a/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__b/.keep b/cdist/test/config/fixtures/type/__b/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__c/.keep b/cdist/test/config/fixtures/type/__c/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__d/.keep b/cdist/test/config/fixtures/type/__d/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__e/.keep b/cdist/test/config/fixtures/type/__e/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__f/.keep b/cdist/test/config/fixtures/type/__f/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__g/.keep b/cdist/test/config/fixtures/type/__g/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__g/manifest b/cdist/test/config/fixtures/type/__g/manifest new file mode 100644 index 00000000..107dbda4 --- /dev/null +++ b/cdist/test/config/fixtures/type/__g/manifest @@ -0,0 +1 @@ +require="__c/c __d/d" __a a diff --git a/cdist/test/config/fixtures/type/__h/.keep b/cdist/test/config/fixtures/type/__h/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__h/manifest b/cdist/test/config/fixtures/type/__h/manifest new file mode 100644 index 00000000..ce3d8fb7 --- /dev/null +++ b/cdist/test/config/fixtures/type/__h/manifest @@ -0,0 +1,3 @@ +# require="__b/b" __a a +require="__j/j" __i i +__j j diff --git a/cdist/test/config/fixtures/type/__i/.keep b/cdist/test/config/fixtures/type/__i/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/config/fixtures/type/__j/.keep b/cdist/test/config/fixtures/type/__j/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 4fd0ed40..51de3180 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -499,6 +499,60 @@ class StdinTestCase(test.CdistTestCase): self.assertEqual(random_string, stdin_saved_by_emulator) +class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase): + + def setUp(self): + self.temp_dir = self.mkdtemp() + handle, self.script = self.mkstemp(dir=self.temp_dir) + os.close(handle) + base_path = self.temp_dir + hostdir = cdist.str_hash(self.target_host[0]) + host_base_path = os.path.join(base_path, hostdir) + + self.local = local.Local( + target_host=self.target_host, + base_root_path=host_base_path, + host_dir_name=hostdir, + exec_path=test.cdist_exec_path, + add_conf_dirs=[conf_dir]) + self.local.create_files_dirs() + + self.manifest = core.Manifest(self.target_host, self.local) + self.env = self.manifest.env_initial_manifest(self.script) + self.env['__cdist_object_marker'] = self.local.object_marker_name + + def tearDown(self): + shutil.rmtree(self.temp_dir) + + def test_object_existing_requirements_req_none(self): + """Test to show dependency resolver warning message.""" + argv = ['__directory', 'spam'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + argv = ['__file', 'eggs'] + self.env['require'] = '__directory/spam' + emu = emulator.Emulator(argv, env=self.env) + emu.run() + argv = ['__file', 'eggs'] + if 'require' in self.env: + del self.env['require'] + emu = emulator.Emulator(argv, env=self.env) + + def test_object_existing_requirements_none_req(self): + """Test to show dependency resolver warning message.""" + argv = ['__directory', 'spam'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + argv = ['__file', 'eggs'] + if 'require' in self.env: + del self.env['require'] + emu = emulator.Emulator(argv, env=self.env) + emu.run() + argv = ['__file', 'eggs'] + self.env['require'] = '__directory/spam' + emu = emulator.Emulator(argv, env=self.env) + + if __name__ == '__main__': import unittest unittest.main() diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 52562e14..9cc28011 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -246,6 +246,35 @@ connection. In this case ssh will disable multiplexing. This limit is controlled with sshd :strong:`MaxSessions` configuration options. For more details refer to :strong:`sshd_config`\ (5). +When requirements for the same object are defined in different manifests (see +example below) in init manifest and in some other type manifest and they differs +then dependency resolver cannot detect dependencies right. This happends because +cdist cannot prepare all objects first and then run objects because some +object can depend on the result of type explorer(s) and explorers are executed +during object run. cdist will detect such case and write warning message. +Example for such a case: + +.. code-block:: sh + + init manifest: + __a a + require="__e/e" __b b + require="__f/f" __c c + __e e + __f f + require="__c/c" __d d + __g g + __h h + + type __g manifest: + require="__c/c __d/d" __a a + + Warning message: + .WARNING: cdisttesthost: Object __a/a already exists with requirements: + /usr/home/darko/ungleich/cdist/cdist/test/config/fixtures/manifest/init-deps-resolver /tmp/tmp.cdist.test.ozagkg54/local/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__g/manifest: set() + /tmp/tmp.cdist.test.ozagkg54/local/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__g/manifest: {'__c/c', '__d/d'} + Dependency resolver could not handle dependencies as expected. + COPYING ------- Copyright \(C) 2011-2013 Nico Schottelius. Free use of this software is From c4e89eb2457d79e0af3ae100367439be668124a3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 8 Oct 2016 11:44:55 +0200 Subject: [PATCH 111/197] Fix spelling. --- docs/src/man1/cdist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 9cc28011..5a30c321 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -248,7 +248,7 @@ options. For more details refer to :strong:`sshd_config`\ (5). When requirements for the same object are defined in different manifests (see example below) in init manifest and in some other type manifest and they differs -then dependency resolver cannot detect dependencies right. This happends because +then dependency resolver cannot detect dependencies right. This happens because cdist cannot prepare all objects first and then run objects because some object can depend on the result of type explorer(s) and explorers are executed during object run. cdist will detect such case and write warning message. From ddd8eab06f182819bb07bebe0d3ba24b3756f23c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 8 Oct 2016 11:45:50 +0200 Subject: [PATCH 112/197] Remove extra dot. --- docs/src/man1/cdist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 5a30c321..c80b3386 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -270,7 +270,7 @@ Example for such a case: require="__c/c __d/d" __a a Warning message: - .WARNING: cdisttesthost: Object __a/a already exists with requirements: + WARNING: cdisttesthost: Object __a/a already exists with requirements: /usr/home/darko/ungleich/cdist/cdist/test/config/fixtures/manifest/init-deps-resolver /tmp/tmp.cdist.test.ozagkg54/local/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__g/manifest: set() /tmp/tmp.cdist.test.ozagkg54/local/759547ff4356de6e3d9e08522b0d0807/data/conf/type/__g/manifest: {'__c/c', '__d/d'} Dependency resolver could not handle dependencies as expected. From aa4c3dc7bda3e3a29c0cb67a503e35fb86955d2b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 10 Oct 2016 12:25:10 +0200 Subject: [PATCH 113/197] Update changelog. --- docs/changelog | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/changelog b/docs/changelog index 61fc93cf..4ea8cddd 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,11 @@ Changelog --------- next: + * Core: Add warning message for faulty dependencies case (Darko Poljak) + * Explorer os_version: Use /etc/os-release instead of /etc/SuSE-release (Daniel Heule) + * Type __package: Call __package_pkg_openbsd on openbsd (Andres Erbsen) + * Type __package_pkg_openbsd: Support --version (Andres Erbsen) + * Type __hostname: Support openbsd (Andres Erbsen) * New type: __firewalld_start: start/stop firewalld and/or enable/disable start on boot (Darko Poljak) * Bugfix __consul_agent: config option was misnamed 'syslog' instead of 'enable_syslog' (Steven Armstrong) From ce4803a201ffe7bdb3526556b8ed086218b83d81 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 12 Oct 2016 16:56:29 +0200 Subject: [PATCH 114/197] telmich -> ungleich --- cdist/conf/type/__cdist/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__cdist/man.rst b/cdist/conf/type/__cdist/man.rst index 72b11e78..9e1c72cb 100644 --- a/cdist/conf/type/__cdist/man.rst +++ b/cdist/conf/type/__cdist/man.rst @@ -30,7 +30,7 @@ username source Select the source from which to clone cdist from. - Defaults to "git://github.com/telmich/cdist.git". + Defaults to "git://github.com/ungleich/cdist.git". branch @@ -47,7 +47,7 @@ EXAMPLES __cdist /home/cdist/cdist # Use alternative source - __cdist --source "git://github.com/telmich/cdist" /home/cdist/cdist + __cdist --source "git://github.com/ungleich/cdist" /home/cdist/cdist AUTHORS From 67d93f3e0a693783bf7af00837d3dd841fc84c62 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 12 Oct 2016 16:59:45 +0200 Subject: [PATCH 115/197] Update changelog. --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 4ea8cddd..5a23cc6e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Documentation: Update no longer existing links (Simon Walter) * Core: Add warning message for faulty dependencies case (Darko Poljak) * Explorer os_version: Use /etc/os-release instead of /etc/SuSE-release (Daniel Heule) * Type __package: Call __package_pkg_openbsd on openbsd (Andres Erbsen) From aceb4ac13d66d71190c2747aea85b1550b2c9b73 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 12 Oct 2016 20:15:07 +0200 Subject: [PATCH 116/197] Update telmich -> ungleich and mailing list refs. --- cdist/conf/type/__cdist/parameter/default/source | 2 +- cdist/conf/type/__cdistmarker/gencode-remote | 2 +- cdist/conf/type/__git/man.rst | 2 +- docs/src/cdist-hacker.rst | 6 +++--- docs/web/cdist/install.mdwn | 2 +- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cdist/conf/type/__cdist/parameter/default/source b/cdist/conf/type/__cdist/parameter/default/source index d669308f..3f8e31ad 100644 --- a/cdist/conf/type/__cdist/parameter/default/source +++ b/cdist/conf/type/__cdist/parameter/default/source @@ -1 +1 @@ -git://github.com/telmich/cdist.git +git://github.com/ungleich/cdist.git diff --git a/cdist/conf/type/__cdistmarker/gencode-remote b/cdist/conf/type/__cdistmarker/gencode-remote index 92ea582b..5e889e52 100755 --- a/cdist/conf/type/__cdistmarker/gencode-remote +++ b/cdist/conf/type/__cdistmarker/gencode-remote @@ -2,7 +2,7 @@ # # Copyright (C) 2011 Daniel Maher (phrawzty+cdist at gmail.com) # -# This file is part of cdist (https://github.com/telmich/cdist/). +# This file is part of cdist. # # cdist is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/cdist/conf/type/__git/man.rst b/cdist/conf/type/__git/man.rst index 64adfd2f..17e9c623 100644 --- a/cdist/conf/type/__git/man.rst +++ b/cdist/conf/type/__git/man.rst @@ -44,7 +44,7 @@ EXAMPLES __git /home/services/dokuwiki --source git://github.com/splitbrain/dokuwiki.git # Checkout cdist, stay on branch 2.1 - __git /home/nico/cdist --source git://github.com/telmich/cdist.git --branch 2.1 + __git /home/nico/cdist --source git://github.com/ungleich/cdist.git --branch 2.1 AUTHORS diff --git a/docs/src/cdist-hacker.rst b/docs/src/cdist-hacker.rst index efc5da4b..d7d6a056 100644 --- a/docs/src/cdist-hacker.rst +++ b/docs/src/cdist-hacker.rst @@ -50,8 +50,8 @@ work nor kill the authors brain: the other needs to be improved. As soon as your work meets these requirements, write a mail -for inclusion to the mailinglist **cdist at cdist -- at -- l.schottelius.org** -or open a pull request at http://github.com/telmich/cdist. +for inclusion to the mailinglist **cdist-configuration-management at googlegroups.com** +or open a pull request at http://github.com/ungleich/cdist. How to submit a new type @@ -77,7 +77,7 @@ The following workflow works fine for most developers .. code-block:: sh # get latest upstream master branch - git clone https://github.com/telmich/cdist.git + git clone https://github.com/ungleich/cdist.git # update if already existing cd cdist; git fetch -v; git merge origin/master diff --git a/docs/web/cdist/install.mdwn b/docs/web/cdist/install.mdwn index cff2d369..0ced26db 100644 --- a/docs/web/cdist/install.mdwn +++ b/docs/web/cdist/install.mdwn @@ -29,7 +29,7 @@ immediately. To install cdist, execute the following commands: - git clone https://github.com/telmich/cdist.git + git clone https://github.com/ungleich/cdist.git cd cdist export PATH=$PATH:$(pwd -P)/bin From 54f752a42a07beba13e970635b822b11aea9d908 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 13 Oct 2016 18:51:43 +0200 Subject: [PATCH 117/197] Release 4.3.2 --- docs/changelog | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docs/changelog b/docs/changelog index 5a23cc6e..1b7de44f 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.3.2: 2016-10-13 * Documentation: Update no longer existing links (Simon Walter) * Core: Add warning message for faulty dependencies case (Darko Poljak) * Explorer os_version: Use /etc/os-release instead of /etc/SuSE-release (Daniel Heule) @@ -9,8 +9,7 @@ next: * Type __package_pkg_openbsd: Support --version (Andres Erbsen) * Type __hostname: Support openbsd (Andres Erbsen) * New type: __firewalld_start: start/stop firewalld and/or enable/disable start on boot (Darko Poljak) - * Bugfix __consul_agent: config option was misnamed 'syslog' instead of - 'enable_syslog' (Steven Armstrong) + * Bugfix __consul_agent: config option was misnamed 'syslog' instead of 'enable_syslog' (Steven Armstrong) 4.3.1: 2016-08-22 * Documentation: Spelling fixes (Darko Poljak) From acf94abe2639d3132489b7c52134f4ccc05be6bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 13 Oct 2016 21:24:29 +0200 Subject: [PATCH 118/197] pep8 --- cdist/install.py | 7 ++++--- scripts/cdist | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cdist/install.py b/cdist/install.py index 4530029a..bec8d24a 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -29,9 +29,10 @@ class Install(cdist.config.Config): """Short name for object list retrieval. In install mode, we only care about install objects. """ - for cdist_object in cdist.core.CdistObject.list_objects(self.local.object_path, - self.local.type_path): + for cdist_object in cdist.core.CdistObject.list_objects( + self.local.object_path, self.local.type_path): if cdist_object.cdist_type.is_install: yield cdist_object else: - self.log.debug("Running in install mode, ignoring non install object: {0}".format(cdist_object)) + self.log.debug("Running in install mode, ignoring non install" + "object: {0}".format(cdist_object)) diff --git a/scripts/cdist b/scripts/cdist index badf0f76..0acfe06c 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -164,7 +164,7 @@ def commandline(): # Install parser['install'] = parser['sub'].add_parser('install', add_help=False, - parents=[parser['config']]) + parents=[parser['config']]) parser['install'].set_defaults(func=cdist.install.Install.commandline) for p in parser: From 750f90db4c4dfbe4bcfb83fb62562e2c0133059e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 13 Oct 2016 21:35:41 +0200 Subject: [PATCH 119/197] Make install command beta. --- cdist/__init__.py | 20 ++++++++++++++------ scripts/cdist | 6 ++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index 9068ae69..b6f5c8cb 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -59,16 +59,24 @@ class UnresolvableRequirementsError(cdist.Error): class CdistBetaRequired(cdist.Error): """Beta functionality is used but beta is not enabled""" - def __init__(self, command, arg): + def __init__(self, command, arg=None): self.command = command self.arg = arg def __str__(self): - err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta is " - "not enabled. If you want to use it please enable beta " - "functionalities by using the -b/--enable-beta command " - "line flag.") - return err_msg.format(self.arg, self.command) + if self.arg is None: + err_msg = ("\'{}\' command is beta, but beta is " + "not enabled. If you want to use it please enable beta " + "functionalities by using the -b/--enable-beta command " + "line flag.") + fmt_args = [self.command, ] + else: + err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta " + "is not enabled. If you want to use it please enable " + "beta functionalities by using the -b/--enable-beta " + "command line flag.") + fmt_args = [self.arg, self.command, ] + return err_msg.format(*fmt_args) class CdistObjectError(Error): diff --git a/scripts/cdist b/scripts/cdist index 0acfe06c..9f8d326d 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -22,6 +22,8 @@ # +# list of beta sub-commands +BETA_COMMANDS = ['install', ] # list of beta arguments for sub-commands BETA_ARGS = { 'config': ['jobs', ], @@ -49,6 +51,10 @@ def check_beta(args_dict): # raise error. if not args_dict['beta']: cmd = args_dict['command'] + # first check if command is beta + if cmd in BETA_COMMANDS: + raise cdist.CdistBetaRequired(cmd) + # then check if command's argument is beta if cmd in BETA_ARGS: for arg in BETA_ARGS[cmd]: if arg in args_dict and args_dict[arg]: From 536a64e56dac0b4c2793c440c551a3f8d0b98ba3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 13 Oct 2016 21:38:34 +0200 Subject: [PATCH 120/197] Add install to cdist man page. --- docs/src/man1/cdist.rst | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index c80b3386..45ce339e 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,7 +11,7 @@ SYNOPSIS :: - cdist [-h] [-d] [-v] [-V] {banner,config,shell} ... + cdist [-h] [-d] [-v] [-V] {banner,config,shell,install} ... cdist banner [-h] [-d] [-v] @@ -20,6 +20,11 @@ SYNOPSIS [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] [host [host ...]] + cdist install [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE] + [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s] + [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] + [host [host ...]] + cdist shell [-h] [-d] [-v] [-s SHELL] @@ -58,9 +63,9 @@ Displays the cdist banner. Useful for printing cdist posters - a must have for every office. -CONFIG ------- -Configure one or more hosts. +CONFIG/INSTALL +-------------- +Configure/install one or more hosts. .. option:: -b, --enable-beta @@ -191,6 +196,8 @@ EXAMPLES usage: __git --source SOURCE [--state STATE] [--branch BRANCH] [--group GROUP] [--owner OWNER] [--mode MODE] object_id + # Install ikq05.ethz.ch with debug enabled + % cdist install -d ikq05.ethz.ch ENVIRONMENT ----------- From ac94d182b6ea2d5676d9a59243399f1e3c78587c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 13 Oct 2016 21:40:09 +0200 Subject: [PATCH 121/197] Add install to bash/zsh completions. --- completions/bash/cdist-completion.bash | 4 ++-- completions/zsh/_cdist | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash index c756fca8..68f45327 100644 --- a/completions/bash/cdist-completion.bash +++ b/completions/bash/cdist-completion.bash @@ -6,7 +6,7 @@ _cdist() prev="${COMP_WORDS[COMP_CWORD-1]}" prevprev="${COMP_WORDS[COMP_CWORD-2]}" opts="-h --help -d --debug -v --verbose -V --version" - cmds="banner shell config" + cmds="banner shell config install" case "${prevprev}" in shell) @@ -35,7 +35,7 @@ _cdist() COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; - config) + config|install) opts="-h --help -d --debug -v --verbose -b --enable-beta \ -c --conf-dir -f --file -i --initial-manifest -j --jobs \ -n --dry-run -o --out-dir -p --parallel -s --sequential \ diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist index 18fda0f0..dc320224 100644 --- a/completions/zsh/_cdist +++ b/completions/zsh/_cdist @@ -11,7 +11,7 @@ _cdist() case $state in opts_cmds) - _arguments '1:Options and commands:(banner config shell -h --help -d --debug -v --verbose -V --version)' + _arguments '1:Options and commands:(banner config shell install -h --help -d --debug -v --verbose -V --version)' ;; *) case $words[2] in @@ -35,7 +35,7 @@ _cdist() ;; esac ;; - config) + config|install) opts=(-h --help -d --debug -v --verbose -b --enable-beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec) compadd "$@" -- $opts ;; From a05ae761a400e35c1f851648c0d74a55ab369899 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 14 Oct 2016 21:16:28 +0200 Subject: [PATCH 122/197] man.text -> man.rst --- .../type/__cdist_preos/{man.text => man.rst} | 28 +++++--- .../type/__chroot_mount/{man.text => man.rst} | 24 +++---- .../__chroot_umount/{man.text => man.rst} | 25 +++++--- .../{man.text => man.rst} | 29 +++++---- .../conf/type/__install_chroot_mount/man.rst | 1 + .../conf/type/__install_chroot_mount/man.text | 1 - .../conf/type/__install_chroot_umount/man.rst | 1 + .../type/__install_chroot_umount/man.text | 1 - .../__install_config/{man.text => man.rst} | 26 ++++---- cdist/conf/type/__install_file/man.rst | 1 + cdist/conf/type/__install_file/man.text | 1 - .../__install_fstab/{man.text => man.rst} | 31 +++++---- .../{man.text => man.rst} | 29 +++++---- cdist/conf/type/__install_mkfs/man.rst | 62 ++++++++++++++++++ cdist/conf/type/__install_mkfs/man.text | 57 ----------------- .../__install_mount/{man.text => man.rst} | 40 ++++++------ .../type/__install_partition_msdos/man.rst | 64 +++++++++++++++++++ .../type/__install_partition_msdos/man.text | 62 ------------------ .../{man.text => man.rst} | 25 +++++--- .../__install_reboot/{man.text => man.rst} | 24 +++---- .../{man.text => man.rst} | 24 +++---- cdist/conf/type/__install_stage/man.rst | 58 +++++++++++++++++ cdist/conf/type/__install_stage/man.text | 58 ----------------- .../__install_umount/{man.text => man.rst} | 24 +++---- 24 files changed, 367 insertions(+), 329 deletions(-) rename cdist/conf/type/__cdist_preos/{man.text => man.rst} (52%) rename cdist/conf/type/__chroot_mount/{man.text => man.rst} (50%) rename cdist/conf/type/__chroot_umount/{man.text => man.rst} (52%) rename cdist/conf/type/__install_bootloader_grub/{man.text => man.rst} (55%) create mode 120000 cdist/conf/type/__install_chroot_mount/man.rst delete mode 120000 cdist/conf/type/__install_chroot_mount/man.text create mode 120000 cdist/conf/type/__install_chroot_umount/man.rst delete mode 120000 cdist/conf/type/__install_chroot_umount/man.text rename cdist/conf/type/__install_config/{man.text => man.rst} (60%) create mode 120000 cdist/conf/type/__install_file/man.rst delete mode 120000 cdist/conf/type/__install_file/man.text rename cdist/conf/type/__install_fstab/{man.text => man.rst} (54%) rename cdist/conf/type/__install_generate_fstab/{man.text => man.rst} (57%) create mode 100644 cdist/conf/type/__install_mkfs/man.rst delete mode 100644 cdist/conf/type/__install_mkfs/man.text rename cdist/conf/type/__install_mount/{man.text => man.rst} (54%) create mode 100644 cdist/conf/type/__install_partition_msdos/man.rst delete mode 100644 cdist/conf/type/__install_partition_msdos/man.text rename cdist/conf/type/__install_partition_msdos_apply/{man.text => man.rst} (52%) rename cdist/conf/type/__install_reboot/{man.text => man.rst} (52%) rename cdist/conf/type/__install_reset_disk/{man.text => man.rst} (50%) create mode 100644 cdist/conf/type/__install_stage/man.rst delete mode 100644 cdist/conf/type/__install_stage/man.text rename cdist/conf/type/__install_umount/{man.text => man.rst} (54%) diff --git a/cdist/conf/type/__cdist_preos/man.text b/cdist/conf/type/__cdist_preos/man.rst similarity index 52% rename from cdist/conf/type/__cdist_preos/man.text rename to cdist/conf/type/__cdist_preos/man.rst index 19caa8e2..8b7d22d6 100644 --- a/cdist/conf/type/__cdist_preos/man.text +++ b/cdist/conf/type/__cdist_preos/man.rst @@ -1,7 +1,5 @@ cdist-type__cdist_preos(7) ========================== -Nico Schottelius - NAME ---- @@ -13,26 +11,38 @@ DESCRIPTION This cdist type creates a directory containing an operating suitable for installation using cdist. + REQUIRED PARAMETERS ------------------- +None + OPTIONAL PARAMETERS ------------------- +None + + EXAMPLES -------- --------------------------------------------------------------------------------- -__cdist_preos /tmp/random_name_for_packaging --------------------------------------------------------------------------------- +.. code-block:: sh + + __cdist_preos /tmp/random_name_for_packaging SEE ALSO -------- -- cdist-type(7) -- cdist-type__cdist(7) +:strong:`cdist-type__cdist`\ (7) + + +AUTHORS +------- +Nico Schottelius COPYING ------- -Copyright \(C) 2015 Nico Schottelius. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2015 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__chroot_mount/man.text b/cdist/conf/type/__chroot_mount/man.rst similarity index 50% rename from cdist/conf/type/__chroot_mount/man.text rename to cdist/conf/type/__chroot_mount/man.rst index adce80d9..5ffd7de5 100644 --- a/cdist/conf/type/__chroot_mount/man.text +++ b/cdist/conf/type/__chroot_mount/man.rst @@ -1,7 +1,5 @@ cdist-type__install_chroot_mount(7) =================================== -Steven Armstrong - NAME ---- @@ -15,28 +13,30 @@ Mount and prepare a chroot for running commands within it. REQUIRED PARAMETERS ------------------- -None. +None OPTIONAL PARAMETERS ------------------- -None. +None EXAMPLES -------- --------------------------------------------------------------------------------- -__install_chroot_mount /path/to/chroot --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_chroot_mount /path/to/chroot -SEE ALSO --------- -- cdist-type(7) +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2012 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2012 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__chroot_umount/man.text b/cdist/conf/type/__chroot_umount/man.rst similarity index 52% rename from cdist/conf/type/__chroot_umount/man.text rename to cdist/conf/type/__chroot_umount/man.rst index a5ca1ef0..a2ea1d9b 100644 --- a/cdist/conf/type/__chroot_umount/man.text +++ b/cdist/conf/type/__chroot_umount/man.rst @@ -1,7 +1,5 @@ cdist-type__install_chroot_umount(7) ==================================== -Steven Armstrong - NAME ---- @@ -15,28 +13,35 @@ Undo what __chroot_mount did. REQUIRED PARAMETERS ------------------- -None. +None OPTIONAL PARAMETERS ------------------- -None. +None EXAMPLES -------- --------------------------------------------------------------------------------- -__install_chroot_umount /path/to/chroot --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_chroot_umount /path/to/chroot SEE ALSO -------- -- cdist-type(7) +:strong:`cdist-type__chroot_mount`\ (7) + + +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2012 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2012 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_bootloader_grub/man.text b/cdist/conf/type/__install_bootloader_grub/man.rst similarity index 55% rename from cdist/conf/type/__install_bootloader_grub/man.text rename to cdist/conf/type/__install_bootloader_grub/man.rst index 858e6a67..625db1d2 100644 --- a/cdist/conf/type/__install_bootloader_grub/man.text +++ b/cdist/conf/type/__install_bootloader_grub/man.rst @@ -1,7 +1,5 @@ cdist-type__install_bootloader_grub(7) ====================================== -Steven Armstrong - NAME ---- @@ -15,33 +13,36 @@ This cdist type allows you to install grub2 bootloader on given disk. REQUIRED PARAMETERS ------------------- -None. +None OPTIONAL PARAMETERS ------------------- -device:: +device The device to install grub to. Defaults to object_id -chroot:: +chroot where to chroot before running grub-install. Defaults to /target. EXAMPLES -------- --------------------------------------------------------------------------------- -__install_bootloader_grub /dev/sda -__install_bootloader_grub /dev/sda --chroot /mnt/foobar --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_bootloader_grub /dev/sda + + __install_bootloader_grub /dev/sda --chroot /mnt/foobar -SEE ALSO --------- -- cdist-type(7) +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_chroot_mount/man.rst b/cdist/conf/type/__install_chroot_mount/man.rst new file mode 120000 index 00000000..3267c3db --- /dev/null +++ b/cdist/conf/type/__install_chroot_mount/man.rst @@ -0,0 +1 @@ +../__chroot_mount/man.rst \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_mount/man.text b/cdist/conf/type/__install_chroot_mount/man.text deleted file mode 120000 index e131fceb..00000000 --- a/cdist/conf/type/__install_chroot_mount/man.text +++ /dev/null @@ -1 +0,0 @@ -../__chroot_mount/man.text \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_umount/man.rst b/cdist/conf/type/__install_chroot_umount/man.rst new file mode 120000 index 00000000..fa34c4b0 --- /dev/null +++ b/cdist/conf/type/__install_chroot_umount/man.rst @@ -0,0 +1 @@ +../__chroot_umount/man.rst \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_umount/man.text b/cdist/conf/type/__install_chroot_umount/man.text deleted file mode 120000 index f615c734..00000000 --- a/cdist/conf/type/__install_chroot_umount/man.text +++ /dev/null @@ -1 +0,0 @@ -../__chroot_umount/man.text \ No newline at end of file diff --git a/cdist/conf/type/__install_config/man.text b/cdist/conf/type/__install_config/man.rst similarity index 60% rename from cdist/conf/type/__install_config/man.text rename to cdist/conf/type/__install_config/man.rst index def0439b..0034e85d 100644 --- a/cdist/conf/type/__install_config/man.text +++ b/cdist/conf/type/__install_config/man.rst @@ -1,7 +1,5 @@ cdist-type__install_config(7) ============================= -Steven Armstrong - NAME ---- @@ -17,31 +15,33 @@ cdist config against the /target chroot on the remote host. REQUIRED PARAMETERS ------------------- -None. +None OPTIONAL PARAMETERS ------------------- -chroot:: +chroot where to chroot before running grub-install. Defaults to /target. EXAMPLES -------- --------------------------------------------------------------------------------- -__install_config +.. code-block:: sh -__install_config --chroot /mnt/somewhere --------------------------------------------------------------------------------- + __install_config + + __install_config --chroot /mnt/somewhere -SEE ALSO --------- -- cdist-type(7) +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_file/man.rst b/cdist/conf/type/__install_file/man.rst new file mode 120000 index 00000000..c937b8af --- /dev/null +++ b/cdist/conf/type/__install_file/man.rst @@ -0,0 +1 @@ +../__file/man.rst \ No newline at end of file diff --git a/cdist/conf/type/__install_file/man.text b/cdist/conf/type/__install_file/man.text deleted file mode 120000 index ba483161..00000000 --- a/cdist/conf/type/__install_file/man.text +++ /dev/null @@ -1 +0,0 @@ -../__file/man.text \ No newline at end of file diff --git a/cdist/conf/type/__install_fstab/man.text b/cdist/conf/type/__install_fstab/man.rst similarity index 54% rename from cdist/conf/type/__install_fstab/man.text rename to cdist/conf/type/__install_fstab/man.rst index 7c509427..5562c139 100644 --- a/cdist/conf/type/__install_fstab/man.text +++ b/cdist/conf/type/__install_fstab/man.rst @@ -1,7 +1,5 @@ cdist-type__install_fstab(7) ============================ -Steven Armstrong - NAME ---- @@ -16,12 +14,12 @@ to the target machine at ${prefix}/etc/fstab. REQUIRED PARAMETERS ------------------- -None. +None OPTIONAL PARAMETERS ------------------- -prefix:: +prefix The prefix under which to generate the /etc/fstab file. Defaults to /target. @@ -29,20 +27,27 @@ prefix:: EXAMPLES -------- --------------------------------------------------------------------------------- -__install_fstab -__install_fstab --prefix /mnt/target --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_fstab + + __install_fstab --prefix /mnt/target SEE ALSO -------- -- cdist-type(7) -- cdist-type__install_mount(7) -- cdist-type__install_generate_fstab(7) +:strong:`cdist-type__install_generate_fstab`\ (7), +:strong:`cdist-type__install_mount`\ (7) + + +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_generate_fstab/man.text b/cdist/conf/type/__install_generate_fstab/man.rst similarity index 57% rename from cdist/conf/type/__install_generate_fstab/man.text rename to cdist/conf/type/__install_generate_fstab/man.rst index d229f4df..b38f8876 100644 --- a/cdist/conf/type/__install_generate_fstab/man.text +++ b/cdist/conf/type/__install_generate_fstab/man.rst @@ -1,7 +1,5 @@ cdist-type__install_generate_fstab(7) ===================================== -Steven Armstrong - NAME ---- @@ -16,37 +14,40 @@ __install_mount definitions. REQUIRED PARAMETERS ------------------- -destination:: +destination The path where to store the generated fstab file. Note that this is a path on the server, where cdist is running, not the target host. OPTIONAL PARAMETERS ------------------- -None. +None BOOLEAN PARAMETERS ------------------- -uuid:: +uuid use UUID instead of device in fstab EXAMPLES -------- --------------------------------------------------------------------------------- -__install_generate_fstab --destination /path/where/you/want/fstab -__install_generate_fstab --uuid --destination /path/where/you/want/fstab --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_generate_fstab --destination /path/where/you/want/fstab + + __install_generate_fstab --uuid --destination /path/where/you/want/fstab -SEE ALSO --------- -- cdist-type(7) +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2012 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2012 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_mkfs/man.rst b/cdist/conf/type/__install_mkfs/man.rst new file mode 100644 index 00000000..6e5c9aa9 --- /dev/null +++ b/cdist/conf/type/__install_mkfs/man.rst @@ -0,0 +1,62 @@ +cdist-type__install_mkfs(7) +=========================== + +NAME +---- +cdist-type__install_mkfs - build a linux file system + + +DESCRIPTION +----------- +This cdist type is a wrapper for the mkfs command. + + +REQUIRED PARAMETERS +------------------- +type + The filesystem type to use. Same as used with mkfs -t. + + +OPTIONAL PARAMETERS +------------------- +device + defaults to object_id + +options + file system-specific options to be passed to the mkfs command + +blocks + the number of blocks to be used for the file system + + +EXAMPLES +-------- + +.. code-block:: sh + + # reiserfs /dev/sda5 + __install_mkfs /dev/sda5 --type reiserfs + + # same thing with explicit device + __install_mkfs whatever --device /dev/sda5 --type reiserfs + + # jfs with journal on /dev/sda2 + __install_mkfs /dev/sda1 --type jfs --options "-j /dev/sda2" + + +SEE ALSO +-------- +:strong:`mkfs`\ (8) + + +AUTHORS +------- +Steven Armstrong + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_mkfs/man.text b/cdist/conf/type/__install_mkfs/man.text deleted file mode 100644 index 3a9a325d..00000000 --- a/cdist/conf/type/__install_mkfs/man.text +++ /dev/null @@ -1,57 +0,0 @@ -cdist-type__install_mkfs(7) -=========================== -Steven Armstrong - - -NAME ----- -cdist-type__install_mkfs - build a linux file system - - -DESCRIPTION ------------ -This cdist type is a wrapper for the mkfs command. - - -REQUIRED PARAMETERS -------------------- -type:: - The filesystem type to use. Same as used with mkfs -t. - - -OPTIONAL PARAMETERS -------------------- -device:: - defaults to object_id - -options:: - file system-specific options to be passed to the mkfs command - -blocks:: - the number of blocks to be used for the file system - - -EXAMPLES --------- - --------------------------------------------------------------------------------- -# reiserfs /dev/sda5 -__install_mkfs /dev/sda5 --type reiserfs -# same thing with explicit device -__install_mkfs whatever --device /dev/sda5 --type reiserfs - -# jfs with journal on /dev/sda2 -__install_mkfs /dev/sda1 --type jfs --options "-j /dev/sda2" --------------------------------------------------------------------------------- - - -SEE ALSO --------- -- cdist-type(7) -- mkfs(8) - - -COPYING -------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_mount/man.text b/cdist/conf/type/__install_mount/man.rst similarity index 54% rename from cdist/conf/type/__install_mount/man.text rename to cdist/conf/type/__install_mount/man.rst index b55cb83e..256cef53 100644 --- a/cdist/conf/type/__install_mount/man.text +++ b/cdist/conf/type/__install_mount/man.rst @@ -1,7 +1,5 @@ cdist-type__install_mount(7) ============================ -Steven Armstrong - NAME ---- @@ -15,24 +13,24 @@ Mounts filesystems in the installer. Collects data to generate /etc/fstab. REQUIRED PARAMETERS ------------------- -device:: +device the device to mount OPTIONAL PARAMETERS ------------------- -dir:: +dir where to mount device. Defaults to object_id. -options:: +options mount options passed to mount(8) and used in /etc/fstab -type:: +type filesystem type passed to mount(8) and used in /etc/fstab. If type is swap, 'dir' is ignored. Defaults to the filesystem used in __install_mkfs for the same 'device'. -prefix:: +prefix the prefix to prepend to 'dir' when mounting in the installer. Defaults to /target. @@ -40,22 +38,28 @@ prefix:: EXAMPLES -------- --------------------------------------------------------------------------------- -__install_mount slash --dir / --device /dev/sda5 --options noatime -require="__install_mount/slash" __install_mount /boot --device /dev/sda1 -__install_mount swap --device /dev/sda2 --type swap -require="__install_mount/slash" __install_mount /tmp --device tmpfs --type tmpfs --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_mount slash --dir / --device /dev/sda5 --options noatime + require="__install_mount/slash" __install_mount /boot --device /dev/sda1 + __install_mount swap --device /dev/sda2 --type swap + require="__install_mount/slash" __install_mount /tmp --device tmpfs --type tmpfs SEE ALSO -------- -- cdist-type(7) -- cdist-type__install_mount_apply(7) -- cdist-type__install_mkfs(7) +:strong:`cdist-type__install_mkfs`\ (7), +:strong:`cdist-type__install_mount_apply` (7) + + +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_partition_msdos/man.rst b/cdist/conf/type/__install_partition_msdos/man.rst new file mode 100644 index 00000000..5ebb9218 --- /dev/null +++ b/cdist/conf/type/__install_partition_msdos/man.rst @@ -0,0 +1,64 @@ +cdist-type__install_partition_msdos(7) +====================================== + +NAME +---- +cdist-type__install_partition_msdos - creates msdos partitions + + +DESCRIPTION +----------- +This cdist type allows you to create msdos paritions. + + +REQUIRED PARAMETERS +------------------- +type + the partition type used in fdisk (such as 82 or 83) or "extended" + + +OPTIONAL PARAMETERS +------------------- +partition + defaults to object_id + +bootable + mark partition as bootable, true or false, defaults to false + +size + the size of the partition (such as 32M or 15G, whole numbers + only), '+' for remaining space, or 'n%' for percentage of remaining + (these should only be used after all specific partition sizes are + specified). Defaults to +. + + +EXAMPLES +-------- + +.. code-block:: sh + + # 128MB, linux, bootable + __install_partition_msdos /dev/sda1 --type 83 --size 128M --bootable true + # 512MB, swap + __install_partition_msdos /dev/sda2 --type 82 --size 512M + # 100GB, extended + __install_partition_msdos /dev/sda3 --type extended --size 100G + # 10GB, linux + __install_partition_msdos /dev/sda5 --type 83 --size 10G + # 50% of the free space of the extended partition, linux + __install_partition_msdos /dev/sda6 --type 83 --size 50% + # rest of the extended partition, linux + __install_partition_msdos /dev/sda7 --type 83 --size + + + +AUTHORS +------- +Steven Armstrong + + +COPYING +------- +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_partition_msdos/man.text b/cdist/conf/type/__install_partition_msdos/man.text deleted file mode 100644 index 82d81ac5..00000000 --- a/cdist/conf/type/__install_partition_msdos/man.text +++ /dev/null @@ -1,62 +0,0 @@ -cdist-type__install_partition_msdos(7) -====================================== -Steven Armstrong - - -NAME ----- -cdist-type__install_partition_msdos - creates msdos partitions - - -DESCRIPTION ------------ -This cdist type allows you to create msdos paritions. - - -REQUIRED PARAMETERS -------------------- -type:: - the partition type used in fdisk (such as 82 or 83) or "extended" - - -OPTIONAL PARAMETERS -------------------- -partition:: - defaults to object_id -bootable:: - mark partition as bootable, true or false, defaults to false -size:: - the size of the partition (such as 32M or 15G, whole numbers - only), '+' for remaining space, or 'n%' for percentage of remaining - (these should only be used after all specific partition sizes are - specified). Defaults to +. - - -EXAMPLES --------- - --------------------------------------------------------------------------------- -# 128MB, linux, bootable -__install_partition_msdos /dev/sda1 --type 83 --size 128M --bootable true -# 512MB, swap -__install_partition_msdos /dev/sda2 --type 82 --size 512M -# 100GB, extended -__install_partition_msdos /dev/sda3 --type extended --size 100G -# 10GB, linux -__install_partition_msdos /dev/sda5 --type 83 --size 10G -# 50% of the free space of the extended partition, linux -__install_partition_msdos /dev/sda6 --type 83 --size 50% -# rest of the extended partition, linux -__install_partition_msdos /dev/sda7 --type 83 --size + --------------------------------------------------------------------------------- - - -SEE ALSO --------- -- cdist-type(7) - - -COPYING -------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_partition_msdos_apply/man.text b/cdist/conf/type/__install_partition_msdos_apply/man.rst similarity index 52% rename from cdist/conf/type/__install_partition_msdos_apply/man.text rename to cdist/conf/type/__install_partition_msdos_apply/man.rst index 5399afb7..80740fde 100644 --- a/cdist/conf/type/__install_partition_msdos_apply/man.text +++ b/cdist/conf/type/__install_partition_msdos_apply/man.rst @@ -1,7 +1,5 @@ cdist-type__install_partition_msdos_apply(7) ============================================ -Steven Armstrong - NAME ---- @@ -20,23 +18,30 @@ None OPTIONAL PARAMETERS ------------------- -None. +None EXAMPLES -------- --------------------------------------------------------------------------------- -__install_partition_msdos_apply --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_partition_msdos_apply SEE ALSO -------- -- cdist-type(7) -- cdist-type__install_partition_msdos_apply(7) +:strong:`cdist-type__install_partition_msdos_apply`\ (7) + + +AUTHORS +------- +Steven Armstrong + COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_reboot/man.text b/cdist/conf/type/__install_reboot/man.rst similarity index 52% rename from cdist/conf/type/__install_reboot/man.text rename to cdist/conf/type/__install_reboot/man.rst index 91aec19a..ecf78ca7 100644 --- a/cdist/conf/type/__install_reboot/man.text +++ b/cdist/conf/type/__install_reboot/man.rst @@ -1,7 +1,5 @@ cdist-type__install_reboot(7) ============================= -Steven Armstrong - NAME ---- @@ -15,29 +13,31 @@ This cdist type allows you to reboot a machine. REQUIRED PARAMETERS ------------------- -None. +None OPTIONAL PARAMETERS ------------------- -options:: +options options to pass to the reboot command. e.g. -f EXAMPLES -------- --------------------------------------------------------------------------------- -__install_reboot --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_reboot -SEE ALSO --------- -- cdist-type(7) +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_reset_disk/man.text b/cdist/conf/type/__install_reset_disk/man.rst similarity index 50% rename from cdist/conf/type/__install_reset_disk/man.text rename to cdist/conf/type/__install_reset_disk/man.rst index 542d68ba..fadeec71 100644 --- a/cdist/conf/type/__install_reset_disk/man.text +++ b/cdist/conf/type/__install_reset_disk/man.rst @@ -1,7 +1,5 @@ cdist-type__install_reset_disk(7) ================================= -Steven Armstrong - NAME ---- @@ -17,27 +15,29 @@ Remove mdadm superblock. REQUIRED PARAMETERS ------------------- -None. +None OPTIONAL PARAMETERS ------------------- -None. +None EXAMPLES -------- --------------------------------------------------------------------------------- -__install_reset_disk /dev/sdb --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_reset_disk /dev/sdb -SEE ALSO --------- -- cdist-type(7) +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2012 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2012 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_stage/man.rst b/cdist/conf/type/__install_stage/man.rst new file mode 100644 index 00000000..6c68c543 --- /dev/null +++ b/cdist/conf/type/__install_stage/man.rst @@ -0,0 +1,58 @@ +cdist-type__install_stage(7) +============================ + +NAME +---- +cdist-type__install_stage - download and unpack a stage file + + +DESCRIPTION +----------- +Downloads a operating system stage using curl and unpacks it to /target +using tar. The stage tarball is expected to be gzip compressed. + + +REQUIRED PARAMETERS +------------------- +uri + The uri from which to fetch the tarball. + Can be anything understood by curl, e.g: + | http://path/to/stage.tgz + | tftp:///path/to/stage.tgz + | file:///local/path/stage.tgz + + +OPTIONAL PARAMETERS +------------------- +target + where to unpack the tarball to. Defaults to /target. + + +BOOLEAN PARAMETERS +------------------ +insecure + run curl in insecure mode so it does not check the servers ssl certificate + + +EXAMPLES +-------- + +.. code-block:: sh + + __install_stage --uri tftp:///path/to/stage.tgz + __install_stage --uri http://path/to/stage.tgz --target /mnt/foobar + __install_stage --uri file:///path/to/stage.tgz --target /target + __install_stage --uri https://path/to/stage.tgz --target /mnt/foobar --insecure + + +AUTHORS +------- +Steven Armstrong + + +COPYING +------- +Copyright \(C) 2011 - 2013 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_stage/man.text b/cdist/conf/type/__install_stage/man.text deleted file mode 100644 index 289c8621..00000000 --- a/cdist/conf/type/__install_stage/man.text +++ /dev/null @@ -1,58 +0,0 @@ -cdist-type__install_stage(7) -============================ -Steven Armstrong - - -NAME ----- -cdist-type__install_stage - download and unpack a stage file - - -DESCRIPTION ------------ -Downloads a operating system stage using curl and unpacks it to /target -using tar. The stage tarball is expected to be gzip compressed. - - -REQUIRED PARAMETERS -------------------- -uri:: - The uri from which to fetch the tarball. - Can be anything understood by curl, e.g: - http://path/to/stage.tgz - tftp:///path/to/stage.tgz - file:///local/path/stage.tgz - - -OPTIONAL PARAMETERS -------------------- -target:: - where to unpack the tarball to. Defaults to /target. - - -BOOLEAN PARAMETERS ------------------- -insecure:: - run curl in insecure mode so it does not check the servers ssl certificate - - -EXAMPLES --------- - --------------------------------------------------------------------------------- -__install_stage --uri tftp:///path/to/stage.tgz -__install_stage --uri http://path/to/stage.tgz --target /mnt/foobar -__install_stage --uri file:///path/to/stage.tgz --target /target -__install_stage --uri https://path/to/stage.tgz --target /mnt/foobar --insecure --------------------------------------------------------------------------------- - - -SEE ALSO --------- -- cdist-type(7) - - -COPYING -------- -Copyright \(C) 2011 - 2013 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__install_umount/man.text b/cdist/conf/type/__install_umount/man.rst similarity index 54% rename from cdist/conf/type/__install_umount/man.text rename to cdist/conf/type/__install_umount/man.rst index 8d9d1f55..59f63449 100644 --- a/cdist/conf/type/__install_umount/man.text +++ b/cdist/conf/type/__install_umount/man.rst @@ -1,7 +1,5 @@ cdist-type__install_umount(7) ============================= -Steven Armstrong - NAME ---- @@ -15,29 +13,31 @@ This cdist type allows you to recursively umount the given target directory. REQUIRED PARAMETERS ------------------- -None. +None OPTIONAL PARAMETERS ------------------- -target:: +target the mount point to umount. Defaults to object_id EXAMPLES -------- --------------------------------------------------------------------------------- -__install_umount /target --------------------------------------------------------------------------------- +.. code-block:: sh + + __install_umount /target -SEE ALSO --------- -- cdist-type(7) +AUTHORS +------- +Steven Armstrong COPYING ------- -Copyright \(C) 2011 Steven Armstrong. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2011 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. From 93c80c9f4d7336fe8f79420ca7bd8cc117933557 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 15 Oct 2016 19:04:14 +0200 Subject: [PATCH 123/197] Remove outdated __cdist_preos. --- cdist/conf/type/__cdist_preos/man.rst | 48 ----------- cdist/conf/type/__cdist_preos/manifest | 79 ------------------- .../__cdist_preos/parameter/default/branch | 1 - .../__cdist_preos/parameter/default/source | 1 - .../__cdist_preos/parameter/default/username | 1 - .../type/__cdist_preos/parameter/optional | 4 - 6 files changed, 134 deletions(-) delete mode 100644 cdist/conf/type/__cdist_preos/man.rst delete mode 100755 cdist/conf/type/__cdist_preos/manifest delete mode 100644 cdist/conf/type/__cdist_preos/parameter/default/branch delete mode 100644 cdist/conf/type/__cdist_preos/parameter/default/source delete mode 100644 cdist/conf/type/__cdist_preos/parameter/default/username delete mode 100644 cdist/conf/type/__cdist_preos/parameter/optional diff --git a/cdist/conf/type/__cdist_preos/man.rst b/cdist/conf/type/__cdist_preos/man.rst deleted file mode 100644 index 8b7d22d6..00000000 --- a/cdist/conf/type/__cdist_preos/man.rst +++ /dev/null @@ -1,48 +0,0 @@ -cdist-type__cdist_preos(7) -========================== - -NAME ----- -cdist-type__cdist - Manage cdist installations - - -DESCRIPTION ------------ -This cdist type creates a directory containing an operating -suitable for installation using cdist. - - -REQUIRED PARAMETERS -------------------- -None - - -OPTIONAL PARAMETERS -------------------- -None - - -EXAMPLES --------- - -.. code-block:: sh - - __cdist_preos /tmp/random_name_for_packaging - - -SEE ALSO --------- -:strong:`cdist-type__cdist`\ (7) - - -AUTHORS -------- -Nico Schottelius - - -COPYING -------- -Copyright \(C) 2015 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. diff --git a/cdist/conf/type/__cdist_preos/manifest b/cdist/conf/type/__cdist_preos/manifest deleted file mode 100755 index 78166b38..00000000 --- a/cdist/conf/type/__cdist_preos/manifest +++ /dev/null @@ -1,79 +0,0 @@ -#!/bin/sh -# -# 2015 Nico Schottelius (nico-cdist at schottelius.org) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# - -destination="/$__object_id" - -os=$(cat "$__global/explorer/os") - -case "$os" in - archlinux) - kernel=/boot/vmlinuz-linux - initramfs=/boot/initramfs-linux-fallback.img - required_pkg="cdrkit syslinux" - ;; - *) - echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 - echo "Please contribute an implementation for it if you can." >&2 - exit 1 - ;; -esac - -# Our root -__directory "$destination" \ - --mode 0755 - -for rootdir in boot bin etc lib; do - require="__directory/$destination" __directory "$destination/$rootdir" \ - --mode 0755 -done - -require="__directory/$destination/etc" __cdistmarker \ - --destination "$destination/etc/cdist-configured" - -for pkg in $required_pkg; do - __package "$pkg" --state present -done - -# Create full dependency chain, because we don't know which file depends on which package -export CDIST_ORDER_DEPENDENCY=1 - -require="__directory/$destination/boot" __file "$destination/boot/linux" \ - --source "$kernel" --mode 0644 - -require="__directory/$destination/boot" __file "$destination/boot/initramfs" \ - --source "$initramfs" --mode 0644 - -require="__directory/$destination/boot" __file "$destination/boot/syslinux.cfg" \ - - - PROMPT 1 - TIMEOUT 50 - DEFAULT arch - - LABEL arch - LINUX ../vmlinuz-linux - APPEND root=/dev/sda2 rw - INITRD ../initramfs-linux.img - - LABEL archfallback - LINUX ../vmlinuz-linux - APPEND root=/dev/sda2 rw - INITRD ../initramfs-linux-fallback.img diff --git a/cdist/conf/type/__cdist_preos/parameter/default/branch b/cdist/conf/type/__cdist_preos/parameter/default/branch deleted file mode 100644 index 1f7391f9..00000000 --- a/cdist/conf/type/__cdist_preos/parameter/default/branch +++ /dev/null @@ -1 +0,0 @@ -master diff --git a/cdist/conf/type/__cdist_preos/parameter/default/source b/cdist/conf/type/__cdist_preos/parameter/default/source deleted file mode 100644 index d669308f..00000000 --- a/cdist/conf/type/__cdist_preos/parameter/default/source +++ /dev/null @@ -1 +0,0 @@ -git://github.com/telmich/cdist.git diff --git a/cdist/conf/type/__cdist_preos/parameter/default/username b/cdist/conf/type/__cdist_preos/parameter/default/username deleted file mode 100644 index a585e141..00000000 --- a/cdist/conf/type/__cdist_preos/parameter/default/username +++ /dev/null @@ -1 +0,0 @@ -cdist diff --git a/cdist/conf/type/__cdist_preos/parameter/optional b/cdist/conf/type/__cdist_preos/parameter/optional deleted file mode 100644 index a5f14343..00000000 --- a/cdist/conf/type/__cdist_preos/parameter/optional +++ /dev/null @@ -1,4 +0,0 @@ -branch -source -username -shell From 75e85379f6f2c6490172c859574569d0fd1fce83 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 27 Oct 2016 18:34:24 +0200 Subject: [PATCH 124/197] Order subcommands alphabetically. --- scripts/cdist | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/cdist b/scripts/cdist index 9f8d326d..68084ca4 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -159,6 +159,11 @@ def commandline(): default=os.environ.get('CDIST_REMOTE_EXEC')) parser['config'].set_defaults(func=cdist.config.Config.commandline) + # Install + parser['install'] = parser['sub'].add_parser('install', add_help=False, + parents=[parser['config']]) + parser['install'].set_defaults(func=cdist.install.Install.commandline) + # Shell parser['shell'] = parser['sub'].add_parser( 'shell', parents=[parser['loglevel']]) @@ -168,11 +173,6 @@ def commandline(): ' should be POSIX compatible shell.')) parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) - # Install - parser['install'] = parser['sub'].add_parser('install', add_help=False, - parents=[parser['config']]) - parser['install'].set_defaults(func=cdist.install.Install.commandline) - for p in parser: parser[p].epilog = ( "Get cdist at http://www.nico.schottelius.org/software/cdist/") From c293a9b2cea5c59a3098d49c3df7e4d2c71a3956 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 1 Nov 2016 08:11:37 +0100 Subject: [PATCH 125/197] Add missing param. --- cdist/install.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/install.py b/cdist/install.py index bec8d24a..b88ad016 100644 --- a/cdist/install.py +++ b/cdist/install.py @@ -30,7 +30,8 @@ class Install(cdist.config.Config): In install mode, we only care about install objects. """ for cdist_object in cdist.core.CdistObject.list_objects( - self.local.object_path, self.local.type_path): + self.local.object_path, self.local.type_path, + self.local.object_marker_name): if cdist_object.cdist_type.is_install: yield cdist_object else: From ca9dd7338adfd371705f27793d8bd20e99762d8d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 2 Nov 2016 13:35:48 +0100 Subject: [PATCH 126/197] Support IPv6 in python code. --- cdist/exec/remote.py | 43 ++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 9c70bdf4..f374262f 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -32,6 +32,43 @@ import cdist import cdist.exec.util as exec_util +# check whether addr is IPv6 +try: + # python 3.3+ + import ipaddress + + def _is_ipv6(addr): + try: + return ipaddress.ip_address(addr).version == 6 + except ValueError: + return False +except ImportError: + # fallback for older python versions + import socket + + def _is_ipv6(addr): + try: + socket.inet_aton(addr) + return False + except socket.error: + pass + try: + socket.inet_pton(socket.AF_INET6, addr) + return True + except socket.error: + pass + return False + + +def _wrap_addr(addr): + """If addr is IPv6 then return addr wrapped between '[' and ']', + otherwise return it intact.""" + if _is_ipv6(addr): + return "".join(("[", addr, "]", )) + else: + return addr + + class DecodeError(cdist.Error): def __init__(self, command): self.command = command @@ -118,12 +155,12 @@ class Remote(object): command = self._copy.split() path = os.path.join(source, f) command.extend([path, '{0}:{1}'.format( - self.target_host[0], destination)]) + _wrap_addr(self.target_host[0]), destination)]) self._run_command(command) else: command = self._copy.split() command.extend([source, '{0}:{1}'.format( - self.target_host[0], destination)]) + _wrap_addr(self.target_host[0]), destination)]) self._run_command(command) def transfer_dir_parallel(self, source, destination, jobs): @@ -145,7 +182,7 @@ class Remote(object): command = self._copy.split() path = os.path.join(source, f) command.extend([path, '{0}:{1}'.format( - self.target_host[0], destination)]) + _wrap_addr(self.target_host[0]), destination)]) commands.append(command) results = [ pool.apply_async(self._run_command, (cmd,)) From 4ddf6557e3d9b269650c35ee7e11040b4e747932 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 2 Nov 2016 13:55:25 +0100 Subject: [PATCH 127/197] IPv6 fix in gencode scripts. --- cdist/conf/type/__file/gencode-local | 11 ++++++++++- cdist/conf/type/__jail_freebsd10/gencode-local | 11 ++++++++++- cdist/conf/type/__jail_freebsd9/gencode-local | 11 ++++++++++- cdist/conf/type/__pf_ruleset/gencode-local | 14 ++++++++++++-- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index ed7482cb..674650b7 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -66,8 +66,17 @@ destination_upload="\$($__remote_exec $__target_host "mktemp $tempfile_template" DONE if [ "$upload_file" ]; then echo upload >> "$__messages_out" + # IPv6 fix + case "${__target_host}" in + *:*) + my_target_host="[${__target_host}]" + ;; + *) + my_target_host="${__target_host}" + ;; + esac cat << DONE -$__remote_copy "$source" "${__target_host}:\$destination_upload" +$__remote_copy "$source" "${my_target_host}:\$destination_upload" DONE fi # move uploaded file into place diff --git a/cdist/conf/type/__jail_freebsd10/gencode-local b/cdist/conf/type/__jail_freebsd10/gencode-local index d4b89730..766692a2 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-local +++ b/cdist/conf/type/__jail_freebsd10/gencode-local @@ -43,7 +43,16 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then - echo "$__remote_copy" "${jailbase}" "$__target_host:${remotebase}" + # IPv6 fix + case "${__target_host}" in + *:*) + my_target_host="[${__target_host}]" + ;; + *) + my_target_host="${__target_host}" + ;; + esac + echo "$__remote_copy" "${jailbase}" "${my_target_host}:${remotebase}" fi # basepresent=NONE fi # state=present diff --git a/cdist/conf/type/__jail_freebsd9/gencode-local b/cdist/conf/type/__jail_freebsd9/gencode-local index 08c7b7bf..35b6f789 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-local +++ b/cdist/conf/type/__jail_freebsd9/gencode-local @@ -39,7 +39,16 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then - echo "$__remote_copy" "${jailbase}" "$__target_host:${remotebase}" + # IPv6 fix + case "${__target_host}" in + *:*) + my_target_host="[${__target_host}]" + ;; + *) + my_target_host="${__target_host}" + ;; + esac + echo "$__remote_copy" "${jailbase}" "${my_target_host}:${remotebase}" fi # basepresent=NONE fi # state=present diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local index c2495509..307c89ec 100644 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -59,12 +59,22 @@ case $uname in ;; esac +# IPv6 fix +case "${__target_host}" in + *:*) + my_target_host="[${__target_host}]" + ;; + *) + my_target_host="${__target_host}" + ;; +esac + if [ -n "${cksum}" ]; then if [ ! "\${currentSum}" = "${cksum}" ]; then - $__remote_copy "${source}" "$__target_host:${rcvar}.new" + $__remote_copy "${source}" "${my_target_host}:${rcvar}.new" fi else # File just doesn't exist yet - $__remote_copy "${source}" "$__target_host:${rcvar}.new" + $__remote_copy "${source}" "${my_target_host}:${rcvar}.new" fi EOF From 9268062de5eab5c01fdb69acdba92d99228b807f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 3 Nov 2016 13:26:50 +0100 Subject: [PATCH 128/197] Fix target_host vars in Code. --- cdist/config.py | 1 + cdist/core/code.py | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index b0131601..b8d0672c 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -285,6 +285,7 @@ class Config(object): host_fqdn = '' target_host = (host, host_name, host_fqdn) + log.debug("target_host: {}".format(target_host)) local = cdist.exec.local.Local( target_host=target_host, diff --git a/cdist/core/code.py b/cdist/core/code.py index cfc1316a..e9e2edf0 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -100,9 +100,7 @@ class Code(object): """ # target_host is tuple (target_host, target_hostname, target_fqdn) def __init__(self, target_host, local, remote): - self.target_host = target_host[0] - self.target_hostname = target_host[1] - self.target_fqdn = target_host[2] + self.target_host = target_host self.local = local self.remote = remote self.env = { From 39c3ac43ec7c278d72e573495ab6b9fbffc33883 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 4 Nov 2016 09:53:00 +0200 Subject: [PATCH 129/197] __package_upgrade_all shouldn't dist-upgrade by default. also add clean. --- .../type/__package_upgrade_all/gencode-remote | 15 +++++++++++++-- .../type/__package_upgrade_all/parameter/boolean | 2 ++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 cdist/conf/type/__package_upgrade_all/parameter/boolean diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index 9dd3ddf6..3e25f45f 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -24,6 +24,10 @@ type="$__object/parameter/type" +apt_clean="$__object/parameter/apt-clean" + +apt_dist_upgrade="$__object/parameter/apt-dist-upgrade" + if [ -f "$type" ]; then type="$(cat "$type")" else @@ -48,8 +52,15 @@ case "$type" in echo "yum --quiet clean all" ;; apt) - echo $aptget dist-upgrade - echo "apt-get --quiet autoclean" + if [ -f "$apt_dist_upgrade" ] + then echo $aptget dist-upgrade + else echo $aptget upgrade + fi + + if [ -f "$apt_clean" ] + then echo "apt-get --quiet clean" + else echo "apt-get --quiet autoclean" + fi ;; pacman) echo "pacman --noprogressbar --noconfirm --sync --sysupgrade" diff --git a/cdist/conf/type/__package_upgrade_all/parameter/boolean b/cdist/conf/type/__package_upgrade_all/parameter/boolean new file mode 100644 index 00000000..7a56a34b --- /dev/null +++ b/cdist/conf/type/__package_upgrade_all/parameter/boolean @@ -0,0 +1,2 @@ +apt-clean +apt-dist-upgrade From 4ef057c65dfb696342d1ca47ee07bd820d2057b9 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Fri, 4 Nov 2016 10:14:42 +0200 Subject: [PATCH 130/197] add docs --- cdist/conf/type/__package_upgrade_all/man.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/cdist/conf/type/__package_upgrade_all/man.rst b/cdist/conf/type/__package_upgrade_all/man.rst index 62cbc43d..e9e2b8ce 100644 --- a/cdist/conf/type/__package_upgrade_all/man.rst +++ b/cdist/conf/type/__package_upgrade_all/man.rst @@ -28,6 +28,15 @@ type * pacman for Arch Linux +BOOLEAN PARAMETERS +------------------ +apt-dist-upgrade + Do dist-upgrade instead of upgrade. + +apt-clean + Clean out the local repository of retrieved package files. + + EXAMPLES -------- From 6ce6c7830bcc0e360349d071b00f2aecbf099dd2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 5 Nov 2016 16:38:49 +0100 Subject: [PATCH 131/197] Better test for IPv6 address. --- cdist/conf/type/__file/gencode-local | 15 +++++++-------- cdist/conf/type/__jail_freebsd10/gencode-local | 15 +++++++-------- cdist/conf/type/__jail_freebsd9/gencode-local | 15 +++++++-------- cdist/conf/type/__pf_ruleset/gencode-local | 15 +++++++-------- 4 files changed, 28 insertions(+), 32 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 674650b7..c9e0fa2c 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -67,14 +67,13 @@ DONE if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix - case "${__target_host}" in - *:*) - my_target_host="[${__target_host}]" - ;; - *) - my_target_host="${__target_host}" - ;; - esac + echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + if [ $? -eq 0 ] + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi cat << DONE $__remote_copy "$source" "${my_target_host}:\$destination_upload" DONE diff --git a/cdist/conf/type/__jail_freebsd10/gencode-local b/cdist/conf/type/__jail_freebsd10/gencode-local index 766692a2..bdd0ffd7 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-local +++ b/cdist/conf/type/__jail_freebsd10/gencode-local @@ -44,14 +44,13 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - case "${__target_host}" in - *:*) - my_target_host="[${__target_host}]" - ;; - *) - my_target_host="${__target_host}" - ;; - esac + echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + if [ $? -eq 0 ] + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi echo "$__remote_copy" "${jailbase}" "${my_target_host}:${remotebase}" fi # basepresent=NONE fi # state=present diff --git a/cdist/conf/type/__jail_freebsd9/gencode-local b/cdist/conf/type/__jail_freebsd9/gencode-local index 35b6f789..ce9b215e 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-local +++ b/cdist/conf/type/__jail_freebsd9/gencode-local @@ -40,14 +40,13 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - case "${__target_host}" in - *:*) - my_target_host="[${__target_host}]" - ;; - *) - my_target_host="${__target_host}" - ;; - esac + echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' + if [ $? -eq 0 ] + then + my_target_host="[${__target_host}]" + else + my_target_host="${__target_host}" + fi echo "$__remote_copy" "${jailbase}" "${my_target_host}:${remotebase}" fi # basepresent=NONE fi # state=present diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local index 307c89ec..d14a6045 100644 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -60,14 +60,13 @@ case $uname in esac # IPv6 fix -case "${__target_host}" in - *:*) - my_target_host="[${__target_host}]" - ;; - *) - my_target_host="${__target_host}" - ;; -esac +echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' +if [ $? -eq 0 ] +then + my_target_host="[${__target_host}]" +else + my_target_host="${__target_host}" +fi if [ -n "${cksum}" ]; then if [ ! "\${currentSum}" = "${cksum}" ]; then From 28f2672c2d667918fe284996fe30823362d7ba43 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 6 Nov 2016 11:27:24 +0100 Subject: [PATCH 132/197] Update changelog. --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 9519f2b3..721bad6a 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Type __package_upgrade_all: do not dist-upgrade by default, add apt-clean and apt-dist-upgrade options (Ander Punnar) * All: Merge install feature from 4.0-pre-not-stable (Darko Poljak) 4.3.2: 2016-10-13 From 6fd9dac14511954ffa946d117ef2d6dbbf4985f5 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 7 Nov 2016 11:19:48 +0200 Subject: [PATCH 133/197] add __apt_mark --- cdist/conf/type/__apt_mark/gencode-remote | 37 +++++++++++++++ cdist/conf/type/__apt_mark/man.rst | 47 +++++++++++++++++++ cdist/conf/type/__apt_mark/parameter/optional | 1 + cdist/conf/type/__apt_mark/parameter/required | 1 + 4 files changed, 86 insertions(+) create mode 100644 cdist/conf/type/__apt_mark/gencode-remote create mode 100644 cdist/conf/type/__apt_mark/man.rst create mode 100644 cdist/conf/type/__apt_mark/parameter/optional create mode 100644 cdist/conf/type/__apt_mark/parameter/required diff --git a/cdist/conf/type/__apt_mark/gencode-remote b/cdist/conf/type/__apt_mark/gencode-remote new file mode 100644 index 00000000..76e87660 --- /dev/null +++ b/cdist/conf/type/__apt_mark/gencode-remote @@ -0,0 +1,37 @@ +#!/bin/sh +# +# 2016 Ander Punnar (cdist at kvlt.ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if [ -f "$__object/parameter/name" ]; then + name="$(cat "$__object/parameter/name")" +else + name="$__object_id" +fi + +state="$(cat "$__object/parameter/state")" + +case "$state" in + auto|manual|hold|unhold) + echo "apt-mark $state $name" + ;; + *) + echo "Unknown state: $state" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__apt_mark/man.rst b/cdist/conf/type/__apt_mark/man.rst new file mode 100644 index 00000000..8daed6e3 --- /dev/null +++ b/cdist/conf/type/__apt_mark/man.rst @@ -0,0 +1,47 @@ +cdist-type__apt_mark(7) +====================== + +NAME +---- +cdist-type__apt_mark - set package state as 'auto', 'manual', 'hold' or 'unhold' + + +DESCRIPTION +----------- +See apt-mark(8) for details. + + +REQUIRED PARAMETERS +------------------- +state + Possible states are 'auto', 'manual', 'hold' and 'unhold'. + + +OPTIONAL PARAMETERS +------------------- +name + If supplied, use the name and not the object id as the package name. + + +EXAMPLES +-------- + +.. code-block:: sh + + # hold package + __apt_mark quagga --state hold + # unhold package + __apt_mark quagga --state unhold + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2016 Ander Punnar. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_mark/parameter/optional b/cdist/conf/type/__apt_mark/parameter/optional new file mode 100644 index 00000000..f121bdbf --- /dev/null +++ b/cdist/conf/type/__apt_mark/parameter/optional @@ -0,0 +1 @@ +name diff --git a/cdist/conf/type/__apt_mark/parameter/required b/cdist/conf/type/__apt_mark/parameter/required new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/cdist/conf/type/__apt_mark/parameter/required @@ -0,0 +1 @@ +state From 6f69cd6a1178d225faa5bbfe80e4e2322e74022f Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 7 Nov 2016 13:36:27 +0200 Subject: [PATCH 134/197] fix man --- cdist/conf/type/__apt_mark/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_mark/man.rst b/cdist/conf/type/__apt_mark/man.rst index 8daed6e3..35ab2677 100644 --- a/cdist/conf/type/__apt_mark/man.rst +++ b/cdist/conf/type/__apt_mark/man.rst @@ -1,5 +1,5 @@ cdist-type__apt_mark(7) -====================== +======================= NAME ---- From 45e45016a7062c632e126ca74aef86e8526f24d7 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 7 Nov 2016 15:45:54 +0200 Subject: [PATCH 135/197] retrieve the mark before with an explorer --- cdist/conf/type/__apt_mark/explorer/state | 27 +++++++++++++++++++++++ cdist/conf/type/__apt_mark/gencode-remote | 14 ++++++++---- 2 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 cdist/conf/type/__apt_mark/explorer/state diff --git a/cdist/conf/type/__apt_mark/explorer/state b/cdist/conf/type/__apt_mark/explorer/state new file mode 100644 index 00000000..3b70003a --- /dev/null +++ b/cdist/conf/type/__apt_mark/explorer/state @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2016 Ander Punnar (cdist at kvlt.ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if [ -f "$__object/parameter/name" ]; then + name="$(cat "$__object/parameter/name")" +else + name="$__object_id" +fi + +apt-mark showhold | grep -q $name && echo hold || echo unhold diff --git a/cdist/conf/type/__apt_mark/gencode-remote b/cdist/conf/type/__apt_mark/gencode-remote index 76e87660..d83e96d9 100644 --- a/cdist/conf/type/__apt_mark/gencode-remote +++ b/cdist/conf/type/__apt_mark/gencode-remote @@ -24,14 +24,20 @@ else name="$__object_id" fi -state="$(cat "$__object/parameter/state")" +state_should="$(cat "$__object/parameter/state")" -case "$state" in +state_is="$(cat "$__object/explorer/state")" + +if [ "$state_should" = "$state_is" ]; then + exit 0 +fi + +case "$state_should" in auto|manual|hold|unhold) - echo "apt-mark $state $name" + echo "apt-mark $state_should $name" ;; *) - echo "Unknown state: $state" >&2 + echo "Unknown state: $state_should" >&2 exit 1 ;; esac From b3cf70ae4297af854ef3d7561d85357cdb048cf8 Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 7 Nov 2016 15:51:50 +0200 Subject: [PATCH 136/197] remove auto and manual states --- cdist/conf/type/__apt_mark/gencode-remote | 2 +- cdist/conf/type/__apt_mark/man.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__apt_mark/gencode-remote b/cdist/conf/type/__apt_mark/gencode-remote index d83e96d9..d97e2c7a 100644 --- a/cdist/conf/type/__apt_mark/gencode-remote +++ b/cdist/conf/type/__apt_mark/gencode-remote @@ -33,7 +33,7 @@ if [ "$state_should" = "$state_is" ]; then fi case "$state_should" in - auto|manual|hold|unhold) + hold|unhold) echo "apt-mark $state_should $name" ;; *) diff --git a/cdist/conf/type/__apt_mark/man.rst b/cdist/conf/type/__apt_mark/man.rst index 35ab2677..7aa2a519 100644 --- a/cdist/conf/type/__apt_mark/man.rst +++ b/cdist/conf/type/__apt_mark/man.rst @@ -3,7 +3,7 @@ cdist-type__apt_mark(7) NAME ---- -cdist-type__apt_mark - set package state as 'auto', 'manual', 'hold' or 'unhold' +cdist-type__apt_mark - set package state as 'hold' or 'unhold' DESCRIPTION @@ -14,7 +14,7 @@ See apt-mark(8) for details. REQUIRED PARAMETERS ------------------- state - Possible states are 'auto', 'manual', 'hold' and 'unhold'. + Either "hold" or "unhold". OPTIONAL PARAMETERS From 35975582f09b2a22e18e2ae71071b80c7d2fc40c Mon Sep 17 00:00:00 2001 From: Ander Punnar Date: Mon, 7 Nov 2016 16:55:51 +0200 Subject: [PATCH 137/197] suppress output --- cdist/conf/type/__apt_mark/gencode-remote | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__apt_mark/gencode-remote b/cdist/conf/type/__apt_mark/gencode-remote index d97e2c7a..14505809 100644 --- a/cdist/conf/type/__apt_mark/gencode-remote +++ b/cdist/conf/type/__apt_mark/gencode-remote @@ -34,7 +34,7 @@ fi case "$state_should" in hold|unhold) - echo "apt-mark $state_should $name" + echo "apt-mark $state_should $name > /dev/null" ;; *) echo "Unknown state: $state_should" >&2 From c4996396c689d5d53a46ec7230fbb31cd97dbac9 Mon Sep 17 00:00:00 2001 From: Kamila Souckova Date: Tue, 22 Nov 2016 18:21:03 +0100 Subject: [PATCH 138/197] __user type: fix for FreeBSD --- cdist/conf/type/__user/gencode-remote | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote index 463fbe49..223d4d46 100755 --- a/cdist/conf/type/__user/gencode-remote +++ b/cdist/conf/type/__user/gencode-remote @@ -104,7 +104,7 @@ if [ "$state" = "present" ]; then if [ $# -gt 0 ]; then echo mod >> "$__messages_out" if [ "$os" = "freebsd" ]; then - echo pw usermod "$@" "$name" + echo pw usermod "$@" -n "$name" else echo usermod "$@" "$name" fi @@ -125,7 +125,7 @@ if [ "$state" = "present" ]; then done if [ "$os" = "freebsd" ]; then - echo pw useradd "$@" "$name" + echo pw useradd "$@" -n "$name" else echo useradd "$@" "$name" fi From 6bc250399671ff51b3c111b4f2cb5ae57e0683da Mon Sep 17 00:00:00 2001 From: Carlos Ortigoza Dempster Date: Fri, 25 Nov 2016 14:15:00 -0400 Subject: [PATCH 139/197] Adding consul 0.7.0 files --- cdist/conf/type/__consul/files/versions/0.7.0/cksum | 1 + cdist/conf/type/__consul/files/versions/0.7.0/source | 1 + 2 files changed, 2 insertions(+) create mode 100644 cdist/conf/type/__consul/files/versions/0.7.0/cksum create mode 100644 cdist/conf/type/__consul/files/versions/0.7.0/source diff --git a/cdist/conf/type/__consul/files/versions/0.7.0/cksum b/cdist/conf/type/__consul/files/versions/0.7.0/cksum new file mode 100644 index 00000000..3bffeedb --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/0.7.0/cksum @@ -0,0 +1 @@ +695240564 24003648 consul diff --git a/cdist/conf/type/__consul/files/versions/0.7.0/source b/cdist/conf/type/__consul/files/versions/0.7.0/source new file mode 100644 index 00000000..ad610fc7 --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/0.7.0/source @@ -0,0 +1 @@ +https://releases.hashicorp.com/consul/0.7.0/consul_0.7.0_linux_amd64.zip From 39f69ddedf8392eef8c30002e47bd4ac25d384a8 Mon Sep 17 00:00:00 2001 From: Carlos Ortigoza Dempster Date: Fri, 25 Nov 2016 14:19:01 -0400 Subject: [PATCH 140/197] Adding consul 0.7.1 files to __consul type --- cdist/conf/type/__consul/files/versions/0.7.1/cksum | 1 + cdist/conf/type/__consul/files/versions/0.7.1/source | 1 + 2 files changed, 2 insertions(+) create mode 100644 cdist/conf/type/__consul/files/versions/0.7.1/cksum create mode 100644 cdist/conf/type/__consul/files/versions/0.7.1/source diff --git a/cdist/conf/type/__consul/files/versions/0.7.1/cksum b/cdist/conf/type/__consul/files/versions/0.7.1/cksum new file mode 100644 index 00000000..476bd9f6 --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/0.7.1/cksum @@ -0,0 +1 @@ +3128343188 28402769 consul diff --git a/cdist/conf/type/__consul/files/versions/0.7.1/source b/cdist/conf/type/__consul/files/versions/0.7.1/source new file mode 100644 index 00000000..6ba2e7bf --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/0.7.1/source @@ -0,0 +1 @@ +https://releases.hashicorp.com/consul/0.7.1/consul_0.7.1_linux_amd64.zip From 835e281c9c8229fca959bb18dfc5ca77dd4ae7d7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 27 Nov 2016 17:02:51 +0100 Subject: [PATCH 141/197] Notice IPv6 address [] notation. --- docs/src/cdist-remote-exec-copy.rst | 5 +++++ docs/src/cdist-type.rst | 6 ++++++ docs/src/man1/cdist.rst | 6 ++++++ 3 files changed, 17 insertions(+) diff --git a/docs/src/cdist-remote-exec-copy.rst b/docs/src/cdist-remote-exec-copy.rst index 882f0bc2..bb818310 100644 --- a/docs/src/cdist-remote-exec-copy.rst +++ b/docs/src/cdist-remote-exec-copy.rst @@ -17,6 +17,11 @@ passing them to cdist with the --remote-exec and/or --remote-copy arguments. For __remote_exec, the custom implementation must behave as if it where ssh. For __remote_copy, it must behave like scp. +Please notice, custom implementations should work like ssh/scp so __remote_copy +must support IPv6 addresses enclosed in square brackets. For __remote_exec you +must take into account that for some options (like -L) IPv6 addresses can be +specified by enclosed in square brackets (see :strong:`ssh`\ (1) and +:strong:`scp`\ (1)). With this simple interface the user can take total control of how cdist interacts with the target when required, while the default implementation diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 62694fd8..1ff82ce0 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -251,6 +251,12 @@ script, you can write to stderr: # Output to be saved by cdist for execution on the target echo "touch /etc/cdist-configured" +Notice: if you use __remote_copy or __remote_exec directly in your scripts +then for IPv6 address with __remote_copy execution you should enclose IPv6 +address in square brackets. The same applies to __remote_exec if it behaves +the same as ssh for some options where colon is a delimiter, as for -L ssh +option (see :strong:`ssh`\ (1) and :strong:`scp`\ (1)). + Variable access from the generated scripts ------------------------------------------ diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 45ce339e..f3cb672c 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -157,6 +157,12 @@ cdist/conf The distribution configuration directory. It contains official types and explorers. This path is relative to cdist installation directory. +NOTES +----- +cdist detects if host is specified by IPv6 address. If so then remote_copy +command is executed with host address enclosed in square brackets +(see :strong:`scp`\ (1)). + EXAMPLES -------- From ed4e18713201f19e1df1bf3f67cbdae1862e8f28 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 27 Nov 2016 17:11:14 +0100 Subject: [PATCH 142/197] Update changelog. --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 721bad6a..b5959d77 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,7 +2,11 @@ Changelog --------- next: + * Type __consul: add source and cksum files for Consul 0.7.0 and 0.7.1 (Carlos Ortigoza) + * Type __user: FreeBSD fix (Kamila Součková) + * New type: __apt_mark (Ander Punnar) * Type __package_upgrade_all: do not dist-upgrade by default, add apt-clean and apt-dist-upgrade options (Ander Punnar) + * Core: fix target_host vars (Darko Poljak) * All: Merge install feature from 4.0-pre-not-stable (Darko Poljak) 4.3.2: 2016-10-13 From 36244b8a11d56106e8a041cc7b066a5debbf826b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Nov 2016 08:02:33 +0100 Subject: [PATCH 143/197] Update changelog. --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index b5959d77..07278241 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Core, types: support IPv6 (Darko Poljak) * Type __consul: add source and cksum files for Consul 0.7.0 and 0.7.1 (Carlos Ortigoza) * Type __user: FreeBSD fix (Kamila Součková) * New type: __apt_mark (Ander Punnar) From ca424a34e6f94261d681a588e6403673b965fc28 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 28 Nov 2016 08:13:59 +0100 Subject: [PATCH 144/197] Fix shell IPv6 testing (for -e). --- cdist/conf/type/__file/gencode-local | 3 +-- cdist/conf/type/__jail_freebsd10/gencode-local | 3 +-- cdist/conf/type/__jail_freebsd9/gencode-local | 3 +-- cdist/conf/type/__pf_ruleset/gencode-local | 3 +-- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index c9e0fa2c..4caa6df6 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -67,8 +67,7 @@ DONE if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix - echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - if [ $? -eq 0 ] + if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__jail_freebsd10/gencode-local b/cdist/conf/type/__jail_freebsd10/gencode-local index bdd0ffd7..8c1687a9 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-local +++ b/cdist/conf/type/__jail_freebsd10/gencode-local @@ -44,8 +44,7 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - if [ $? -eq 0 ] + if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__jail_freebsd9/gencode-local b/cdist/conf/type/__jail_freebsd9/gencode-local index ce9b215e..67420a6f 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-local +++ b/cdist/conf/type/__jail_freebsd9/gencode-local @@ -40,8 +40,7 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' - if [ $? -eq 0 ] + if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local index d14a6045..2db2ae06 100644 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -60,8 +60,7 @@ case $uname in esac # IPv6 fix -echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' -if [ $? -eq 0 ] +if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') then my_target_host="[${__target_host}]" else From f473b671b1b885ec8c91258e447dff86801d5ed0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 29 Nov 2016 07:43:53 +0100 Subject: [PATCH 145/197] Update AUTHORS in cdist man page. --- docs/src/man1/cdist.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index f3cb672c..ff09ea2f 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -245,7 +245,8 @@ The following exit values shall be returned: AUTHORS ------- -Nico Schottelius +Originally written by Nico Schottelius +and Steven Armstrong . CAVEATS ------- From e7c3c157245d6a4d9aa9c36b92c9b768e50517b9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 30 Nov 2016 23:19:53 +0100 Subject: [PATCH 146/197] Deprecate -d, make -v log level counter. --- docs/src/man1/cdist.rst | 5 +++-- scripts/cdist | 30 ++++++++++++++++++++++-------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index ff09ea2f..08c49446 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -46,11 +46,12 @@ All commands accept the following options: .. option:: -d, --debug - Set log level to debug + Set log level to debug (deprecated, use -vvv instead) .. option:: -v, --verbose - Set log level to info, be more verbose + Increase log level, be more verbose. The order of levels from lowest to + greatest are: ERROR, WARNING, INFO, DEBUG. .. option:: -V, --version diff --git a/scripts/cdist b/scripts/cdist index 68084ca4..22acb033 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -21,6 +21,8 @@ # # +import collections +import logging # list of beta sub-commands BETA_COMMANDS = ['install', ] @@ -61,6 +63,15 @@ def check_beta(args_dict): raise cdist.CdistBetaRequired(cmd, arg) +_verbosity_level = { + 0: logging.ERROR, + 1: logging.WARNING, + 2: logging.INFO, +} +_verbosity_level = collections.defaultdict( + lambda: logging.DEBUG, _verbosity_level) + + def commandline(): """Parse command line""" import argparse @@ -78,11 +89,14 @@ def commandline(): # Options _all_ parsers have in common parser['loglevel'] = argparse.ArgumentParser(add_help=False) parser['loglevel'].add_argument( - '-d', '--debug', help='Set log level to debug', + '-d', '--debug', + help=('Set log level to debug (deprecated, use -vvv instead)'), action='store_true', default=False) parser['loglevel'].add_argument( - '-v', '--verbose', help='Set log level to info, be more verbose', - action='store_true', default=False) + '-v', '--verbose', + help=('Increase log level, be more verbose. The order of levels ' + 'from lowest to greatest are: ERROR, WARNING, INFO, DEBUG.'), + action='count', default=0) # Main subcommand parser parser['main'] = argparse.ArgumentParser( @@ -179,11 +193,12 @@ def commandline(): args = parser['main'].parse_args(sys.argv[1:]) - # Loglevels are handled globally in here and debug wins over verbose - if args.verbose: - logging.root.setLevel(logging.INFO) + # Loglevels are handled globally in here if args.debug: - logging.root.setLevel(logging.DEBUG) + log.warning("-d/--debug is deprecated, use -vvv instead") + args.verbose = 3 + + logging.root.setLevel(_verbosity_level[args.verbose]) log.debug(args) log.info("version %s" % cdist.VERSION) @@ -219,7 +234,6 @@ if __name__ == "__main__": exit_code = 0 try: - import logging import os import re import cdist From b81950741560acf12bd4258c61777cf6f6dd28b9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 30 Nov 2016 23:25:31 +0100 Subject: [PATCH 147/197] Better document -v option. --- docs/src/man1/cdist.rst | 5 +++-- scripts/cdist | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 08c49446..4907da55 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -50,8 +50,9 @@ All commands accept the following options: .. option:: -v, --verbose - Increase log level, be more verbose. The order of levels from lowest to - greatest are: ERROR, WARNING, INFO, DEBUG. + Increase log level, be more verbose. Use it more than onceto increase + log level. The order of levels from the lowest to the greatest are: + ERROR, WARNING, INFO, DEBUG. .. option:: -V, --version diff --git a/scripts/cdist b/scripts/cdist index 22acb033..7e755f4c 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -94,8 +94,9 @@ def commandline(): action='store_true', default=False) parser['loglevel'].add_argument( '-v', '--verbose', - help=('Increase log level, be more verbose. The order of levels ' - 'from lowest to greatest are: ERROR, WARNING, INFO, DEBUG.'), + help=('Increase log level, be more verbose. Use it more than once ' + 'to increase log level. The order of levels from the lowest ' + 'to the greatest are: ERROR, WARNING, INFO, DEBUG.'), action='count', default=0) # Main subcommand parser From ab71ab621d20849fa0924bfcd2f846fced5a738f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 30 Nov 2016 23:26:53 +0100 Subject: [PATCH 148/197] Fix typo. --- docs/src/man1/cdist.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 4907da55..4970994a 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -50,7 +50,7 @@ All commands accept the following options: .. option:: -v, --verbose - Increase log level, be more verbose. Use it more than onceto increase + Increase log level, be more verbose. Use it more than once to increase log level. The order of levels from the lowest to the greatest are: ERROR, WARNING, INFO, DEBUG. From 8e3281aa7c5cf83c1180f139dbcbcc59e8878268 Mon Sep 17 00:00:00 2001 From: Tomas Pospisek Date: Thu, 1 Dec 2016 09:09:57 +0100 Subject: [PATCH 149/197] rewrite man page in rst Only minimal changes needed. This was done to satisfy darko-poljak's request here: https://github.com/ungleich/cdist/pull/354#issuecomment-263492801 --- .../type/__package_dpkg/{man.text => man.rst} | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) rename cdist/conf/type/__package_dpkg/{man.text => man.rst} (62%) diff --git a/cdist/conf/type/__package_dpkg/man.text b/cdist/conf/type/__package_dpkg/man.rst similarity index 62% rename from cdist/conf/type/__package_dpkg/man.text rename to cdist/conf/type/__package_dpkg/man.rst index ae98be99..2af69341 100644 --- a/cdist/conf/type/__package_dpkg/man.text +++ b/cdist/conf/type/__package_dpkg/man.rst @@ -1,7 +1,5 @@ cdist-type__package_dpkg(7) ========================== -Tomas Pospisek - NAME ---- @@ -18,27 +16,29 @@ The object given to __package_dpkg must be the name of the deb package. REQUIRED PARAMETERS ------------------- -source:: +source path to the *.deb package EXAMPLES -------- --------------------------------------------------------------------------------- -# Install foo and bar packages -__package_dpkg --source /tmp/foo_0.1_all.deb foo_0.1_all.deb -__package_dpkg --source $__type/files/bar_1.4.deb bar_1.4.deb --------------------------------------------------------------------------------- +.. code-block:: sh + + # Install foo and bar packages + __package_dpkg --source /tmp/foo_0.1_all.deb foo_0.1_all.deb + __package_dpkg --source $__type/files/bar_1.4.deb bar_1.4.deb SEE ALSO -------- -- cdist-type(7) -- cdist-type__package(7) +:strong:`cdist-type`\ (7), :strong:`cdist-type__package`\ (7) +AUTHORS +------- +Tomas Pospisek COPYING ------- Copyright \(C) 2013 Tomas Pospisek. Free use of this software is granted under the terms of the GNU General Public License version 3 (GPLv3). -This type is based on __package_apt +This type is based on __package_apt. From 4514bf1bd49e616f179811dfa1f4d7d92efef0d9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 1 Dec 2016 09:14:51 +0100 Subject: [PATCH 150/197] greatest -> highest --- docs/src/man1/cdist.rst | 2 +- scripts/cdist | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 4970994a..5daedcd4 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -51,7 +51,7 @@ All commands accept the following options: .. option:: -v, --verbose Increase log level, be more verbose. Use it more than once to increase - log level. The order of levels from the lowest to the greatest are: + log level. The order of levels from the lowest to the highest are: ERROR, WARNING, INFO, DEBUG. .. option:: -V, --version diff --git a/scripts/cdist b/scripts/cdist index 7e755f4c..4ff64bb5 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -96,7 +96,7 @@ def commandline(): '-v', '--verbose', help=('Increase log level, be more verbose. Use it more than once ' 'to increase log level. The order of levels from the lowest ' - 'to the greatest are: ERROR, WARNING, INFO, DEBUG.'), + 'to the highest are: ERROR, WARNING, INFO, DEBUG.'), action='count', default=0) # Main subcommand parser From 60906c0228d8f844fd31a4434fd5969ae6fe77a3 Mon Sep 17 00:00:00 2001 From: Tomas Pospisek Date: Thu, 1 Dec 2016 09:27:56 +0100 Subject: [PATCH 151/197] rewrite man page in rst Only minimal changes needed. This was done to satisfy darko-poljak's request here: https://github.com/ungleich/cdist/pull/360#issuecomment-263491151 --- .../{man.text => man.rst} | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) rename cdist/conf/type/__postgres_extension/{man.text => man.rst} (58%) diff --git a/cdist/conf/type/__postgres_extension/man.text b/cdist/conf/type/__postgres_extension/man.rst similarity index 58% rename from cdist/conf/type/__postgres_extension/man.text rename to cdist/conf/type/__postgres_extension/man.rst index 6d722d68..39aa0837 100644 --- a/cdist/conf/type/__postgres_extension/man.text +++ b/cdist/conf/type/__postgres_extension/man.rst @@ -1,7 +1,5 @@ cdist-type__postgres_extension(7) ================================= -Tomas Pospisek - NAME ---- @@ -16,35 +14,42 @@ The object you need to pass to __postgres_extension consists of the database name and the extension name joined by a colon in the following form: +.. code-block:: + dbname:extension f.ex. +.. code-block:: + rails_test:unaccent OPTIONAL PARAMETERS ------------------- -state:: - Either "present" or "absent", defaults to "present" +state + either "present" or "absent", defaults to "present" EXAMPLES -------- --------------------------------------------------------------------------------- -__postgres_extension rails_test:unaccent -__postgres_extension --present rails_test:unaccent -__postgres_extension --absent rails_test:unaccent --------------------------------------------------------------------------------- +.. code-block:: sh + + __postgres_extension rails_test:unaccent + __postgres_extension --present rails_test:unaccent + __postgres_extension --absent rails_test:unaccent SEE ALSO -------- -- cdist-type(7) -- cdist-type__postgres_database(7) -- http://www.postgresql.org/docs/current/static/sql-createextension.html +:strong:`cdist-type`\ (7), :strong:`cdist-type__postgre_database`\ (7) +Postgres "Create Extension" documentation at: . + +AUTHOR +------- +Tomas Pospisek COPYING ------- From 267d8239c1b43a1485f5e2bbb1d8cc8bc040bfbc Mon Sep 17 00:00:00 2001 From: Tomas Pospisek Date: Thu, 1 Dec 2016 09:50:05 +0100 Subject: [PATCH 152/197] remove reference to cdist_type, use GPL3+ as requested by darko-poljak here: https://github.com/ungleich/cdist/pull/360#issuecomment-264110087 --- cdist/conf/type/__postgres_extension/man.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__postgres_extension/man.rst b/cdist/conf/type/__postgres_extension/man.rst index 39aa0837..ead51c83 100644 --- a/cdist/conf/type/__postgres_extension/man.rst +++ b/cdist/conf/type/__postgres_extension/man.rst @@ -43,7 +43,7 @@ EXAMPLES SEE ALSO -------- -:strong:`cdist-type`\ (7), :strong:`cdist-type__postgre_database`\ (7) +:strong:`cdist-type__postgre_database`\ (7) Postgres "Create Extension" documentation at: . @@ -53,5 +53,7 @@ Tomas Pospisek COPYING ------- -Copyright \(C) 2014 Tomas Pospisek. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2014 Tomas Pospisek. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. From 6bcfdec1545e862339ae0457e6ecd77593e8ca44 Mon Sep 17 00:00:00 2001 From: Tomas Pospisek Date: Thu, 1 Dec 2016 09:54:27 +0100 Subject: [PATCH 153/197] remove reference to cdist_type, use GPL3+ as requested by darko-poljak here: https://github.com/ungleich/cdist/pull/360#issuecomment-264110087 --- cdist/conf/type/__package_dpkg/man.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cdist/conf/type/__package_dpkg/man.rst b/cdist/conf/type/__package_dpkg/man.rst index 2af69341..fbd5b4b1 100644 --- a/cdist/conf/type/__package_dpkg/man.rst +++ b/cdist/conf/type/__package_dpkg/man.rst @@ -31,7 +31,7 @@ EXAMPLES SEE ALSO -------- -:strong:`cdist-type`\ (7), :strong:`cdist-type__package`\ (7) +:strong:`cdist-type__package`\ (7) AUTHORS ------- @@ -39,6 +39,8 @@ Tomas Pospisek COPYING ------- -Copyright \(C) 2013 Tomas Pospisek. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +Copyright \(C) 2013 Tomas Pospisek. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. This type is based on __package_apt. From 57d83be05452a59939b203398d4fa05c4a16c284 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 1 Dec 2016 15:40:34 +0100 Subject: [PATCH 154/197] Update changelog. --- docs/changelog | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 07278241..78c4069e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,9 +2,11 @@ Changelog --------- next: + * Core: deprecate -d option and make -v option log level counter (Darko Poljak) + * New type: __postgres_extension (Tomas Pospisek) * Core, types: support IPv6 (Darko Poljak) * Type __consul: add source and cksum files for Consul 0.7.0 and 0.7.1 (Carlos Ortigoza) - * Type __user: FreeBSD fix (Kamila Součková) + * Type __user: FreeBSD fix (Kamila Souckova) * New type: __apt_mark (Ander Punnar) * Type __package_upgrade_all: do not dist-upgrade by default, add apt-clean and apt-dist-upgrade options (Ander Punnar) * Core: fix target_host vars (Darko Poljak) From 4370efdbb86910bb060cb003550347252904ba67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Posp=C3=AD=C5=A1ek?= Date: Fri, 2 Dec 2016 09:39:53 +0100 Subject: [PATCH 155/197] Update man.rst fix title as requested here https://github.com/ungleich/cdist/pull/354#issuecomment-264117501 --- cdist/conf/type/__package_dpkg/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__package_dpkg/man.rst b/cdist/conf/type/__package_dpkg/man.rst index fbd5b4b1..65a695b5 100644 --- a/cdist/conf/type/__package_dpkg/man.rst +++ b/cdist/conf/type/__package_dpkg/man.rst @@ -1,5 +1,5 @@ cdist-type__package_dpkg(7) -========================== +=========================== NAME ---- From 3054bae8c2cd847b29416027ecfd38235309b4ab Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Fri, 2 Dec 2016 14:34:01 +0100 Subject: [PATCH 156/197] Added __docker type from asteven, implemented debian support --- cdist/conf/type/__docker/man.text | 51 ++++++++++++++ cdist/conf/type/__docker/manifest | 81 ++++++++++++++++++++++ cdist/conf/type/__docker/parameter/boolean | 1 + cdist/conf/type/__docker/singleton | 0 4 files changed, 133 insertions(+) create mode 100644 cdist/conf/type/__docker/man.text create mode 100755 cdist/conf/type/__docker/manifest create mode 100644 cdist/conf/type/__docker/parameter/boolean create mode 100644 cdist/conf/type/__docker/singleton diff --git a/cdist/conf/type/__docker/man.text b/cdist/conf/type/__docker/man.text new file mode 100644 index 00000000..566c2f4c --- /dev/null +++ b/cdist/conf/type/__docker/man.text @@ -0,0 +1,51 @@ +cdist-type__docker(7) +===================== +Steven Armstrong + + +NAME +---- +cdist-type__docker - install docker-engine + + +DESCRIPTION +----------- +Installs latest docker-engine package from dockerproject.org. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +experimentel:: + Install the experimentel docker-engine package instead of the latest stable release. + + +EXAMPLES +-------- + +-------------------------------------------------------------------------------- +__docker + +# experimentel +__docker --experimental +-------------------------------------------------------------------------------- + + +SEE ALSO +-------- +- cdist-type(7) + + +COPYING +------- +Copyright \(C) 2016 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest new file mode 100755 index 00000000..ba13b3e4 --- /dev/null +++ b/cdist/conf/type/__docker/manifest @@ -0,0 +1,81 @@ +#!/bin/sh +# +# 2016 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") + +case "$os" in + centos) + component="main" + if [ -f "$__object/parameter/experimental" ]; then + component="experimental" + fi + export CDIST_ORDER_DEPENDENCY=on + __yum_repo docker \ + --name 'Docker Repository' \ + --baseurl "https://yum.dockerproject.org/repo/$component/centos/\$releasever/" \ + --enabled \ + --gpgcheck \ + --gpgkey 'https://yum.dockerproject.org/gpg' + __package docker-engine + unset CDIST_ORDER_DEPENDENCY + ;; + ubuntu) + component="main" + if [ -f "$__object/parameter/experimental" ]; then + component="experimental" + fi + __package apparmor + __package ca-certificates + __package apt-transport-https + __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D + export CDIST_ORDER_DEPENDENCY=on + __apt_source docker \ + --uri https://apt.dockerproject.org/repo \ + --distribution "ubuntu-$(cat "$__global/explorer/lsb_codename")" \ + --component "$component" + __package docker-engine + unset CDIST_ORDER_DEPENDENCY + ;; + debian) + component="main" + if [ -f "$__object/parameter/experimental" ]; then + component="experimental" + fi + + __package apt-transport-https + __package ca-certificates + __package gnupg2 + __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D + export CDIST_ORDER_DEPENDENCY=on + __apt_source docker \ + --uri https://apt.dockerproject.org/repo \ + --distribution "debian-$(cat "$__global/explorer/lsb_codename")" \ + --component "$component" + __package docker-engine + unset CDIST_ORDER_DEPENDENCY + + ;; + *) + echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker/parameter/boolean b/cdist/conf/type/__docker/parameter/boolean new file mode 100644 index 00000000..9839eb20 --- /dev/null +++ b/cdist/conf/type/__docker/parameter/boolean @@ -0,0 +1 @@ +experimental diff --git a/cdist/conf/type/__docker/singleton b/cdist/conf/type/__docker/singleton new file mode 100644 index 00000000..e69de29b From 0fae040f77bc080a9f03f078495f10b09802e4f7 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 09:52:49 +0100 Subject: [PATCH 157/197] Update changelog: release 4.4.0. --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 78c4069e..d9939bf1 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.4.0: 2016-12-03 * Core: deprecate -d option and make -v option log level counter (Darko Poljak) * New type: __postgres_extension (Tomas Pospisek) * Core, types: support IPv6 (Darko Poljak) From d07ad486512896043caebeeb2bc19ae7a22d6233 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 10:21:00 +0100 Subject: [PATCH 158/197] Add reference to beta docs. --- docs/web/cdist/documentation.mdwn | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/web/cdist/documentation.mdwn b/docs/web/cdist/documentation.mdwn index 56c2798e..6479a4bb 100644 --- a/docs/web/cdist/documentation.mdwn +++ b/docs/web/cdist/documentation.mdwn @@ -6,4 +6,7 @@ have a look at [all versions](/software/cdist/man). You can also view [speeches about cdist](/software/cdist/speeches). +Checking out beta? Find the docs here: +[beta documentation](/software/cdist/man/beta). + [[!tag cdist unix]] From 8c53ce78f5cf8607b544d53d51da77cbfe7f6dc4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 10:46:49 +0100 Subject: [PATCH 159/197] Started the good, the bad and the ugly - code cleanup. --- cdist/config.py | 65 ++-------------------------------------- cdist/util/ipaddr.py | 57 +++++++++++++++++++++++++++++++++++ cdist/util/remoteutil.py | 50 +++++++++++++++++++++++++++++++ 3 files changed, 110 insertions(+), 62 deletions(-) create mode 100644 cdist/util/ipaddr.py create mode 100644 cdist/util/remoteutil.py diff --git a/cdist/config.py b/cdist/config.py index b8d0672c..855aaade 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -34,38 +34,10 @@ import cdist import cdist.exec.local import cdist.exec.remote +import cdist.util.ipaddr as ipaddr from cdist import core - - -def inspect_ssh_mux_opts(): - """Inspect whether or not ssh supports multiplexing options. - - Return string containing multiplexing options if supported. - If ControlPath is supported then placeholder for that path is - specified and can be used for final string formatting. - For example, this function can return string: - "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}". - Then it can be formatted: - mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path'). - """ - import subprocess - - wanted_mux_opts = { - "ControlPath": "{}", - "ControlMaster": "auto", - "ControlPersist": "125", - } - mux_opts = " ".join([" -o {}={}".format( - x, wanted_mux_opts[x]) for x in wanted_mux_opts]) - try: - subprocess.check_output("ssh {}".format(mux_opts), - stderr=subprocess.STDOUT, shell=True) - except subprocess.CalledProcessError as e: - subproc_output = e.output.decode().lower() - if "bad configuration option" in subproc_output: - return "" - return mux_opts +from cdist.util.remoteutil import inspect_ssh_mux_opts class Config(object): @@ -253,38 +225,7 @@ class Config(object): log.debug("remote_copy for host \"{}\": {}".format( host, remote_copy)) - try: - # getaddrinfo returns a list of 5-tuples: - # (family, type, proto, canonname, sockaddr) - # where sockaddr is: - # (address, port) for AF_INET, - # (address, port, flow_info, scopeid) for AF_INET6 - ip_addr = socket.getaddrinfo( - host, None, type=socket.SOCK_STREAM)[0][4][0] - # gethostbyaddr returns triple - # (hostname, aliaslist, ipaddrlist) - host_name = socket.gethostbyaddr(ip_addr)[0] - log.debug("derived host_name for host \"{}\": {}".format( - host, host_name)) - except (socket.gaierror, socket.herror) as e: - log.warn("Could not derive host_name for {}" - ", $host_name will be empty. Error is: {}".format( - host, e)) - # in case of error provide empty value - host_name = '' - - try: - host_fqdn = socket.getfqdn(host) - log.debug("derived host_fqdn for host \"{}\": {}".format( - host, host_fqdn)) - except socket.herror as e: - log.warn("Could not derive host_fqdn for {}" - ", $host_fqdn will be empty. Error is: {}".format( - host, e)) - # in case of error provide empty value - host_fqdn = '' - - target_host = (host, host_name, host_fqdn) + target_host = ipaddr.resolve_target_addresses(host) log.debug("target_host: {}".format(target_host)) local = cdist.exec.local.Local( diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py new file mode 100644 index 00000000..7c3c037a --- /dev/null +++ b/cdist/util/ipaddr.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import socket +import logging + + +def resolve_target_addresses(host): + log = logging.getLogger(host) + try: + # getaddrinfo returns a list of 5-tuples: + # (family, type, proto, canonname, sockaddr) + # where sockaddr is: + # (address, port) for AF_INET, + # (address, port, flow_info, scopeid) for AF_INET6 + ip_addr = socket.getaddrinfo( + host, None, type=socket.SOCK_STREAM)[0][4][0] + # gethostbyaddr returns triple + # (hostname, aliaslist, ipaddrlist) + host_name = socket.gethostbyaddr(ip_addr)[0] + log.debug("derived host_name for host \"{}\": {}".format( + host, host_name)) + except (socket.gaierror, socket.herror) as e: + log.warn("Could not derive host_name for {}" + ", $host_name will be empty. Error is: {}".format(host, e)) + # in case of error provide empty value + host_name = '' + + try: + host_fqdn = socket.getfqdn(host) + log.debug("derived host_fqdn for host \"{}\": {}".format( + host, host_fqdn)) + except socket.herror as e: + log.warn("Could not derive host_fqdn for {}" + ", $host_fqdn will be empty. Error is: {}".format(host, e)) + # in case of error provide empty value + host_fqdn = '' + + return (host, host_name, host_fqdn) diff --git a/cdist/util/remoteutil.py b/cdist/util/remoteutil.py new file mode 100644 index 00000000..c18d6705 --- /dev/null +++ b/cdist/util/remoteutil.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +def inspect_ssh_mux_opts(): + """Inspect whether or not ssh supports multiplexing options. + + Return string containing multiplexing options if supported. + If ControlPath is supported then placeholder for that path is + specified and can be used for final string formatting. + For example, this function can return string: + "-o ControlMaster=auto -o ControlPersist=125 -o ControlPath={}". + Then it can be formatted: + mux_opts_string.format('/tmp/tmpxxxxxx/ssh-control-path'). + """ + import subprocess + + wanted_mux_opts = { + "ControlPath": "{}", + "ControlMaster": "auto", + "ControlPersist": "125", + } + mux_opts = " ".join([" -o {}={}".format( + x, wanted_mux_opts[x]) for x in wanted_mux_opts]) + try: + subprocess.check_output("ssh {}".format(mux_opts), + stderr=subprocess.STDOUT, shell=True) + except subprocess.CalledProcessError as e: + subproc_output = e.output.decode().lower() + if "bad configuration option" in subproc_output: + return "" + return mux_opts From b24adcbe202fdb1d59e6ccdea08a39a012e8c244 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 14:03:24 +0100 Subject: [PATCH 160/197] Fix changelog style. --- docs/changelog | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/changelog b/docs/changelog index d9939bf1..168282aa 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,14 +2,14 @@ Changelog --------- 4.4.0: 2016-12-03 - * Core: deprecate -d option and make -v option log level counter (Darko Poljak) + * Core: Deprecate -d option and make -v option log level counter (Darko Poljak) * New type: __postgres_extension (Tomas Pospisek) - * Core, types: support IPv6 (Darko Poljak) - * Type __consul: add source and cksum files for Consul 0.7.0 and 0.7.1 (Carlos Ortigoza) + * Core, types: Support IPv6 (Darko Poljak) + * Type __consul: Add source and cksum files for Consul 0.7.0 and 0.7.1 (Carlos Ortigoza) * Type __user: FreeBSD fix (Kamila Souckova) * New type: __apt_mark (Ander Punnar) - * Type __package_upgrade_all: do not dist-upgrade by default, add apt-clean and apt-dist-upgrade options (Ander Punnar) - * Core: fix target_host vars (Darko Poljak) + * Type __package_upgrade_all: Do not dist-upgrade by default, add apt-clean and apt-dist-upgrade options (Ander Punnar) + * Core: Correct target_host var in code.py (Darko Poljak) * All: Merge install feature from 4.0-pre-not-stable (Darko Poljak) 4.3.2: 2016-10-13 @@ -20,7 +20,7 @@ Changelog * Type __package_pkg_openbsd: Support --version (Andres Erbsen) * Type __hostname: Support openbsd (Andres Erbsen) * New type: __firewalld_start: start/stop firewalld and/or enable/disable start on boot (Darko Poljak) - * Bugfix __consul_agent: config option was misnamed 'syslog' instead of 'enable_syslog' (Steven Armstrong) + * Bugfix __consul_agent: Config option was misnamed 'syslog' instead of 'enable_syslog' (Steven Armstrong) 4.3.1: 2016-08-22 * Documentation: Spelling fixes (Darko Poljak) From e6fc74c081423c8095a3d1e8e72d8ce24699f436 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 18:12:38 +0100 Subject: [PATCH 161/197] ugly -> bad --- cdist/config.py | 111 ++++++++++++++++++------------------------- cdist/exec/remote.py | 31 +----------- cdist/hostsource.py | 72 ++++++++++++++++++++++++++++ cdist/util/ipaddr.py | 26 ++++++++++ 4 files changed, 146 insertions(+), 94 deletions(-) create mode 100644 cdist/hostsource.py diff --git a/cdist/config.py b/cdist/config.py index 855aaade..b1a120ca 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -22,15 +22,14 @@ import logging import os -import shutil import sys import time -import pprint import itertools import tempfile import socket import cdist +import cdist.hostsource import cdist.exec.local import cdist.exec.remote @@ -61,55 +60,17 @@ class Config(object): self.local.create_files_dirs() self.remote.create_files_dirs() - @staticmethod - def hostfile_process_line(line): - """Return host from read line or None if no host present.""" - if not line: - return None - # remove comment if present - comment_index = line.find('#') - if comment_index >= 0: - host = line[:comment_index] - else: - host = line - # remove leading and trailing whitespaces - host = host.strip() - # skip empty lines - if host: - return host - else: - return None - @staticmethod def hosts(source): - """Yield hosts from source. - Source can be a sequence or filename (stdin if \'-\'). - In case of filename each line represents one host. - """ - if isinstance(source, str): - import fileinput - try: - for host in fileinput.input(files=(source)): - host = Config.hostfile_process_line(host) - if host: - yield host - except (IOError, OSError, UnicodeError) as e: - raise cdist.Error( - "Error reading hosts from file \'{}\': {}".format( - source, e)) - else: - if source: - for host in source: - yield host + try: + yield from cdist.hostsource.HostSource(source)() + except (IOError, OSError, UnicodeError) as e: + raise cdist.Error( + "Error reading hosts from \'{}\': {}".format( + source, e)) @classmethod - def commandline(cls, args): - """Configure remote system""" - import multiprocessing - - # FIXME: Refactor relict - remove later - log = logging.getLogger("cdist") - + def _check_and_prepare_args(cls, args): if args.manifest == '-' and args.hostfile == '-': raise cdist.Error(("Cannot read both, manifest and host file, " "from stdin")) @@ -134,10 +95,6 @@ class Config(object): import atexit atexit.register(lambda: os.remove(initial_manifest_temp_path)) - process = {} - failed_hosts = [] - time_start = time.time() - # default remote cmd patterns args.remote_exec_pattern = None args.remote_copy_pattern = None @@ -154,10 +111,29 @@ class Config(object): if args_dict['remote_copy'] is None: args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts + @classmethod + def _base_root_path(cls, args): if args.out_path: base_root_path = args.out_path else: base_root_path = tempfile.mkdtemp() + return base_root_path + + @classmethod + def commandline(cls, args): + """Configure remote system""" + import multiprocessing + + # FIXME: Refactor relict - remove later + log = logging.getLogger("cdist") + + cls._check_and_prepare_args(args) + + process = {} + failed_hosts = [] + time_start = time.time() + + base_root_path = cls._base_root_path(args) hostcnt = 0 for host in itertools.chain(cls.hosts(args.host), @@ -199,6 +175,24 @@ class Config(object): raise cdist.Error("Failed to configure the following hosts: " + " ".join(failed_hosts)) + @classmethod + def _resolve_remote_cmds(cls, args, host_base_path): + control_path = os.path.join(host_base_path, "ssh-control-path") + # If we constructed patterns for remote commands then there is + # placeholder for ssh ControlPath, format it and we have unique + # ControlPath for each host. + # + # If not then use args.remote_exec/copy that user specified. + if args.remote_exec_pattern: + remote_exec = args.remote_exec_pattern.format(control_path) + else: + remote_exec = args.remote_exec + if args.remote_copy_pattern: + remote_copy = args.remote_copy_pattern.format(control_path) + else: + remote_copy = args.remote_copy + return (remote_exec, remote_copy, ) + @classmethod def onehost(cls, host, host_base_path, host_dir_name, args, parallel): """Configure ONE system""" @@ -206,20 +200,7 @@ class Config(object): log = logging.getLogger(host) try: - control_path = os.path.join(host_base_path, "ssh-control-path") - # If we constructed patterns for remote commands then there is - # placeholder for ssh ControlPath, format it and we have unique - # ControlPath for each host. - # - # If not then use args.remote_exec/copy that user specified. - if args.remote_exec_pattern: - remote_exec = args.remote_exec_pattern.format(control_path) - else: - remote_exec = args.remote_exec - if args.remote_copy_pattern: - remote_copy = args.remote_copy_pattern.format(control_path) - else: - remote_copy = args.remote_copy + remote_exec, remote_copy = cls._resolve_remote_cmds(host_base_path) log.debug("remote_exec for host \"{}\": {}".format( host, remote_exec)) log.debug("remote_copy for host \"{}\": {}".format( diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index f374262f..74a33f73 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -30,40 +30,13 @@ import multiprocessing import cdist import cdist.exec.util as exec_util - - -# check whether addr is IPv6 -try: - # python 3.3+ - import ipaddress - - def _is_ipv6(addr): - try: - return ipaddress.ip_address(addr).version == 6 - except ValueError: - return False -except ImportError: - # fallback for older python versions - import socket - - def _is_ipv6(addr): - try: - socket.inet_aton(addr) - return False - except socket.error: - pass - try: - socket.inet_pton(socket.AF_INET6, addr) - return True - except socket.error: - pass - return False +import cdist.util.ipaddr as ipaddr def _wrap_addr(addr): """If addr is IPv6 then return addr wrapped between '[' and ']', otherwise return it intact.""" - if _is_ipv6(addr): + if ipaddr.is_ipv6(addr): return "".join(("[", addr, "]", )) else: return addr diff --git a/cdist/hostsource.py b/cdist/hostsource.py new file mode 100644 index 00000000..8170536c --- /dev/null +++ b/cdist/hostsource.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import fileinput + + +class HostSource(object): + """ + Host source object. + Source can be a sequence or filename (stdin if \'-\'). + In case of filename each line represents one host. + """ + def __init__(self, source): + self.source = source + + def _process_file_line(self, line): + """Return host from read line or None if no host present.""" + if not line: + return None + # remove comment if present + comment_index = line.find('#') + if comment_index >= 0: + host = line[:comment_index] + else: + host = line + # remove leading and trailing whitespaces + host = host.strip() + # skip empty lines + if host: + return host + else: + return None + + def _hosts_from_sequence(self): + for host in self.source: + yield host + + def _hosts_from_file(self): + for line in fileinput.input(files=(self.source)): + host = self._process_file_line(line) + if host: + yield host + + def hosts(self): + if not source: + return + + if isinstance(self.source, str): + yield from self._hosts_from_file() + else: + yield from self._hosts_from_sequence() + + def __call__(self): + yield from self.hosts() diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index 7c3c037a..71477682 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -55,3 +55,29 @@ def resolve_target_addresses(host): host_fqdn = '' return (host, host_name, host_fqdn) + + +# check whether addr is IPv6 +try: + # python 3.3+ + import ipaddress + + def is_ipv6(addr): + try: + return ipaddress.ip_address(addr).version == 6 + except ValueError: + return False +except ImportError: + # fallback for older python versions + def is_ipv6(addr): + try: + socket.inet_aton(addr) + return False + except socket.error: + pass + try: + socket.inet_pton(socket.AF_INET6, addr) + return True + except socket.error: + pass + return False From fc18e0f99cc57a2813bfe9c577566cfc5e4096ac Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Sat, 3 Dec 2016 18:14:58 +0100 Subject: [PATCH 162/197] migrated man.text -> man.rst --- .../conf/type/__docker/{man.text => man.rst} | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) rename cdist/conf/type/__docker/{man.text => man.rst} (71%) diff --git a/cdist/conf/type/__docker/man.text b/cdist/conf/type/__docker/man.rst similarity index 71% rename from cdist/conf/type/__docker/man.text rename to cdist/conf/type/__docker/man.rst index 566c2f4c..88786ad7 100644 --- a/cdist/conf/type/__docker/man.text +++ b/cdist/conf/type/__docker/man.rst @@ -1,7 +1,5 @@ cdist-type__docker(7) ===================== -Steven Armstrong - NAME ---- @@ -25,24 +23,23 @@ None. BOOLEAN PARAMETERS ------------------ -experimentel:: +experimentel Install the experimentel docker-engine package instead of the latest stable release. EXAMPLES -------- --------------------------------------------------------------------------------- -__docker +.. code-block:: sh + __docker -# experimentel -__docker --experimental --------------------------------------------------------------------------------- + # experimentel + __docker --experimental -SEE ALSO --------- -- cdist-type(7) +AUTHORS +------- +Steven Armstrong COPYING From d0f5d2c45967429d86002e5526b2bb60cadd3ed6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 18:24:37 +0100 Subject: [PATCH 163/197] ugly -> bad --- cdist/argparse.py | 209 ++++++++++++++++++++++++++++++++++++++++++++++ scripts/cdist | 176 ++------------------------------------ 2 files changed, 218 insertions(+), 167 deletions(-) create mode 100644 cdist/argparse.py mode change 100755 => 100644 scripts/cdist diff --git a/cdist/argparse.py b/cdist/argparse.py new file mode 100644 index 00000000..cef3dd5f --- /dev/null +++ b/cdist/argparse.py @@ -0,0 +1,209 @@ +import argparse +import cdist +import multiprocessing +import os +import logging +import collections + + +# list of beta sub-commands +BETA_COMMANDS = ['install', ] +# list of beta arguments for sub-commands +BETA_ARGS = { + 'config': ['jobs', ], +} +EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/" +# Parser others can reuse +parser = None + + +_verbosity_level = { + 0: logging.ERROR, + 1: logging.WARNING, + 2: logging.INFO, +} +_verbosity_level = collections.defaultdict( + lambda: logging.DEBUG, _verbosity_level) + + +def add_beta_command(cmd): + if cmd not in BETA_COMMANDS: + BETA_COMMANDS.append(cmd) + + +def add_beta_arg(cmd, arg): + if cmd in BETA_ARGS: + if arg not in BETA_ARGS[cmd]: + BETA_ARGS[cmd].append(arg) + else: + BETA_ARGS[cmd] = [arg, ] + + +def check_beta(args_dict): + if 'beta' not in args_dict: + args_dict['beta'] = False + # Check only if beta is not enabled: if beta option is specified then + # raise error. + if not args_dict['beta']: + cmd = args_dict['command'] + # first check if command is beta + if cmd in BETA_COMMANDS: + raise cdist.CdistBetaRequired(cmd) + # then check if some command's argument is beta + if cmd in BETA_ARGS: + for arg in BETA_ARGS[cmd]: + if arg in args_dict and args_dict[arg]: + raise cdist.CdistBetaRequired(cmd, arg) + + +def check_positive_int(value): + import argparse + + try: + val = int(value) + except ValueError: + raise argparse.ArgumentTypeError( + "{} is invalid int value".format(value)) + if val <= 0: + raise argparse.ArgumentTypeError( + "{} is invalid positive int value".format(val)) + return val + + +def get_parsers(): + global parser + + # Construct parser others can reuse + if parser: + return parser + else: + parser = {} + # Options _all_ parsers have in common + parser['loglevel'] = argparse.ArgumentParser(add_help=False) + parser['loglevel'].add_argument( + '-d', '--debug', + help=('Set log level to debug (deprecated, use -vvv instead)'), + action='store_true', default=False) + parser['loglevel'].add_argument( + '-v', '--verbose', + help=('Increase log level, be more verbose. Use it more than once ' + 'to increase log level. The order of levels from the lowest ' + 'to the highest are: ERROR, WARNING, INFO, DEBUG.'), + action='count', default=0) + + parser['beta'] = argparse.ArgumentParser(add_help=False) + parser['beta'].add_argument( + '-b', '--beta', + help=('Enable beta functionalities. ' + 'Can also be enabled using CDIST_BETA env var.'), + action='store_true', dest='beta', + default='CDIST_BETA' in os.environ) + + # Main subcommand parser + parser['main'] = argparse.ArgumentParser( + description='cdist ' + cdist.VERSION, parents=[parser['loglevel']]) + parser['main'].add_argument( + '-V', '--version', help='Show version', action='version', + version='%(prog)s ' + cdist.VERSION) + parser['sub'] = parser['main'].add_subparsers( + title="Commands", dest="command") + + # Banner + parser['banner'] = parser['sub'].add_parser( + 'banner', parents=[parser['loglevel']]) + parser['banner'].set_defaults(func=cdist.banner.banner) + + # Config + parser['config_main'] = argparse.ArgumentParser(add_help=False) + parser['config_main'].add_argument( + '-c', '--conf-dir', + help=('Add configuration directory (can be repeated, ' + 'last one wins)'), action='append') + parser['config_main'].add_argument( + '-i', '--initial-manifest', + help='path to a cdist manifest or \'-\' to read from stdin.', + dest='manifest', required=False) + parser['config_main'].add_argument( + '-j', '--jobs', nargs='?', + type=check_positive_int, + help=('Specify the maximum number of parallel jobs, currently ' + 'only global explorers are supported'), + action='store', dest='jobs', + const=multiprocessing.cpu_count()) + parser['config_main'].add_argument( + '-n', '--dry-run', + help='do not execute code', action='store_true') + parser['config_main'].add_argument( + '-o', '--out-dir', + help='directory to save cdist output in', dest="out_path") + + # remote-copy and remote-exec defaults are environment variables + # if set; if not then None - these will be futher handled after + # parsing to determine implementation default + parser['config_main'].add_argument( + '--remote-copy', + help='Command to use for remote copy (should behave like scp)', + action='store', dest='remote_copy', + default=os.environ.get('CDIST_REMOTE_COPY')) + parser['config_main'].add_argument( + '--remote-exec', + help=('Command to use for remote execution ' + '(should behave like ssh)'), + action='store', dest='remote_exec', + default=os.environ.get('CDIST_REMOTE_EXEC')) + + # Config + parser['config_args'] = argparse.ArgumentParser(add_help=False) + parser['config_args'].add_argument( + 'host', nargs='*', help='host(s) to operate on') + parser['config_args'].add_argument( + '-f', '--file', + help=('Read additional hosts to operate on from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read hosts from stdin.'), + dest='hostfile', required=False) + parser['config_args'].add_argument( + '-p', '--parallel', + help='operate on multiple hosts in parallel', + action='store_true', dest='parallel') + parser['config_args'].add_argument( + '-s', '--sequential', + help='operate on multiple hosts sequentially (default)', + action='store_false', dest='parallel') + parser['config'] = parser['sub'].add_parser( + 'config', parents=[parser['loglevel'], parser['beta'], + parser['config_main'], + parser['config_args']]) + parser['config'].set_defaults(func=cdist.config.Config.commandline) + + # Install + parser['install'] = parser['sub'].add_parser('install', add_help=False, + parents=[parser['config']]) + parser['install'].set_defaults(func=cdist.install.Install.commandline) + + # Shell + parser['shell'] = parser['sub'].add_parser( + 'shell', parents=[parser['loglevel']]) + parser['shell'].add_argument( + '-s', '--shell', + help=('Select shell to use, defaults to current shell. Used shell' + ' should be POSIX compatible shell.')) + parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) + + for p in parser: + parser[p].epilog = EPILOG + + return parser + + +def handle_loglevel(args): + if args.debug: + retval = "-d/--debug is deprecated, use -vvv instead" + args.verbose = 3 + else: + retval = None + + logging.root.setLevel(_verbosity_level[args.verbose]) + + return retval diff --git a/scripts/cdist b/scripts/cdist old mode 100755 new mode 100644 index 4ff64bb5..498091b8 --- a/scripts/cdist +++ b/scripts/cdist @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- # -# 2010-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2010-2016 Nico Schottelius (nico-cdist at schottelius.org) # 2016 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. @@ -24,182 +24,25 @@ import collections import logging -# list of beta sub-commands -BETA_COMMANDS = ['install', ] -# list of beta arguments for sub-commands -BETA_ARGS = { - 'config': ['jobs', ], -} - - -def check_positive_int(value): - import argparse - - try: - val = int(value) - except ValueError as e: - raise argparse.ArgumentTypeError( - "{} is invalid int value".format(value)) - if val <= 0: - raise argparse.ArgumentTypeError( - "{} is invalid positive int value".format(val)) - return val - - -def check_beta(args_dict): - if 'beta' not in args_dict: - args_dict['beta'] = False - # Check only if beta is not enabled: if beta option is specified then - # raise error. - if not args_dict['beta']: - cmd = args_dict['command'] - # first check if command is beta - if cmd in BETA_COMMANDS: - raise cdist.CdistBetaRequired(cmd) - # then check if command's argument is beta - if cmd in BETA_ARGS: - for arg in BETA_ARGS[cmd]: - if arg in args_dict and args_dict[arg]: - raise cdist.CdistBetaRequired(cmd, arg) - - -_verbosity_level = { - 0: logging.ERROR, - 1: logging.WARNING, - 2: logging.INFO, -} -_verbosity_level = collections.defaultdict( - lambda: logging.DEBUG, _verbosity_level) - def commandline(): """Parse command line""" - import argparse + import cdist.argparse import cdist.banner import cdist.config import cdist.install import cdist.shell import shutil import os - import multiprocessing - - # Construct parser others can reuse - parser = {} - # Options _all_ parsers have in common - parser['loglevel'] = argparse.ArgumentParser(add_help=False) - parser['loglevel'].add_argument( - '-d', '--debug', - help=('Set log level to debug (deprecated, use -vvv instead)'), - action='store_true', default=False) - parser['loglevel'].add_argument( - '-v', '--verbose', - help=('Increase log level, be more verbose. Use it more than once ' - 'to increase log level. The order of levels from the lowest ' - 'to the highest are: ERROR, WARNING, INFO, DEBUG.'), - action='count', default=0) - - # Main subcommand parser - parser['main'] = argparse.ArgumentParser( - description='cdist ' + cdist.VERSION, parents=[parser['loglevel']]) - parser['main'].add_argument( - '-V', '--version', help='Show version', action='version', - version='%(prog)s ' + cdist.VERSION) - parser['sub'] = parser['main'].add_subparsers( - title="Commands", dest="command") - - # Banner - parser['banner'] = parser['sub'].add_parser( - 'banner', parents=[parser['loglevel']]) - parser['banner'].set_defaults(func=cdist.banner.banner) - - # Config - parser['config'] = parser['sub'].add_parser( - 'config', parents=[parser['loglevel']]) - parser['config'].add_argument( - 'host', nargs='*', help='host(s) to operate on') - parser['config'].add_argument( - '-b', '--enable-beta', - help=('Enable beta functionalities. Beta functionalities ' - 'include the following options: -j/--jobs.'), - action='store_true', dest='beta', default=False) - parser['config'].add_argument( - '-c', '--conf-dir', - help=('Add configuration directory (can be repeated, ' - 'last one wins)'), action='append') - parser['config'].add_argument( - '-f', '--file', - help=('Read additional hosts to operate on from specified file ' - 'or from stdin if \'-\' (each host on separate line). ' - 'If no host or host file is specified then, by default, ' - 'read hosts from stdin.'), - dest='hostfile', required=False) - parser['config'].add_argument( - '-i', '--initial-manifest', - help='Path to a cdist manifest or \'-\' to read from stdin.', - dest='manifest', required=False) - parser['config'].add_argument( - '-j', '--jobs', nargs='?', type=check_positive_int, - help=('Specify the maximum number of parallel jobs, currently ' - 'only global explorers are supported (currently in beta'), - action='store', dest='jobs', - const=multiprocessing.cpu_count()) - parser['config'].add_argument( - '-n', '--dry-run', - help='Do not execute code', action='store_true') - parser['config'].add_argument( - '-o', '--out-dir', - help='Directory to save cdist output in', dest="out_path") - parser['config'].add_argument( - '-p', '--parallel', - help='Operate on multiple hosts in parallel', - action='store_true', dest='parallel') - parser['config'].add_argument( - '-s', '--sequential', - help='Operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') - # remote-copy and remote-exec defaults are environment variables - # if set; if not then None - these will be futher handled after - # parsing to determine implementation default - parser['config'].add_argument( - '--remote-copy', - help='Command to use for remote copy (should behave like scp)', - action='store', dest='remote_copy', - default=os.environ.get('CDIST_REMOTE_COPY')) - parser['config'].add_argument( - '--remote-exec', - help=('Command to use for remote execution ' - '(should behave like ssh)'), - action='store', dest='remote_exec', - default=os.environ.get('CDIST_REMOTE_EXEC')) - parser['config'].set_defaults(func=cdist.config.Config.commandline) - - # Install - parser['install'] = parser['sub'].add_parser('install', add_help=False, - parents=[parser['config']]) - parser['install'].set_defaults(func=cdist.install.Install.commandline) - - # Shell - parser['shell'] = parser['sub'].add_parser( - 'shell', parents=[parser['loglevel']]) - parser['shell'].add_argument( - '-s', '--shell', - help=('Select shell to use, defaults to current shell. Used shell' - ' should be POSIX compatible shell.')) - parser['shell'].set_defaults(func=cdist.shell.Shell.commandline) - - for p in parser: - parser[p].epilog = ( - "Get cdist at http://www.nico.schottelius.org/software/cdist/") + parser = cdist.argparse.get_parsers() args = parser['main'].parse_args(sys.argv[1:]) # Loglevels are handled globally in here - if args.debug: - log.warning("-d/--debug is deprecated, use -vvv instead") - args.verbose = 3 - - logging.root.setLevel(_verbosity_level[args.verbose]) + retval = cdist.argparse.handle_loglevel(args) + if retval: + log.warning(retval) log.debug(args) log.info("version %s" % cdist.VERSION) @@ -219,17 +62,16 @@ def commandline(): parser['main'].print_help() sys.exit(0) - check_beta(vars(args)) + cdist.argparse.check_beta(vars(args)) args.func(args) if __name__ == "__main__": - # Sys is needed for sys.exit() import sys cdistpythonversion = '3.2' if sys.version < cdistpythonversion: - print('Python >= ' + cdistpythonversion + - ' is required on the source host.', file=sys.stderr) + print('Python >= {} is required on the source host.'.format( + cdistpythonversion), file=sys.stderr) sys.exit(1) exit_code = 0 From 2999f1269873245733a612aac82c6e768eba7663 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 3 Dec 2016 18:29:10 +0100 Subject: [PATCH 164/197] chmod +x --- scripts/cdist | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/cdist diff --git a/scripts/cdist b/scripts/cdist old mode 100644 new mode 100755 From 609977b7ff8d5659622f3fff8494b8fd8b9aa427 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 4 Dec 2016 20:27:42 +0100 Subject: [PATCH 165/197] ugly->bad --- cdist/__init__.py | 4 +- cdist/core/explorer.py | 11 ++--- cdist/exec/remote.py | 80 +++++++++++++++++---------------- cdist/test/exec/remote.py | 4 +- docs/src/cdist-reference.rst.sh | 3 ++ docs/src/man1/cdist.rst | 3 ++ 6 files changed, 54 insertions(+), 51 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index b6f5c8cb..c142230c 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -68,13 +68,13 @@ class CdistBetaRequired(cdist.Error): err_msg = ("\'{}\' command is beta, but beta is " "not enabled. If you want to use it please enable beta " "functionalities by using the -b/--enable-beta command " - "line flag.") + "line flag or setting CDIST_BETA env var.") fmt_args = [self.command, ] else: err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta " "is not enabled. If you want to use it please enable " "beta functionalities by using the -b/--enable-beta " - "command line flag.") + "command line flag or setting CDIST_BETA env var.") fmt_args = [self.arg, self.command, ] return err_msg.format(*fmt_args) diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index ef85431c..23996240 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -149,14 +149,9 @@ class Explorer(object): def transfer_global_explorers(self): """Transfer the global explorers to the remote side.""" self.remote.mkdir(self.remote.global_explorer_path) - if self.jobs is None: - self.remote.transfer(self.local.global_explorer_path, - self.remote.global_explorer_path) - else: - self.remote.transfer_dir_parallel( - self.local.global_explorer_path, - self.remote.global_explorer_path, - self.jobs) + self.remote.transfer(self.local.global_explorer_path, + self.remote.global_explorer_path, + self.jobs) self.remote.run(["chmod", "0700", "%s/*" % (self.remote.global_explorer_path)]) diff --git a/cdist/exec/remote.py b/cdist/exec/remote.py index 74a33f73..440aafa7 100644 --- a/cdist/exec/remote.py +++ b/cdist/exec/remote.py @@ -118,57 +118,59 @@ class Remote(object): self.log.debug("Remote mkdir: %s", path) self.run(["mkdir", "-p", path]) - def transfer(self, source, destination): + def transfer(self, source, destination, jobs=None): """Transfer a file or directory to the remote side.""" self.log.debug("Remote transfer: %s -> %s", source, destination) self.rmdir(destination) if os.path.isdir(source): self.mkdir(destination) - for f in glob.glob1(source, '*'): - command = self._copy.split() - path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - self._run_command(command) + if jobs: + self._transfer_dir_parallel(source, destination, jobs) + else: + self._transfer_dir_sequential(source, destination) + elif jobs: + raise cdist.Error("Source {} is not a directory".format(source)) else: command = self._copy.split() command.extend([source, '{0}:{1}'.format( _wrap_addr(self.target_host[0]), destination)]) self._run_command(command) - def transfer_dir_parallel(self, source, destination, jobs): - """Transfer a directory to the remote side in parallel mode.""" - self.log.debug("Remote transfer: %s -> %s", source, destination) - self.rmdir(destination) - if os.path.isdir(source): - self.mkdir(destination) - self.log.info("Remote transfer in {} parallel jobs".format( - jobs)) - self.log.debug("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for parallel " - "remote transfer")) - with multiprocessing.Pool(jobs) as pool: - self.log.debug("Starting async for parallel transfer") - commands = [] - for f in glob.glob1(source, '*'): - command = self._copy.split() - path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - commands.append(command) - results = [ - pool.apply_async(self._run_command, (cmd,)) - for cmd in commands - ] + def _transfer_dir_sequential(self, source, destination): + for f in glob.glob1(source, '*'): + command = self._copy.split() + path = os.path.join(source, f) + command.extend([path, '{0}:{1}'.format( + _wrap_addr(self.target_host[0]), destination)]) + self._run_command(command) - self.log.debug("Waiting async results for parallel transfer") - for r in results: - r.get() # self._run_command returns None - self.log.debug(("Multiprocessing for parallel transfer " - "finished")) - else: - raise cdist.Error("Source {} is not a directory".format(source)) + def _transfer_dir_parallel(self, source, destination, jobs): + """Transfer a directory to the remote side in parallel mode.""" + self.log.info("Remote transfer in {} parallel jobs".format( + jobs)) + self.log.debug("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + self.log.debug(("Starting multiprocessing Pool for parallel " + "remote transfer")) + with multiprocessing.Pool(jobs) as pool: + self.log.debug("Starting async for parallel transfer") + commands = [] + for f in glob.glob1(source, '*'): + command = self._copy.split() + path = os.path.join(source, f) + command.extend([path, '{0}:{1}'.format( + _wrap_addr(self.target_host[0]), destination)]) + commands.append(command) + results = [ + pool.apply_async(self._run_command, (cmd,)) + for cmd in commands + ] + + self.log.debug("Waiting async results for parallel transfer") + for r in results: + r.get() # self._run_command returns None + self.log.debug(("Multiprocessing for parallel transfer " + "finished")) def run_script(self, script, env=None, return_output=False): """Run the given script with the given environment on the remote side. diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py index 45dabb18..371d17e3 100644 --- a/cdist/test/exec/remote.py +++ b/cdist/test/exec/remote.py @@ -136,8 +136,8 @@ class RemoteTestCase(test.CdistTestCase): source_file_name = os.path.split(source_file)[-1] filenames.append(source_file_name) target = self.mkdtemp(dir=self.temp_dir) - self.remote.transfer_dir_parallel(source, target, - multiprocessing.cpu_count()) + self.remote.transfer(source, target, + multiprocessing.cpu_count()) # test if the payload files are in the target directory for filename in filenames: self.assertTrue(os.path.isfile(os.path.join(target, filename))) diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index 97b22473..4b94b858 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -273,4 +273,7 @@ CDIST_REMOTE_EXEC CDIST_REMOTE_COPY Use this command for remote copy (should behave like scp). + +CDIST_BETA + Enable beta functionalities. eof diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 5daedcd4..08c856b1 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -236,6 +236,9 @@ CDIST_REMOTE_EXEC CDIST_REMOTE_COPY Use this command for remote copy (should behave like scp). +CDIST_BETA + Enable beta functionalities. + EXIT STATUS ----------- The following exit values shall be returned: From 1ee6c2e7b1dbfe25d26f30a6648ce4d7f5336e1f Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Mon, 5 Dec 2016 07:57:30 +0100 Subject: [PATCH 166/197] added line after 33, changed experimentel to experimental --- cdist/conf/type/__docker/man.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker/man.rst b/cdist/conf/type/__docker/man.rst index 88786ad7..80088983 100644 --- a/cdist/conf/type/__docker/man.rst +++ b/cdist/conf/type/__docker/man.rst @@ -23,7 +23,7 @@ None. BOOLEAN PARAMETERS ------------------ -experimentel +experimental Install the experimentel docker-engine package instead of the latest stable release. @@ -31,6 +31,7 @@ EXAMPLES -------- .. code-block:: sh + __docker # experimentel From eb56c6ef59118a30987aa44683644f808a650b6d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 5 Dec 2016 08:53:48 +0100 Subject: [PATCH 167/197] experimentel -> experimental --- cdist/conf/type/__docker/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__docker/man.rst b/cdist/conf/type/__docker/man.rst index 80088983..42e71af5 100644 --- a/cdist/conf/type/__docker/man.rst +++ b/cdist/conf/type/__docker/man.rst @@ -24,7 +24,7 @@ None. BOOLEAN PARAMETERS ------------------ experimental - Install the experimentel docker-engine package instead of the latest stable release. + Install the experimental docker-engine package instead of the latest stable release. EXAMPLES @@ -34,7 +34,7 @@ EXAMPLES __docker - # experimentel + # experimental __docker --experimental From 0f175bc65a184a89f6b82556bbf71e7c3f3c1ef5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 5 Dec 2016 08:53:59 +0100 Subject: [PATCH 168/197] Update changelog. --- docs/changelog | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/changelog b/docs/changelog index 168282aa..ea3c74c2 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,10 @@ Changelog --------- +next: + * New type: __docker (Steven Armstrong) + * New type: __package_dpkg (Tomas Pospisek) + 4.4.0: 2016-12-03 * Core: Deprecate -d option and make -v option log level counter (Darko Poljak) * New type: __postgres_extension (Tomas Pospisek) From 3e763e9e6c9c07398df10510b11e1167f755d764 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 7 Dec 2016 18:36:19 +0100 Subject: [PATCH 169/197] list -> set for beta commands and args --- cdist/argparse.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index cef3dd5f..04f6e6a4 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -6,11 +6,11 @@ import logging import collections -# list of beta sub-commands -BETA_COMMANDS = ['install', ] -# list of beta arguments for sub-commands +# set of beta sub-commands +BETA_COMMANDS = set(('install', )) +# set of beta arguments for sub-commands BETA_ARGS = { - 'config': ['jobs', ], + 'config': set(('jobs', )), } EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/" # Parser others can reuse @@ -27,8 +27,7 @@ _verbosity_level = collections.defaultdict( def add_beta_command(cmd): - if cmd not in BETA_COMMANDS: - BETA_COMMANDS.append(cmd) + BETA_COMMANDS.add(cmd) def add_beta_arg(cmd, arg): @@ -36,7 +35,7 @@ def add_beta_arg(cmd, arg): if arg not in BETA_ARGS[cmd]: BETA_ARGS[cmd].append(arg) else: - BETA_ARGS[cmd] = [arg, ] + BETA_ARGS[cmd] = set((arg, )) def check_beta(args_dict): From 341de216a63e8ae2922fa3aa375026a65d8f52e3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 7 Dec 2016 18:39:43 +0100 Subject: [PATCH 170/197] Fix missing vars. --- cdist/config.py | 3 ++- cdist/hostsource.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cdist/config.py b/cdist/config.py index b1a120ca..6b57e7bf 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -200,7 +200,8 @@ class Config(object): log = logging.getLogger(host) try: - remote_exec, remote_copy = cls._resolve_remote_cmds(host_base_path) + remote_exec, remote_copy = cls._resolve_remote_cmds( + args, host_base_path) log.debug("remote_exec for host \"{}\": {}".format( host, remote_exec)) log.debug("remote_copy for host \"{}\": {}".format( diff --git a/cdist/hostsource.py b/cdist/hostsource.py index 8170536c..9c2c0616 100644 --- a/cdist/hostsource.py +++ b/cdist/hostsource.py @@ -60,7 +60,7 @@ class HostSource(object): yield host def hosts(self): - if not source: + if not self.source: return if isinstance(self.source, str): From d17c517a0c220213ea343bf1926a6ff4d8d7dc37 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 7 Dec 2016 22:43:53 +0100 Subject: [PATCH 171/197] enable-beta -> beta --- cdist/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/__init__.py b/cdist/__init__.py index c142230c..6ea02d41 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -67,13 +67,13 @@ class CdistBetaRequired(cdist.Error): if self.arg is None: err_msg = ("\'{}\' command is beta, but beta is " "not enabled. If you want to use it please enable beta " - "functionalities by using the -b/--enable-beta command " + "functionalities by using the -b/--beta command " "line flag or setting CDIST_BETA env var.") fmt_args = [self.command, ] else: err_msg = ("\'{}\' argument of \'{}\' command is beta, but beta " "is not enabled. If you want to use it please enable " - "beta functionalities by using the -b/--enable-beta " + "beta functionalities by using the -b/--beta " "command line flag or setting CDIST_BETA env var.") fmt_args = [self.arg, self.command, ] return err_msg.format(*fmt_args) From 6d765c1ff728098d5570118015bbb21fe0ea4e0f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 7 Dec 2016 23:43:03 +0100 Subject: [PATCH 172/197] enable-beta -> beta --- completions/bash/cdist-completion.bash | 2 +- completions/zsh/_cdist | 2 +- docs/src/man1/cdist.rst | 7 ++++--- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash index 68f45327..1c4226c2 100644 --- a/completions/bash/cdist-completion.bash +++ b/completions/bash/cdist-completion.bash @@ -36,7 +36,7 @@ _cdist() return 0 ;; config|install) - opts="-h --help -d --debug -v --verbose -b --enable-beta \ + opts="-h --help -d --debug -v --verbose -b --beta \ -c --conf-dir -f --file -i --initial-manifest -j --jobs \ -n --dry-run -o --out-dir -p --parallel -s --sequential \ --remote-copy --remote-exec" diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist index dc320224..001356d4 100644 --- a/completions/zsh/_cdist +++ b/completions/zsh/_cdist @@ -36,7 +36,7 @@ _cdist() esac ;; config|install) - opts=(-h --help -d --debug -v --verbose -b --enable-beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec) + opts=(-h --help -d --debug -v --verbose -b --beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec) compadd "$@" -- $opts ;; *) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 08c856b1..55901300 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -69,10 +69,11 @@ CONFIG/INSTALL -------------- Configure/install one or more hosts. -.. option:: -b, --enable-beta +.. option:: -b, --beta - Enable beta functionalities. Beta functionalities include the - following options: -j/--jobs. + Enable beta functionalities. + + Can also be enabled using CDIST_BETA env var. .. option:: -c CONF_DIR, --conf-dir CONF_DIR From b0911ab87d2444a1cfad050ec48c308abc332d4b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 8 Dec 2016 20:04:17 +0100 Subject: [PATCH 173/197] Improve __apt_mark (Ander Punnar from beta) --- .../conf/type/__apt_mark/explorer/apt_version | 31 +++++++++++++++++++ .../__apt_mark/explorer/package_installed | 30 ++++++++++++++++++ cdist/conf/type/__apt_mark/gencode-remote | 13 ++++++++ docs/changelog | 1 + 4 files changed, 75 insertions(+) create mode 100644 cdist/conf/type/__apt_mark/explorer/apt_version create mode 100644 cdist/conf/type/__apt_mark/explorer/package_installed diff --git a/cdist/conf/type/__apt_mark/explorer/apt_version b/cdist/conf/type/__apt_mark/explorer/apt_version new file mode 100644 index 00000000..32a0a58f --- /dev/null +++ b/cdist/conf/type/__apt_mark/explorer/apt_version @@ -0,0 +1,31 @@ +#!/bin/sh +# +# 2016 Ander Punnar (cdist at kvlt.ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +apt_version_is=$(dpkg-query --show --showformat '${Version}' apt) + +# from APT changelog: +# apt (0.8.14.2) UNRELEASED; urgency=low +# provide a 'dpkg --set-selections' wrapper to set/release holds + +apt_version_should=0.8.14.2 + +dpkg --compare-versions $apt_version_should le $apt_version_is \ + && echo 0 \ + || echo 1 diff --git a/cdist/conf/type/__apt_mark/explorer/package_installed b/cdist/conf/type/__apt_mark/explorer/package_installed new file mode 100644 index 00000000..c78ac3a9 --- /dev/null +++ b/cdist/conf/type/__apt_mark/explorer/package_installed @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2016 Ander Punnar (cdist at kvlt.ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if [ -f "$__object/parameter/name" ]; then + name="$(cat "$__object/parameter/name")" +else + name="$__object_id" +fi + +dpkg-query --show --showformat '${Status}' $name 2>/dev/null \ + | grep -q 'ok installed' \ + && echo 0 \ + || echo 1 diff --git a/cdist/conf/type/__apt_mark/gencode-remote b/cdist/conf/type/__apt_mark/gencode-remote index 14505809..c1ac58b3 100644 --- a/cdist/conf/type/__apt_mark/gencode-remote +++ b/cdist/conf/type/__apt_mark/gencode-remote @@ -24,6 +24,19 @@ else name="$__object_id" fi +apt_version="$(cat "$__object/explorer/apt_version")" + +if [ "$apt_version" != '0' ]; then + echo 'APT version not supported' >&2 + exit 1 +fi + +package_installed="$(cat "$__object/explorer/package_installed")" + +if [ "$package_installed" != '0' ]; then + exit 0 +fi + state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" diff --git a/docs/changelog b/docs/changelog index ea3c74c2..41fc1ca9 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Type __apt_mark: check supported apt version and if package is installed (Ander Punnar) * New type: __docker (Steven Armstrong) * New type: __package_dpkg (Tomas Pospisek) From a9cd6dc7c2f3ccb18a1623a1b05c51001da62a78 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 8 Dec 2016 20:08:08 +0100 Subject: [PATCH 174/197] Fix changelog style --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 41fc1ca9..f0b296cc 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,7 +2,7 @@ Changelog --------- next: - * Type __apt_mark: check supported apt version and if package is installed (Ander Punnar) + * Type __apt_mark: Check supported apt version and if package is installed (Ander Punnar) * New type: __docker (Steven Armstrong) * New type: __package_dpkg (Tomas Pospisek) From 7abb96b48d38ae7ed60981bdaa93300d7bb641fc Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Sun, 11 Dec 2016 22:12:44 +0100 Subject: [PATCH 175/197] Created __docker_compose type for cdist Features: - Install __docker_compose --- .../conf/type/__docker_compose/gencode-remote | 5 +++ cdist/conf/type/__docker_compose/man.rst | 45 +++++++++++++++++++ cdist/conf/type/__docker_compose/manifest | 24 ++++++++++ cdist/conf/type/__docker_compose/singleton | 0 4 files changed, 74 insertions(+) create mode 100644 cdist/conf/type/__docker_compose/gencode-remote create mode 100644 cdist/conf/type/__docker_compose/man.rst create mode 100644 cdist/conf/type/__docker_compose/manifest create mode 100644 cdist/conf/type/__docker_compose/singleton diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote new file mode 100644 index 00000000..825dc17a --- /dev/null +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -0,0 +1,5 @@ +# Download docker-compose file +echo 'curl -L "https://github.com/docker/compose/releases/download/1.9.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose' + +# Change permissions +echo 'chmod +x /usr/local/bin/docker-compose' diff --git a/cdist/conf/type/__docker_compose/man.rst b/cdist/conf/type/__docker_compose/man.rst new file mode 100644 index 00000000..2eb1b964 --- /dev/null +++ b/cdist/conf/type/__docker_compose/man.rst @@ -0,0 +1,45 @@ +cdist-type__docker_compose(7) +============================= + +NAME +---- +cdist-type__docker_compose - install docker-compose + + +DESCRIPTION +----------- +Installs latest docker-compose package from dockerproject.org. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __docker_compose + + +AUTHORS +------- +Dominique Roux + + +COPYING +------- +Copyright \(C) 2016 Dominique Roux. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__docker_compose/manifest b/cdist/conf/type/__docker_compose/manifest new file mode 100644 index 00000000..2ff80aca --- /dev/null +++ b/cdist/conf/type/__docker_compose/manifest @@ -0,0 +1,24 @@ +#!/bin/sh +# +# 2016 Dominique Roux (dominique.roux at ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +# Needed packages +__docker +__package curl diff --git a/cdist/conf/type/__docker_compose/singleton b/cdist/conf/type/__docker_compose/singleton new file mode 100644 index 00000000..e69de29b From afd0d8c8c824b1c7d921d68bc7997323111c8bfd Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Mon, 12 Dec 2016 09:09:48 +0100 Subject: [PATCH 176/197] Added / Changed license to GPLv3+ --- .../conf/type/__docker_compose/gencode-remote | 20 +++++++++++++++++++ cdist/conf/type/__docker_compose/man.rst | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote index 825dc17a..93609172 100644 --- a/cdist/conf/type/__docker_compose/gencode-remote +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -1,3 +1,23 @@ +#!/bin/sh +# +# 2016 Dominique Roux (dominique.roux at ungleich.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + # Download docker-compose file echo 'curl -L "https://github.com/docker/compose/releases/download/1.9.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose' diff --git a/cdist/conf/type/__docker_compose/man.rst b/cdist/conf/type/__docker_compose/man.rst index 2eb1b964..4e57b156 100644 --- a/cdist/conf/type/__docker_compose/man.rst +++ b/cdist/conf/type/__docker_compose/man.rst @@ -42,4 +42,4 @@ Dominique Roux COPYING ------- Copyright \(C) 2016 Dominique Roux. Free use of this software is -granted under the terms of the GNU General Public License version 3 (GPLv3). +granted under the terms of the GNU General Public License version 3 or later (GPLv3+). From db50e2e9e28d8782d5bf87078547413babc0caa5 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Mon, 12 Dec 2016 19:43:15 +0100 Subject: [PATCH 177/197] Added parameter --version to define the docker-compose version --- cdist/conf/type/__docker_compose/gencode-remote | 5 ++++- cdist/conf/type/__docker_compose/parameter/default/version | 1 + cdist/conf/type/__docker_compose/parameter/optional | 1 + 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 cdist/conf/type/__docker_compose/parameter/default/version create mode 100644 cdist/conf/type/__docker_compose/parameter/optional diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote index 93609172..f22823d3 100644 --- a/cdist/conf/type/__docker_compose/gencode-remote +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -18,8 +18,11 @@ # along with cdist. If not, see . # +# Variables +version="$(cat "$__object/parameter/version")" + # Download docker-compose file -echo 'curl -L "https://github.com/docker/compose/releases/download/1.9.0/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose' +echo 'curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose' # Change permissions echo 'chmod +x /usr/local/bin/docker-compose' diff --git a/cdist/conf/type/__docker_compose/parameter/default/version b/cdist/conf/type/__docker_compose/parameter/default/version new file mode 100644 index 00000000..f8e233b2 --- /dev/null +++ b/cdist/conf/type/__docker_compose/parameter/default/version @@ -0,0 +1 @@ +1.9.0 diff --git a/cdist/conf/type/__docker_compose/parameter/optional b/cdist/conf/type/__docker_compose/parameter/optional new file mode 100644 index 00000000..088eda41 --- /dev/null +++ b/cdist/conf/type/__docker_compose/parameter/optional @@ -0,0 +1 @@ +version From f180cbcb09d42400d110ef605d44b02fbc34327b Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Tue, 13 Dec 2016 08:09:14 +0100 Subject: [PATCH 178/197] documented --version parameter --- cdist/conf/type/__docker_compose/man.rst | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker_compose/man.rst b/cdist/conf/type/__docker_compose/man.rst index 4e57b156..29c47fbf 100644 --- a/cdist/conf/type/__docker_compose/man.rst +++ b/cdist/conf/type/__docker_compose/man.rst @@ -18,7 +18,8 @@ None. OPTIONAL PARAMETERS ------------------- -None. +version + Define docker_compose version. BOOLEAN PARAMETERS @@ -33,6 +34,9 @@ EXAMPLES __docker_compose + # Install version 1.9.0-rc4 + __docker_compose --version 1.9.0-rc4 + AUTHORS ------- From d6f972057e7ed5b357a1709627fef4605b2ead1c Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Tue, 13 Dec 2016 08:16:24 +0100 Subject: [PATCH 179/197] updated documentation --- cdist/conf/type/__docker_compose/man.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cdist/conf/type/__docker_compose/man.rst b/cdist/conf/type/__docker_compose/man.rst index 29c47fbf..720a306e 100644 --- a/cdist/conf/type/__docker_compose/man.rst +++ b/cdist/conf/type/__docker_compose/man.rst @@ -8,7 +8,7 @@ cdist-type__docker_compose - install docker-compose DESCRIPTION ----------- -Installs latest docker-compose package from dockerproject.org. +Installs docker-compose package. REQUIRED PARAMETERS @@ -19,7 +19,7 @@ None. OPTIONAL PARAMETERS ------------------- version - Define docker_compose version. + Define docker_compose version, defaults to "1.9.0" BOOLEAN PARAMETERS From d597b64705a7c53908b87a6fdb3cb12cad5ebbdc Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 13 Dec 2016 11:41:04 +0100 Subject: [PATCH 180/197] Update changelog. --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index f0b296cc..68be920e 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * New type: __docker_compose (Dominique Roux) * Type __apt_mark: Check supported apt version and if package is installed (Ander Punnar) * New type: __docker (Steven Armstrong) * New type: __package_dpkg (Tomas Pospisek) From d9c4a062eeefa3fd275d801127819c1f4ca62f7d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 14 Dec 2016 09:08:43 +0100 Subject: [PATCH 181/197] Update changelog. --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 68be920e..ec85c50d 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Type __cron: no '# marker' for raw_command due to cron security (Daniel Heule) * New type: __docker_compose (Dominique Roux) * Type __apt_mark: Check supported apt version and if package is installed (Ander Punnar) * New type: __docker (Steven Armstrong) From 2b45405898d8adb3b89e9005d21fc7bf69642191 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 16 Dec 2016 08:19:41 +0100 Subject: [PATCH 182/197] Update docs for types that used man.rst as symbolic links. --- cdist/conf/type/__chroot_mount/man.rst | 6 +- cdist/conf/type/__chroot_umount/man.rst | 8 +- .../conf/type/__install_chroot_mount/man.rst | 43 ++++++- .../conf/type/__install_chroot_umount/man.rst | 48 +++++++- cdist/conf/type/__install_file/man.rst | 113 +++++++++++++++++- docs/changelog | 1 + docs/src/cdist-type.rst | 25 ++++ 7 files changed, 234 insertions(+), 10 deletions(-) mode change 120000 => 100644 cdist/conf/type/__install_chroot_mount/man.rst mode change 120000 => 100644 cdist/conf/type/__install_chroot_umount/man.rst mode change 120000 => 100644 cdist/conf/type/__install_file/man.rst diff --git a/cdist/conf/type/__chroot_mount/man.rst b/cdist/conf/type/__chroot_mount/man.rst index 5ffd7de5..0d7cdce3 100644 --- a/cdist/conf/type/__chroot_mount/man.rst +++ b/cdist/conf/type/__chroot_mount/man.rst @@ -1,9 +1,9 @@ -cdist-type__install_chroot_mount(7) +cdist-type__chroot_mount(7) =================================== NAME ---- -cdist-type__install_chroot_mount - mount a chroot +cdist-type__chroot_mount - mount a chroot DESCRIPTION @@ -26,7 +26,7 @@ EXAMPLES .. code-block:: sh - __install_chroot_mount /path/to/chroot + __chroot_mount /path/to/chroot AUTHORS diff --git a/cdist/conf/type/__chroot_umount/man.rst b/cdist/conf/type/__chroot_umount/man.rst index a2ea1d9b..ff116da5 100644 --- a/cdist/conf/type/__chroot_umount/man.rst +++ b/cdist/conf/type/__chroot_umount/man.rst @@ -1,9 +1,9 @@ -cdist-type__install_chroot_umount(7) -==================================== +cdist-type__chroot_umount(7) +============================ NAME ---- -cdist-type__install_chroot_umount - unmount a chroot mounted by __chroot_mount +cdist-type__chroot_umount - unmount a chroot mounted by __chroot_mount DESCRIPTION @@ -26,7 +26,7 @@ EXAMPLES .. code-block:: sh - __install_chroot_umount /path/to/chroot + __chroot_umount /path/to/chroot SEE ALSO diff --git a/cdist/conf/type/__install_chroot_mount/man.rst b/cdist/conf/type/__install_chroot_mount/man.rst deleted file mode 120000 index 3267c3db..00000000 --- a/cdist/conf/type/__install_chroot_mount/man.rst +++ /dev/null @@ -1 +0,0 @@ -../__chroot_mount/man.rst \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_mount/man.rst b/cdist/conf/type/__install_chroot_mount/man.rst new file mode 100644 index 00000000..4054c4c4 --- /dev/null +++ b/cdist/conf/type/__install_chroot_mount/man.rst @@ -0,0 +1,42 @@ +cdist-type__install_chroot_mount(7) +=================================== + +NAME +---- +cdist-type__install_chroot_mount - mount a chroot with install command + + +DESCRIPTION +----------- +Mount and prepare a chroot for running commands within it. + + +REQUIRED PARAMETERS +------------------- +None + + +OPTIONAL PARAMETERS +------------------- +None + + +EXAMPLES +-------- + +.. code-block:: sh + + __install_chroot_mount /path/to/chroot + + +AUTHORS +------- +Steven Armstrong + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_chroot_umount/man.rst b/cdist/conf/type/__install_chroot_umount/man.rst deleted file mode 120000 index fa34c4b0..00000000 --- a/cdist/conf/type/__install_chroot_umount/man.rst +++ /dev/null @@ -1 +0,0 @@ -../__chroot_umount/man.rst \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_umount/man.rst b/cdist/conf/type/__install_chroot_umount/man.rst new file mode 100644 index 00000000..2e020c01 --- /dev/null +++ b/cdist/conf/type/__install_chroot_umount/man.rst @@ -0,0 +1,47 @@ +cdist-type__install_chroot_umount(7) +==================================== + +NAME +---- +cdist-type__install_chroot_umount - unmount a chroot mounted by __install_chroot_mount + + +DESCRIPTION +----------- +Undo what __install_chroot_mount did. + + +REQUIRED PARAMETERS +------------------- +None + + +OPTIONAL PARAMETERS +------------------- +None + + +EXAMPLES +-------- + +.. code-block:: sh + + __install_chroot_umount /path/to/chroot + + +SEE ALSO +-------- +:strong:`cdist-type__install_chroot_mount`\ (7) + + +AUTHORS +------- +Steven Armstrong + + +COPYING +------- +Copyright \(C) 2012 Steven Armstrong. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_file/man.rst b/cdist/conf/type/__install_file/man.rst deleted file mode 120000 index c937b8af..00000000 --- a/cdist/conf/type/__install_file/man.rst +++ /dev/null @@ -1 +0,0 @@ -../__file/man.rst \ No newline at end of file diff --git a/cdist/conf/type/__install_file/man.rst b/cdist/conf/type/__install_file/man.rst new file mode 100644 index 00000000..c5409167 --- /dev/null +++ b/cdist/conf/type/__install_file/man.rst @@ -0,0 +1,112 @@ +cdist-type__install_file(7) +=========================== + +NAME +---- +cdist-type__install_file - Manage files with install command. + + +DESCRIPTION +----------- +This cdist type allows you to create files, remove files and set file +attributes on the target. + +If the file already exists on the target, then if it is a: + +regular file, and state is: + present + replace it with the source file if they are not equal + exists + do nothing +symlink + replace it with the source file +directory + replace it with the source file + +In any case, make sure that the file attributes are as specified. + + +REQUIRED PARAMETERS +------------------- +None. + +OPTIONAL PARAMETERS +------------------- +state + 'present', 'absent' or 'exists', defaults to 'present' where: + + present + the file is exactly the one from source + absent + the file does not exist + exists + the file from source but only if it doesn't already exist + +group + Group to chgrp to. + +mode + Unix permissions, suitable for chmod. + +owner + User to chown to. + +source + If supplied, copy this file from the host running cdist to the target. + If not supplied, an empty file or directory will be created. + If source is '-' (dash), take what was written to stdin as the file content. + +MESSAGES +-------- +chgrp + Changed group membership +chown + Changed owner +chmod + Changed mode +create + Empty file was created (no --source specified) +remove + File exists, but state is absent, file will be removed by generated code. +upload + File was uploaded + + +EXAMPLES +-------- + +.. code-block:: sh + + # Create /etc/cdist-configured as an empty file + __install_file /etc/cdist-configured + # The same thing + __install_file /etc/cdist-configured --state present + # Use __file from another type + __install_file /etc/issue --source "$__type/files/archlinux" --state present + # Delete existing file + __install_file /etc/cdist-configured --state absent + # Supply some more settings + __install_file /etc/shadow --source "$__type/files/shadow" \ + --owner root --group shadow --mode 0640 \ + --state present + # Provide a default file, but let the user change it + __install_file /home/frodo/.bashrc --source "/etc/skel/.bashrc" \ + --state exists \ + --owner frodo --mode 0600 + # Take file content from stdin + __install_file /tmp/whatever --owner root --group root --mode 644 --source - << DONE + Here goes the content for /tmp/whatever + DONE + + +AUTHORS +------- +Nico Schottelius + + +COPYING +------- +Copyright \(C) 2011-2013 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/docs/changelog b/docs/changelog index ec85c50d..a84ebf10 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Documentation: Update docs for types that used man.rst as symbolic links (Darko Poljak) * Type __cron: no '# marker' for raw_command due to cron security (Daniel Heule) * New type: __docker_compose (Dominique Roux) * Type __apt_mark: Check supported apt version and if package is installed (Ander Punnar) diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 1ff82ce0..2c5f9f6a 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -51,6 +51,19 @@ Example: __myfancysingleton --colour green +Config types +------------ +By default types are used with config command. These are types that are not +flagged by any known command flag. If a type is marked then it will be skipped +with config command. + + +Install types +------------- +If a type is flagged with 'install' flag then it is used only with install command. +With other commands, i.e. config, these types are skipped if used. + + How to write a new type ----------------------- A type consists of @@ -209,6 +222,18 @@ As you can see, the object ID is omitted, because it does not make any sense, if your type can be used only once. +Install - type with install command +----------------------------------- +If you want a type to be used with install command, you must mark it as +install: create the (empty) file "install" in your type directory: + +.. code-block:: sh + + touch cdist/conf/type/__install_NAME/install + +With other commands, i.e. config, it will be skipped if used. + + The type explorers ------------------ If a type needs to explore specific details, it can provide type specific From 7868165e912581a23f4ec6761bb648a61826ca95 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 16 Dec 2016 20:27:00 +0100 Subject: [PATCH 183/197] Update changelog. --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index a84ebf10..425a32f3 100644 --- a/docs/changelog +++ b/docs/changelog @@ -3,7 +3,7 @@ Changelog next: * Documentation: Update docs for types that used man.rst as symbolic links (Darko Poljak) - * Type __cron: no '# marker' for raw_command due to cron security (Daniel Heule) + * Type __cron: Remove '# marker' for raw_command due to cron security (Daniel Heule) * New type: __docker_compose (Dominique Roux) * Type __apt_mark: Check supported apt version and if package is installed (Ander Punnar) * New type: __docker (Steven Armstrong) From 4d106550148210fefa4982ae6b8f7e8c7b657ec4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 17 Dec 2016 09:46:44 +0100 Subject: [PATCH 184/197] release++ --- docs/changelog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog b/docs/changelog index 425a32f3..d5519f25 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,7 +1,7 @@ Changelog --------- -next: +4.4.1: 2016-12-17 * Documentation: Update docs for types that used man.rst as symbolic links (Darko Poljak) * Type __cron: Remove '# marker' for raw_command due to cron security (Daniel Heule) * New type: __docker_compose (Dominique Roux) From 7a5244bf7c5b4fb486de5c69b31e772626d4c5ad Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Wed, 21 Dec 2016 23:03:24 +0100 Subject: [PATCH 185/197] Bugfixed __docker_compose: If docker-compose is already running, curl won't override the binary I add an if the file does not exist before download --- cdist/conf/type/__docker_compose/gencode-remote | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote index f22823d3..3f2fb940 100644 --- a/cdist/conf/type/__docker_compose/gencode-remote +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -22,7 +22,6 @@ version="$(cat "$__object/parameter/version")" # Download docker-compose file -echo 'curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose' - + echo 'if [ ! -e /usr/local/bin/docker-compose ]; then curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose ; fi' # Change permissions echo 'chmod +x /usr/local/bin/docker-compose' From 6fa7bfbfb57dfd29dfaac333f9e172ac6d23aaf1 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Tue, 27 Dec 2016 13:20:57 +0100 Subject: [PATCH 186/197] Better bugfix: - Download the docker-compose binary first to /tmp then move it to its target location --- cdist/conf/type/__docker_compose/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote index 3f2fb940..3424ed6d 100644 --- a/cdist/conf/type/__docker_compose/gencode-remote +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -22,6 +22,7 @@ version="$(cat "$__object/parameter/version")" # Download docker-compose file - echo 'if [ ! -e /usr/local/bin/docker-compose ]; then curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose ; fi' + echo 'curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' + echo 'mv /tmp/docker-compose /usr/local/bin/docker-compose' # Change permissions echo 'chmod +x /usr/local/bin/docker-compose' From 95b92627aa84bf73744724f02fbffd367c2c6751 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Thu, 29 Dec 2016 14:36:46 +0100 Subject: [PATCH 187/197] Added --state absent functionality to docker types Changed types: __docker __docker_compose --- cdist/conf/type/__docker/manifest | 56 ++++++++++--------- .../type/__docker/parameter/default/state | 1 + cdist/conf/type/__docker/parameter/optional | 1 + .../conf/type/__docker_compose/gencode-remote | 13 +++-- cdist/conf/type/__docker_compose/manifest | 11 +++- .../__docker_compose/parameter/default/state | 1 + .../type/__docker_compose/parameter/optional | 1 + 7 files changed, 51 insertions(+), 33 deletions(-) create mode 100644 cdist/conf/type/__docker/parameter/default/state create mode 100644 cdist/conf/type/__docker/parameter/optional create mode 100644 cdist/conf/type/__docker_compose/parameter/default/state diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest index ba13b3e4..1f473afb 100755 --- a/cdist/conf/type/__docker/manifest +++ b/cdist/conf/type/__docker/manifest @@ -20,38 +20,39 @@ os=$(cat "$__global/explorer/os") +state=$(cat "$__object/parameter/state") case "$os" in - centos) - component="main" + centos) + component="main" + if [ -f "$__object/parameter/experimental" ]; then + component="experimental" + fi + __yum_repo docker \ + --name 'Docker Repository' \ + --baseurl "https://yum.dockerproject.org/repo/$component/centos/\$releasever/" \ + --enabled \ + --gpgcheck 1 \ + --gpgkey 'https://yum.dockerproject.org/gpg' \ + --state ${state} + require="__yum_repo/docker" __package docker-engine --state ${state} + ;; + ubuntu) + component="main" if [ -f "$__object/parameter/experimental" ]; then component="experimental" fi - export CDIST_ORDER_DEPENDENCY=on - __yum_repo docker \ - --name 'Docker Repository' \ - --baseurl "https://yum.dockerproject.org/repo/$component/centos/\$releasever/" \ - --enabled \ - --gpgcheck \ - --gpgkey 'https://yum.dockerproject.org/gpg' - __package docker-engine - unset CDIST_ORDER_DEPENDENCY - ;; - ubuntu) - component="main" - if [ -f "$__object/parameter/experimental" ]; then - component="experimental" - fi - __package apparmor - __package ca-certificates - __package apt-transport-https - __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D + __package apparmor --state ${state} + __package ca-certificates --state ${state} + __package apt-transport-https --state ${state} + __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D --state ${state} export CDIST_ORDER_DEPENDENCY=on __apt_source docker \ --uri https://apt.dockerproject.org/repo \ --distribution "ubuntu-$(cat "$__global/explorer/lsb_codename")" \ + --state ${state} \ --component "$component" - __package docker-engine + __package docker-engine --state ${state} unset CDIST_ORDER_DEPENDENCY ;; debian) @@ -60,16 +61,17 @@ case "$os" in component="experimental" fi - __package apt-transport-https - __package ca-certificates - __package gnupg2 - __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D + __package apt-transport-https --state ${state} + __package ca-certificates --state ${state} + __package gnupg2 --state ${state} + __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D --state ${state} export CDIST_ORDER_DEPENDENCY=on __apt_source docker \ --uri https://apt.dockerproject.org/repo \ --distribution "debian-$(cat "$__global/explorer/lsb_codename")" \ + --state ${state} \ --component "$component" - __package docker-engine + __package docker-engine --state ${state} unset CDIST_ORDER_DEPENDENCY ;; diff --git a/cdist/conf/type/__docker/parameter/default/state b/cdist/conf/type/__docker/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker/parameter/optional b/cdist/conf/type/__docker/parameter/optional new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/cdist/conf/type/__docker/parameter/optional @@ -0,0 +1 @@ +state diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote index 3424ed6d..bd1ad452 100644 --- a/cdist/conf/type/__docker_compose/gencode-remote +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -20,9 +20,12 @@ # Variables version="$(cat "$__object/parameter/version")" +state="$(cat "$__object/parameter/state")" -# Download docker-compose file - echo 'curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' - echo 'mv /tmp/docker-compose /usr/local/bin/docker-compose' -# Change permissions -echo 'chmod +x /usr/local/bin/docker-compose' +if [ ${state} = "present" ]; then + # Download docker-compose file + echo 'curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' + echo 'mv /tmp/docker-compose /usr/local/bin/docker-compose' + # Change permissions + echo 'chmod +x /usr/local/bin/docker-compose' +fi diff --git a/cdist/conf/type/__docker_compose/manifest b/cdist/conf/type/__docker_compose/manifest index 2ff80aca..113e87e9 100644 --- a/cdist/conf/type/__docker_compose/manifest +++ b/cdist/conf/type/__docker_compose/manifest @@ -19,6 +19,15 @@ # # +state="$(cat "$__object/parameter/state")" + # Needed packages -__docker +__docker --state ${state} __package curl + +if [ ${state} = "absent" ]; then + __file /usr/local/bin/docker-compose --state absent +elif [ ${state} != "present" -a ${state} != "absent" ]; then + echo "Unknown state: $state_should" >&2 + exit 1 +fi diff --git a/cdist/conf/type/__docker_compose/parameter/default/state b/cdist/conf/type/__docker_compose/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_compose/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_compose/parameter/optional b/cdist/conf/type/__docker_compose/parameter/optional index 088eda41..4d595ed7 100644 --- a/cdist/conf/type/__docker_compose/parameter/optional +++ b/cdist/conf/type/__docker_compose/parameter/optional @@ -1 +1,2 @@ +state version From 54a58abcaa23357d14f3825b5e31ee121a7b1ec2 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Thu, 29 Dec 2016 14:47:29 +0100 Subject: [PATCH 188/197] Changed man.rst of __docker and __docker_compose - Added state parameter for both types in the man.rst - Changed --state absent behavior of __docker_compose -- only remove docker-compose binary not whole docker --- cdist/conf/type/__docker/man.rst | 9 ++++++++- cdist/conf/type/__docker_compose/man.rst | 7 +++++++ cdist/conf/type/__docker_compose/manifest | 10 +++++----- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/cdist/conf/type/__docker/man.rst b/cdist/conf/type/__docker/man.rst index 42e71af5..70b92cc7 100644 --- a/cdist/conf/type/__docker/man.rst +++ b/cdist/conf/type/__docker/man.rst @@ -26,17 +26,24 @@ BOOLEAN PARAMETERS experimental Install the experimental docker-engine package instead of the latest stable release. +state + 'present' or 'absent', defaults to 'present' + EXAMPLES -------- .. code-block:: sh + # Install docker __docker - # experimental + # Install experimental __docker --experimental + # Remove docker + __docker --state absent + AUTHORS ------- diff --git a/cdist/conf/type/__docker_compose/man.rst b/cdist/conf/type/__docker_compose/man.rst index 720a306e..d54b3370 100644 --- a/cdist/conf/type/__docker_compose/man.rst +++ b/cdist/conf/type/__docker_compose/man.rst @@ -21,6 +21,9 @@ OPTIONAL PARAMETERS version Define docker_compose version, defaults to "1.9.0" +state + 'present' or 'absent', defaults to 'present' + BOOLEAN PARAMETERS ------------------ @@ -32,11 +35,15 @@ EXAMPLES .. code-block:: sh + # Install docker-compose __docker_compose # Install version 1.9.0-rc4 __docker_compose --version 1.9.0-rc4 + # Remove docker-compose + __docker_compose --state absent + AUTHORS ------- diff --git a/cdist/conf/type/__docker_compose/manifest b/cdist/conf/type/__docker_compose/manifest index 113e87e9..3ab27c63 100644 --- a/cdist/conf/type/__docker_compose/manifest +++ b/cdist/conf/type/__docker_compose/manifest @@ -22,12 +22,12 @@ state="$(cat "$__object/parameter/state")" # Needed packages -__docker --state ${state} -__package curl - -if [ ${state} = "absent" ]; then +if [ ${state} = "present" ]; then + __docker + __package curl +elif [ ${state} = "absent" ]; then __file /usr/local/bin/docker-compose --state absent -elif [ ${state} != "present" -a ${state} != "absent" ]; then +else echo "Unknown state: $state_should" >&2 exit 1 fi From 0dbe9e14286428437052fa7c9c96ef5335a11b82 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Sat, 31 Dec 2016 11:03:28 +0100 Subject: [PATCH 189/197] updated man.rst --- cdist/conf/type/__docker_compose/man.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cdist/conf/type/__docker_compose/man.rst b/cdist/conf/type/__docker_compose/man.rst index d54b3370..38366157 100644 --- a/cdist/conf/type/__docker_compose/man.rst +++ b/cdist/conf/type/__docker_compose/man.rst @@ -9,6 +9,8 @@ cdist-type__docker_compose - install docker-compose DESCRIPTION ----------- Installs docker-compose package. +State 'absent' will not remove docker binary itself +only docker-compose binary will be removed REQUIRED PARAMETERS From 07906451b23abfe4c100d7413d6f362e183cff1d Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Sat, 31 Dec 2016 11:06:06 +0100 Subject: [PATCH 190/197] updated man.rst --- cdist/conf/type/__docker_compose/man.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker_compose/man.rst b/cdist/conf/type/__docker_compose/man.rst index 38366157..7386e737 100644 --- a/cdist/conf/type/__docker_compose/man.rst +++ b/cdist/conf/type/__docker_compose/man.rst @@ -9,7 +9,7 @@ cdist-type__docker_compose - install docker-compose DESCRIPTION ----------- Installs docker-compose package. -State 'absent' will not remove docker binary itself +State 'absent' will not remove docker binary itself, only docker-compose binary will be removed From 4742913244ce33c7910b0b20224dba3439292147 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Mon, 2 Jan 2017 11:48:07 +0100 Subject: [PATCH 191/197] fixed type --- cdist/conf/type/__docker_compose/manifest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdist/conf/type/__docker_compose/manifest b/cdist/conf/type/__docker_compose/manifest index 3ab27c63..559375ef 100644 --- a/cdist/conf/type/__docker_compose/manifest +++ b/cdist/conf/type/__docker_compose/manifest @@ -28,6 +28,6 @@ if [ ${state} = "present" ]; then elif [ ${state} = "absent" ]; then __file /usr/local/bin/docker-compose --state absent else - echo "Unknown state: $state_should" >&2 + echo "Unknown state: ${state}" >&2 exit 1 fi From 0ad767fa293b80960656d2fe2f3ec40b348de454 Mon Sep 17 00:00:00 2001 From: Daniel Heule Date: Wed, 11 Jan 2017 16:44:17 +0100 Subject: [PATCH 192/197] fix filter for new cron on sles12 sp2 --- cdist/conf/type/__cron/gencode-remote | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote index f5c1cc62..3c3ed6b3 100644 --- a/cdist/conf/type/__cron/gencode-remote +++ b/cdist/conf/type/__cron/gencode-remote @@ -3,6 +3,7 @@ # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2013 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Thomas Oettli (otho at sfs.biz) +# 2017 Daniel Heule (hda at sfs.biz) # # This file is part of cdist. # @@ -57,7 +58,7 @@ state_should="$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" # These are the old markers prefix="#cdist:__cron/$__object_id" suffix="#/cdist:__cron/$__object_id" -filter="^# DO NOT EDIT THIS FILE|^# \(.* installed on |^# \(Cron version V" +filter="^# DO NOT EDIT THIS FILE|^# \(.* installed on |^# \(Cron version V|^# \(Cronie version .\..\)$" cat << DONE crontab -u $user -l 2>/dev/null | grep -v -E "$filter" | awk -v prefix="$prefix" -v suffix="$suffix" ' { From 86678b5beb0a292273947c250796035de0814758 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 11 Jan 2017 21:32:52 +0100 Subject: [PATCH 193/197] Cleanup __user_groups oldusermod explorer. --- cdist/conf/type/__user_groups/explorer/oldusermod | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/cdist/conf/type/__user_groups/explorer/oldusermod b/cdist/conf/type/__user_groups/explorer/oldusermod index 248d3922..6ef25b13 100644 --- a/cdist/conf/type/__user_groups/explorer/oldusermod +++ b/cdist/conf/type/__user_groups/explorer/oldusermod @@ -18,12 +18,4 @@ # along with cdist. If not, see . # -os="$($__explorer/os)" - -if [ "$os" = "netbsd" ]; then - echo netbsd -elif [ "$os" = "freebsd" ]; then - echo freebsd -else - usermod --help | grep -q -- '-A group' && echo true || echo false -fi +usermod --help | grep -q -- '-A group' && echo true || echo false From 61c45e7eaba64fea0de259c5b3099418aff68cd3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 14 Jan 2017 19:54:52 +0100 Subject: [PATCH 194/197] Update changelog. --- docs/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog b/docs/changelog index d5519f25..3ffb0c68 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,6 +1,12 @@ Changelog --------- +next: + * Type __user_groups: Support FreeBSD (Andres Erbsen) + * Type __cron: Fix filter for new cron on sles12 sp2 (Daniel Heule) + * Type __docker: Support absent state (Dominique Roux) + * Type __docker_compose: Support absent state (Dominique Roux) + 4.4.1: 2016-12-17 * Documentation: Update docs for types that used man.rst as symbolic links (Darko Poljak) * Type __cron: Remove '# marker' for raw_command due to cron security (Daniel Heule) From 2087f7a28aa477a95e69b45a002402a3829bba7c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Jan 2017 15:22:25 +0100 Subject: [PATCH 195/197] Un-suppress manifest stdout. --- cdist/core/manifest.py | 6 ++++-- cdist/exec/local.py | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index a16e9346..574884a0 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -138,7 +138,8 @@ class Manifest(object): message_prefix = "initialmanifest" self.local.run_script(initial_manifest, env=self.env_initial_manifest(initial_manifest), - message_prefix=message_prefix) + message_prefix=message_prefix, + save_output=False) def env_type_manifest(self, cdist_object): type_manifest = os.path.join(self.local.type_path, @@ -163,4 +164,5 @@ class Manifest(object): if os.path.isfile(type_manifest): self.local.run_script(type_manifest, env=self.env_type_manifest(cdist_object), - message_prefix=message_prefix) + message_prefix=message_prefix, + save_output=False) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index 93301063..ec7a6ee8 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -212,7 +212,7 @@ class Local(object): try: if save_output: output, errout = exec_util.call_get_output(command, env=env) - self.log.debug("Local stdout: {}".format(output)) + self.log.info("Local stdout: {}".format(output)) # Currently, stderr is not captured. # self.log.debug("Local stderr: {}".format(errout)) if return_output: @@ -231,7 +231,7 @@ class Local(object): message.merge_messages() def run_script(self, script, env=None, return_output=False, - message_prefix=None): + message_prefix=None, save_output=True): """Run the given script with the given environment. Return the output as a string. @@ -240,7 +240,7 @@ class Local(object): command.append(script) return self.run(command=command, env=env, return_output=return_output, - message_prefix=message_prefix) + message_prefix=message_prefix, save_output=save_output) def save_cache(self): destination = os.path.join(self.cache_path, self.hostdir) From 707f220f779fd13fda631bc8c1ed6c3d246bc9f4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 21 Jan 2017 17:23:50 +0100 Subject: [PATCH 196/197] Update changelog. --- docs/changelog | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/changelog b/docs/changelog index 3ffb0c68..71ca7782 100644 --- a/docs/changelog +++ b/docs/changelog @@ -2,6 +2,7 @@ Changelog --------- next: + * Core: Fix suppression of manifests' outputs (Darko Poljak) * Type __user_groups: Support FreeBSD (Andres Erbsen) * Type __cron: Fix filter for new cron on sles12 sp2 (Daniel Heule) * Type __docker: Support absent state (Dominique Roux) From b03bed242572fad1931cc90f5bd36bd452d2dff0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 28 Jan 2017 13:49:19 +0100 Subject: [PATCH 197/197] Better describe -v option. --- cdist/argparse.py | 8 +++++--- docs/src/man1/cdist.rst | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cdist/argparse.py b/cdist/argparse.py index 04f6e6a4..e2a13c38 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -85,9 +85,11 @@ def get_parsers(): action='store_true', default=False) parser['loglevel'].add_argument( '-v', '--verbose', - help=('Increase log level, be more verbose. Use it more than once ' - 'to increase log level. The order of levels from the lowest ' - 'to the highest are: ERROR, WARNING, INFO, DEBUG.'), + help=('Increase the verbosity level. Every instance of -v ' + 'increments the verbosity level by one. Its default value is ' + '0. There are 4 levels of verbosity. The order of levels ' + 'from the lowest to the highest are: ERROR (0), WARNING (1), ' + 'INFO (2) and DEBUG (3 or higher).'), action='count', default=0) parser['beta'] = argparse.ArgumentParser(add_help=False) diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index 55901300..adabe052 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -50,9 +50,10 @@ All commands accept the following options: .. option:: -v, --verbose - Increase log level, be more verbose. Use it more than once to increase - log level. The order of levels from the lowest to the highest are: - ERROR, WARNING, INFO, DEBUG. + Increase the verbosity level. Every instance of -v increments the verbosity + level by one. Its default value is 0. There are 4 levels of verbosity. The + order of levels from the lowest to the highest are: ERROR (0), WARNING (1), + INFO (2) and DEBUG (3 or higher). .. option:: -V, --version