Merge pull request #550 from ungleich/archiving

Add archiving support

cdist/argparse.py

@@ -10,7 +10,7 @@ import collections
 BETA_COMMANDS = set(('install', 'inventory', ))
 # set of beta arguments for sub-commands
 BETA_ARGS = {
-    'config': set(('jobs', 'tag', 'all_tagged_hosts', )),
+    'config': set(('jobs', 'tag', 'all_tagged_hosts', 'use_archiving', )),
 }
 EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
 # Parser others can reuse
@@ -164,6 +164,15 @@ def get_parsers():
     parser['config_main'].add_argument(
            '-o', '--out-dir',
            help='directory to save cdist output in', dest="out_path")
+    parser['config_main'].add_argument(
+           '-R', '--use-archiving', nargs='?',
+           choices=('tar', 'tgz', 'tbz2', 'txz',),
+           help=('Operate by using archiving with compression where '
+                 'apropriate. Supported values are: tar - tar archive, '
+                 'tgz - gzip tar archive (the default), '
+                 'tbz2 - bzip2 tar archive and txz - lzma tar archive.'),
+           action='store', dest='use_archiving',
+           const='tgz')
 
     # remote-copy and remote-exec defaults are environment variables
     # if set; if not then None - these will be futher handled after

cdist/autil.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+#
+# 2017 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+
+import cdist
+import tarfile
+import os
+import glob
+import tempfile
+
+
+_ARCHIVING_MODES = {
+    'tar': '',
+    'tgz': 'gz',
+    'tbz2': 'bz2',
+    'txz': 'xz',
+}
+
+
+_UNARCHIVE_OPT = {
+    'tar': None,
+    'tgz': '-z',
+    'tbz2': '-j',
+    'txz': '-J',
+}
+
+
+# Archiving will be enabled if directory contains more than FILES_LIMIT files.
+FILES_LIMIT = 1
+
+
+def get_extract_option(mode):
+    return _UNARCHIVE_OPT[mode]
+
+
+def tar(source, mode="tgz"):
+    if mode not in _ARCHIVING_MODES:
+        raise cdist.Error("Unsupported archiving mode {}.".format(mode))
+
+    files = glob.glob1(source, '*')
+    fcnt = len(files)
+    if fcnt <= FILES_LIMIT:
+        return None, fcnt
+
+    tarmode = 'w:{}'.format(_ARCHIVING_MODES[mode])
+    _, tarpath = tempfile.mkstemp(suffix='.' + mode)
+    with tarfile.open(tarpath, tarmode, dereference=True) as tar:
+        if os.path.isdir(source):
+            for f in files:
+                tar.add(os.path.join(source, f), arcname=f)
+        else:
+            tar.add(source)
+    return tarpath, fcnt

cdist/config.py

@@ -302,7 +302,8 @@ class Config(object):
                 remote_exec=remote_exec,
                 remote_copy=remote_copy,
                 base_path=args.remote_out_path,
-                quiet_mode=args.quiet)
+                quiet_mode=args.quiet,
+                archiving_mode=args.use_archiving)
 
             c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs)
             c.run()

cdist/exec/remote.py

@@ -63,7 +63,8 @@ class Remote(object):
                  remote_exec,
                  remote_copy,
                  base_path=None,
-                 quiet_mode=None):
+                 quiet_mode=None,
+                 archiving_mode=None):
         self.target_host = target_host
         self._exec = remote_exec
         self._copy = remote_copy
@@ -73,6 +74,7 @@ class Remote(object):
         else:
             self.base_path = "/var/lib/cdist"
         self.quiet_mode = quiet_mode
+        self.archiving_mode = archiving_mode
 
         self.conf_path = os.path.join(self.base_path, "conf")
         self.object_path = os.path.join(self.base_path, "object")
@@ -111,6 +113,11 @@ class Remote(object):
         self.run(["chmod", "0700", self.base_path])
         self.mkdir(self.conf_path)
 
+    def rmfile(self, path):
+        """Remove file on the remote side."""
+        self.log.trace("Remote rm: %s", path)
+        self.run(["rm", "-f",  path])
+
     def rmdir(self, path):
         """Remove directory on the remote side."""
         self.log.trace("Remote rmdir: %s", path)
@@ -121,23 +128,76 @@ class Remote(object):
         self.log.trace("Remote mkdir: %s", path)
         self.run(["mkdir", "-p", path])
 
+    def extract_archive(self, path, mode):
+        """Extract archive path on the remote side."""
+        import cdist.autil as autil
+
+        self.log.trace("Remote extract archive: %s", path)
+        command = ["tar", "-x", "-m", "-C", ]
+        directory = os.path.dirname(path)
+        command.append(directory)
+        xopt = autil.get_extract_option(mode)
+        if xopt:
+            command.append(xopt)
+        command.append("-f")
+        command.append(path)
+        self.run(command)
+
+    def _transfer_file(self, source, destination):
+        command = self._copy.split()
+        command.extend([source, '{0}:{1}'.format(
+            _wrap_addr(self.target_host[0]), destination)])
+        self._run_command(command)
+
     def transfer(self, source, destination, jobs=None):
         """Transfer a file or directory to the remote side."""
         self.log.trace("Remote transfer: %s -> %s", source, destination)
         # self.rmdir(destination)
         if os.path.isdir(source):
             self.mkdir(destination)
-            if jobs:
-                self._transfer_dir_parallel(source, destination, jobs)
-            else:
-                self._transfer_dir_sequential(source, destination)
+            used_archiving = False
+            if self.archiving_mode:
+                self.log.trace("Remote transfer in archiving mode")
+                import cdist.autil as autil
+
+                # create archive
+                tarpath, fcnt = autil.tar(source, self.archiving_mode)
+                if tarpath is None:
+                    self.log.trace(("Files count {} is lower than {} limit, "
+                                    "skipping archiving").format(
+                                        fcnt, autil.FILES_LIMIT))
+                else:
+                    self.log.trace(("Archiving mode, tarpath: %s, file count: "
+                                    "%s"), tarpath, fcnt)
+                    # get archive name
+                    tarname = os.path.basename(tarpath)
+                    self.log.trace("Archiving mode tarname: %s", tarname)
+                    # archive path at the remote
+                    desttarpath = os.path.join(destination, tarname)
+                    self.log.trace(
+                        "Archiving mode desttarpath: %s", desttarpath)
+                    # transfer archive to the remote side
+                    self.log.trace("Archiving mode: transfering")
+                    self._transfer_file(tarpath, desttarpath)
+                    # extract archive at the remote
+                    self.log.trace("Archiving mode: extracting")
+                    self.extract_archive(desttarpath, self.archiving_mode)
+                    # remove remote archive
+                    self.log.trace("Archiving mode: removing remote archive")
+                    self.rmfile(desttarpath)
+                    # remove local archive
+                    self.log.trace("Archiving mode: removing local archive")
+                    os.remove(tarpath)
+                    used_archiving = True
+            if not used_archiving:
+                if jobs:
+                    self._transfer_dir_parallel(source, destination, jobs)
+                else:
+                    self._transfer_dir_sequential(source, destination)
         elif jobs:
             raise cdist.Error("Source {} is not a directory".format(source))
         else:
-            command = self._copy.split()
-            command.extend([source, '{0}:{1}'.format(
-                _wrap_addr(self.target_host[0]), destination)])
-            self._run_command(command)
+            self._transfer_file(source, destination)
 
     def _transfer_dir_commands(self, source, destination):
         for f in glob.glob1(source, '*'):

cdist/test/autil/__init__.py

@@ -0,0 +1,57 @@
+# -*- coding: utf-8 -*-
+#
+# 2017 Darko Poljak (darko.poljak at gmail.com)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+from cdist import test
+import cdist.autil as autil
+import os
+import os.path as op
+import tarfile
+
+
+my_dir = op.abspath(op.dirname(__file__))
+fixtures = op.join(my_dir, 'fixtures')
+explorers_path = op.join(fixtures, 'explorer')
+
+
+class AUtilTestCase(test.CdistTestCase):
+    def test_tar(self):
+        test_modes = {
+            'tar': 'r:',
+            'tgz': 'r:gz',
+            'tbz2': 'r:bz2',
+            'txz': 'r:xz',
+        }
+        source = explorers_path
+        for mode in test_modes:
+            tarpath = autil.tar(source, mode)
+            self.assertIsNotNone(tarpath)
+            fcnt = 0
+            with tarfile.open(tarpath, test_modes[mode]) as tar:
+                for tarinfo in tar:
+                    fcnt += 1
+            os.remove(tarpath)
+            self.assertGreater(fcnt, 0)
+
+
+if __name__ == "__main__":
+    import unittest
+
+    unittest.main()

cdist/test/autil/fixtures/explorer/cpu_cores

@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+# 2014 Daniel Heule  (hda at sfs.biz)
+# 2014 Thomas Oettli (otho at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# FIXME: other system types (not linux ...)
+
+os=$("$__explorer/os")
+case "$os" in
+    "macosx")
+        echo "$(sysctl -n hw.physicalcpu)"
+    ;;
+
+    *)
+        if [ -r /proc/cpuinfo ]; then
+            cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)"
+            if [ ${cores} -eq 0 ]; then
+                cores="1"
+            fi
+            echo "$cores"
+        fi
+    ;;
+esac

cdist/test/autil/fixtures/explorer/cpu_sockets

@@ -0,0 +1,40 @@
+#!/bin/sh
+#
+# 2014 Daniel Heule  (hda at sfs.biz)
+# 2014 Thomas Oettli (otho at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# FIXME: other system types (not linux ...)
+
+os=$("$__explorer/os")
+case "$os" in
+    "macosx")
+        echo "$(system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}')"
+    ;;
+
+    *)
+    if [ -r /proc/cpuinfo ]; then
+        sockets="$(grep "physical id" /proc/cpuinfo | sort | uniq | wc -l)"
+        if [ ${sockets} -eq 0 ]; then
+            sockets="$(cat /proc/cpuinfo | grep "processor" | wc -l)"
+        fi
+        echo "${sockets}"
+    fi
+    ;;
+esac

cdist/test/autil/fixtures/explorer/disks

@@ -0,0 +1,2 @@
+cd /dev
+echo sd? hd? vd?

cdist/test/autil/fixtures/explorer/hostname

@@ -0,0 +1,25 @@
+#!/bin/sh
+#
+# 2010-2014 Nico Schottelius (nico-cdist at schottelius.org)
+# 2012 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+if command -v uname >/dev/null; then
+   uname -n
+fi

cdist/test/autil/fixtures/explorer/init

@@ -0,0 +1,35 @@
+#!/bin/sh
+#
+# 2016 Daniel Heule (hda at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# Returns the process name of pid 1 ( normaly the init system )
+# for example at linux this value is "init" or "systemd" in most cases
+#
+
+uname_s="$(uname -s)"
+
+case "$uname_s" in
+    Linux|FreeBSD)
+        ps -o comm= -p 1 || true
+    ;;
+    *)
+        # return a empty string as unknown value
+        echo ""
+    ;;
+esac

cdist/test/autil/fixtures/explorer/interfaces

@@ -0,0 +1,51 @@
+#!/bin/sh
+#
+# 2012 Sébastien Gross <seb•ɑƬ•chezwam•ɖɵʈ•org>
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# List all network interfaces in explorer/ifaces. One interface per line.
+#
+# If your OS is not supported please provide a ifconfig output
+#
+
+# Use ip, if available
+if command -v ip >/dev/null; then
+    ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p'
+    exit 0
+fi
+
+if ! command -v ifconfig >/dev/null; then
+   # no ifconfig, nothing we could do
+   exit 0
+fi
+
+uname_s="$(uname -s)"
+REGEXP='s/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p'
+
+case "$uname_s" in
+    Darwin)
+	ifconfig -a | sed -n -E "$REGEXP"
+	;;
+    Linux|*BSD)
+	ifconfig -a | sed -n -r "$REGEXP"
+	;;
+    *)
+	echo "Unsupported ifconfig output for $uname_s" >&2
+	exit 1
+	;;
+esac

cdist/test/autil/fixtures/explorer/kernel_name

@@ -0,0 +1 @@
+uname -s

cdist/test/autil/fixtures/explorer/lsb_codename

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+case "$($__explorer/os)" in
+   openwrt)
+      (. /etc/openwrt_release && echo "$DISTRIB_CODENAME")
+   ;;
+   *)
+      lsb_release=$(command -v lsb_release)
+      if [ -x "$lsb_release" ]; then
+         $lsb_release --short --codename
+      fi
+   ;;
+esac

cdist/test/autil/fixtures/explorer/lsb_description

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+case "$($__explorer/os)" in
+   openwrt)
+      (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION")
+   ;;
+   *)
+      lsb_release=$(command -v lsb_release)
+      if [ -x "$lsb_release" ]; then
+         $lsb_release --short --description
+      fi
+   ;;
+esac

cdist/test/autil/fixtures/explorer/lsb_id

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+case "$($__explorer/os)" in
+   openwrt)
+      (. /etc/openwrt_release && echo "$DISTRIB_ID")
+   ;;
+   *)
+      lsb_release=$(command -v lsb_release)
+      if [ -x "$lsb_release" ]; then
+         $lsb_release --short --id
+      fi
+   ;;
+esac

cdist/test/autil/fixtures/explorer/lsb_release

@@ -0,0 +1,33 @@
+#!/bin/sh
+#
+# 2011 Steven Armstrong (steven-cdist at armstrong.cc)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+case "$($__explorer/os)" in
+   openwrt)
+      (. /etc/openwrt_release && echo "$DISTRIB_RELEASE")
+   ;;
+   *)
+      lsb_release=$(command -v lsb_release)
+      if [ -x "$lsb_release" ]; then
+         $lsb_release --short --release
+      fi
+   ;;
+esac

cdist/test/autil/fixtures/explorer/machine

@@ -0,0 +1,27 @@
+#!/bin/sh
+#
+# 2010-2011 Andi Brönnimann (andi-cdist at v-net.ch)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# All os variables are lower case
+#
+#
+
+if command -v uname 2>&1 >/dev/null; then
+   uname -m
+fi

cdist/test/autil/fixtures/explorer/machine_type

@@ -0,0 +1,66 @@
+#!/bin/sh
+#
+# 2014 Daniel Heule  (hda at sfs.biz)
+# 2014 Thomas Oettli (otho at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# FIXME: other system types (not linux ...)
+
+if [ -d "/proc/vz" -a ! -d "/proc/bc" ]; then
+    echo openvz
+    exit
+fi
+
+if [ -e "/proc/1/environ" ] &&
+    cat "/proc/1/environ" | tr '\000' '\n' | grep -Eiq '^container='; then
+    echo lxc
+    exit
+fi
+
+if [ -r /proc/cpuinfo ]; then
+    # this should only exist on virtual guest machines,
+    # tested on vmware, xen, kvm
+    if grep -q "hypervisor" /proc/cpuinfo; then
+        # this file is aviable in xen guest systems
+        if [ -r /sys/hypervisor/type ]; then
+            if grep -q -i "xen" /sys/hypervisor/type; then
+                echo virtual_by_xen
+                exit 
+            fi
+        else
+            if [ -r /sys/class/dmi/id/product_name ]; then
+                if grep -q -i 'vmware' /sys/class/dmi/id/product_name; then
+                    echo "virtual_by_vmware"
+                    exit
+                elif grep -q -i 'bochs' /sys/class/dmi/id/product_name; then
+                    echo "virtual_by_kvm"
+                    exit 
+                elif grep -q -i 'virtualbox' /sys/class/dmi/id/product_name; then
+                    echo "virtual_by_virtualbox"
+                    exit 
+                fi
+            fi
+        fi
+        echo "virtual_by_unknown"
+    else
+        echo "physical"
+    fi
+else
+    echo "unknown"
+fi

cdist/test/autil/fixtures/explorer/memory

@@ -0,0 +1,36 @@
+#!/bin/sh
+#
+# 2014 Daniel Heule  (hda at sfs.biz)
+# 2014 Thomas Oettli (otho at sfs.biz)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+# FIXME: other system types (not linux ...)
+
+os=$("$__explorer/os")
+case "$os" in
+    "macosx")
+        echo "$(sysctl -n hw.memsize)/1024" | bc
+    ;;
+
+    *)
+    if [ -r /proc/meminfo ]; then
+        grep "MemTotal:" /proc/meminfo | awk '{print $2}'
+    fi
+    ;;
+esac

cdist/test/autil/fixtures/explorer/os

@@ -0,0 +1,143 @@
+#!/bin/sh
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# All os variables are lower case.  Keep this file in alphabetical
+# order by os variable except in cases where order otherwise matters,
+# in which case keep the primary os and its derivatives together in
+# a block (see Debian and Redhat examples below).
+#
+
+if grep -q ^Amazon /etc/system-release 2>/dev/null; then
+   echo amazon
+   exit 0
+fi
+
+if [ -f /etc/arch-release ]; then
+   echo archlinux
+   exit 0
+fi
+
+if [ -f /etc/cdist-preos ]; then
+   echo cdist-preos
+   exit 0
+fi
+
+if [ -d /gnu/store ]; then
+   echo guixsd
+   exit 0
+fi
+
+### Debian and derivatives
+if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then
+   echo ubuntu
+   exit 0
+fi
+
+if [ -f /etc/debian_version ]; then
+   echo debian
+   exit 0
+fi
+
+if [ -f /etc/devuan_version ]; then
+   echo devuan
+   exit 0
+fi
+###
+
+if [ -f /etc/gentoo-release ]; then
+   echo gentoo
+   exit 0
+fi
+
+if [ -f /etc/openwrt_version ]; then
+    echo openwrt
+    exit 0
+fi
+
+if [ -f /etc/owl-release ]; then
+   echo owl
+   exit 0
+fi
+
+### Redhat and derivatives
+if grep -q ^Scientific /etc/redhat-release 2>/dev/null; then
+    echo scientific
+    exit 0
+fi
+
+if grep -q ^CentOS /etc/redhat-release 2>/dev/null; then
+    echo centos
+    exit 0
+fi
+
+if grep -q ^Fedora /etc/redhat-release 2>/dev/null; then
+   echo fedora
+   exit 0
+fi
+
+if grep -q ^Mitel /etc/redhat-release 2>/dev/null; then
+   echo mitel
+   exit 0
+fi
+
+if [ -f /etc/redhat-release ]; then
+   echo redhat
+   exit 0
+fi
+###
+
+if [ -f /etc/SuSE-release ]; then
+   echo suse
+   exit 0
+fi
+
+if [ -f /etc/slackware-version ]; then
+   echo slackware
+   exit 0
+fi
+
+uname_s="$(uname -s)"
+
+# Assume there is no tr on the client -> do lower case ourselves
+case "$uname_s" in
+   Darwin)
+      echo macosx
+      exit 0
+   ;;
+   NetBSD)
+      echo netbsd
+      exit 0
+   ;;
+   FreeBSD)
+      echo freebsd
+      exit 0
+   ;;
+   OpenBSD)
+      echo openbsd
+      exit 0
+   ;;
+   SunOS)
+      echo solaris
+      exit 0
+   ;;
+esac
+
+echo "Unknown OS" >&2
+exit 1

cdist/test/autil/fixtures/explorer/os_version

@@ -0,0 +1,73 @@
+#!/bin/sh
+#
+# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+# All os variables are lower case
+#
+#
+
+case "$($__explorer/os)" in
+   amazon)
+      cat /etc/system-release
+   ;;
+   archlinux)
+      # empty, but well...
+      cat /etc/arch-release
+   ;;
+   debian)
+      cat /etc/debian_version
+   ;;
+   devuan)
+      cat /etc/devuan_version
+   ;;
+   fedora)
+      cat /etc/fedora-release
+   ;;
+   gentoo)
+      cat /etc/gentoo-release
+   ;;
+   macosx)
+      sw_vers -productVersion
+   ;;
+   *bsd|solaris)
+      uname -r
+   ;;
+   openwrt)
+      cat /etc/openwrt_version
+   ;;
+   owl)
+      cat /etc/owl-release
+   ;;
+   redhat|centos|mitel|scientific)
+      cat /etc/redhat-release
+   ;;
+   slackware)
+      cat /etc/slackware-version
+   ;;
+   suse)
+      if [ -f /etc/os-release ]; then
+        cat /etc/os-release
+      else
+        cat /etc/SuSE-release
+      fi
+   ;;
+   ubuntu)
+      lsb_release -sr
+   ;;
+esac

cdist/test/autil/fixtures/explorer/runlevel

@@ -0,0 +1,26 @@
+#!/bin/sh
+#
+# 2012 Nico Schottelius (nico-cdist at schottelius.org)
+#
+# This file is part of cdist.
+#
+# cdist is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# cdist is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with cdist. If not, see <http://www.gnu.org/licenses/>.
+#
+#
+
+set +e
+executable=$(command -v runlevel)
+if [ -x "$executable" ]; then
+   "$executable" | awk '{ print $2 }'
+fi