Merge branch '2.1'

Signed-off-by: Nico Schottelius <nico@brief.schottelius.org>

Conflicts:
	docs/changelog

.gitignore

@@ -2,20 +2,27 @@
 .*.swp
 
 # Ignore generated manpages
-doc/man/.marker
-doc/man/man1/*.1
-doc/man/man7/*.7
-doc/man/man*/*.html
-doc/man/man*/*.xml
-doc/man/man7/cdist-type__*.text
-doc/man/man7/cdist-reference.text
-doc/man/man*/docbook-xsl.css
+docs/man/.marker
+docs/man/man1/*.1
+docs/man/man7/*.7
+docs/man/man*/*.html
+docs/man/man*/*.xml
+docs/man/man*/docbook-xsl.css
+docs/man/man7/cdist-type__*.text
+docs/man/man7/cdist-reference.text
 
 # Ignore cdist cache for version control
 /cache/
 
-# Python / cache
+# Python: cache, distutils, distribution in general
 __pycache__/
+MANIFEST
+dist/
+cdist/version.py
 
-# Is static and will never be included
-lib/cdist/version_static.py
+# Packaging: Archlinux
+/PKGBUILD
+/cdist-*.pkg.tar.xz
+/cdist-*.tar.gz
+/pkg
+/src

.version

@@ -1 +1 @@
-2.0.14
+2.1.0-pre1

MANIFEST.in

@@ -0,0 +1,3 @@
+include docs/changelog
+recursive-include docs/gfx *.png *.text
+recursive-include docs *.text *.html *.1 *.7

PKGBUILD.in

@@ -0,0 +1,26 @@
+#!/bin/sh
+
+version=$(git describe)
+outfile=${0%.in}
+
+cat << eof > "${outfile}"
+pkgname=cdist
+pkgver=$version
+pkgrel=1
+pkgdesc='A Usable Configuration Management System"'
+arch=('any')
+url='http://www.nico.schottelius.org/software/cdist/'
+license=('GPL3')
+depends=('python>=3.2.0')
+source=("http://pypi.python.org/packages/source/c/cdist/cdist-\${pkgver}.tar.gz")
+
+package() {
+    cd cdist-\${pkgver}
+    python3 setup.py build install --root="\${pkgdir}"
+    mv "\${pkgdir}"/usr/bin/cdist.py "\${pkgdir}"/usr/bin/cdist
+
+    #install -Dm644 offlineimap.1 "\${pkgdir}"/usr/share/man/man1/offlineimap.1
+}
+eof
+
+makepkg -g >> "${outfile}"

bin/cdist

@@ -1,7 +1,7 @@
-#!/usr/bin/env python3
+#!/bin/sh
 # -*- coding: utf-8 -*-
 #
-# 2010-2012 Nico Schottelius (nico-cdist at schottelius.org)
+# 2012 Nico Schottelius (nico-cdist at schottelius.org)
 #
 # This file is part of cdist.
 #
@@ -20,229 +20,10 @@
 #
 #
 
-def commandline():
-    """Parse command line"""
-    import argparse
+# Wrapper for real script to allow execution from checkout
+dir=${0%/*}
 
-    import cdist.banner
-    import cdist.config
-    import cdist.install
-
-    # Construct parser others can reuse
-    parser = {}
-    # Options _all_ parsers have in common
-    parser['loglevel'] = argparse.ArgumentParser(add_help=False)
-    parser['loglevel'].add_argument('-d', '--debug',
-        help='Set log level to debug', action='store_true',
-        default=False)
-    parser['loglevel'].add_argument('-v', '--verbose',
-        help='Set log level to info, be more verbose',
-        action='store_true', default=False)
-
-    # Main subcommand parser
-    parser['main'] = argparse.ArgumentParser(description='cdist ' + cdist.VERSION,
-        parents=[parser['loglevel']])
-    parser['main'].add_argument('-V', '--version',
-        help='Show version', action='version',
-        version='%(prog)s ' + cdist.VERSION)
-    parser['sub'] = parser['main'].add_subparsers(title="Commands")
-
-    # Banner
-    parser['banner'] = parser['sub'].add_parser('banner', 
-        parents=[parser['loglevel']])
-    parser['banner'].set_defaults(func=cdist.banner.banner)
-
-    # Config and install (common stuff)
-    parser['configinstall'] = argparse.ArgumentParser(add_help=False)
-    parser['configinstall'].add_argument('host', nargs='+',
-        help='one or more hosts to operate on')
-    parser['configinstall'].add_argument('-c', '--cdist-home',
-         help='Change cdist home (default: .. from bin directory)',
-         action='store')
-    parser['configinstall'].add_argument('-i', '--initial-manifest', 
-         help='Path to a cdist manifest or \'-\' to read from stdin.',
-         dest='manifest', required=False)
-    parser['configinstall'].add_argument('-p', '--parallel',
-         help='Operate on multiple hosts in parallel',
-         action='store_true', dest='parallel')
-    parser['configinstall'].add_argument('-s', '--sequential',
-         help='Operate on multiple hosts sequentially (default)',
-         action='store_false', dest='parallel')
-
-    parser['configinstall'].add_argument('--remote-copy',
-         help='Command to use for remote copy (should behave like scp)',
-         action='store', dest='remote_copy',
-         default="scp -o User=root -q")
-    parser['configinstall'].add_argument('--remote-exec',
-         help='Command to use for remote execution (should behave like ssh)',
-         action='store', dest='remote_exec',
-         default="ssh -o User=root -q")
-
-    # Config
-    parser['config'] = parser['sub'].add_parser('config',
-        parents=[parser['loglevel'], parser['configinstall']])
-    parser['config'].set_defaults(func=config)
-
-    # Install
-    # 20120525/sar: commented until it actually does something
-    #parser['install'] = parser['sub'].add_parser('install',
-    #    parents=[parser['loglevel'], parser['configinstall']])
-    #parser['install'].set_defaults(func=install)
-
-    for p in parser:
-        parser[p].epilog = "Get cdist at http://www.nico.schottelius.org/software/cdist/"
-
-    args = parser['main'].parse_args(sys.argv[1:])
-
-    # Loglevels are handled globally in here and debug wins over verbose
-    if args.verbose:
-        logging.root.setLevel(logging.INFO)
-    if args.debug:
-        logging.root.setLevel(logging.DEBUG)
-
-    log.debug(args)
-    args.func(args)
-
-def config(args):
-    configinstall(args, mode=cdist.config.Config)
-
-def install(args):
-    configinstall(args, mode=cdist.install.Install)
-
-def configinstall(args, mode):
-    """Configure or install remote system"""
-    import multiprocessing
-    import time
-
-    initial_manifest_tempfile = None
-    if args.manifest == '-':
-        # read initial manifest from stdin
-        import tempfile
-        try:
-            handle, initial_manifest_temp_path = tempfile.mkstemp(prefix='cdist.stdin.')
-            with os.fdopen(handle, 'w') as fd:
-                fd.write(sys.stdin.read())
-        except (IOError, OSError) as e:
-            raise cdist.Error("Creating tempfile for stdin data failed: %s" % e)
-
-        args.manifest = initial_manifest_temp_path
-        import atexit
-        atexit.register(lambda: os.remove(initial_manifest_temp_path))
-
-    process = {}
-    failed_hosts = []
-    time_start = time.time()
-
-    for host in args.host:
-        if args.parallel:
-            log.debug("Creating child process for %s", host)
-            process[host] = multiprocessing.Process(target=configinstall_onehost, args=(host, args, mode, True))
-            process[host].start()
-        else:
-            try:
-                configinstall_onehost(host, args, mode, parallel=False)
-            except cdist.Error as e:
-                failed_hosts.append(host)
-
-    # Catch errors in parallel mode when joining
-    if args.parallel:
-        for host in process.keys():
-            log.debug("Joining process %s", host)
-            process[host].join()
-
-            if not process[host].exitcode == 0:
-                failed_hosts.append(host)
-
-    time_end = time.time()
-    log.info("Total processing time for %s host(s): %s", len(args.host),
-                (time_end - time_start))
-
-    if len(failed_hosts) > 0:
-        raise cdist.Error("Failed to deploy to the following hosts: " + 
-            " ".join(failed_hosts))
-
-def configinstall_onehost(host, args, mode, parallel):
-    """Configure or install ONE remote system"""
-
-    try:
-        import cdist.context
-
-        context = cdist.context.Context(
-            target_host=host,
-            remote_copy=args.remote_copy,
-            remote_exec=args.remote_exec,
-            initial_manifest=args.manifest,
-            base_path=args.cdist_home,
-            exec_path=sys.argv[0],
-            debug=args.debug)
-
-        c = mode(context)
-        c.deploy_and_cleanup()
-        context.cleanup()
-
-    except cdist.Error as e:
-        # We are running in our own process here, need to sys.exit!
-        if parallel:
-            log.error(e)
-            sys.exit(1)
-        else:
-            raise
-
-    except KeyboardInterrupt:
-        # Ignore in parallel mode, we are existing anyway
-        if parallel:
-            sys.exit(0)
-        # Pass back to controlling code in sequential mode
-        else:
-            raise
-
-def emulator():
-    """Prepare and run emulator"""
-    import cdist.emulator
-    emulator = cdist.emulator.Emulator(sys.argv)
-    return emulator.run()
-
-if __name__ == "__main__":
-    # Sys is needed for sys.exit()
-    import sys
-
-    cdistpythonversion = '3.2'
-    if sys.version < cdistpythonversion:
-        print('Cdist requires Python >= ' + cdistpythonversion +
-            ' on the source host.', file=sys.stderr)
-        sys.exit(1)
-
-
-    exit_code = 0
-
-    try:
-        import logging
-        import os
-        import re
-
-        # Ensure our /lib/ is included into PYTHON_PATH
-        sys.path.insert(0, os.path.abspath(
-                os.path.join(os.path.dirname(os.path.realpath(__file__)), '../lib')))
-
-        # And now import our stuff
-        import cdist
-
-        log = logging.getLogger("cdist")
-
-        logging.basicConfig(format='%(levelname)s: %(message)s')
-
-        if re.match("__", os.path.basename(sys.argv[0])):
-            emulator()
-        else:
-            commandline()
-
-    except KeyboardInterrupt:
-        pass
-
-    except cdist.Error as e:
-        log.error(e)
-        exit_code = 1
-
-    # Determine exit code by return value of function
-
-    sys.exit(exit_code)
+# Ensure version is present - the bundled/shipped version contains a static version,
+# the git version contains a dynamic version
+"$dir/../build" version
+PYTHONPATH="${dir}/../" "$dir/../scripts/cdist" "$@"

build

@@ -26,6 +26,7 @@
 # exit on any error
 #set -e
 
+basedir=${0%/*}
 version=$(git describe)
 
 # Manpage and HTML
@@ -39,10 +40,13 @@ WEBMAN=$WEBBASE/man/$version
 WEBPAGE=${WEBBASE}.mdwn
 
 # Documentation
-MANDIR=doc/man
+MANDIR=docs/man
 MAN1DSTDIR=${MANDIR}/man1
 MAN7DSTDIR=${MANDIR}/man7
-SPEECHESDIR=doc/speeches
+SPEECHESDIR=docs/speeches
+
+# Change to checkout directory
+cd "$basedir"
 
 case "$1" in
     man)
@@ -74,8 +78,8 @@ case "$1" in
     ;;
 
     mantype)
-	   for mansrc in conf/type/*/man.text; do
-         dst="$(echo $mansrc | sed -e 's;conf/;cdist-;'  -e 's;/;;' -e 's;/man;;' -e 's;^;doc/man/man7/;')"
+        for mansrc in cdist/conf/type/*/man.text; do
+            dst="$(echo $mansrc | sed -e 's;cdist/conf/;cdist-;'  -e 's;/;;' -e 's;/man;;' -e 's;^;docs/man/man7/;')"
             ln -sf "../../../$mansrc" "$dst"
         done
     ;;
@@ -84,8 +88,152 @@ case "$1" in
         ${MANDIR}/cdist-reference.text.sh
     ;;
 
-   release)
-        ./doc/dev/releasechecklist
+    dist)
+        set -e
+        # Do the checks
+        $0 dist-check
+
+        # Git changes - everything depends on this
+        $0 dist-tag
+        $0 dist-branch-merge
+
+        # Pypi first - is the base for others
+        $0 dist-pypi
+        
+        # Archlinux depends on successful pypi ;-)
+        $0 dist-archlinux
+
+        # Update website (includes documentation)
+        $0 web
+
+        $0 pub
+
+        $0 dist-freecode
+
+        $0 dist-post
+    ;;
+
+    changelog-version)
+        # get version from changelog and ensure it's not already present
+        grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//'
+    ;;
+
+    dist-check)
+        set -e
+        echo "Verifying documentation building works ..."
+        $0 clean 
+        $0 man
+
+        changelog_version=$($0 changelog-version)
+        echo "Target version from changelog: $changelog_version"
+
+        if git show --quiet $changelog_version >/dev/null 2>&1; then
+            echo "Version $changelog_version already exists, aborting."
+            exit 1
+        fi
+
+        # verify date in changelog
+        date_today="$(date +%Y-%m-%d)"
+        date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
+
+        if [ "$date_today" != "$date_changelog" ]; then
+            echo "Date in changelog is not today"
+            echo "Changelog: $date_changelog"
+            exit 1
+        fi
+
+    ;;
+
+    dist-post)
+        cat << notes
+
+    To be done manually...
+    
+        - freecode release
+        - blog entry
+        - linkedin entry
+        - mailinglist update
+notes
+
+    ;;
+
+    dist-tag)
+        version=$($0 changelog-version)
+        # add tag
+        printf "Enter tag description for %s> " "$version"
+        read tagmessage
+        git tag "$version" -m "$tagmessage"
+    ;;
+
+    dist-branch-merge)
+        version=$($0 changelog-version)
+        target_branch=${version%\.*}
+        current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+        if [ "$target_branch" = "$current_branch" ]; then
+            echo "Skipping merge, already on destination branch"
+        else
+            printf "Press enter to git merge $current_branch into \"$target_branch\" > "
+            read prompt
+            git checkout "$target_branch"
+            git merge "$current_branch"
+            git checkout "$current_branch"
+        fi
+   ;;
+
+    dist-archlinux)
+        $0 dist-archlinux-makepkg
+        $0 dist-archlinux-aur-upload
+    ;;
+
+    dist-archlinux-makepkg)
+        ./PKGBUILD.in
+        makepkg -c --source
+    ;;
+
+    dist-archlinux-aur-upload)
+        version=$($0 changelog-version)
+        tar=cdist-${version}-1.src.tar.gz
+        burp -c system "$tar"
+    ;;
+
+    dist-freecode)
+        version=$($0 changelog-version)
+        api_token=$(awk '/machine freecode login/ { print $8 }' ~/.netrc)
+
+        printf "Enter tag list for freecode release %s> " "$version"
+        read taglist
+
+        printf "Enter changelog for freecode release %s> " "$version"
+        read changelog
+
+        echo "Submit preview"
+        cat << eof
+tag_list = $taglist
+changelog = $changelog
+version = $version
+eof
+        printf "Press enter to submit to freecode> "
+        read dummy
+
+        cat << eof | cfreecode-api release-add cdist
+        {
+            "auth_code": "$api_token",
+            "release": {
+                "tag_list": "$taglist",
+                "version": "$version",
+                "changelog": "$changelog",
+                "hidden_from_frontpage": false
+            }
+        }
+eof
+
+    ;;
+
+    dist-pypi)
+        $0 man
+        $0 version
+        python3 setup.py sdist upload
     ;;
 
     speeches)
@@ -108,11 +256,6 @@ case "$1" in
         rm -rf ${WEBMAN}
         mkdir -p ${WEBMAN}/man1 ${WEBMAN}/man7
 
-      # old stuff
-      # rm -rf ${WEBDIR}/${WEBBASE}/speeches && mkdir ${WEBDIR}/${WEBBASE}/speeches
-      # cp ${SPEECHESDIR}/*.pdf ${WEBDIR}/${WEBBASE}/speeches
-      # git describe > ${WEBDIR}/${WEBBASE}/man/VERSION
-
         cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${WEBMAN}/man1 
         cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${WEBMAN}/man7
         
@@ -134,18 +277,34 @@ case "$1" in
     ;;
 
     clean)
-      rm -f ${MAN7DSTDIR}/cdist-reference.text lib/cdist/version.py
+        rm -f ${MAN7DSTDIR}/cdist-reference.text
+
         find "${MANDIR}" -mindepth 2 -type l \
             -o -name "*.1" \
             -o -name "*.7" \
             -o -name "*.html" \
             -o -name "*.xml" \
         | xargs rm -f
+
+        find * -name __pycache__  | xargs rm -rf
+    ;;
+    clean-dist)
+        rm -f cdist/version.py MANIFEST PKGBUILD
+        rm -rf cache/ dist/
+
+        # Archlinux
+        rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
+        rm -rf pkg/ src/
+    ;;
+
+    very-clean)
+        $0 clean
+        $0 clean-dist
     ;;
 
     test)
         shift # skip t
-      export PYTHONPATH=$PYTHONPATH:$(pwd -P)/lib
+        export PYTHONPATH="$(pwd -P)"
 
         if [ $# -lt 1 ]; then
             python3 -m cdist.test
@@ -154,16 +313,8 @@ case "$1" in
         fi
     ;;
 
-    version-dynamic)
-        cd lib/cdist/
-        ln -sf version_dynamic.py version.py
-    ;;
-
-    version-dist)
-        version=$(cat .version)
-        cd lib/cdist/
-        echo "VERSION=\"$version\"" > version_static.py
-        ln -sf version_static.py version.py
+    version)
+        echo "VERSION=\"$version\"" > cdist/version.py
     ;;
 
     *)
@@ -180,4 +331,5 @@ case "$1" in
         echo "Unknown target, \"$1\"" >&2
         exit 1
     ;;
+
 esac

cdist/conf/manifest/sample-from-distribution

@@ -0,0 +1,62 @@
+## #
+## # Sample manifest from cdist distribution
+## #
+## 
+## # Every machine becomes a marker, so sysadmins know that automatic
+## # configurations are happening
+## __file /etc/cdist-configured
+## __cdistmarker
+## 
+## case "$__target_host" in
+##    # Everybody has this
+##    localhost)
+##       require="__file/etc/cdist-configured" __link /tmp/cdist-testfile \
+##          --source /etc/cdist-configured  --type symbolic
+##       require="__directory/tmp/cdist-test-dir" __file /tmp/cdist-test-dir/test-file \
+##          --mode 0750 --owner nobody --group root
+##       __directory /tmp/cdist-test-dir --mode 4777
+## 
+##       require="__file/etc/cdist-configured __link/tmp/cdist-testfile" \
+##          __file /tmp/cdist-another-testfile
+## 
+##    ;;
+## 
+##    #
+##    # Use an alias in /etc/hosts for localhost to use these hosts:
+##    # 
+##    # 127.0.0.1    localhost.localdomain   localhost cdist-archlinux
+##    #
+##    cdist-archlinux)
+##       # This is the specific package type for pacman
+##       __package_pacman zsh --state installed
+##       
+##       # The __package type autoselect the right type based on the os
+##       __package vim --state installed
+## 
+##       # If the type is a singleton, it does not take an object id
+##       __issue
+##    ;;
+##    # This is how it would look like on gentoo
+##    cdist-gentoo)
+##       # Same stuff for gentoo
+##       __package tree --state installed
+##    ;;
+## 
+##    cdist-debian)
+##       __package_apt atop --state installed
+##       __package apache2 --state removed
+##    ;;
+## 
+##    cdist-redhat)
+##       __issue
+##       __motd
+##    ;;
+## 
+##    # Real machines may be used with their hostname or fqdn,
+##    # depending on how you call cdist
+##    # ...
+##    # ;;
+##    # machine.example.org)
+##    # ...
+##    # ;;
+## esac

cdist/conf/type/__directory/gencode-remote

@@ -18,7 +18,6 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-# Check state and exit if nothing needs to be done
 state_should="present"
 [ -f "$__object/parameter/state" ] && state_should="$(cat "$__object/parameter/state")"
 state_is="$(cat "$__object/explorer/state")"
@@ -27,9 +26,9 @@ state_is="$(cat "$__object/explorer/state")"
 destination="/$__object_id"
 
 mkdiropt=""
-grep yes "$__object/parameter/parents" >/dev/null 2>&1 && mkdiropt="-p"
+[ -f "$__object/parameter/parents" ] && mkdiropt="-p"
 recursive=""
-grep yes "$__object/parameter/recursive" >/dev/null 2>&1 && recursive="-R"
+[ -f "$__object/parameter/recursive" ] && recursive="-R"
 
 case "$state_should" in
     present)

cdist/conf/type/__directory/man.text

@@ -32,13 +32,15 @@ mode::
 owner::
    User to chown to.
 
+
+BOOLEAN PARAMETERS
+------------------
 parents::
-   Whether to create parents as well (mkdir -p behaviour). Must be yes or no.
+   Whether to create parents as well (mkdir -p behaviour)
 
 recursive::
    If supplied the chgrp and chown call will run recursively.
    This does *not* influence the behaviour of chmod.
-   Must be yes or no.
 
 
 EXAMPLES
@@ -55,13 +57,18 @@ __directory /tmp/foobar --state absent
 __directory /etc --owner root --group root --mode 0755
 
 # Create nfs service directory, including parents
-__directory /home/services/nfs --parents yes
+__directory /home/services/nfs --parents
 
 # Change permissions recursively
-__directory /home/services --recursive yes --owner root --group root
+__directory /home/services --recursive --owner root --group root
 
 # Setup a temp directory
 __directory /local --mode 1777
+
+# Take it all
+__directory /home/services/kvm --recursive --parents \
+    --owner root --group root --mode 0755 --state present
+
 --------------------------------------------------------------------------------
 
 

cdist/conf/type/__directory/parameter/boolean

@@ -0,0 +1,2 @@
+parents
+recursive

cdist/conf/type/__directory/parameter/optional

@@ -2,5 +2,3 @@ state
 group
 mode
 owner
-parents
-recursive