Merge branch 'master' of github.com:telmich/cdist

cdist/conf/type/__consul/files/versions/0.8.1/cksum

@@ -0,0 +1 @@
+283033689 36101209 consul

cdist/conf/type/__consul/files/versions/0.8.1/source

@@ -0,0 +1 @@
+https://releases.hashicorp.com/consul/0.8.1/consul_0.8.1_linux_amd64.zip

cdist/conf/type/__consul/manifest

@@ -23,7 +23,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   scientific|centos|redhat|ubuntu|debian|archlinux|gentoo)
+   scientific|centos|redhat|ubuntu|debian|devuan|archlinux|gentoo)
       # any linux should work
       :
    ;;

cdist/conf/type/__consul_agent/manifest

@@ -23,7 +23,7 @@
 os=$(cat "$__global/explorer/os")
 
 case "$os" in
-   scientific|centos|debian|redhat|ubuntu)
+   scientific|centos|debian|devuan|redhat|ubuntu)
       # whitelist safeguard
       :
    ;;
@@ -215,7 +215,11 @@ case "$os" in
         esac
     ;;
 
-   ubuntu)
+    devuan)
-      init_upstart
+        init_sysvinit debian
-   ;;
+    ;;
+
+    ubuntu)
+        init_upstart
+    ;;
 esac

cdist/conf/type/__consul_service/man.rst

@@ -24,6 +24,9 @@ OPTIONAL PARAMETERS
 check-interval
    the interval in which the script given with --check-script should be run
 
+check-http
+   the URL to check for HTTP 200-ish status every --check-interval
+
 check-script
    the shell command to run every --check-interval
 

cdist/conf/type/__consul_service/manifest

@@ -32,6 +32,10 @@ if [ -f "$__object/parameter/check-script" -a ! -f "$__object/parameter/check-in
    echo "When using --check-script you must also define --check-interval" >&2
    exit 1
 fi
+if [ -f "$__object/parameter/check-http" -a ! -f "$__object/parameter/check-interval" ]; then
+   echo "When using --check-http you must also define --check-interval" >&2
+   exit 1
+fi
 
 # Generate json config file
 (
@@ -52,6 +56,12 @@ for param in $(ls "$__object/parameter/"); do
          printf '         "ttl": "%s"\n' "$(cat "$__object/parameter/check-ttl")"
          printf '      }\n'
       ;;
+      check-http)
+         printf '     ,"check": {\n'
+         printf '         "http": "%s"\n' "$(cat "$__object/parameter/check-http")"
+         printf '         ,"interval": "%s"\n' "$(cat "$__object/parameter/check-interval")"
+         printf '      }\n'
+      ;;
       tag)
          # create json array from newline delimited file
          tags="$(awk '{printf "\""$1"\","}' "$__object/parameter/tag")"

cdist/conf/type/__consul_service/parameter/optional

@@ -1,3 +1,4 @@
+check-http
 check-interval
 check-script
 check-ttl

cdist/conf/type/__daemontools/files/init.d-svscan

@@ -0,0 +1,63 @@
+#!/bin/bash
+### BEGIN INIT INFO
+# Provides:          svscan
+# Required-Start:
+# Required-Stop:
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: svscan
+# Description:       djb svscan
+### END INIT INFO
+# from https://gist.githubusercontent.com/pacojp/5766990/raw/2ed009ab19515afc9e58291b636d673c5ca864b3/init.d.svscan
+# written by Adam McKenna <adam@debian.org>
+# edited by Kamila Součková <kamila@ksp.sk>
+
+export PATH=$PATH:/usr/local/bin
+
+l=/var/log/svscan
+
+if [ ! -d $l ]; then
+  mkdir $l
+  chown daemon $l
+fi
+
+case "$1" in
+    start)
+        echo -n "Starting daemontools: "
+        if [ ! `pidof svscan` ]; then
+                echo -n "svscan "
+                env - PATH="$PATH" svscan /service 2>&1 | setuidgid daemon multilog t /var/log/svscan &
+                echo "."
+        else
+                echo "already running."
+        fi
+        ;;
+    stop)
+        echo -n "Stopping daemontools: "
+        if [ `pidof svscan` ]; then
+                echo -n "svscan"
+                while [ `pidof svscan` ]; do
+                     kill `pidof svscan`
+                     echo -n "."
+                done
+        fi
+        echo -n " services"
+        for i in `ls -d /service/*`; do
+                svc -dx $i
+                echo -n "."
+        done
+        echo -n " logging "
+        for i in `ls -d /service/*/log`; do
+                svc -dx $i
+                echo -n "."
+        done
+        echo ""
+        ;;
+    restart|force-reload)
+        $0 stop
+        $0 start
+        ;;
+    *)
+        echo 'Usage: /etc/init.d/svscan {start|stop|restart|force-reload}'
+        exit 1
+esac

cdist/conf/type/__daemontools/man.rst

@@ -0,0 +1,49 @@
+cdist-type__daemontools(7)
+==========================
+
+NAME
+----
+cdist-type__daemontools - Install daemontools
+
+
+DESCRIPTION
+-----------
+Install djb daemontools and (optionally) an init script.
+
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+
+OPTIONAL PARAMETERS
+-------------------
+from-package
+   Package to install. Must be compatible with the original daemontools. Example: daemontools-encore. Default: daemontools.
+
+BOOLEAN PARAMETERS
+------------------
+install-init-script
+   Add an init script and set it to start on boot. Default yes.
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    __daemontools --from-package daemontools-encore  # if you prefer
+
+SEE ALSO
+--------
+:strong:`cdist-type__daemontools_service`\ (7)
+
+AUTHORS
+-------
+Kamila Součková <kamila--@--ksp.sk>
+
+COPYING
+-------
+Copyright \(C) 2017 Kamila Součková. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__daemontools/manifest

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+pkg=$(cat "$__object/parameter/from-package")
+
+__package $pkg
+
+if [ -f "$__object/parameter/install-init-script" ]; then
+	init=$(cat "$__global/explorer/init")
+	case $init in
+		init)
+			__config_file /etc/init.d/svscan --mode 755 --source "$__type/files/init.d-svscan"
+			require="$require __config_file/etc/init.d/svscan" __start_on_boot svscan
+			require="$require __start_on_boot/svscan" __process svscan --start 'service svscan start'
+		;;
+		*)
+			echo "Your init system ($init) is not supported by this type. Submit a patch at github.com/ungleich/cdist!"
+			exit 1
+		;;
+	esac
+fi

cdist/conf/type/__daemontools/parameter/boolean

@@ -0,0 +1 @@
+install-init-script

cdist/conf/type/__daemontools/parameter/default/from-package

@@ -0,0 +1 @@
+daemontools

cdist/conf/type/__daemontools/parameter/optional

@@ -0,0 +1 @@
+from-package

cdist/conf/type/__daemontools_service/explorer/svc

@@ -0,0 +1 @@
+command -v svc

cdist/conf/type/__daemontools_service/man.rst

@@ -0,0 +1,72 @@
+cdist-type__daemontools_service(7)
+==================================
+
+NAME
+----
+cdist-type__daemontools_service - Create a daemontools-compatible service dir.
+
+
+DESCRIPTION
+-----------
+Create a directory structure compatible with daemontools-like service management.
+
+Note that svc must be present on the target system.
+
+The object ID will be used as the service name.
+
+REQUIRED PARAMETERS
+-------------------
+None.
+
+OPTIONAL PARAMETERS
+-------------------
+run
+   Command to run. exec-ing and stderr redirection will be added. One of run, run-file must be specified.
+
+   Example: `my-program`
+
+run-file
+   File to save as <servicedir>/run. One of run, run-file must be specified.
+
+   Example:
+
+.. code-block:: sh
+
+    #!/bin/sh
+    exec 2>&1
+    exec my_program
+
+
+log-run
+   Command to run for log consumption. Default: `multilog t ./main`
+
+servicedir
+   Directory to install into. Default: `/service`
+
+BOOLEAN PARAMETERS
+------------------
+None.
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    require="__daemontools" __daemontools_service prometheus --run "setuidgid prometheus $GOBIN/prometheus $FLAGS"
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__daemontools`\ (7)
+
+
+AUTHORS
+-------
+Kamila Součková <kamila--@--ksp.sk>
+
+COPYING
+-------
+Copyright \(C) 2017 Kamila Součková. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__daemontools_service/manifest

@@ -0,0 +1,38 @@
+#!/bin/sh
+
+RUN_PREFIX="#!/bin/sh
+exec 2>&1
+exec "  # mind the space :D
+
+name=$__object_id
+servicedir=$(cat "$__object/parameter/servicedir")
+run=$(cat "$__object/parameter/run")
+runfile=$(cat "$__object/parameter/run-file")
+logrun=$(cat "$__object/parameter/log-run")
+
+svc=$(cat "$__type/explorer/svc")
+
+if [ -z "$svc" ]; then
+	echo "svc not found! Install daemontools first: see __daemontools"
+	exit 1
+fi
+
+badusage() {
+	echo "__daemontools_service/$__object_id: exactly one of --run, --run-file must be set" >&2
+	exit 1
+}
+
+[ -z "$run$runfile" ] && badusage
+[ -n "$run" ] && [ -n "$runfile" ] && badusage
+
+__directory $servicedir/$name/log/main --parents
+
+echo "$RUN_PREFIX$run" | require="__directory/$servicedir/$name/log/main" __config_file "$servicedir/$name/run" \
+	--onchange "svc -t '$servicedir/$name' 2>/dev/null" \
+	--mode 755 \
+	--source "${runfile:--}"
+
+echo "$RUN_PREFIX$logrun" | require="__directory/$servicedir/$name/log/main" __config_file $servicedir/$name/log/run \
+	--onchange "svc -t '$servicedir/$name/log' 2>/dev/null" \
+	--mode 755 \
+	--source "-"

cdist/conf/type/__daemontools_service/parameter/default/log-run

@@ -0,0 +1 @@
+multilog t ./main

cdist/conf/type/__daemontools_service/parameter/default/servicedir

@@ -0,0 +1 @@
+/service

cdist/conf/type/__daemontools_service/parameter/optional

@@ -0,0 +1,4 @@
+log-run
+run
+run-file
+servicedir

cdist/conf/type/__prometheus_alertmanager/man.rst

@@ -0,0 +1,66 @@
+cdist-type__prometheus_alertmanager(7)
+======================================
+
+NAME
+----
+cdist-type__prometheus_alertmanager - install Alertmanager
+
+
+DESCRIPTION
+-----------
+Install and configure Prometheus Alertmanager (https://prometheus.io/docs/alerting/alertmanager/).
+
+
+REQUIRED PARAMETERS
+-------------------
+config
+   Alertmanager configuration file. It will be saved as /etc/alertmanager/alertmanager.yml on the target.
+listen-address
+   Passed as web.listen-address.
+
+
+OPTIONAL PARAMETERS
+-------------------
+storage-path
+   Where to put data. Default: /data/alertmanager. (Directory will be created if needed.)
+
+
+BOOLEAN PARAMETERS
+------------------
+with-daemontools
+   Create a daemontools service directory under /service/prometheus. Default: yes.
+   Note: If you do not use this, Alertmanager will not be launched, and will not reload config on change.
+   If you use this, daemontools (or something compatible) must be installed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    ALERTPORT=9093
+
+    __daemontools
+    __golang_from_vendor --version 1.8.1  # required for prometheus and many exporters
+
+    require="__daemontools __golang_from_vendor" __prometheus_alertmanager \
+      --config "$__manifest/files/alertmanager.yml" \
+      --storage-path /data/alertmanager \
+      --listen-address "[::]:$ALERTPORT"
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__prometheus_server`\ (7), :strong:`cdist-type__daemontools`\ (7),
+Prometheus alerting documentation: https://prometheus.io/docs/alerting/overview/
+
+AUTHORS
+-------
+Kamila Součková <kamila--@--ksp.sk>
+
+COPYING
+-------
+Copyright \(C) 2017 Kamila Součková. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__prometheus_alertmanager/manifest

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+GOBIN=/opt/gocode/bin  # where to find go binaries
+CONF_DIR=/etc/prometheus
+LOGLEVEL=info
+CONF=$CONF_DIR/alertmanager.yml
+
+### Prometheus server #######################################################
+
+config="$(cat "$__object/parameter/config")"
+storage_path="$(cat "$__object/parameter/storage-path")"
+listen_address="$(cat "$__object/parameter/listen-address")"
+
+ONCHANGE=""
+if [ -f "$__object/parameter/with-daemontools" ]; then
+	__daemontools_service prometheus --run "setuidgid prometheus $GOBIN/prometheus $REAL_FLAGS"
+	ONCHANGE="svc -h /service/prometheus"
+fi
+
+FLAGS="config.file '$CONF'
+storage.path '$storage_path'
+web.listen-address '$listen_address'
+log.level $LOGLEVEL"
+
+REAL_FLAGS="$(echo "$FLAGS" | sed -nE 's/^([^#]+).*/ --\1 \\/p')"
+
+__go_get github.com/prometheus/alertmanager/cmd/...
+
+__user prometheus --system
+__directory "$storage_path" --owner prometheus
+__directory "$CONF_DIR" --owner prometheus
+
+__daemontools_service alertmanager --run "setuidgid prometheus $GOBIN/alertmanager $REAL_FLAGS"
+
+require="$require __directory/$storage_path" \
+__config_file $CONF \
+    --source $config \
+    --group prometheus --mode 640 \
+    --onchange "$ONCHANGE"

cdist/conf/type/__prometheus_alertmanager/parameter/default/storage-path

@@ -0,0 +1 @@
+/data/alertmanager

cdist/conf/type/__prometheus_alertmanager/parameter/optional

@@ -0,0 +1 @@
+storage-path

cdist/conf/type/__prometheus_alertmanager/parameter/required

@@ -0,0 +1,2 @@
+config
+listen-address

cdist/conf/type/__prometheus_server/man.rst

@@ -0,0 +1,78 @@
+cdist-type__prometheus_server(7)
+================================
+
+NAME
+----
+cdist-type__prometheus_server - install Prometheus
+
+
+DESCRIPTION
+-----------
+Install and configure Prometheus (https://prometheus.io/).
+
+
+REQUIRED PARAMETERS
+-------------------
+config
+   Prometheus configuration file. It will be saved as /etc/prometheus/prometheus.yml on the target.
+listen-address
+   Passed as web.listen-address.
+alertmanager-url
+   Passed as alertmanager.url
+
+
+OPTIONAL PARAMETERS
+-------------------
+retention-days
+   How long to keep data. Default: 30
+rule-files
+   Path to rule files. They will be installed under /etc/prometheus/<filename>. You need to include `rule_files: [/etc/prometheus/<your-pattern>]` in the config file if you use this.
+storage-path
+   Where to put data. Default: /data/prometheus. (Directory will be created if needed.)
+target-heap-size
+   Passed as storage.local.target-heap-size. Default: 1/2 of RAM.
+
+
+BOOLEAN PARAMETERS
+------------------
+with-daemontools
+   Create a daemontools service directory under /service/prometheus. Default: yes.
+   Note: If you do not use this, Prometheus will not be launched, and will not reload config on change.
+   If you use this, daemontools (or something compatible) must be installed.
+
+
+EXAMPLES
+--------
+
+.. code-block:: sh
+
+    PROMPORT=9090
+    ALERTPORT=9093
+
+    __daemontools
+    __golang_from_vendor --version 1.8.1  # required for prometheus and many exporters
+
+    require="__daemontools __golang_from_vendor" __prometheus_server \
+        --config "$__manifest/files/prometheus.yml" \
+        --retention-days 14 \
+        --storage-path /data/prometheus \
+        --listen-address "[::]:$PROMPORT" \
+        --rule-files "$__manifest/files/*.rules" \
+        --alertmanager-url "http://monitoring1.node.consul:$ALERTPORT,http://monitoring2.node.consul:$ALERTPORT"
+
+
+SEE ALSO
+--------
+:strong:`cdist-type__prometheus_alertmanager`\ (7), :strong:`cdist-type__daemontools`\ (7),
+Prometheus documentation: https://prometheus.io/docs/introduction/overview/
+
+AUTHORS
+-------
+Kamila Součková <kamila--@--ksp.sk>
+
+COPYING
+-------
+Copyright \(C) 2017 Kamila Součková. You can redistribute it
+and/or modify it under the terms of the GNU General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.

cdist/conf/type/__prometheus_server/manifest

@@ -0,0 +1,57 @@
+#!/bin/sh
+
+GOBIN=/opt/gocode/bin  # where to find go binaries
+CONF_DIR=/etc/prometheus
+CONF=$CONF_DIR/prometheus.yml
+LOGLEVEL=info
+
+config="$(cat "$__object/parameter/config")"
+retention_days="$(cat "$__object/parameter/retention-days")"
+storage_path="$(cat "$__object/parameter/storage-path")"
+listen_address="$(cat "$__object/parameter/listen-address")"
+alertmanager_url="$(cat "$__object/parameter/alertmanager-url")"
+target_heap_size="$(cat "$__object/parameter/target-heap-size")"
+rule_files="$(cat "$__object/parameter/rule-files")"
+
+# explorer in kB => convert; by default we go with 1/2 RAM
+[ "$target_heap_size" = "auto" ] && target_heap_size=$(($(cat $__global/explorer/memory)*1024/2))
+
+ONCHANGE=""
+if [ -f "$__object/parameter/with-daemontools" ]; then
+	__daemontools_service prometheus --run "setuidgid prometheus $GOBIN/prometheus $REAL_FLAGS"
+	ONCHANGE="&& svc -h /service/prometheus"
+fi
+
+FLAGS="config.file '$CONF'
+storage.local.path '$storage_path'
+storage.local.target-heap-size $(($target_heap_size))  # in bytes; should be 2/3 of available memory because it may be hungry
+storage.local.retention $(($retention_days*24))h  # golang doesn't have days :D
+web.listen-address '$listen_address'
+alertmanager.url '$alertmanager_url'
+log.level $LOGLEVEL"
+
+REAL_FLAGS="$(echo "$FLAGS" | sed -nE 's/^([^#]+).*/ --\1 \\/p')"
+
+__go_get github.com/prometheus/prometheus/cmd/...
+
+__user prometheus --system
+__directory "$storage_path" --owner prometheus
+__directory "$CONF_DIR" --owner prometheus
+
+
+
+require="$require __directory/$storage_path" \
+__config_file $CONF \
+	--source $config \
+	--group prometheus --mode 640 \
+	--onchange "$GOBIN/promtool check-config $CONF $ONCHANGE"
+
+for file in $rule_files; do
+	dest=$CONF_DIR/$(basename $file)
+	require="$require __directory/$CONF_DIR" \
+	__config_file "$dest" \
+		--source "$file" \
+		--owner prometheus \
+		--onchange "$GOBIN/promtool check-rules '$dest' $ONCHANGE"
+done
+

cdist/conf/type/__prometheus_server/parameter/boolean

@@ -0,0 +1 @@
+with-daemontools

cdist/conf/type/__prometheus_server/parameter/default/retention-days

@@ -0,0 +1 @@
+30

cdist/conf/type/__prometheus_server/parameter/default/storage-path

@@ -0,0 +1 @@
+/data/prometheus

cdist/conf/type/__prometheus_server/parameter/default/target-heap-size

@@ -0,0 +1 @@
+auto

cdist/conf/type/__prometheus_server/parameter/optional

@@ -0,0 +1,4 @@
+target-heap-size
+retention-days
+rule-files
+storage-path

cdist/conf/type/__prometheus_server/parameter/required

@@ -0,0 +1,3 @@
+alertmanager-url
+config
+listen-address

cdist/config.py

@@ -27,6 +27,8 @@ import time
 import itertools
 import tempfile
 import socket
+import atexit
+import shutil
 
 import cdist
 import cdist.hostsource
@@ -92,7 +94,6 @@ class Config(object):
                                    "failed: %s" % e))
 
             args.manifest = initial_manifest_temp_path
-            import atexit
             atexit.register(lambda: os.remove(initial_manifest_temp_path))
 
         # default remote cmd patterns
@@ -176,8 +177,15 @@ class Config(object):
                               " ".join(failed_hosts))
 
     @classmethod
-    def _resolve_remote_cmds(cls, args, host_base_path):
+    def _resolve_ssh_control_path(cls):
-        control_path = os.path.join(host_base_path, "ssh-control-path")
+        base_path = tempfile.mkdtemp()
+        control_path = os.path.join(base_path, "s")
+        atexit.register(lambda: shutil.rmtree(base_path))
+        return control_path
+
+    @classmethod
+    def _resolve_remote_cmds(cls, args):
+        control_path = cls._resolve_ssh_control_path()
         # If we constructed patterns for remote commands then there is
         # placeholder for ssh ControlPath, format it and we have unique
         # ControlPath for each host.
@@ -200,8 +208,7 @@ class Config(object):
         log = logging.getLogger(host)
 
         try:
-            remote_exec, remote_copy = cls._resolve_remote_cmds(
+            remote_exec, remote_copy = cls._resolve_remote_cmds(args)
-                args, host_base_path)
             log.debug("remote_exec for host \"{}\": {}".format(
                 host, remote_exec))
             log.debug("remote_copy for host \"{}\": {}".format(

docs/changelog

@@ -7,6 +7,10 @@ next:
 	* Explorer kernel_name: uname -s (Kamila Součková)
 	* Type __sysctl: Add devuan support (Nico Schottelius)
 	* Type __start_on_boot: Add devuan support (Nico Schottelius)
+	* Core: Shorten ssh control path (Darko Poljak)
+	* Type __consul: Add new version and add http check (Kamila Součková)
+	* New types: __daemontools and __daemontools_service (Kamila Součková)
+	* New types: __prometheus_server and __prometheus_alertmanager (Kamila Součková)
 
 4.4.2: 2017-03-08
 	* Core: Fix suppression of manifests' outputs (Darko Poljak)