Merge branch 'master' of github.com:telmich/cdist

This commit is contained in:
Nico Schottelius 2017-06-03 18:16:25 +02:00
commit 3a7013d7a1
42 changed files with 548 additions and 10 deletions

View file

@ -0,0 +1 @@
283033689 36101209 consul

View file

@ -0,0 +1 @@
https://releases.hashicorp.com/consul/0.8.1/consul_0.8.1_linux_amd64.zip

View file

@ -23,7 +23,7 @@
os=$(cat "$__global/explorer/os") os=$(cat "$__global/explorer/os")
case "$os" in case "$os" in
scientific|centos|redhat|ubuntu|debian|archlinux|gentoo) scientific|centos|redhat|ubuntu|debian|devuan|archlinux|gentoo)
# any linux should work # any linux should work
: :
;; ;;

View file

@ -23,7 +23,7 @@
os=$(cat "$__global/explorer/os") os=$(cat "$__global/explorer/os")
case "$os" in case "$os" in
scientific|centos|debian|redhat|ubuntu) scientific|centos|debian|devuan|redhat|ubuntu)
# whitelist safeguard # whitelist safeguard
: :
;; ;;
@ -215,7 +215,11 @@ case "$os" in
esac esac
;; ;;
ubuntu) devuan)
init_upstart init_sysvinit debian
;; ;;
ubuntu)
init_upstart
;;
esac esac

View file

@ -24,6 +24,9 @@ OPTIONAL PARAMETERS
check-interval check-interval
the interval in which the script given with --check-script should be run the interval in which the script given with --check-script should be run
check-http
the URL to check for HTTP 200-ish status every --check-interval
check-script check-script
the shell command to run every --check-interval the shell command to run every --check-interval

View file

@ -32,6 +32,10 @@ if [ -f "$__object/parameter/check-script" -a ! -f "$__object/parameter/check-in
echo "When using --check-script you must also define --check-interval" >&2 echo "When using --check-script you must also define --check-interval" >&2
exit 1 exit 1
fi fi
if [ -f "$__object/parameter/check-http" -a ! -f "$__object/parameter/check-interval" ]; then
echo "When using --check-http you must also define --check-interval" >&2
exit 1
fi
# Generate json config file # Generate json config file
( (
@ -52,6 +56,12 @@ for param in $(ls "$__object/parameter/"); do
printf ' "ttl": "%s"\n' "$(cat "$__object/parameter/check-ttl")" printf ' "ttl": "%s"\n' "$(cat "$__object/parameter/check-ttl")"
printf ' }\n' printf ' }\n'
;; ;;
check-http)
printf ' ,"check": {\n'
printf ' "http": "%s"\n' "$(cat "$__object/parameter/check-http")"
printf ' ,"interval": "%s"\n' "$(cat "$__object/parameter/check-interval")"
printf ' }\n'
;;
tag) tag)
# create json array from newline delimited file # create json array from newline delimited file
tags="$(awk '{printf "\""$1"\","}' "$__object/parameter/tag")" tags="$(awk '{printf "\""$1"\","}' "$__object/parameter/tag")"

View file

@ -1,3 +1,4 @@
check-http
check-interval check-interval
check-script check-script
check-ttl check-ttl

View file

@ -0,0 +1,63 @@
#!/bin/bash
### BEGIN INIT INFO
# Provides: svscan
# Required-Start:
# Required-Stop:
# Default-Start: 2 3 4 5
# Default-Stop: 0 1 6
# Short-Description: svscan
# Description: djb svscan
### END INIT INFO
# from https://gist.githubusercontent.com/pacojp/5766990/raw/2ed009ab19515afc9e58291b636d673c5ca864b3/init.d.svscan
# written by Adam McKenna <adam@debian.org>
# edited by Kamila Součková <kamila@ksp.sk>
export PATH=$PATH:/usr/local/bin
l=/var/log/svscan
if [ ! -d $l ]; then
mkdir $l
chown daemon $l
fi
case "$1" in
start)
echo -n "Starting daemontools: "
if [ ! `pidof svscan` ]; then
echo -n "svscan "
env - PATH="$PATH" svscan /service 2>&1 | setuidgid daemon multilog t /var/log/svscan &
echo "."
else
echo "already running."
fi
;;
stop)
echo -n "Stopping daemontools: "
if [ `pidof svscan` ]; then
echo -n "svscan"
while [ `pidof svscan` ]; do
kill `pidof svscan`
echo -n "."
done
fi
echo -n " services"
for i in `ls -d /service/*`; do
svc -dx $i
echo -n "."
done
echo -n " logging "
for i in `ls -d /service/*/log`; do
svc -dx $i
echo -n "."
done
echo ""
;;
restart|force-reload)
$0 stop
$0 start
;;
*)
echo 'Usage: /etc/init.d/svscan {start|stop|restart|force-reload}'
exit 1
esac

View file

@ -0,0 +1,49 @@
cdist-type__daemontools(7)
==========================
NAME
----
cdist-type__daemontools - Install daemontools
DESCRIPTION
-----------
Install djb daemontools and (optionally) an init script.
REQUIRED PARAMETERS
-------------------
None.
OPTIONAL PARAMETERS
-------------------
from-package
Package to install. Must be compatible with the original daemontools. Example: daemontools-encore. Default: daemontools.
BOOLEAN PARAMETERS
------------------
install-init-script
Add an init script and set it to start on boot. Default yes.
EXAMPLES
--------
.. code-block:: sh
__daemontools --from-package daemontools-encore # if you prefer
SEE ALSO
--------
:strong:`cdist-type__daemontools_service`\ (7)
AUTHORS
-------
Kamila Součková <kamila--@--ksp.sk>
COPYING
-------
Copyright \(C) 2017 Kamila Součková. You can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

View file

@ -0,0 +1,20 @@
#!/bin/sh
pkg=$(cat "$__object/parameter/from-package")
__package $pkg
if [ -f "$__object/parameter/install-init-script" ]; then
init=$(cat "$__global/explorer/init")
case $init in
init)
__config_file /etc/init.d/svscan --mode 755 --source "$__type/files/init.d-svscan"
require="$require __config_file/etc/init.d/svscan" __start_on_boot svscan
require="$require __start_on_boot/svscan" __process svscan --start 'service svscan start'
;;
*)
echo "Your init system ($init) is not supported by this type. Submit a patch at github.com/ungleich/cdist!"
exit 1
;;
esac
fi

View file

@ -0,0 +1 @@
install-init-script

View file

@ -0,0 +1 @@
daemontools

View file

@ -0,0 +1 @@
from-package

View file

View file

@ -0,0 +1 @@
command -v svc

View file

@ -0,0 +1,72 @@
cdist-type__daemontools_service(7)
==================================
NAME
----
cdist-type__daemontools_service - Create a daemontools-compatible service dir.
DESCRIPTION
-----------
Create a directory structure compatible with daemontools-like service management.
Note that svc must be present on the target system.
The object ID will be used as the service name.
REQUIRED PARAMETERS
-------------------
None.
OPTIONAL PARAMETERS
-------------------
run
Command to run. exec-ing and stderr redirection will be added. One of run, run-file must be specified.
Example: `my-program`
run-file
File to save as <servicedir>/run. One of run, run-file must be specified.
Example:
.. code-block:: sh
#!/bin/sh
exec 2>&1
exec my_program
log-run
Command to run for log consumption. Default: `multilog t ./main`
servicedir
Directory to install into. Default: `/service`
BOOLEAN PARAMETERS
------------------
None.
EXAMPLES
--------
.. code-block:: sh
require="__daemontools" __daemontools_service prometheus --run "setuidgid prometheus $GOBIN/prometheus $FLAGS"
SEE ALSO
--------
:strong:`cdist-type__daemontools`\ (7)
AUTHORS
-------
Kamila Součková <kamila--@--ksp.sk>
COPYING
-------
Copyright \(C) 2017 Kamila Součková. You can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

View file

@ -0,0 +1,38 @@
#!/bin/sh
RUN_PREFIX="#!/bin/sh
exec 2>&1
exec " # mind the space :D
name=$__object_id
servicedir=$(cat "$__object/parameter/servicedir")
run=$(cat "$__object/parameter/run")
runfile=$(cat "$__object/parameter/run-file")
logrun=$(cat "$__object/parameter/log-run")
svc=$(cat "$__type/explorer/svc")
if [ -z "$svc" ]; then
echo "svc not found! Install daemontools first: see __daemontools"
exit 1
fi
badusage() {
echo "__daemontools_service/$__object_id: exactly one of --run, --run-file must be set" >&2
exit 1
}
[ -z "$run$runfile" ] && badusage
[ -n "$run" ] && [ -n "$runfile" ] && badusage
__directory $servicedir/$name/log/main --parents
echo "$RUN_PREFIX$run" | require="__directory/$servicedir/$name/log/main" __config_file "$servicedir/$name/run" \
--onchange "svc -t '$servicedir/$name' 2>/dev/null" \
--mode 755 \
--source "${runfile:--}"
echo "$RUN_PREFIX$logrun" | require="__directory/$servicedir/$name/log/main" __config_file $servicedir/$name/log/run \
--onchange "svc -t '$servicedir/$name/log' 2>/dev/null" \
--mode 755 \
--source "-"

View file

@ -0,0 +1 @@
multilog t ./main

View file

@ -0,0 +1 @@
/service

View file

@ -0,0 +1,4 @@
log-run
run
run-file
servicedir

View file

@ -0,0 +1,66 @@
cdist-type__prometheus_alertmanager(7)
======================================
NAME
----
cdist-type__prometheus_alertmanager - install Alertmanager
DESCRIPTION
-----------
Install and configure Prometheus Alertmanager (https://prometheus.io/docs/alerting/alertmanager/).
REQUIRED PARAMETERS
-------------------
config
Alertmanager configuration file. It will be saved as /etc/alertmanager/alertmanager.yml on the target.
listen-address
Passed as web.listen-address.
OPTIONAL PARAMETERS
-------------------
storage-path
Where to put data. Default: /data/alertmanager. (Directory will be created if needed.)
BOOLEAN PARAMETERS
------------------
with-daemontools
Create a daemontools service directory under /service/prometheus. Default: yes.
Note: If you do not use this, Alertmanager will not be launched, and will not reload config on change.
If you use this, daemontools (or something compatible) must be installed.
EXAMPLES
--------
.. code-block:: sh
ALERTPORT=9093
__daemontools
__golang_from_vendor --version 1.8.1 # required for prometheus and many exporters
require="__daemontools __golang_from_vendor" __prometheus_alertmanager \
--config "$__manifest/files/alertmanager.yml" \
--storage-path /data/alertmanager \
--listen-address "[::]:$ALERTPORT"
SEE ALSO
--------
:strong:`cdist-type__prometheus_server`\ (7), :strong:`cdist-type__daemontools`\ (7),
Prometheus alerting documentation: https://prometheus.io/docs/alerting/overview/
AUTHORS
-------
Kamila Součková <kamila--@--ksp.sk>
COPYING
-------
Copyright \(C) 2017 Kamila Součková. You can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

View file

@ -0,0 +1,39 @@
#!/bin/sh
GOBIN=/opt/gocode/bin # where to find go binaries
CONF_DIR=/etc/prometheus
LOGLEVEL=info
CONF=$CONF_DIR/alertmanager.yml
### Prometheus server #######################################################
config="$(cat "$__object/parameter/config")"
storage_path="$(cat "$__object/parameter/storage-path")"
listen_address="$(cat "$__object/parameter/listen-address")"
ONCHANGE=""
if [ -f "$__object/parameter/with-daemontools" ]; then
__daemontools_service prometheus --run "setuidgid prometheus $GOBIN/prometheus $REAL_FLAGS"
ONCHANGE="svc -h /service/prometheus"
fi
FLAGS="config.file '$CONF'
storage.path '$storage_path'
web.listen-address '$listen_address'
log.level $LOGLEVEL"
REAL_FLAGS="$(echo "$FLAGS" | sed -nE 's/^([^#]+).*/ --\1 \\/p')"
__go_get github.com/prometheus/alertmanager/cmd/...
__user prometheus --system
__directory "$storage_path" --owner prometheus
__directory "$CONF_DIR" --owner prometheus
__daemontools_service alertmanager --run "setuidgid prometheus $GOBIN/alertmanager $REAL_FLAGS"
require="$require __directory/$storage_path" \
__config_file $CONF \
--source $config \
--group prometheus --mode 640 \
--onchange "$ONCHANGE"

View file

@ -0,0 +1 @@
/data/alertmanager

View file

@ -0,0 +1 @@
storage-path

View file

@ -0,0 +1,2 @@
config
listen-address

View file

@ -0,0 +1,78 @@
cdist-type__prometheus_server(7)
================================
NAME
----
cdist-type__prometheus_server - install Prometheus
DESCRIPTION
-----------
Install and configure Prometheus (https://prometheus.io/).
REQUIRED PARAMETERS
-------------------
config
Prometheus configuration file. It will be saved as /etc/prometheus/prometheus.yml on the target.
listen-address
Passed as web.listen-address.
alertmanager-url
Passed as alertmanager.url
OPTIONAL PARAMETERS
-------------------
retention-days
How long to keep data. Default: 30
rule-files
Path to rule files. They will be installed under /etc/prometheus/<filename>. You need to include `rule_files: [/etc/prometheus/<your-pattern>]` in the config file if you use this.
storage-path
Where to put data. Default: /data/prometheus. (Directory will be created if needed.)
target-heap-size
Passed as storage.local.target-heap-size. Default: 1/2 of RAM.
BOOLEAN PARAMETERS
------------------
with-daemontools
Create a daemontools service directory under /service/prometheus. Default: yes.
Note: If you do not use this, Prometheus will not be launched, and will not reload config on change.
If you use this, daemontools (or something compatible) must be installed.
EXAMPLES
--------
.. code-block:: sh
PROMPORT=9090
ALERTPORT=9093
__daemontools
__golang_from_vendor --version 1.8.1 # required for prometheus and many exporters
require="__daemontools __golang_from_vendor" __prometheus_server \
--config "$__manifest/files/prometheus.yml" \
--retention-days 14 \
--storage-path /data/prometheus \
--listen-address "[::]:$PROMPORT" \
--rule-files "$__manifest/files/*.rules" \
--alertmanager-url "http://monitoring1.node.consul:$ALERTPORT,http://monitoring2.node.consul:$ALERTPORT"
SEE ALSO
--------
:strong:`cdist-type__prometheus_alertmanager`\ (7), :strong:`cdist-type__daemontools`\ (7),
Prometheus documentation: https://prometheus.io/docs/introduction/overview/
AUTHORS
-------
Kamila Součková <kamila--@--ksp.sk>
COPYING
-------
Copyright \(C) 2017 Kamila Součková. You can redistribute it
and/or modify it under the terms of the GNU General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.

View file

@ -0,0 +1,57 @@
#!/bin/sh
GOBIN=/opt/gocode/bin # where to find go binaries
CONF_DIR=/etc/prometheus
CONF=$CONF_DIR/prometheus.yml
LOGLEVEL=info
config="$(cat "$__object/parameter/config")"
retention_days="$(cat "$__object/parameter/retention-days")"
storage_path="$(cat "$__object/parameter/storage-path")"
listen_address="$(cat "$__object/parameter/listen-address")"
alertmanager_url="$(cat "$__object/parameter/alertmanager-url")"
target_heap_size="$(cat "$__object/parameter/target-heap-size")"
rule_files="$(cat "$__object/parameter/rule-files")"
# explorer in kB => convert; by default we go with 1/2 RAM
[ "$target_heap_size" = "auto" ] && target_heap_size=$(($(cat $__global/explorer/memory)*1024/2))
ONCHANGE=""
if [ -f "$__object/parameter/with-daemontools" ]; then
__daemontools_service prometheus --run "setuidgid prometheus $GOBIN/prometheus $REAL_FLAGS"
ONCHANGE="&& svc -h /service/prometheus"
fi
FLAGS="config.file '$CONF'
storage.local.path '$storage_path'
storage.local.target-heap-size $(($target_heap_size)) # in bytes; should be 2/3 of available memory because it may be hungry
storage.local.retention $(($retention_days*24))h # golang doesn't have days :D
web.listen-address '$listen_address'
alertmanager.url '$alertmanager_url'
log.level $LOGLEVEL"
REAL_FLAGS="$(echo "$FLAGS" | sed -nE 's/^([^#]+).*/ --\1 \\/p')"
__go_get github.com/prometheus/prometheus/cmd/...
__user prometheus --system
__directory "$storage_path" --owner prometheus
__directory "$CONF_DIR" --owner prometheus
require="$require __directory/$storage_path" \
__config_file $CONF \
--source $config \
--group prometheus --mode 640 \
--onchange "$GOBIN/promtool check-config $CONF $ONCHANGE"
for file in $rule_files; do
dest=$CONF_DIR/$(basename $file)
require="$require __directory/$CONF_DIR" \
__config_file "$dest" \
--source "$file" \
--owner prometheus \
--onchange "$GOBIN/promtool check-rules '$dest' $ONCHANGE"
done

View file

@ -0,0 +1 @@
with-daemontools

View file

@ -0,0 +1 @@
/data/prometheus

View file

@ -0,0 +1,4 @@
target-heap-size
retention-days
rule-files
storage-path

View file

@ -0,0 +1,3 @@
alertmanager-url
config
listen-address

View file

@ -27,6 +27,8 @@ import time
import itertools import itertools
import tempfile import tempfile
import socket import socket
import atexit
import shutil
import cdist import cdist
import cdist.hostsource import cdist.hostsource
@ -92,7 +94,6 @@ class Config(object):
"failed: %s" % e)) "failed: %s" % e))
args.manifest = initial_manifest_temp_path args.manifest = initial_manifest_temp_path
import atexit
atexit.register(lambda: os.remove(initial_manifest_temp_path)) atexit.register(lambda: os.remove(initial_manifest_temp_path))
# default remote cmd patterns # default remote cmd patterns
@ -176,8 +177,15 @@ class Config(object):
" ".join(failed_hosts)) " ".join(failed_hosts))
@classmethod @classmethod
def _resolve_remote_cmds(cls, args, host_base_path): def _resolve_ssh_control_path(cls):
control_path = os.path.join(host_base_path, "ssh-control-path") base_path = tempfile.mkdtemp()
control_path = os.path.join(base_path, "s")
atexit.register(lambda: shutil.rmtree(base_path))
return control_path
@classmethod
def _resolve_remote_cmds(cls, args):
control_path = cls._resolve_ssh_control_path()
# If we constructed patterns for remote commands then there is # If we constructed patterns for remote commands then there is
# placeholder for ssh ControlPath, format it and we have unique # placeholder for ssh ControlPath, format it and we have unique
# ControlPath for each host. # ControlPath for each host.
@ -200,8 +208,7 @@ class Config(object):
log = logging.getLogger(host) log = logging.getLogger(host)
try: try:
remote_exec, remote_copy = cls._resolve_remote_cmds( remote_exec, remote_copy = cls._resolve_remote_cmds(args)
args, host_base_path)
log.debug("remote_exec for host \"{}\": {}".format( log.debug("remote_exec for host \"{}\": {}".format(
host, remote_exec)) host, remote_exec))
log.debug("remote_copy for host \"{}\": {}".format( log.debug("remote_copy for host \"{}\": {}".format(

View file

@ -7,6 +7,10 @@ next:
* Explorer kernel_name: uname -s (Kamila Součková) * Explorer kernel_name: uname -s (Kamila Součková)
* Type __sysctl: Add devuan support (Nico Schottelius) * Type __sysctl: Add devuan support (Nico Schottelius)
* Type __start_on_boot: Add devuan support (Nico Schottelius) * Type __start_on_boot: Add devuan support (Nico Schottelius)
* Core: Shorten ssh control path (Darko Poljak)
* Type __consul: Add new version and add http check (Kamila Součková)
* New types: __daemontools and __daemontools_service (Kamila Součková)
* New types: __prometheus_server and __prometheus_alertmanager (Kamila Součková)
4.4.2: 2017-03-08 4.4.2: 2017-03-08
* Core: Fix suppression of manifests' outputs (Darko Poljak) * Core: Fix suppression of manifests' outputs (Darko Poljak)