Merge branch 'consul_agent' into 'master'

Refactor __consul_* for use with distribution packages

See merge request ungleich-public/cdist!837

cdist/conf/type/__consul_agent/man.rst

@@ -116,6 +116,9 @@ verify-incoming
 verify-outgoing
    enforce the use of TLS and verify the peers authenticity on outgoing connections
 
+use-distribution-package
+   uses distribution package instead of upstream binary
+
 
 EXAMPLES
 --------

cdist/conf/type/__consul_agent/manifest

@@ -2,6 +2,7 @@
 #
 # 2015 Steven Armstrong (steven-cdist at armstrong.cc)
 # 2015-2019 Nico Schottelius (nico-cdist at schottelius.org)
+# 2019 Timothée Floure (timothee.floure at ungleich.ch)
 #
 # This file is part of cdist.
 #
@@ -19,133 +20,64 @@
 # along with cdist. If not, see <http://www.gnu.org/licenses/>.
 #
 
-
 os=$(cat "$__global/explorer/os")
 
-case "$os" in
+###
-   alpine|scientific|centos|debian|devuan|redhat|ubuntu)
+# Type parameters.
-      # whitelist safeguard
-      :
-   ;;
-   *)
-      echo "Your operating system ($os) is currently not supported by this type (${__type##*/})." >&2
-      echo "Please contribute an implementation for it if you can." >&2
-      exit 1
-   ;;
-esac
 
 state="$(cat "$__object/parameter/state")"
 user="$(cat "$__object/parameter/user")"
 group="$(cat "$__object/parameter/group")"
+release=$(cat "$__global/explorer/lsb_release")
+if [ -f "$__object/parameter/use-distribution-package" ]; then
+  use_distribution_package=1
+fi
+
+###
+# Those are default that might be overriden by os-specific logic.
+
 data_dir="/var/lib/consul"
 conf_dir="/etc/consul/conf.d"
 conf_file="config.json"
+tls_dir="$conf_dir/tls"
 
-# FIXME: there has got to be a better way to handle the dependencies in this case
+###
-case "$state" in
+# Sane deployment, based on distribution package when available.
-   present)
+
-      __group "$group" --system --state "$state"
+distribution_setup () {
-      require="__group/$group" \
+  case "$os" in
-         __user "$user" --system --gid "$group" \
+     debian)
-            --home "$data_dir" --state "$state"
+       # consul is only available starting Debian 10 (buster).
-      export require="__user/consul"
+       # See https://packages.debian.org/buster/consul
-   ;;
+       if [ "$release" -lt 10 ]; then
-   absent)
+         echo "Consul is not available for your debian release." >&2
-      echo "Sorry, state=absent currently not supported :-(" >&2
+         echo "Please use the 'manual' (i.e. non-package) installation or \
+           upgrade the target system." >&2
          exit 1
-      require="$__object_name" \
+       fi
-         __user "$user" --system --gid "$group" --state "$state"
-      require="__user/$user" \
-         __group "$group" --system --state "$state"
-   ;;
-esac
 
-__directory /etc/consul \
+       # Override previously defined environment to match debian packaging.
-   --owner root --group "$group" --mode 750 --state "$state"
+       conf_dir='/etc/consul.d'
-require="__directory/etc/consul" \
+       user='consul'
-   __directory "$conf_dir" \
+       group='consul'
-      --owner root --group "$group" --mode 750 --state "$state"
-
-if [ -f "$__object/parameter/ca-file-source" ] || [ -f "$__object/parameter/cert-file-source" ] || [ -f "$__object/parameter/key-file-source" ]; then
-   # create directory for ssl certs
-   require="__directory/etc/consul" \
-      __directory /etc/consul/ssl \
-         --owner root --group "$group" --mode 750 --state "$state"
-fi
-
-__directory "$data_dir" \
-   --owner "$user" --group "$group" --mode 770 --state "$state"
-
-
-# Generate json config file
-(
-echo "{"
-
-# parameters we define ourself
-printf '   "data_dir": "%s"\n' "$data_dir"
-
-cd "$__object/parameter/"
-for param in *; do
-   case "$param" in
-      state|user|group|json-config) continue ;;
-      ca-file-source|cert-file-source|key-file-source)
-         source="$(cat "$__object/parameter/$param")"
-         destination="/etc/consul/ssl/${source##*/}"
-         require="__directory/etc/consul/ssl" \
-            __file "$destination" \
-               --owner root --group consul --mode 640 \
-               --source "$source" \
-               --state "$state"
-         key="$(echo "${param%-*}" | tr '-' '_')"
-         printf '   ,"%s": "%s"\n' "$key" "$destination"
-      ;;
-      disable-remote-exec|disable-update-check|leave-on-terminate|rejoin-after-leave|server|enable-syslog|verify-incoming|verify-outgoing)
-         # handle boolean parameters
-         key="$(echo "$param" | tr '-' '_')"
-         printf '   ,"%s": true\n' "$key"
-      ;;
-      retry-join)
-         # join multiple parameters into json array
-         retry_join="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join")"
-         # remove trailing ,
-         printf '   ,"retry_join": [%s]\n' "${retry_join%*,}"
-      ;;
-      retry-join-wan)
-         # join multiple parameters into json array over wan
-         retry_join_wan="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join-wan")"
-         # remove trailing ,
-         printf '   ,"retry_join_wan": [%s]\n' "${retry_join_wan%*,}"
-      ;;
-      bootstrap-expect)
-         # integer key=value parameters
-         key="$(echo "$param" | tr '-' '_')"
-         printf '   ,"%s": %s\n' "$key" "$(cat "$__object/parameter/$param")"
      ;;
      *)
-         # string key=value parameters
+        echo "Your operating system ($os) is currently not supported with the \
-         key="$(echo "$param" | tr '-' '_')"
+          --use-distribution-package flag (${__type##*/})." >&2
-         printf '   ,"%s": "%s"\n' "$key" "$(cat "$__object/parameter/$param")"
+        echo "Please use non-package installation or contribute an \
+          implementation for if you can." >&2
+        exit 1
      ;;
   esac
-done
+
-if [ -f "$__object/parameter/json-config" ]; then
+  # Install consul package.
-   json_config="$(cat "$__object/parameter/json-config")"
+  __package consul --state "$state"
-   if [ "$json_config" = "-" ]; then
+
-      json_config="$__object/stdin"
+  export config_deployment_requires="__package/consul"
-   fi
+}
-   # remove leading and trailing whitespace and commas from first and last line
+
-   # indent each line with 3 spaces for consistency
+###
-   json=$(sed -e 's/^[ \t]*/   /' -e '1s/^[ \t,]*//' -e '$s/[ \t,]*$//' "$json_config")
+# LEGACY manual deployment, kept for compatibility reasons.
-   printf '   ,%s\n' "$json"
-fi
-echo "}"
-) | \
-require="__directory${conf_dir}" \
-   __config_file "${conf_dir}/${conf_file}" \
-      --owner root --group "$group" --mode 640 \
-      --state "$state" \
-      --onchange 'service consul status >/dev/null && service consul reload || true' \
-      --source -
 
 init_sysvinit()
 {
@@ -179,8 +111,43 @@ init_upstart()
     require="__file/etc/init/consul.conf" __start_on_boot consul
 }
 
-# Install init script to start on boot
+manual_setup () {
-case "$os" in
+  case "$os" in
+     alpine|scientific|centos|debian|devuan|redhat|ubuntu)
+        # whitelist safeguard
+        :
+     ;;
+     *)
+        echo "Your operating system ($os) is currently not supported by this \
+          type (${__type##*/})." >&2
+        echo "Please contribute an implementation for it if you can." >&2
+        exit 1
+     ;;
+  esac
+
+  # FIXME: there has got to be a better way to handle the dependencies in this case
+  case "$state" in
+     present)
+        __group "$group" --system --state "$state"
+        require="__group/$group" __user "$user" \
+          --system --gid "$group" --home "$data_dir" --state "$state"
+     ;;
+     *)
+        echo "The $state state is not (yet?) supported by this type." >&2
+        exit 1
+     ;;
+  esac
+
+  # Create data directory.
+  require="__user/consul" __directory "$data_dir" \
+    --owner "$user" --group "$group" --mode 770 --state "$state"
+
+  # Create config directory.
+  require="__user/consul" __directory "$conf_dir" \
+    --parents --owner root --group "$group" --mode 750 --state "$state"
+
+  # Install init script to start on boot
+  case "$os" in
       devuan)
           init_sysvinit debian
           ;;
@@ -222,4 +189,108 @@ case "$os" in
       ubuntu)
           init_upstart
           ;;
-esac
+  esac
+
+  config_deployment_requires="__user/consul __directory/$conf_dir"
+}
+
+###
+# Trigger requested installation method.
+if [ $use_distribution_package ]; then
+  distribution_setup
+else
+  manual_setup
+fi
+
+###
+# Install TLS certificates.
+
+if [ -f "$__object/parameter/ca-file-source" ] || \
+   [ -f "$__object/parameter/cert-file-source" ] || \
+   [ -f "$__object/parameter/key-file-source" ]; then
+
+   requires="$config_deployment_requires" __directory $tls_dir \
+     --owner root --group "$group" --mode 750 --state "$state"
+
+   # Append to service restart requirements.
+   restart_requires="$restart_requires __directory/$conf_dir/tls"
+fi
+
+###
+# Generate and deploy configuration.
+
+json_configuration=$(
+  echo "{"
+
+  # parameters we define ourself
+  printf '   "data_dir": "%s"\n' "$data_dir"
+
+  cd "$__object/parameter/"
+  for param in *; do
+     case "$param" in
+        state|user|group|json-config|use-distribution-package) continue ;;
+        ca-file-source|cert-file-source|key-file-source)
+           source="$(cat "$__object/parameter/$param")"
+           destination="$tls_dir/${source##*/}"
+           require="__directory/$tls_dir" \
+              __file "$destination" \
+                 --owner root --group consul --mode 640 \
+                 --source "$source" \
+                 --state "$state"
+           key="$(echo "${param%-*}" | tr '-' '_')"
+           printf '   ,"%s": "%s"\n' "$key" "$destination"
+        ;;
+        disable-remote-exec|disable-update-check|leave-on-terminate\
+          |rejoin-after-leave|server|enable-syslog|verify-incoming|verify-outgoing)
+           # handle boolean parameters
+           key="$(echo "$param" | tr '-' '_')"
+           printf '   ,"%s": true\n' "$key"
+        ;;
+        retry-join)
+           # join multiple parameters into json array
+           retry_join="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join")"
+           # remove trailing ,
+           printf '   ,"retry_join": [%s]\n' "${retry_join%*,}"
+        ;;
+        retry-join-wan)
+           # join multiple parameters into json array over wan
+           retry_join_wan="$(awk '{printf "\""$1"\","}' "$__object/parameter/retry-join-wan")"
+           # remove trailing ,
+           printf '   ,"retry_join_wan": [%s]\n' "${retry_join_wan%*,}"
+        ;;
+        bootstrap-expect)
+           # integer key=value parameters
+           key="$(echo "$param" | tr '-' '_')"
+           printf '   ,"%s": %s\n' "$key" "$(cat "$__object/parameter/$param")"
+        ;;
+        *)
+           # string key=value parameters
+           key="$(echo "$param" | tr '-' '_')"
+           printf '   ,"%s": "%s"\n' "$key" "$(cat "$__object/parameter/$param")"
+        ;;
+     esac
+  done
+  if [ -f "$__object/parameter/json-config" ]; then
+     json_config="$(cat "$__object/parameter/json-config")"
+     if [ "$json_config" = "-" ]; then
+        json_config="$__object/stdin"
+     fi
+     # remove leading and trailing whitespace and commas from first and last line
+     # indent each line with 3 spaces for consistency
+     json=$(sed -e 's/^[ \t]*/   /' -e '1s/^[ \t,]*//' -e '$s/[ \t,]*$//' "$json_config")
+     printf '   ,%s\n' "$json"
+  fi
+  echo "}"
+)
+echo "$json_configuration" | require="$config_deployment_requires" \
+  __file "$conf_dir/$conf_file" \
+      --owner root --group "$group" --mode 640 \
+      --state "$state" \
+      --source -
+
+# Set configuration deployment as requirement for service restart.
+restart_requires="__file/$conf_dir/$conf_file"
+
+###
+# Restart consul agent after everything else.
+require="$restart_requires" __service consul --action restart

cdist/conf/type/__consul_agent/parameter/boolean

@@ -6,3 +6,4 @@ server
 enable-syslog
 verify-incoming
 verify-outgoing
+use-distribution-package

cdist/conf/type/__consul_check/explorer/conf-dir

@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir

cdist/conf/type/__consul_check/manifest

@@ -19,7 +19,7 @@
 #
 
 name="$(cat "$__object/parameter/name" 2>/dev/null || echo "$__object_id")"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="check_${name}.json"
 state="$(cat "$__object/parameter/state")"
 

cdist/conf/type/__consul_service/explorer/conf-dir

@@ -0,0 +1,15 @@
+# Determine the configuration directory used by consul.
+
+check_dir () {
+  if [ -d "$1" ]; then
+    printf '%s' "$1"
+    exit
+  fi
+}
+
+check_dir '/etc/consul/conf.d'
+check_dir '/etc/consul.d'
+check_dir '/etc/consul'
+
+echo 'Could not determine consul configuration dir. Exiting.' >&2
+exit 1

cdist/conf/type/__consul_service/manifest

@@ -19,7 +19,7 @@
 #
 
 name="$(cat "$__object/parameter/name" 2>/dev/null || echo "$__object_id")"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="service_${name}.json"
 state="$(cat "$__object/parameter/state")"
 
@@ -45,7 +45,7 @@ printf '      "name": "%s"\n' "$name"
 cd "$__object/parameter/"
 for param in *; do
    case "$param" in
-      state|name|check-interval) continue ;;
+      state|name|check-interval|conf-dir) continue ;;
       check-script)
          printf '     ,"check": {\n'
          printf '         "script": "%s"\n' "$(cat "$__object/parameter/check-script")"
@@ -86,7 +86,6 @@ echo "   }"
 # end json file
 echo "}"
 ) | \
-require="__directory${conf_dir}" \
    __config_file "${conf_dir}/${conf_file}" \
       --owner root --group consul --mode 640 \
       --state "$state" \

cdist/conf/type/__consul_watch_checks/explorer/conf-dir

@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir

cdist/conf/type/__consul_watch_checks/manifest

@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 

cdist/conf/type/__consul_watch_event/explorer/conf-dir

@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir

cdist/conf/type/__consul_watch_event/manifest

@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 

cdist/conf/type/__consul_watch_key/explorer/conf-dir

@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir

cdist/conf/type/__consul_watch_key/manifest

@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 

cdist/conf/type/__consul_watch_keyprefix/explorer/conf-dir

@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir

cdist/conf/type/__consul_watch_keyprefix/manifest

@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 

cdist/conf/type/__consul_watch_nodes/explorer/conf-dir

@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir

cdist/conf/type/__consul_watch_nodes/manifest

@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 

cdist/conf/type/__consul_watch_service/explorer/conf-dir

@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir

cdist/conf/type/__consul_watch_service/manifest

@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"
 

cdist/conf/type/__consul_watch_services/explorer/conf-dir

@@ -0,0 +1 @@
+../../__consul_service/explorer/conf-dir

cdist/conf/type/__consul_watch_services/manifest

@@ -20,7 +20,7 @@
 
 cdist_type="${__type##*/}"
 watch_type="${cdist_type##*_}"
-conf_dir="/etc/consul/conf.d"
+conf_dir=$(cat "$__object/explorer/conf-dir")
 conf_file="watch_${watch_type}_${__object_id}.json"
 state="$(cat "$__object/parameter/state")"