diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..45c10d7b --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +.gitignore export-ignore +.gitattributes export-ignore +.gitkeep export-ignore +docs/speeches export-ignore +docs/video export-ignore +docs/src/man7 export-ignore +bin/build-helper export-ignore +README-maintainers export-ignore diff --git a/.gitignore b/.gitignore index 4258c2eb..4b80b425 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,22 @@ # -vim -.*.swp +# Swap +[._]*.s[a-v][a-z] +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim + +# Temporary +.netrwhist +*~ +*.tmp +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ # Ignore generated manpages docs/src/.marker @@ -11,18 +28,23 @@ docs/src/cdist-reference.rst # Ignore cdist cache for version control /cache/ +# Ignore inventory basedir +cdist/inventory/ + # Python: cache, distutils, distribution in general __pycache__/ *.pyc MANIFEST dist/ cdist/version.py +cdist.egg-info/ # sphinx build dirs, cache _build/ docs/dist # Ignore temp files used for signing +cdist-*.tar cdist-*.tar.gz cdist-*.tar.gz.asc diff --git a/Makefile b/Makefile index 417140c5..fa3327d1 100644 --- a/Makefile +++ b/Makefile @@ -18,31 +18,30 @@ # # -helper=./bin/build-helper +.PHONY: help +help: + @echo "Please use \`make ' where is one of" + @echo "man build only man user documentation" + @echo "html build only html user documentation" + @echo "docs build both man and html user documentation" + @echo "dotman build man pages for types in your ~/.cdist directory" + @echo "speeches build speeches pdf files" + @echo "install install in the system site-packages directory" + @echo "install-user install in the user site-packages directory" + @echo "docs-clean clean documentation" + @echo "clean clean" -DOCS_SRC_DIR=docs/src -SPEECHDIR=docs/speeches -TYPEDIR=cdist/conf/type - -WEBSRCDIR=docs/web - -WEBDIR=$$HOME/vcs/www.nico.schottelius.org -WEBBLOG=$(WEBDIR)/blog -WEBBASE=$(WEBDIR)/software/cdist -WEBPAGE=$(WEBBASE).mdwn - -CHANGELOG_VERSION=$(shell $(helper) changelog-version) -CHANGELOG_FILE=docs/changelog - -PYTHON_VERSION=cdist/version.py +DOCS_SRC_DIR=./docs/src +SPEECHDIR=./docs/speeches +TYPEDIR=./cdist/conf/type SPHINXM=make -C $(DOCS_SRC_DIR) man SPHINXH=make -C $(DOCS_SRC_DIR) html SPHINXC=make -C $(DOCS_SRC_DIR) clean + ################################################################################ # Manpages # -MAN1DSTDIR=$(DOCS_SRC_DIR)/man1 MAN7DSTDIR=$(DOCS_SRC_DIR)/man7 # Manpages #1: Types @@ -54,6 +53,7 @@ MANTYPES=$(subst /man.rst,.rst,$(MANTYPEPREFIX)) # Link manpage: do not create man.html but correct named file $(MAN7DSTDIR)/cdist-type%.rst: $(TYPEDIR)/%/man.rst + mkdir -p $(MAN7DSTDIR) ln -sf "../../../$^" $@ # Manpages #2: reference @@ -63,11 +63,16 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh $(DOCSREF): $(DOCSREFSH) $(DOCSREFSH) +version: + @[ -f "cdist/version.py" ] || { \ + printf "Missing 'cdist/version.py', please generate it first.\n" && exit 1; \ + } + # Manpages #3: generic part -man: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION) +man: version $(MANTYPES) $(DOCSREF) $(SPHINXM) -html: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION) +html: version $(MANTYPES) $(DOCSREF) $(SPHINXH) docs: man html @@ -75,24 +80,6 @@ docs: man html docs-clean: $(SPHINXC) -# Manpages #5: release part -MANWEBDIR=$(WEBBASE)/man/$(CHANGELOG_VERSION) -HTMLBUILDDIR=docs/dist/html - -docs-dist: html - rm -rf "${MANWEBDIR}" - mkdir -p "${MANWEBDIR}" - # mkdir -p "${MANWEBDIR}/man1" "${MANWEBDIR}/man7" - # cp ${MAN1DSTDIR}/*.html ${MAN1DSTDIR}/*.css ${MANWEBDIR}/man1 - # cp ${MAN7DSTDIR}/*.html ${MAN7DSTDIR}/*.css ${MANWEBDIR}/man7 - cp -R ${HTMLBUILDDIR}/* ${MANWEBDIR} - cd ${MANWEBDIR} && git add . && git commit -m "cdist manpages update: $(CHANGELOG_VERSION)" || true - -man-latest-link: web-pub - # Fix ikiwiki, which does not like symlinks for pseudo security - ssh staticweb.ungleich.ch \ - "cd /home/services/www/nico/nico.schottelius.org/www/software/cdist/man/ && rm -f latest && ln -sf "$(CHANGELOG_VERSION)" latest" - # Manpages: .cdist Types DOT_CDIST_PATH=${HOME}/.cdist DOTMAN7DSTDIR=$(MAN7DSTDIR) @@ -105,8 +92,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX)) $(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst ln -sf "$^" $@ -# Manpages #3: generic part -dotman: $(DOTMANTYPES) +dotman: version $(DOTMANTYPES) $(SPHINXM) ################################################################################ @@ -114,7 +100,6 @@ dotman: $(DOTMANTYPES) # SPEECHESOURCES=$(SPEECHDIR)/*.tex SPEECHES=$(SPEECHESOURCES:.tex=.pdf) -SPEECHESWEBDIR=$(WEBBASE)/speeches # Create speeches and ensure Toc is up-to-date $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex @@ -124,128 +109,26 @@ $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex speeches: $(SPEECHES) -speeches-dist: speeches - rm -rf "${SPEECHESWEBDIR}" - mkdir -p "${SPEECHESWEBDIR}" - cp ${SPEECHES} "${SPEECHESWEBDIR}" - cd ${SPEECHESWEBDIR} && git add . && git commit -m "cdist speeches updated" || true - ################################################################################ -# Website +# Misc # - -BLOGFILE=$(WEBBLOG)/cdist-$(CHANGELOG_VERSION)-released.mdwn - -$(BLOGFILE): $(CHANGELOG_FILE) - $(helper) blog $(CHANGELOG_VERSION) $(BLOGFILE) - -web-blog: $(BLOGFILE) - -web-doc: - # Go to top level, because of cdist.mdwn - rsync -av "$(WEBSRCDIR)/" "${WEBBASE}/.." - cd "${WEBBASE}/.." && git add cdist* && git commit -m "cdist doc update" cdist* || true - -web-dist: web-blog web-doc - -web-pub: web-dist docs-dist speeches-dist - cd "${WEBDIR}" && make pub - -web-release-all: man-latest-link -web-release-all-no-latest: web-pub - -################################################################################ -# Release: Mailinglist -# -ML_FILE=.lock-ml - -# Only send mail once - lock until new changelog things happened -$(ML_FILE): $(CHANGELOG_FILE) - $(helper) ml-release $(CHANGELOG_VERSION) - touch $@ - -ml-release: $(ML_FILE) - - -################################################################################ -# pypi -# -PYPI_FILE=.pypi-release -$(PYPI_FILE): man $(PYTHON_VERSION) - python3 setup.py sdist upload - touch $@ - -pypi-release: $(PYPI_FILE) -################################################################################ -# archlinux -# -ARCHLINUX_FILE=.lock-archlinux -ARCHLINUXTAR=cdist-$(CHANGELOG_VERSION)-1.src.tar.gz - -$(ARCHLINUXTAR): PKGBUILD - umask 022; mkaurball - -PKGBUILD: PKGBUILD.in $(PYTHON_VERSION) - ./PKGBUILD.in $(CHANGELOG_VERSION) - -$(ARCHLINUX_FILE): $(ARCHLINUXTAR) $(PYTHON_VERSION) - burp -c system $(ARCHLINUXTAR) - touch $@ - -archlinux-release: $(ARCHLINUX_FILE) - -################################################################################ -# Release -# - -$(PYTHON_VERSION): .git/refs/heads/master - $(helper) version - -# Code that is better handled in a shell script -check-%: - $(helper) $@ - -release: - $(helper) $@ - -################################################################################ -# Cleanup -# - -clean: +clean: docs-clean rm -f $(DOCS_SRC_DIR)/cdist-reference.rst find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \ | xargs rm -f - make -C $(DOCS_SRC_DIR) clean - find * -name __pycache__ | xargs rm -rf - # Archlinux - rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz - rm -rf pkg/ src/ - - rm -f MANIFEST PKGBUILD - rm -rf dist/ - - # Signed release - rm -f cdist-*.tar.gz - rm -f cdist-*.tar.gz.asc - -distclean: clean - rm -f cdist/version.py + # distutils + rm -rf ./build ################################################################################ -# Misc +# install # -# The pub is Nico's "push to all git remotes" way ("make pub") -pub: - git push --mirror +install: + python3 setup.py install -test: - $(helper) $@ - -pep8: - $(helper) $@ +install-user: + python3 setup.py install --user diff --git a/PKGBUILD.in b/PKGBUILD.in index c967249d..c0188e68 100755 --- a/PKGBUILD.in +++ b/PKGBUILD.in @@ -9,7 +9,7 @@ pkgver=$version pkgrel=1 pkgdesc='A Usable Configuration Management System"' arch=('any') -url='http://www.nico.schottelius.org/software/cdist/' +url='https://www.cdi.st/' license=('GPL3') depends=('python>=3.2.0') source=("http://pypi.python.org/packages/source/c/cdist/cdist-\${pkgver}.tar.gz") diff --git a/README b/README index a67e25e3..caf2dac8 100644 --- a/README +++ b/README @@ -3,4 +3,5 @@ cdist cdist is a usable configuration management system. -For the web documentation have a look at docs/web/. +For the web documentation have a look at https://www.cdi.st/ +or at docs/src for reStructuredText manual. diff --git a/README-maintainers b/README-maintainers new file mode 100644 index 00000000..af57f475 --- /dev/null +++ b/README-maintainers @@ -0,0 +1,4 @@ +Maintainers should use ./bin/build-helper script. + +Makefile is intended for end users. It can be used for non-maintaining +targets that can be run from pure source (without git repository). diff --git a/bin/build-helper b/bin/build-helper index 46b139d1..9a776491 100755 --- a/bin/build-helper +++ b/bin/build-helper @@ -1,6 +1,7 @@ #!/bin/sh # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -18,17 +19,66 @@ # along with cdist. If not, see . # # -# This file contains the heavy lifting found usually in the Makefile +# This file contains the heavy lifting found usually in the Makefile. # -basedir=${0%/*}/../ -# Change to checkout directory -cd "$basedir" +usage() { + printf "usage: %s TARGET [TARGET-ARGS...] + Available targets: + changelog-changes + changelog-version + check-date + check-unittest + ml-release + archlinux-release + pypi-release + release-git-tag + sign-git-release + release + test + test-remote + pycodestyle + pep8 + check-pycodestyle + shellcheck-global-explorers + shellcheck-type-explorers + shellcheck-manifests + shellcheck-local-gencodes + shellcheck-remote-gencodes + shellcheck-scripts + shellcheck-gencodes + shellcheck-types + shellcheck + shellcheck-type-files + shellcheck-with-files + shellcheck-build-helper + check-shellcheck + version-branch + version + target-version + clean + distclean\n" "$1" +} -version=$(git describe) +basename="${0##*/}" + +if [ $# -lt 1 ] +then + usage "${basename}" + exit 1 +fi option=$1; shift +SHELLCHECKCMD="shellcheck -s sh -f gcc -x" +# Skip SC2154 for variables starting with __ since such variables are cdist +# environment variables. +SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]' + +# Change to checkout directory +basedir="${0%/*}/../" +cd "$basedir" + case "$option" in changelog-changes) if [ "$#" -eq 1 ]; then @@ -66,8 +116,8 @@ case "$option" in date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //') if [ "$date_today" != "$date_changelog" ]; then - echo "Date in changelog is not today" - echo "Changelog: $date_changelog" + printf "Date in changelog is not today\n" + printf "Changelog date: %s\n" "${date_changelog}" exit 1 fi ;; @@ -76,54 +126,17 @@ case "$option" in "$0" test ;; - blog) - version=$1; shift - blogfile=$1; shift - dir=${blogfile%/*} - file=${blogfile##*/} - - - cat << eof > "$blogfile" -[[!meta title="Cdist $version released"]] - -Here's a short overview about the changes found in version ${version}: - -eof - - $0 changelog-changes "$version" >> "$blogfile" - - cat << eof >> "$blogfile" -For more information visit the [[cdist homepage|software/cdist]]. - -[[!tag cdist config unix]] -eof - cd "$dir" - git add "$file" - # Allow git commit to fail if there are no changes - git commit -m "cdist blog update: $version" "$blogfile" || true - ;; - ml-release) if [ $# -ne 1 ]; then - echo "$0 ml-release version" >&2 + printf "%s ml-release version\n" "$0" >&2 exit 1 fi version=$1; shift - to_a=cdist - to_d=l.schottelius.org - to=${to_a}@${to_d} - - from_a=nico-cdist - from_d=schottelius.org - from=${from_a}@${from_d} - ( cat << eof -From: Nico -telmich- Schottelius <$from> -To: cdist mailing list <$to> -Subject: cdist $version released +Subject: cdist $version has been released Hello .*, @@ -134,25 +147,41 @@ eof "$0" changelog-changes "$version" cat << eof -Cheers, - -Nico - --- -Automatisation at its best level. With cdist. eof - ) | /usr/sbin/sendmail -f "$from" "$to" + ) > mailinglist.tmp ;; + archlinux-release) + if [ $# -ne 1 ]; then + printf "%s archlinux-release version\n" "$0" >&2 + exit 1 + fi + version=$1; shift + + ARCHLINUXTAR="cdist-${version}-1.src.tar.gz" + ./PKGBUILD.in "${version}" + umask 022 + mkaurball + burp -c system "${ARCHLINUXTAR}" + ;; + + pypi-release) + # Ensure that pypi release has the right version + "$0" version + + make docs-clean + make docs + python3 setup.py sdist upload + ;; release-git-tag) target_version=$($0 changelog-version) - if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then - echo "Tag for $target_version exists, aborting" + if git rev-parse --verify "refs/tags/${target_version}" 2>/dev/null; then + printf "Tag for %s exists, aborting\n" "${target_version}" exit 1 fi - printf "Enter tag description for ${target_version}: " - read tagmessage + printf "Enter tag description for %s: " "${target_version}" + read -r tagmessage # setup for signed tags: # gpg --fulL-gen-key @@ -170,7 +199,8 @@ eof # gpg --verify # gpg --no-default-keyring --keyring --verify # Ensure gpg-agent is running. - export GPG_TTY=$(tty) + GPG_TTY=$(tty) + export GPG_TTY gpg-agent git tag -s "$target_version" -m "$tagmessage" @@ -180,14 +210,14 @@ eof sign-git-release) if [ $# -lt 2 ] then - printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n" + printf "usage: %s sign-git-release TAG TOKEN [ARCHIVE]\n" "$0" printf " if ARCHIVE is not specified then it is created\n" exit 1 fi tag="$1" if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1 then - printf "Tag \"${tag}\" not found.\n" + printf "Tag \"%s\" not found.\n" "${tag}" exit 1 fi token="$2" @@ -195,44 +225,53 @@ eof then archivename="$3" else - archivename="cdist-${tag}.tar.gz" + archivename="cdist-${tag}.tar" git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \ || exit 1 + # make sure target version is generated + "$0" target-version + tar -x -f "${archivename}" || exit 1 + cp cdist/version.py "cdist-${tag}/cdist/version.py" || exit 1 + tar -c -f "${archivename}" "cdist-${tag}/" || exit 1 + rm -r -f "cdist-${tag}/" + gzip "${archivename}" || exit 1 + archivename="${archivename}.gz" fi gpg --armor --detach-sign "${archivename}" || exit 1 - # make github release - curl -H "Authorization: token ${token}" \ - --request POST \ - --data "{ \"tag_name\":\"${tag}\", \ - \"target_commitish\":\"master\", \ - \"name\": \"${tag}\", \ - \"body\":\"${tag}\", \ - \"draft\":false, \ - \"prerelease\": false}" \ - "https://api.github.com/repos/ungleich/cdist/releases" || exit 1 + project="ungleich-public%2Fcdist" + sed_cmd='s/^.*"markdown":"\([^"]*\)".*$/\1/' - # get release ID - repoid=$(curl "https://api.github.com/repos/ungleich/cdist/releases/tags/${tag}" \ - | python3 -c 'import json; import sys; print(json.loads(sys.stdin.read())["id"])') \ - || exit 1 + # upload archive + response_archive=$(curl -f -X POST \ + --http1.1 \ + -H "PRIVATE-TOKEN: ${token}" \ + -F "file=@${archivename}" \ + "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \ + | sed "${sed_cmd}") || exit 1 - # upload archive and then signature - curl -H "Authorization: token ${token}" \ - -H "Accept: application/vnd.github.manifold-preview" \ - -H "Content-Type: application/x-gtar" \ - --data-binary @${archivename} \ - "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}" \ - || exit 1 - curl -H "Authorization: token ${token}" \ - -H "Accept: application/vnd.github.manifold-preview" \ - -H "Content-Type: application/pgp-signature" \ - --data-binary @${archivename}.asc \ - "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}.asc" \ + # upload archive signature + response_archive_sig=$(curl -f -X POST \ + --http1.1 \ + -H "PRIVATE-TOKEN: ${token}" \ + -F "file=@${archivename}.asc" \ + "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \ + | sed "${sed_cmd}") || exit 1 + + # make release + changelog=$("$0" changelog-changes "$1" | sed 's/^[[:space:]]*//') + release_notes=$( + printf "%s\n\n%s\n\n**Changelog**\n\n%s\n" \ + "${response_archive}" "${response_archive_sig}" "${changelog}" + ) + curl -f -X POST \ + -H "PRIVATE-TOKEN: ${token}" \ + -F "description=${release_notes}" \ + "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}/release" \ || exit 1 # remove generated files (archive and asc) - if [ $# -eq 2] + if [ $# -eq 2 ] then rm -f "${archivename}" fi @@ -244,29 +283,30 @@ eof target_version=$($0 changelog-version) target_branch=$($0 version-branch) - echo "Beginning release process for $target_version" + printf "Beginning release process for %s\n" "${target_version}" # First check everything is sane "$0" check-date "$0" check-unittest - "$0" check-pep8 + "$0" check-pycodestyle + "$0" check-shellcheck # Generate version file to be included in packaging "$0" target-version # Ensure the git status is clean, else abort if ! git diff-index --name-only --exit-code HEAD ; then - echo "Unclean tree, see files above, aborting" + printf "Unclean tree, see files above, aborting.\n" exit 1 fi # Ensure we are on the master branch masterbranch=yes if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then - echo "Releases are happening from the master branch, aborting" + printf "Releases are happening from the master branch, aborting.\n" - echo "Enter the magic word to release anyway" - read magicword + printf "Enter the magic word to release anyway:" + read -r magicword if [ "$magicword" = "iknowwhatido" ]; then masterbranch=no @@ -277,7 +317,7 @@ eof if [ "$masterbranch" = yes ]; then # Ensure version branch exists - if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then + if ! git rev-parse --verify "refs/heads/${target_branch}" 2>/dev/null; then git branch "$target_branch" fi @@ -295,20 +335,12 @@ eof make docs-clean make docs - # Generate speeches (indirect check if they build) - make speeches - ############################################################# # Everything green, let's do the release # Tag the current commit "$0" release-git-tag - # sign git tag - printf "Enter github authentication token: " - read token - "$0" sign-git-release "${target_version}" "${token}" - # Also merge back the version branch if [ "$masterbranch" = yes ]; then git checkout master @@ -316,41 +348,41 @@ eof fi # Publish git changes - make pub - - # publish man, speeches, website - if [ "$masterbranch" = yes ]; then - make web-release-all - else - make web-release-all-no-latest - fi - - # Ensure that pypi release has the right version - "$0" version + # if you want to have mirror locally then uncomment this and comment below + # git push --mirror + git push + # push also new branch and set up tracking + git push -u origin "${target_branch}" + # fi # Create and publish package for pypi - make pypi-release + "$0" pypi-release - # Archlinux release is based on pypi - make archlinux-release + # sign git tag + printf "Enter upstream repository authentication token: " + read -r token + "$0" sign-git-release "${target_version}" "${token}" # Announce change on ML - make ml-release + "$0" ml-release "${target_version}" cat << eof Manual steps post release: - - - linkedin - - hackernews - - reddit + - cdist-web + - send mail body generated in mailinglist.tmp and inform Dmitry for deb - twitter - eof - ;; test) - export PYTHONPATH="$(pwd -P)" + if [ ! -f "cdist/version.py" ] + then + printf "cdist/version.py is missing, generate it first.\n" + exit 1 + fi + + PYTHONPATH="$(pwd -P)" + export PYTHONPATH if [ $# -lt 1 ]; then python3 -m cdist.test @@ -359,18 +391,31 @@ eof fi ;; - pep8) - pep8 "${basedir}" "${basedir}/scripts/cdist" | less + test-remote) + if [ ! -f "cdist/version.py" ] + then + printf "cdist/version.py is missing, generate it first.\n" + exit 1 + fi + + PYTHONPATH="$(pwd -P)" + export PYTHONPATH + + python3 -m cdist.test.exec.remote ;; - check-pep8) - "$0" pep8 - echo "Please review pep8 report." + pycodestyle|pep8) + pycodestyle "${basedir}" "${basedir}/scripts/cdist" | less + ;; + + check-pycodestyle) + "$0" pycodestyle + printf "\\nPlease review pycodestyle report.\\n" while true do - echo "Continue (yes/no)?" + printf "Continue (yes/no)?\n" any= - read any + read -r any case "$any" in yes) break @@ -379,7 +424,83 @@ eof exit 1 ;; *) - echo "Please answer with 'yes' or 'no' explicitly." + printf "Please answer with 'yes' or 'no' explicitly.\n" + ;; + esac + done + ;; + + shellcheck-global-explorers) + find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + ;; + + shellcheck-type-explorers) + find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + ;; + + shellcheck-manifests) + find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + ;; + + shellcheck-local-gencodes) + find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + ;; + + shellcheck-remote-gencodes) + find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + ;; + + shellcheck-scripts) + ${SHELLCHECKCMD} scripts/cdist-dump scripts/cdist-new-type || exit 0 + ;; + + shellcheck-gencodes) + "$0" shellcheck-local-gencodes + "$0" shellcheck-remote-gencodes + ;; + + shellcheck-types) + "$0" shellcheck-type-explorers + "$0" shellcheck-manifests + "$0" shellcheck-gencodes + ;; + + shellcheck) + "$0" shellcheck-global-explorers + "$0" shellcheck-types + "$0" shellcheck-scripts + ;; + + shellcheck-type-files) + find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0 + ;; + + shellcheck-with-files) + "$0" shellcheck + "$0" shellcheck-type-files + ;; + + shellcheck-build-helper) + ${SHELLCHECKCMD} ./bin/build-helper + ;; + + check-shellcheck) + "$0" shellcheck + printf "\\nPlease review shellcheck report.\\n" + while true + do + printf "Continue (yes/no)?\n" + any= + read -r any + case "$any" in + yes) + break + ;; + no) + exit 1 + ;; + *) + printf "Please answer with 'yes' or 'no' explicitly.\n" ;; esac done @@ -390,16 +511,39 @@ eof ;; version) - echo "VERSION = \"$(git describe)\"" > cdist/version.py + printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py ;; target-version) target_version=$($0 changelog-version) - echo "VERSION = \"${target_version}\"" > cdist/version.py + printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py ;; + clean) + make clean + + # Archlinux + rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz + rm -rf pkg/ src/ + + rm -f MANIFEST PKGBUILD + rm -rf dist/ + + # Signed release + rm -f cdist-*.tar.gz + rm -f cdist-*.tar.gz.asc + + # Temp files + rm -f ./*.tmp + ;; + + distclean) + "$0" clean + rm -f cdist/version.py + ;; *) - echo "Unknown helper target $@ - aborting" + printf "Unknown target: '%s'.\n" "${option}" >&2 + usage "${basename}" exit 1 ;; diff --git a/bin/build-helper.freebsd b/bin/build-helper.freebsd deleted file mode 100755 index 183129db..00000000 --- a/bin/build-helper.freebsd +++ /dev/null @@ -1,468 +0,0 @@ -#!/bin/sh -# -# 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) -# 2016 Darko Poljak (darko.poljak at gmail.com) -# -# This file is part of cdist. -# -# cdist is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# cdist is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with cdist. If not, see . -# -# -# This file contains the heavy lifting found usually in the Makefile -# - -# vars for make -helper=$0 - -basedir=${0%/*}/../ -# run_as is used to check how the script is called (by $0 value) -# currently supported sufixes for $0 are: -# .freebsd - run as freebsd -basename=${0##*/} -run_as=${basename#*.} -case "$run_as" in - freebsd) - to_a=cdist-configuration-management - to_d=googlegroups.com - from_a=darko.poljak - from_d=gmail.com - ml_name="Darko Poljak" - ml_sig_name="Darko" - - # vars for make - WEBDIR=../vcs/www.nico.schottelius.org - ;; - *) - to_a=cdist - to_d=l.schottelius.org - from_a=nico-cdist - from_d=schottelius.org - ml_name="Nico -telmich- Schottelius" - ml_sig_name="Nico" - - # vars for make - WEBDIR=$$HOME/vcs/www.nico.schottelius.org - ;; -esac - -# Change to checkout directory -cd "$basedir" - -version=$(git describe) - -option=$1; shift - -case "$option" in - print-make-vars) - printf "helper: ${helper}\n" - printf "WEBDIR: ${WEBDIR}\n" - ;; - print-runas) - printf "run_as: $run_as\n" - ;; - changelog-changes) - if [ "$#" -eq 1 ]; then - start=$1 - else - start="[[:digit:]]" - fi - - end="[[:digit:]]" - - awk -F: "BEGIN { start=0 } - { - if(start == 0) { - if (\$0 ~ /^$start/) { - start = 1 - } - } else { - if (\$0 ~ /^$end/) { - exit - } else { - print \$0 - } - } - }" "$basedir/docs/changelog" - ;; - - changelog-version) - # get version from changelog - grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/:.*//' - ;; - - check-date) - # verify date in changelog is today - date_today="$(date +%Y-%m-%d)" - date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //') - - if [ "$date_today" != "$date_changelog" ]; then - echo "Date in changelog is not today" - echo "Changelog: $date_changelog" - exit 1 - fi - ;; - - check-unittest) - "$0" test - ;; - - blog) - version=$1; shift - blogfile=$1; shift - dir=${blogfile%/*} - file=${blogfile##*/} - - - cat << eof > "$blogfile" -[[!meta title="Cdist $version released"]] - -Here's a short overview about the changes found in version ${version}: - -eof - - $0 changelog-changes "$version" >> "$blogfile" - - cat << eof >> "$blogfile" -For more information visit the [[cdist homepage|software/cdist]]. - -[[!tag cdist config unix]] -eof - cd "$dir" - git add "$file" - # Allow git commit to fail if there are no changes - git commit -m "cdist blog update: $version" "$blogfile" || true - ;; - - ml-release) - if [ $# -ne 1 ]; then - echo "$0 ml-release version" >&2 - exit 1 - fi - - version=$1; shift - - to=${to_a}@${to_d} - from=${from_a}@${from_d} - - ( - cat << eof -From: ${ml_name} <$from> -To: cdist mailing list <$to> -Subject: cdist $version released - -Hello .*, - -cdist $version has been released with the following changes: - -eof - - "$0" changelog-changes "$version" - cat << eof - -Cheers, - -${ml_sig_name} - --- -Automatisation at its best level. With cdist. -eof - ) | /usr/sbin/sendmail -f "$from" "$to" - ;; - - release-git-tag) - target_version=$($0 changelog-version) - if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then - echo "Tag for $target_version exists, aborting" - exit 1 - fi - printf "Enter tag description for ${target_version}: " - read tagmessage - - # setup for signed tags: - # gpg --fulL-gen-key - # gpg --list-secret-keys --keyid-format LONG - # git config --local user.signingkey - # for exporting pub key: - # gpg --armor --export > pubkey.asc - # gpg --output pubkey.gpg --export - # show tag with signature - # git show - # verify tag signature - # git tag -v - # - # gpg verify signature - # gpg --verify - # gpg --no-default-keyring --keyring --verify - # Ensure gpg-agent is running. - export GPG_TTY=$(tty) - gpg-agent - - git tag -s "$target_version" -m "$tagmessage" - git push --tags - ;; - - sign-git-release) - if [ $# -lt 2 ] - then - printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n" - printf " if ARCHIVE is not specified then it is created\n" - exit 1 - fi - tag="$1" - if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1 - then - printf "Tag \"${tag}\" not found.\n" - exit 1 - fi - token="$2" - if [ $# -gt 2 ] - then - archivename="$3" - else - archivename="cdist-${tag}.tar.gz" - git archive --prefix="cdist-${tag}/" -o "${archivename}" "${tag}" \ - || exit 1 - fi - gpg --armor --detach-sign "${archivename}" || exit 1 - - # make github release - curl -H "Authorization: token ${token}" \ - --request POST \ - --data "{ \"tag_name\":\"${tag}\", \ - \"target_commitish\":\"master\", \ - \"name\": \"${tag}\", \ - \"body\":\"${tag}\", \ - \"draft\":false, \ - \"prerelease\": false}" \ - "https://api.github.com/repos/ungleich/cdist/releases" || exit 1 - - # get release ID - repoid=$(curl "https://api.github.com/repos/ungleich/cdist/releases/tags/${tag}" \ - | python3 -c 'import json; import sys; print(json.loads(sys.stdin.read())["id"])') \ - || exit 1 - - # upload archive and then signature - curl -H "Authorization: token ${token}" \ - -H "Accept: application/vnd.github.manifold-preview" \ - -H "Content-Type: application/x-gtar" \ - --data-binary @${archivename} \ - "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}" \ - || exit 1 - curl -H "Authorization: token ${token}" \ - -H "Accept: application/vnd.github.manifold-preview" \ - -H "Content-Type: application/pgp-signature" \ - --data-binary @${archivename}.asc \ - "https://uploads.github.com/repos/ungleich/cdist/releases/${repoid}/assets?name=${archivename}.asc" \ - || exit 1 - - # remove generated files (archive and asc) - if [ $# -eq 2] - then - rm -f "${archivename}" - fi - rm -f "${archivename}.asc" - ;; - - release) - set -e - target_version=$($0 changelog-version) - target_branch=$($0 version-branch) - - echo "Beginning release process for $target_version" - - # First check everything is sane - "$0" check-date - "$0" check-unittest - "$0" check-pep8 - - # Generate version file to be included in packaging - "$0" target-version - - # Ensure the git status is clean, else abort - if ! git diff-index --name-only --exit-code HEAD ; then - echo "Unclean tree, see files above, aborting" - exit 1 - fi - - # Ensure we are on the master branch - masterbranch=yes - if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then - echo "Releases are happening from the master branch, aborting" - - echo "Enter the magic word to release anyway" - read magicword - - if [ "$magicword" = "iknowwhatido" ]; then - masterbranch=no - else - exit 1 - fi - fi - - if [ "$masterbranch" = yes ]; then - # Ensure version branch exists - if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then - git branch "$target_branch" - fi - - # Merge master branch into version branch - git checkout "$target_branch" - git merge master - fi - - # Verify that after the merge everything works - "$0" check-date - "$0" check-unittest - - # Generate documentation (man and html) - # First, clean old generated docs - make helper=${helper} WEBDIR=${WEBDIR} docs-clean - make helper=${helper} WEBDIR=${WEBDIR} docs - - # Generate speeches (indirect check if they build) - make helper=${helper} WEBDIR=${WEBDIR} speeches - - ############################################################# - # Everything green, let's do the release - - # Tag the current commit - "$0" release-git-tag - - # sign git tag - printf "Enter github authentication token: " - read token - "$0" sign-git-release "${target_version}" "${token}" - - # Also merge back the version branch - if [ "$masterbranch" = yes ]; then - git checkout master - git merge "$target_branch" - fi - - # Publish git changes - case "$run_as" in - freebsd) - # if we are not Nico :) then just push, no mirror - git push - # push also new branch and set up tracking - git push -u origin "${target_branch}" - ;; - *) - make helper=${helper} WEBDIR=${WEBDIR} pub - ;; - esac - - # publish man, speeches, website - if [ "$masterbranch" = yes ]; then - make helper=${helper} WEBDIR=${WEBDIR} web-release-all - else - make helper=${helper} WEBDIR=${WEBDIR} web-release-all-no-latest - fi - - # Ensure that pypi release has the right version - "$0" version - - # Create and publish package for pypi - make helper=${helper} WEBDIR=${WEBDIR} pypi-release - - case "$run_as" in - freebsd) - ;; - *) - # Archlinux release is based on pypi - make archlinux-release - ;; - esac - - # Announce change on ML - make helper=${helper} WEBDIR=${WEBDIR} ml-release - - cat << eof -Manual steps post release: - - - linkedin - - hackernews - - reddit - - twitter - -eof - - case "$run_as" in - freebsd) - cat < cdist/version.py - ;; - - target-version) - target_version=$($0 changelog-version) - echo "VERSION = \"${target_version}\"" > cdist/version.py - ;; - - *) - echo "Unknown helper target $@ - aborting" - exit 1 - ;; - -esac diff --git a/cdist/__init__.py b/cdist/__init__.py index 6ea02d41..c673b3ba 100644 --- a/cdist/__init__.py +++ b/cdist/__init__.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- # # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2017 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -20,9 +21,9 @@ # import os -import subprocess import hashlib +import cdist.log import cdist.version VERSION = cdist.version.VERSION @@ -42,8 +43,9 @@ BANNER = """ "P' "" "" """ -REMOTE_COPY = "scp -o User=root" +REMOTE_COPY = "scp -o User=root -q" REMOTE_EXEC = "ssh -o User=root" +REMOTE_CMDS_CLEANUP_PATTERN = "ssh -o User=root -O exit -S {}" class Error(Exception): @@ -79,18 +81,159 @@ class CdistBetaRequired(cdist.Error): return err_msg.format(*fmt_args) -class CdistObjectError(Error): - """Something went wrong with an object""" +class CdistEntityError(Error): + """Something went wrong while executing cdist entity""" + def __init__(self, entity_name, entity_params, stdout_paths, + stderr_paths, subject=''): + self.entity_name = entity_name + self.entity_params = entity_params + self.stderr_paths = stderr_paths + self.stdout_paths = stdout_paths + if isinstance(subject, Error): + self.original_error = subject + else: + self.original_error = None + self.message = str(subject) - def __init__(self, cdist_object, message): - self.name = cdist_object.name - self.source = " ".join(cdist_object.source) - self.message = message + def _stdpath(self, stdpaths, header_name): + result = {} + for name, path in stdpaths: + if name not in result: + result[name] = [] + try: + if os.path.exists(path) and os.path.getsize(path) > 0: + output = [] + label_begin = name + ":" + header_name + output.append(label_begin) + output.append('\n') + output.append('-' * len(label_begin)) + output.append('\n') + with open(path, 'r') as fd: + output.append(fd.read()) + output.append('\n') + result[name].append(''.join(output)) + except UnicodeError as ue: + result[name].append(('Cannot output {}:{} due to: {}.\n' + 'You can try to read the error file "{}"' + ' yourself.').format( + name, header_name, ue, path)) + return result + + def _stderr(self): + return self._stdpath(self.stderr_paths, 'stderr') + + def _stdout(self): + return self._stdpath(self.stdout_paths, 'stdout') + + def _update_dict_list(self, target, source): + for x in source: + if x not in target: + target[x] = [] + target[x].extend(source[x]) + + @property + def std_streams(self): + std_dict = {} + self._update_dict_list(std_dict, self._stdout()) + self._update_dict_list(std_dict, self._stderr()) + return std_dict def __str__(self): - return '%s: %s (defined at %s)' % (self.name, - self.message, - self.source) + output = [] + output.append(self.message) + output.append('\n\n') + header = "Error processing " + self.entity_name + under_header = '=' * len(header) + output.append(header) + output.append('\n') + output.append(under_header) + output.append('\n') + for param_name, param_value in self.entity_params: + output.append(param_name + ': ' + str(param_value)) + output.append('\n') + output.append('\n') + for x in self.std_streams: + output.append(''.join(self.std_streams[x])) + return ''.join(output) + + +class CdistObjectError(CdistEntityError): + """Something went wrong while working on a specific cdist object""" + def __init__(self, cdist_object, subject=''): + params = [ + ('name', cdist_object.name, ), + ('path', cdist_object.absolute_path, ), + ('source', " ".join(cdist_object.source), ), + ('type', os.path.realpath( + cdist_object.cdist_type.absolute_path), ), + ] + stderr_paths = [] + for stderr_name in os.listdir(cdist_object.stderr_path): + stderr_path = os.path.join(cdist_object.stderr_path, + stderr_name) + stderr_paths.append((stderr_name, stderr_path, )) + stdout_paths = [] + for stdout_name in os.listdir(cdist_object.stdout_path): + stdout_path = os.path.join(cdist_object.stdout_path, + stdout_name) + stdout_paths.append((stdout_name, stdout_path, )) + super().__init__("object '{}'".format(cdist_object.name), + params, stdout_paths, stderr_paths, subject) + + +class CdistObjectExplorerError(CdistEntityError): + """ + Something went wrong while working on a specific + cdist object explorer + """ + def __init__(self, cdist_object, explorer_name, explorer_path, + stderr_path, subject=''): + params = [ + ('object name', cdist_object.name, ), + ('object path', cdist_object.absolute_path, ), + ('object source', " ".join(cdist_object.source), ), + ('object type', os.path.realpath( + cdist_object.cdist_type.absolute_path), ), + ('explorer name', explorer_name, ), + ('explorer path', explorer_path, ), + ] + stdout_paths = [] + stderr_paths = [ + ('remote', stderr_path, ), + ] + super().__init__("explorer '{}' of object '{}'".format( + explorer_name, cdist_object.name), params, stdout_paths, + stderr_paths, subject) + + +class InitialManifestError(CdistEntityError): + """Something went wrong while executing initial manifest""" + def __init__(self, initial_manifest, stdout_path, stderr_path, subject=''): + params = [ + ('path', initial_manifest, ), + ] + stdout_paths = [ + ('init', stdout_path, ), + ] + stderr_paths = [ + ('init', stderr_path, ), + ] + super().__init__('initial manifest', params, stdout_paths, + stderr_paths, subject) + + +class GlobalExplorerError(CdistEntityError): + """Something went wrong while executing global explorer""" + def __init__(self, name, path, stderr_path, subject=''): + params = [ + ('name', name, ), + ('path', path, ), + ] + stderr_paths = [ + ('remote', stderr_path, ), + ] + super().__init__("global explorer '{}'".format(name), + params, [], stderr_paths, subject) def file_to_list(filename): @@ -114,3 +257,15 @@ def str_hash(s): return hashlib.md5(s.encode('utf-8')).hexdigest() else: raise Error("Param should be string") + + +def home_dir(): + if 'HOME' in os.environ: + home = os.environ['HOME'] + if home: + rv = os.path.join(home, ".cdist") + else: + rv = None + else: + rv = None + return rv diff --git a/cdist/argparse.py b/cdist/argparse.py index 045c12bc..421d1b54 100644 --- a/cdist/argparse.py +++ b/cdist/argparse.py @@ -1,29 +1,47 @@ import argparse import cdist import multiprocessing -import os import logging import collections +import functools +import cdist.configuration # set of beta sub-commands -BETA_COMMANDS = set(('install', )) +BETA_COMMANDS = set(('install', 'inventory', )) # set of beta arguments for sub-commands BETA_ARGS = { - 'config': set(('jobs', )), + 'config': set(('tag', 'all_tagged_hosts', 'use_archiving', )), } -EPILOG = "Get cdist at http://www.nico.schottelius.org/software/cdist/" +EPILOG = "Get cdist at https://code.ungleich.ch/ungleich-public/cdist" # Parser others can reuse parser = None +_verbosity_level_off = -2 _verbosity_level = { - 0: logging.ERROR, - 1: logging.WARNING, - 2: logging.INFO, + _verbosity_level_off: logging.OFF, + -1: logging.ERROR, + 0: logging.WARNING, + 1: logging.INFO, + 2: logging.VERBOSE, + 3: logging.DEBUG, + 4: logging.TRACE, } + + +# Generate verbosity level constants: +# VERBOSE_OFF, VERBOSE_ERROR, VERBOSE_WARNING, VERBOSE_INFO, VERBOSE_VERBOSE, +# VERBOSE_DEBUG, VERBOSE_TRACE. +this_globals = globals() +for level in _verbosity_level: + const = 'VERBOSE_' + logging.getLevelName(_verbosity_level[level]) + this_globals[const] = level + + +# All verbosity levels above 4 are TRACE. _verbosity_level = collections.defaultdict( - lambda: logging.DEBUG, _verbosity_level) + lambda: logging.TRACE, _verbosity_level) def add_beta_command(cmd): @@ -55,17 +73,15 @@ def check_beta(args_dict): raise cdist.CdistBetaRequired(cmd, arg) -def check_positive_int(value): - import argparse - +def check_lower_bounded_int(value, lower_bound, name): try: val = int(value) except ValueError: raise argparse.ArgumentTypeError( "{} is invalid int value".format(value)) - if val <= 0: + if val < lower_bound: raise argparse.ArgumentTypeError( - "{} is invalid positive int value".format(val)) + "{} is invalid {} value".format(val, name)) return val @@ -80,31 +96,43 @@ def get_parsers(): # Options _all_ parsers have in common parser['loglevel'] = argparse.ArgumentParser(add_help=False) parser['loglevel'].add_argument( - '-d', '--debug', - help=('Set log level to debug (deprecated, use -vvv instead)'), + '-l', '--log-level', metavar='LOGLEVEL', + type=functools.partial(check_lower_bounded_int, lower_bound=-1, + name="log level"), + help=('Set the specified verbosity level. ' + 'The levels, in order from the lowest to the highest, are: ' + 'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) ' + 'TRACE (4 or higher). If used along with -v then -v ' + 'increases last set value and -l overwrites last set ' + 'value.'), + action='store', dest='verbose', required=False) + parser['loglevel'].add_argument( + '-q', '--quiet', + help='Quiet mode: disables logging, including WARNING and ERROR.', action='store_true', default=False) parser['loglevel'].add_argument( '-v', '--verbose', help=('Increase the verbosity level. Every instance of -v ' 'increments the verbosity level by one. Its default value ' - 'is 0. There are 4 levels of verbosity. The order of levels ' - 'from the lowest to the highest are: ERROR (0), ' - 'WARNING (1), INFO (2) and DEBUG (3 or higher).'), - action='count', default=0) + 'is 0 which includes ERROR and WARNING levels. ' + 'The levels, in order from the lowest to the highest, are: ' + 'ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) ' + 'TRACE (4 or higher). If used along with -l then -l ' + 'overwrites last set value and -v increases last set ' + 'value.'), + action='count', default=None) parser['beta'] = argparse.ArgumentParser(add_help=False) parser['beta'].add_argument( '-b', '--beta', - help=('Enable beta functionalities. ' - 'Can also be enabled using CDIST_BETA env var.'), - action='store_true', dest='beta', - default='CDIST_BETA' in os.environ) + help=('Enable beta functionality. '), + action='store_true', dest='beta', default=None) # Main subcommand parser parser['main'] = argparse.ArgumentParser( - description='cdist ' + cdist.VERSION, parents=[parser['loglevel']]) + description='cdist ' + cdist.VERSION) parser['main'].add_argument( - '-V', '--version', help='Show version', action='version', + '-V', '--version', help='Show version.', action='version', version='%(prog)s ' + cdist.VERSION) parser['sub'] = parser['main'].add_subparsers( title="Commands", dest="command") @@ -114,68 +142,147 @@ def get_parsers(): 'banner', parents=[parser['loglevel']]) parser['banner'].set_defaults(func=cdist.banner.banner) + parser['inventory_common'] = argparse.ArgumentParser(add_help=False) + parser['inventory_common'].add_argument( + '-I', '--inventory', + help=('Use specified custom inventory directory. ' + 'Inventory directory is set up by the following rules: ' + 'if cdist configuration resolves this value then specified ' + 'directory is used, ' + 'if HOME env var is set then ~/.cdist/inventory is ' + 'used, otherwise distribution inventory directory is used.'), + dest="inventory_dir", required=False) + + parser['common'] = argparse.ArgumentParser(add_help=False) + parser['common'].add_argument( + '-g', '--config-file', + help=('Use specified custom configuration file.'), + dest="config_file", required=False) + # Config parser['config_main'] = argparse.ArgumentParser(add_help=False) + parser['config_main'].add_argument( + '-4', '--force-ipv4', + help=('Force to use IPv4 addresses only. No influence for custom' + ' remote commands.'), + action='store_const', dest='force_ipv', const=4) + parser['config_main'].add_argument( + '-6', '--force-ipv6', + help=('Force to use IPv6 addresses only. No influence for custom' + ' remote commands.'), + action='store_const', dest='force_ipv', const=6) + parser['config_main'].add_argument( + '-C', '--cache-path-pattern', + help=('Specify custom cache path pattern. If ' + 'it is not set then default hostdir is used.'), + dest='cache_path_pattern', + default=None) parser['config_main'].add_argument( '-c', '--conf-dir', help=('Add configuration directory (can be repeated, ' - 'last one wins)'), action='append') + 'last one wins).'), action='append') parser['config_main'].add_argument( '-i', '--initial-manifest', - help='path to a cdist manifest or \'-\' to read from stdin.', + help='Path to a cdist manifest or \'-\' to read from stdin.', dest='manifest', required=False) parser['config_main'].add_argument( '-j', '--jobs', nargs='?', - type=check_positive_int, - help=('Specify the maximum number of parallel jobs. Global' - 'explorers, object prepare and object run are supported' - '(currently in beta'), + type=functools.partial(check_lower_bounded_int, lower_bound=1, + name="positive int"), + help=('Operate in parallel in specified maximum number of jobs. ' + 'Global explorers, object prepare and object run are ' + 'supported. Without argument CPU count is used by default. '), action='store', dest='jobs', const=multiprocessing.cpu_count()) parser['config_main'].add_argument( '-n', '--dry-run', - help='do not execute code', action='store_true') + help='Do not execute code.', action='store_true') parser['config_main'].add_argument( '-o', '--out-dir', - help='directory to save cdist output in', dest="out_path") + help='Directory to save cdist output in.', dest="out_path") + parser['config_main'].add_argument( + '-P', '--timestamp', + help=('Timestamp log messages with the current local date and time ' + 'in the format: YYYYMMDDHHMMSS.us.'), + action='store_true', dest='timestamp') + parser['config_main'].add_argument( + '-R', '--use-archiving', nargs='?', + choices=('tar', 'tgz', 'tbz2', 'txz',), + help=('Operate by using archiving with compression where ' + 'appropriate. Supported values are: tar - tar archive, ' + 'tgz - gzip tar archive (the default), ' + 'tbz2 - bzip2 tar archive and txz - lzma tar archive. ' + 'Currently in beta.'), + action='store', dest='use_archiving', + const='tgz') # remote-copy and remote-exec defaults are environment variables # if set; if not then None - these will be futher handled after # parsing to determine implementation default + parser['config_main'].add_argument( + '-r', '--remote-out-dir', + help='Directory to save cdist output in on the target host.', + dest="remote_out_path") parser['config_main'].add_argument( '--remote-copy', - help='Command to use for remote copy (should behave like scp)', + help='Command to use for remote copy (should behave like scp).', action='store', dest='remote_copy', - default=os.environ.get('CDIST_REMOTE_COPY')) + default=None) parser['config_main'].add_argument( '--remote-exec', help=('Command to use for remote execution ' - '(should behave like ssh)'), + '(should behave like ssh).'), action='store', dest='remote_exec', - default=os.environ.get('CDIST_REMOTE_EXEC')) + default=None) + parser['config_main'].add_argument( + '-S', '--disable-saving-output-streams', + help='Disable saving output streams.', + action='store_false', dest='save_output_streams', default=True) # Config parser['config_args'] = argparse.ArgumentParser(add_help=False) parser['config_args'].add_argument( - 'host', nargs='*', help='host(s) to operate on') + '-A', '--all-tagged', + help=('Use all hosts present in tags db. Currently in beta.'), + action="store_true", dest="all_tagged_hosts", default=False) + parser['config_args'].add_argument( + '-a', '--all', + help=('List hosts that have all specified tags, ' + 'if -t/--tag is specified.'), + action="store_true", dest="has_all_tags", default=False) parser['config_args'].add_argument( '-f', '--file', - help=('Read additional hosts to operate on from specified file ' - 'or from stdin if \'-\' (each host on separate line). ' - 'If no host or host file is specified then, by default, ' - 'read hosts from stdin.'), + help=('Read specified file for a list of additional hosts to ' + 'operate on or if \'-\' is given, read stdin (one host per ' + 'line). If no host or host file is specified then, by ' + 'default, read hosts from stdin.'), dest='hostfile', required=False) parser['config_args'].add_argument( - '-p', '--parallel', - help='operate on multiple hosts in parallel', - action='store_true', dest='parallel') + '-p', '--parallel', nargs='?', metavar='HOST_MAX', + type=functools.partial(check_lower_bounded_int, lower_bound=1, + name="positive int"), + help=('Operate on multiple hosts in parallel for specified maximum ' + 'hosts at a time. Without argument CPU count is used by ' + 'default.'), + action='store', dest='parallel', + const=multiprocessing.cpu_count()) parser['config_args'].add_argument( '-s', '--sequential', - help='operate on multiple hosts sequentially (default)', - action='store_false', dest='parallel') + help='Operate on multiple hosts sequentially (default).', + action='store_const', dest='parallel', const=0) + parser['config_args'].add_argument( + '-t', '--tag', + help=('Host is specified by tag, not hostname/address; ' + 'list all hosts that contain any of specified tags. ' + 'Currently in beta.'), + dest='tag', required=False, action="store_true", default=False) + parser['config_args'].add_argument( + 'host', nargs='*', help='Host(s) to operate on.') parser['config'] = parser['sub'].add_parser( 'config', parents=[parser['loglevel'], parser['beta'], + parser['common'], parser['config_main'], + parser['inventory_common'], parser['config_args']]) parser['config'].set_defaults(func=cdist.config.Config.commandline) @@ -184,6 +291,137 @@ def get_parsers(): parents=[parser['config']]) parser['install'].set_defaults(func=cdist.install.Install.commandline) + # Inventory + parser['inventory'] = parser['sub'].add_parser('inventory') + parser['invsub'] = parser['inventory'].add_subparsers( + title="Inventory commands", dest="subcommand") + + parser['add-host'] = parser['invsub'].add_parser( + 'add-host', parents=[parser['loglevel'], parser['beta'], + parser['common'], + parser['inventory_common']]) + parser['add-host'].add_argument( + 'host', nargs='*', help='Host(s) to add.') + parser['add-host'].add_argument( + '-f', '--file', + help=('Read additional hosts to add from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read from stdin.'), + dest='hostfile', required=False) + + parser['add-tag'] = parser['invsub'].add_parser( + 'add-tag', parents=[parser['loglevel'], parser['beta'], + parser['common'], + parser['inventory_common']]) + parser['add-tag'].add_argument( + 'host', nargs='*', + help='List of host(s) for which tags are added.') + parser['add-tag'].add_argument( + '-f', '--file', + help=('Read additional hosts to add tags from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read from stdin. If no tags/tagfile nor hosts/hostfile' + ' are specified then tags are read from stdin and are' + ' added to all hosts.'), + dest='hostfile', required=False) + parser['add-tag'].add_argument( + '-T', '--tag-file', + help=('Read additional tags to add from specified file ' + 'or from stdin if \'-\' (each tag on separate line). ' + 'If no tag or tag file is specified then, by default, ' + 'read from stdin. If no tags/tagfile nor hosts/hostfile' + ' are specified then tags are read from stdin and are' + ' added to all hosts.'), + dest='tagfile', required=False) + parser['add-tag'].add_argument( + '-t', '--taglist', + help=("Tag list to be added for specified host(s), comma separated" + " values."), + dest="taglist", required=False) + + parser['del-host'] = parser['invsub'].add_parser( + 'del-host', parents=[parser['loglevel'], parser['beta'], + parser['common'], + parser['inventory_common']]) + parser['del-host'].add_argument( + 'host', nargs='*', help='Host(s) to delete.') + parser['del-host'].add_argument( + '-a', '--all', help=('Delete all hosts.'), + dest='all', required=False, action="store_true", default=False) + parser['del-host'].add_argument( + '-f', '--file', + help=('Read additional hosts to delete from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read from stdin.'), + dest='hostfile', required=False) + + parser['del-tag'] = parser['invsub'].add_parser( + 'del-tag', parents=[parser['loglevel'], parser['beta'], + parser['common'], + parser['inventory_common']]) + parser['del-tag'].add_argument( + 'host', nargs='*', + help='List of host(s) for which tags are deleted.') + parser['del-tag'].add_argument( + '-a', '--all', + help=('Delete all tags for specified host(s).'), + dest='all', required=False, action="store_true", default=False) + parser['del-tag'].add_argument( + '-f', '--file', + help=('Read additional hosts to delete tags for from specified ' + 'file or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'read from stdin. If no tags/tagfile nor hosts/hostfile' + ' are specified then tags are read from stdin and are' + ' deleted from all hosts.'), + dest='hostfile', required=False) + parser['del-tag'].add_argument( + '-T', '--tag-file', + help=('Read additional tags from specified file ' + 'or from stdin if \'-\' (each tag on separate line). ' + 'If no tag or tag file is specified then, by default, ' + 'read from stdin. If no tags/tagfile nor' + ' hosts/hostfile are specified then tags are read from' + ' stdin and are added to all hosts.'), + dest='tagfile', required=False) + parser['del-tag'].add_argument( + '-t', '--taglist', + help=("Tag list to be deleted for specified host(s), " + "comma separated values."), + dest="taglist", required=False) + + parser['list'] = parser['invsub'].add_parser( + 'list', parents=[parser['loglevel'], parser['beta'], + parser['common'], + parser['inventory_common']]) + parser['list'].add_argument( + 'host', nargs='*', help='Host(s) to list.') + parser['list'].add_argument( + '-a', '--all', + help=('List hosts that have all specified tags, ' + 'if -t/--tag is specified.'), + action="store_true", dest="has_all_tags", default=False) + parser['list'].add_argument( + '-f', '--file', + help=('Read additional hosts to list from specified file ' + 'or from stdin if \'-\' (each host on separate line). ' + 'If no host or host file is specified then, by default, ' + 'list all.'), dest='hostfile', required=False) + parser['list'].add_argument( + '-H', '--host-only', help=('Suppress tags listing.'), + action="store_true", dest="list_only_host", default=False) + parser['list'].add_argument( + '-t', '--tag', + help=('Host is specified by tag, not hostname/address; ' + 'list all hosts that contain any of specified tags.'), + action="store_true", default=False) + + parser['inventory'].set_defaults( + func=cdist.inventory.Inventory.commandline) + # Shell parser['shell'] = parser['sub'].add_parser( 'shell', parents=[parser['loglevel']]) @@ -200,12 +438,31 @@ def get_parsers(): def handle_loglevel(args): - if args.debug: - retval = "-d/--debug is deprecated, use -vvv instead" - args.verbose = 3 - else: - retval = None + if hasattr(args, 'quiet') and args.quiet: + args.verbose = _verbosity_level_off logging.root.setLevel(_verbosity_level[args.verbose]) - return retval + +def parse_and_configure(argv, singleton=True): + parser = get_parsers() + parser_args = parser['main'].parse_args(argv) + try: + cfg = cdist.configuration.Configuration(parser_args, + singleton=singleton) + args = cfg.get_args() + except ValueError as e: + raise cdist.Error(str(e)) + # Loglevels are handled globally in here + handle_loglevel(args) + + log = logging.getLogger("cdist") + + log.verbose("version %s" % cdist.VERSION) + log.trace('command line args: {}'.format(cfg.command_line_args)) + log.trace('configuration: {}'.format(cfg.get_config())) + log.trace('configured args: {}'.format(args)) + + check_beta(vars(args)) + + return parser, cfg diff --git a/cdist/autil.py b/cdist/autil.py new file mode 100644 index 00000000..d16d147e --- /dev/null +++ b/cdist/autil.py @@ -0,0 +1,71 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import cdist +import tarfile +import os +import glob +import tempfile + + +_ARCHIVING_MODES = { + 'tar': '', + 'tgz': 'gz', + 'tbz2': 'bz2', + 'txz': 'xz', +} + + +_UNARCHIVE_OPT = { + 'tar': None, + 'tgz': '-z', + 'tbz2': '-j', + 'txz': '-J', +} + + +# Archiving will be enabled if directory contains more than FILES_LIMIT files. +FILES_LIMIT = 1 + + +def get_extract_option(mode): + return _UNARCHIVE_OPT[mode] + + +def tar(source, mode="tgz"): + if mode not in _ARCHIVING_MODES: + raise cdist.Error("Unsupported archiving mode {}.".format(mode)) + + files = glob.glob1(source, '*') + fcnt = len(files) + if fcnt <= FILES_LIMIT: + return None, fcnt + + tarmode = 'w:{}'.format(_ARCHIVING_MODES[mode]) + _, tarpath = tempfile.mkstemp(suffix='.' + mode) + with tarfile.open(tarpath, tarmode, dereference=True) as tar: + if os.path.isdir(source): + for f in files: + tar.add(os.path.join(source, f), arcname=f) + else: + tar.add(source) + return tarpath, fcnt diff --git a/cdist/banner.py b/cdist/banner.py index edfa72e8..da4dea5d 100644 --- a/cdist/banner.py +++ b/cdist/banner.py @@ -20,8 +20,6 @@ # import logging -import sys - import cdist log = logging.getLogger(__name__) diff --git a/cdist/conf/explorer/cpu_cores b/cdist/conf/explorer/cpu_cores index 7f7a955e..a52bddac 100755 --- a/cdist/conf/explorer/cpu_cores +++ b/cdist/conf/explorer/cpu_cores @@ -25,13 +25,17 @@ os=$("$__explorer/os") case "$os" in "macosx") - echo "$(sysctl -n hw.physicalcpu)" + sysctl -n hw.physicalcpu + ;; + + "openbsd") + sysctl -n hw.ncpuonline ;; *) if [ -r /proc/cpuinfo ]; then cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)" - if [ ${cores} -eq 0 ]; then + if [ "${cores}" -eq 0 ]; then cores="1" fi echo "$cores" diff --git a/cdist/conf/explorer/cpu_sockets b/cdist/conf/explorer/cpu_sockets index 8a8194df..a32e2f00 100755 --- a/cdist/conf/explorer/cpu_sockets +++ b/cdist/conf/explorer/cpu_sockets @@ -25,14 +25,14 @@ os=$("$__explorer/os") case "$os" in "macosx") - echo "$(system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}')" + system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}' ;; *) if [ -r /proc/cpuinfo ]; then - sockets="$(grep "physical id" /proc/cpuinfo | sort | uniq | wc -l)" - if [ ${sockets} -eq 0 ]; then - sockets="$(cat /proc/cpuinfo | grep "processor" | wc -l)" + sockets="$(grep "physical id" /proc/cpuinfo | sort -u | wc -l)" + if [ "${sockets}" -eq 0 ]; then + sockets="$(grep -c "processor" /proc/cpuinfo)" fi echo "${sockets}" fi diff --git a/cdist/conf/explorer/disks b/cdist/conf/explorer/disks old mode 100644 new mode 100755 index 52fef81e..87a6b5c6 --- a/cdist/conf/explorer/disks +++ b/cdist/conf/explorer/disks @@ -1,2 +1,27 @@ -cd /dev -echo sd? hd? vd? +#!/bin/sh + +uname_s="$(uname -s)" + +case "${uname_s}" in + FreeBSD) + sysctl -n kern.disks + ;; + OpenBSD|NetBSD) + sysctl -n hw.disknames | grep -Eo '[lsw]d[0-9]+' | xargs + ;; + Linux) + if command -v lsblk > /dev/null + then + # exclude ram disks, floppies and cdroms + # https://www.kernel.org/doc/Documentation/admin-guide/devices.txt + lsblk -e 1,2,11 -dno name | xargs + else + printf "Don't know how to list disks for %s operating system without lsblk, if you can please submit a patch\n" "${uname_s}" >&2 + fi + ;; + *) + printf "Don't know how to list disks for %s operating system, if you can please submit a patch\n" "${uname_s}" >&2 + ;; +esac + +exit 0 diff --git a/cdist/conf/explorer/init b/cdist/conf/explorer/init index 2693a0d3..a8a7857e 100755 --- a/cdist/conf/explorer/init +++ b/cdist/conf/explorer/init @@ -1,6 +1,7 @@ #!/bin/sh # # 2016 Daniel Heule (hda at sfs.biz) +# Copyright 2017, Philippe Gregoire # # This file is part of cdist. # @@ -25,7 +26,10 @@ uname_s="$(uname -s)" case "$uname_s" in - Linux|FreeBSD) + Linux) + (pgrep -P0 -l | awk '/^1[ \t]/ {print $2;}') || true + ;; + FreeBSD|OpenBSD) ps -o comm= -p 1 || true ;; *) diff --git a/cdist/conf/explorer/interfaces b/cdist/conf/explorer/interfaces index c1f2a57a..55287971 100755 --- a/cdist/conf/explorer/interfaces +++ b/cdist/conf/explorer/interfaces @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2012 Sébastien Gross +# 2019 Ander Punnar (ander-at-kvlt-dot-ee) # # This file is part of cdist. # @@ -17,35 +17,14 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -# List all network interfaces in explorer/ifaces. One interface per line. -# -# If your OS is not supported please provide a ifconfig output -# -# Use ip, if available -if command -v ip >/dev/null; then +if command -v ip > /dev/null +then ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p' - exit 0 + +elif command -v ifconfig > /dev/null +then + ifconfig -a \ + | sed -n -E 's/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p' \ + | sort -u fi - -if ! command -v ifconfig >/dev/null; then - # no ifconfig, nothing we could do - exit 0 -fi - -uname_s="$(uname -s)" -REGEXP='s/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p' - -case "$uname_s" in - Darwin) - ifconfig -a | sed -n -E "$REGEXP" - ;; - Linux|*BSD) - ifconfig -a | sed -n -r "$REGEXP" - ;; - *) - echo "Unsupported ifconfig output for $uname_s" >&2 - exit 1 - ;; -esac diff --git a/cdist/conf/explorer/is-freebsd-jail b/cdist/conf/explorer/is-freebsd-jail new file mode 100755 index 00000000..010917f5 --- /dev/null +++ b/cdist/conf/explorer/is-freebsd-jail @@ -0,0 +1,2 @@ +#!/bin/sh +sysctl -n security.jail.jailed 2>/dev/null | grep "1" || true diff --git a/cdist/conf/explorer/kernel_name b/cdist/conf/explorer/kernel_name old mode 100644 new mode 100755 index 98ebac2a..1f9cfca4 --- a/cdist/conf/explorer/kernel_name +++ b/cdist/conf/explorer/kernel_name @@ -1 +1,2 @@ +#!/bin/sh uname -s diff --git a/cdist/conf/explorer/lsb_codename b/cdist/conf/explorer/lsb_codename index eebd3e0f..26bb8e3d 100755 --- a/cdist/conf/explorer/lsb_codename +++ b/cdist/conf/explorer/lsb_codename @@ -20,8 +20,9 @@ # set +e -case "$($__explorer/os)" in +case "$("$__explorer/os")" in openwrt) + # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") ;; *) diff --git a/cdist/conf/explorer/lsb_description b/cdist/conf/explorer/lsb_description index 23f45421..b1009627 100755 --- a/cdist/conf/explorer/lsb_description +++ b/cdist/conf/explorer/lsb_description @@ -20,8 +20,9 @@ # set +e -case "$($__explorer/os)" in +case "$("$__explorer/os")" in openwrt) + # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") ;; *) diff --git a/cdist/conf/explorer/lsb_id b/cdist/conf/explorer/lsb_id index 9754eb63..82ff9977 100755 --- a/cdist/conf/explorer/lsb_id +++ b/cdist/conf/explorer/lsb_id @@ -20,8 +20,9 @@ # set +e -case "$($__explorer/os)" in +case "$("$__explorer/os")" in openwrt) + # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_ID") ;; *) diff --git a/cdist/conf/explorer/lsb_release b/cdist/conf/explorer/lsb_release index 35b5547c..5ebfff1a 100755 --- a/cdist/conf/explorer/lsb_release +++ b/cdist/conf/explorer/lsb_release @@ -20,8 +20,9 @@ # set +e -case "$($__explorer/os)" in +case "$("$__explorer/os")" in openwrt) + # shellcheck disable=SC1091 (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") ;; *) diff --git a/cdist/conf/explorer/machine b/cdist/conf/explorer/machine index d4a0e106..7ecb67e3 100755 --- a/cdist/conf/explorer/machine +++ b/cdist/conf/explorer/machine @@ -22,6 +22,6 @@ # # -if command -v uname 2>&1 >/dev/null; then +if command -v uname >/dev/null 2>&1 ; then uname -m fi diff --git a/cdist/conf/explorer/machine_type b/cdist/conf/explorer/machine_type index eb3c9d36..bb21f69c 100755 --- a/cdist/conf/explorer/machine_type +++ b/cdist/conf/explorer/machine_type @@ -22,13 +22,13 @@ # FIXME: other system types (not linux ...) -if [ -d "/proc/vz" -a ! -d "/proc/bc" ]; then +if [ -d "/proc/vz" ] && [ ! -d "/proc/bc" ]; then echo openvz exit fi if [ -e "/proc/1/environ" ] && - cat "/proc/1/environ" | tr '\000' '\n' | grep -Eiq '^container='; then + tr '\000' '\n' < "/proc/1/environ" | grep -Eiq '^container='; then echo lxc exit fi @@ -56,6 +56,20 @@ if [ -r /proc/cpuinfo ]; then exit fi fi + + if [ -r /sys/class/dmi/id/sys_vendor ]; then + if grep -q -i 'qemu' /sys/class/dmi/id/sys_vendor; then + echo "virtual_by_kvm" + exit + fi + fi + + if [ -r /sys/class/dmi/id/chassis_vendor ]; then + if grep -q -i 'qemu' /sys/class/dmi/id/chassis_vendor; then + echo "virtual_by_kvm" + exit + fi + fi fi echo "virtual_by_unknown" else diff --git a/cdist/conf/explorer/memory b/cdist/conf/explorer/memory index 05db865f..4e3efff8 100755 --- a/cdist/conf/explorer/memory +++ b/cdist/conf/explorer/memory @@ -2,6 +2,7 @@ # # 2014 Daniel Heule (hda at sfs.biz) # 2014 Thomas Oettli (otho at sfs.biz) +# Copyright 2017, Philippe Gregoire # # This file is part of cdist. # @@ -28,6 +29,10 @@ case "$os" in echo "$(sysctl -n hw.memsize)/1024" | bc ;; + "openbsd") + echo "$(sysctl -n hw.physmem) / 1048576" | bc + ;; + *) if [ -r /proc/meminfo ]; then grep "MemTotal:" /proc/meminfo | awk '{print $2}' diff --git a/cdist/conf/explorer/os b/cdist/conf/explorer/os index 094685ea..d522300c 100755 --- a/cdist/conf/explorer/os +++ b/cdist/conf/explorer/os @@ -1,6 +1,7 @@ #!/bin/sh # # 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# Copyright 2017, Philippe Gregoire # # This file is part of cdist. # @@ -50,15 +51,17 @@ if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then exit 0 fi +# devuan ascii has both devuan_version and debian_version, so we need to check devuan_version first! +if [ -f /etc/devuan_version ]; then + echo devuan + exit 0 +fi + if [ -f /etc/debian_version ]; then echo debian exit 0 fi -if [ -f /etc/devuan_version ]; then - echo devuan - exit 0 -fi ### if [ -f /etc/gentoo-release ]; then @@ -139,5 +142,12 @@ case "$uname_s" in ;; esac +if [ -f /etc/os-release ]; then + # already lowercase, according to: + # https://www.freedesktop.org/software/systemd/man/os-release.html + awk -F= '/^ID=/ {print $2;}' /etc/os-release + exit 0 +fi + echo "Unknown OS" >&2 exit 1 diff --git a/cdist/conf/type/__install_reboot/manifest b/cdist/conf/explorer/os_release old mode 100755 new mode 100644 similarity index 78% rename from cdist/conf/type/__install_reboot/manifest rename to cdist/conf/explorer/os_release index fab80a1e..cfc01004 --- a/cdist/conf/type/__install_reboot/manifest +++ b/cdist/conf/explorer/os_release @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2018 Adam Dej (dejko.a at gmail.com) # # This file is part of cdist. # @@ -17,7 +17,10 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# -# set defaults -options="$(cat "$__object/parameter/options" 2>/dev/null \ - || echo "" | tee "$__object/parameter/options")" +# See os-release(5) and http://0pointer.de/blog/projects/os-release + +set +e + +cat /etc/os-release || cat /usr/lib/os-release || true diff --git a/cdist/conf/explorer/os_version b/cdist/conf/explorer/os_version index 380782cc..4c41695b 100755 --- a/cdist/conf/explorer/os_version +++ b/cdist/conf/explorer/os_version @@ -22,7 +22,7 @@ # # -case "$($__explorer/os)" in +case "$("$__explorer/os")" in amazon) cat /etc/system-release ;; diff --git a/cdist/conf/type/__acl/explorer/acl_is b/cdist/conf/type/__acl/explorer/acl_is new file mode 100755 index 00000000..a693c023 --- /dev/null +++ b/cdist/conf/type/__acl/explorer/acl_is @@ -0,0 +1,31 @@ +#!/bin/sh -e +# +# 2018 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +[ ! -e "/$__object_id" ] && exit 0 + +if ! command -v getfacl > /dev/null +then + echo 'getfacl not available' >&2 + exit 1 +fi + +getfacl "/$__object_id" 2>/dev/null \ + | grep -Eo '^(default:)?(user|group|(mask|other):):[^:][[:graph:]]+' \ + || true diff --git a/cdist/conf/type/__acl/explorer/checks b/cdist/conf/type/__acl/explorer/checks new file mode 100755 index 00000000..70bb0412 --- /dev/null +++ b/cdist/conf/type/__acl/explorer/checks @@ -0,0 +1,39 @@ +#!/bin/sh -e +# +# 2019 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# TODO check if filesystem has ACL turned on etc + +if [ -f "$__object/parameter/acl" ] +then + grep -E '^(default:)?(user|group):' "$__object/parameter/acl" \ + | while read -r acl + do + param="$( echo "$acl" | awk -F: '{print $(NF-2)}' )" + check="$( echo "$acl" | awk -F: '{print $(NF-1)}' )" + + [ "$param" = 'user' ] && db=passwd || db="$param" + + if ! getent "$db" "$check" > /dev/null + then + echo "missing $param '$check'" >&2 + exit 1 + fi + done +fi diff --git a/cdist/conf/type/__acl/explorer/file_is b/cdist/conf/type/__acl/explorer/file_is new file mode 100755 index 00000000..096cffd1 --- /dev/null +++ b/cdist/conf/type/__acl/explorer/file_is @@ -0,0 +1,31 @@ +#!/bin/sh -e +# +# 2018 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if [ -e "/$__object_id" ] +then + if [ -d "/$__object_id" ] + then echo directory + elif [ -f "/$__object_id" ] + then echo regular + else echo other + fi +else + echo missing +fi diff --git a/cdist/conf/type/__acl/gencode-remote b/cdist/conf/type/__acl/gencode-remote new file mode 100755 index 00000000..6dab4d09 --- /dev/null +++ b/cdist/conf/type/__acl/gencode-remote @@ -0,0 +1,126 @@ +#!/bin/sh -e +# +# 2018 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +file_is="$( cat "$__object/explorer/file_is" )" + +[ "$file_is" = 'missing' ] && [ -z "$__cdist_dry_run" ] && exit 0 + +os="$( cat "$__global/explorer/os" )" + +acl_path="/$__object_id" + +acl_is="$( cat "$__object/explorer/acl_is" )" + +if [ -f "$__object/parameter/acl" ] +then + acl_should="$( cat "$__object/parameter/acl" )" +elif + [ -f "$__object/parameter/user" ] \ + || [ -f "$__object/parameter/group" ] \ + || [ -f "$__object/parameter/mask" ] \ + || [ -f "$__object/parameter/other" ] +then + acl_should="$( for param in user group mask other + do + [ ! -f "$__object/parameter/$param" ] && continue + + echo "$param" | grep -Eq 'mask|other' && sep=:: || sep=: + + echo "$param$sep$( cat "$__object/parameter/$param" )" + done )" +else + echo 'no parameters set' >&2 + exit 1 +fi + +if [ -f "$__object/parameter/default" ] +then + acl_should="$( echo "$acl_should" \ + | sed 's/^default://' \ + | sort -u \ + | sed 's/\(.*\)/default:\1\n\1/' )" +fi + +if [ "$file_is" = 'regular' ] \ + && echo "$acl_should" | grep -Eq '^default:' +then + # only directories can have default ACLs, + # but instead of error, + # let's just remove default entries + acl_should="$( echo "$acl_should" | grep -Ev '^default:' )" +fi + +if echo "$acl_should" | awk -F: '{ print $NF }' | grep -Fq 'X' +then + [ "$file_is" = 'directory' ] && rep=x || rep=- + + acl_should="$( echo "$acl_should" | sed "s/\\(.*\\)X/\\1$rep/" )" +fi + +setfacl_exec='setfacl' + +if [ -f "$__object/parameter/recursive" ] +then + if echo "$os" | grep -Fq 'freebsd' + then + echo "$os setfacl do not support recursive operations" >&2 + else + setfacl_exec="$setfacl_exec -R" + fi +fi + +if [ -f "$__object/parameter/remove" ] +then + echo "$acl_is" | while read -r acl + do + # skip wanted ACL entries which already exist + # and skip mask and other entries, because we + # can't actually remove them, but only change. + if echo "$acl_should" | grep -Eq "^$acl" \ + || echo "$acl" | grep -Eq '^(default:)?(mask|other)' + then continue + fi + + if echo "$os" | grep -Fq 'freebsd' + then + remove="$acl" + else + remove="$( echo "$acl" | sed 's/:...$//' )" + fi + + echo "$setfacl_exec -x \"$remove\" \"$acl_path\"" + echo "removed '$remove'" >> "$__messages_out" + done +fi + +for acl in $acl_should +do + if ! echo "$acl_is" | grep -Eq "^$acl" + then + if echo "$os" | grep -Fq 'freebsd' \ + && echo "$acl" | grep -Eq '^default:' + then + echo "setting default ACL in $os is currently not supported" >&2 + else + echo "$setfacl_exec -m \"$acl\" \"$acl_path\"" + echo "added '$acl'" >> "$__messages_out" + fi + fi +done diff --git a/cdist/conf/type/__acl/man.rst b/cdist/conf/type/__acl/man.rst new file mode 100644 index 00000000..85e946ce --- /dev/null +++ b/cdist/conf/type/__acl/man.rst @@ -0,0 +1,85 @@ +cdist-type__acl(7) +================== + +NAME +---- +cdist-type__acl - Set ACL entries + + +DESCRIPTION +----------- +Fully supported and tested on Linux (ext4 filesystem), partial support for FreeBSD. + +See ``setfacl`` and ``acl`` manpages for more details. + + +REQUIRED MULTIPLE PARAMETERS +---------------------------- +acl + Set ACL entry following ``getfacl`` output syntax. + + +BOOLEAN PARAMETERS +------------------ +default + Set all ACL entries as default too. + Only directories can have default ACLs. + Setting default ACL in FreeBSD is currently not supported. + +recursive + Make ``setfacl`` recursive (Linux only), but not ``getfacl`` in explorer. + +remove + Remove undefined ACL entries. + ``mask`` and ``other`` entries can't be removed, but only changed. + + +DEPRECATED PARAMETERS +--------------------- +Parameters ``user``, ``group``, ``mask`` and ``other`` are deprecated and they +will be removed in future versions. Please use ``acl`` parameter instead. + + +EXAMPLES +-------- + +.. code-block:: sh + + __acl /srv/project \ + --default \ + --recursive \ + --remove \ + --acl user:alice:rwx \ + --acl user:bob:r-x \ + --acl group:project-group:rwx \ + --acl group:some-other-group:r-x \ + --acl mask::r-x \ + --acl other::r-x + + # give Alice read-only access to subdir, + # but don't allow her to see parent content. + + __acl /srv/project2 \ + --remove \ + --acl default:group:secret-project:rwx \ + --acl group:secret-project:rwx \ + --acl user:alice:--x + + __acl /srv/project2/subdir \ + --default \ + --remove \ + --acl group:secret-project:rwx \ + --acl user:alice:r-x + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2018 Ander Punnar. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__acl/parameter/boolean b/cdist/conf/type/__acl/parameter/boolean new file mode 100644 index 00000000..8b96693f --- /dev/null +++ b/cdist/conf/type/__acl/parameter/boolean @@ -0,0 +1,3 @@ +recursive +default +remove diff --git a/cdist/conf/type/__acl/parameter/deprecated/group b/cdist/conf/type/__acl/parameter/deprecated/group new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/group @@ -0,0 +1 @@ +see manual for details diff --git a/cdist/conf/type/__acl/parameter/deprecated/mask b/cdist/conf/type/__acl/parameter/deprecated/mask new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/mask @@ -0,0 +1 @@ +see manual for details diff --git a/cdist/conf/type/__acl/parameter/deprecated/other b/cdist/conf/type/__acl/parameter/deprecated/other new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/other @@ -0,0 +1 @@ +see manual for details diff --git a/cdist/conf/type/__acl/parameter/deprecated/user b/cdist/conf/type/__acl/parameter/deprecated/user new file mode 100644 index 00000000..94e14159 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/deprecated/user @@ -0,0 +1 @@ +see manual for details diff --git a/cdist/conf/type/__acl/parameter/optional b/cdist/conf/type/__acl/parameter/optional new file mode 100644 index 00000000..4b32086b --- /dev/null +++ b/cdist/conf/type/__acl/parameter/optional @@ -0,0 +1,2 @@ +mask +other diff --git a/cdist/conf/type/__acl/parameter/optional_multiple b/cdist/conf/type/__acl/parameter/optional_multiple new file mode 100644 index 00000000..95c25d55 --- /dev/null +++ b/cdist/conf/type/__acl/parameter/optional_multiple @@ -0,0 +1,3 @@ +acl +user +group diff --git a/cdist/conf/type/__apt_default_release/man.rst b/cdist/conf/type/__apt_default_release/man.rst new file mode 100644 index 00000000..0277a06f --- /dev/null +++ b/cdist/conf/type/__apt_default_release/man.rst @@ -0,0 +1,46 @@ +cdist-type__apt_default_release(7) +================================== + +NAME +---- +cdist-type__apt_default_release - Configure the default release for apt + + +DESCRIPTION +----------- +Configure the default release for apt, using the APT::Default-Release +configuration value. + +REQUIRED PARAMETERS +------------------- +release + The value to set APT::Default-Release to. + + This can contain release name, codename or release version. Examples: + 'stable', 'testing', 'unstable', 'stretch', 'buster', '4.0', '5.0*'. + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __apt_default_release --release stretch + + +AUTHORS +------- +Matthijs Kooijman + + +COPYING +------- +Copyright \(C) 2017 Matthijs Kooijman. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_default_release/manifest b/cdist/conf/type/__apt_default_release/manifest new file mode 100755 index 00000000..1232efb5 --- /dev/null +++ b/cdist/conf/type/__apt_default_release/manifest @@ -0,0 +1,41 @@ +#!/bin/sh -e +# +# 2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2017 Matthijs Kooijman (matthijs at stdin.nl) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + + +os=$(cat "$__global/explorer/os") +release="$(cat "$__object/parameter/release")" + +case "$os" in + ubuntu|debian|devuan) + __file /etc/apt/apt.conf.d/99-default-release \ + --owner root --group root --mode 644 \ + --source - << DONE +APT::Default-Release "$release"; +DONE + ;; + *) + cat >&2 << DONE +The developer of this type (${__type##*/}) did not think your operating system +($os) would have any use for it. If you think otherwise please submit a patch. +DONE + exit 1 + ;; +esac diff --git a/cdist/conf/type/__apt_default_release/parameter/required b/cdist/conf/type/__apt_default_release/parameter/required new file mode 100644 index 00000000..d7025695 --- /dev/null +++ b/cdist/conf/type/__apt_default_release/parameter/required @@ -0,0 +1 @@ +release diff --git a/cdist/conf/type/__apt_default_release/singleton b/cdist/conf/type/__apt_default_release/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__apt_key/explorer/state b/cdist/conf/type/__apt_key/explorer/state index f7940741..38f1bd3c 100755 --- a/cdist/conf/type/__apt_key/explorer/state +++ b/cdist/conf/type/__apt_key/explorer/state @@ -27,6 +27,18 @@ else keyid="$__object_id" fi -apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ - && echo present \ - || echo absent +keydir="$(cat "$__object/parameter/keydir")" +keyfile="$keydir/$__object_id.gpg" + +if [ -d "$keydir" ] +then + if [ -f "$keyfile" ] + then echo present + else echo absent + fi +else + # fallback to deprecated apt-key + apt-key export "$keyid" | head -n 1 | grep -Fqe "BEGIN PGP PUBLIC KEY BLOCK" \ + && echo present \ + || echo absent +fi diff --git a/cdist/conf/type/__apt_key/gencode-remote b/cdist/conf/type/__apt_key/gencode-remote index c6ead91c..47c8bb49 100755 --- a/cdist/conf/type/__apt_key/gencode-remote +++ b/cdist/conf/type/__apt_key/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -31,12 +31,84 @@ if [ "$state_should" = "$state_is" ]; then exit 0 fi +keydir="$(cat "$__object/parameter/keydir")" +keyfile="$keydir/$__object_id.gpg" + case "$state_should" in present) keyserver="$(cat "$__object/parameter/keyserver")" - echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\"" + + if [ -f "$__object/parameter/uri" ]; then + uri="$(cat "$__object/parameter/uri")" + + if [ -d "$keydir" ]; then + cat << EOF + +curl -s -L \\ + -o "$keyfile" \\ + "$uri" + +if grep -Fq 'BEGIN PGP PUBLIC KEY BLOCK' \\ + "$keyfile" +then + cat "$keyfile" \\ + | gpg --export > "$keyfile" +fi + +EOF + else + # fallback to deprecated apt-key + echo "curl -s -L '$uri' | apt-key add -" + fi + elif [ -d "$keydir" ]; then + tmp='/tmp/cdist_apt_key_tmp' + + # we need to kill gpg after 30 seconds, because gpg + # can get stuck if keyserver is not responding. + # exporting env var and not exit 1, + # because we need to clean up and kill dirmngr. + cat << EOF + +mkdir -m 700 -p "$tmp" + +if timeout 30s \\ + gpg --homedir "$tmp" \\ + --keyserver "$keyserver" \\ + --recv-keys "$keyid" +then + gpg --homedir "$tmp" \\ + --export "$keyid" \\ + > "$keyfile" +else + export GPG_GOT_STUCK=1 +fi + +GNUPGHOME="$tmp" gpgconf --kill dirmngr + +rm -rf "$tmp" + +if [ -n "\$GPG_GOT_STUCK" ] +then + echo "GPG GOT STUCK - no response from keyserver after 30 seconds" >&2 + exit 1 +fi + +EOF + else + # fallback to deprecated apt-key + echo "apt-key adv --keyserver \"$keyserver\" --recv-keys \"$keyid\"" + fi + + echo "added '$keyid'" >> "$__messages_out" ;; absent) - echo "apt-key del \"$keyid\"" + if [ -f "$keyfile" ]; then + echo "rm '$keyfile'" + else + # fallback to deprecated apt-key + echo "apt-key del \"$keyid\"" + fi + + echo "removed '$keyid'" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__apt_key/man.rst b/cdist/conf/type/__apt_key/man.rst index 9009877e..234bc715 100644 --- a/cdist/conf/type/__apt_key/man.rst +++ b/cdist/conf/type/__apt_key/man.rst @@ -28,6 +28,12 @@ keyserver the keyserver from which to fetch the key. If omitted the default set in ./parameter/default/keyserver is used. +keydir + key save location, defaults to ``/etc/apt/trusted.pgp.d`` + +uri + the URI from which to download the key + EXAMPLES -------- @@ -47,15 +53,20 @@ EXAMPLES # same thing with other keyserver __apt_key UbuntuArchiveKey --keyid 437D05B5 --keyserver keyserver.ubuntu.com + # download key from the internet + __apt_key rabbitmq \ + --uri http://www.rabbitmq.com/rabbitmq-signing-key-public.asc + AUTHORS ------- Steven Armstrong +Ander Punnar COPYING ------- -Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the +Copyright \(C) 2011-2019 Steven Armstrong and Ander Punnar. You can +redistribute it and/or modify it under the terms of the GNU General Public +License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_key/manifest b/cdist/conf/type/__apt_key/manifest new file mode 100755 index 00000000..010357cd --- /dev/null +++ b/cdist/conf/type/__apt_key/manifest @@ -0,0 +1,8 @@ +#!/bin/sh -e + +__package gnupg + +if [ -f "$__object/parameter/uri" ] +then __package curl +else __package dirmngr +fi diff --git a/cdist/conf/type/__apt_key/parameter/default/keydir b/cdist/conf/type/__apt_key/parameter/default/keydir new file mode 100644 index 00000000..190eb2de --- /dev/null +++ b/cdist/conf/type/__apt_key/parameter/default/keydir @@ -0,0 +1 @@ +/etc/apt/trusted.gpg.d diff --git a/cdist/conf/type/__apt_key/parameter/optional b/cdist/conf/type/__apt_key/parameter/optional index 18cf2586..de647375 100644 --- a/cdist/conf/type/__apt_key/parameter/optional +++ b/cdist/conf/type/__apt_key/parameter/optional @@ -1,3 +1,5 @@ state keyid keyserver +keydir +uri diff --git a/cdist/conf/type/__apt_key_uri/explorer/state b/cdist/conf/type/__apt_key_uri/explorer/state index 15d6e653..6f607607 100755 --- a/cdist/conf/type/__apt_key_uri/explorer/state +++ b/cdist/conf/type/__apt_key_uri/explorer/state @@ -27,6 +27,6 @@ else name="$__object_id" fi -apt-key list | grep -Fqe "$name" \ +apt-key list 2> /dev/null | grep -Fqe "$name" \ && echo present \ || echo absent diff --git a/cdist/conf/type/__apt_key_uri/gencode-remote b/cdist/conf/type/__apt_key_uri/gencode-remote index 078b8695..229b6564 100755 --- a/cdist/conf/type/__apt_key_uri/gencode-remote +++ b/cdist/conf/type/__apt_key_uri/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2014 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__apt_key_uri/manifest b/cdist/conf/type/__apt_key_uri/manifest index 8dddde56..bf7b267d 100755 --- a/cdist/conf/type/__apt_key_uri/manifest +++ b/cdist/conf/type/__apt_key_uri/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013-2014 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__apt_mark/explorer/apt_version b/cdist/conf/type/__apt_mark/explorer/apt_version old mode 100644 new mode 100755 index 32a0a58f..7bb90cc2 --- a/cdist/conf/type/__apt_mark/explorer/apt_version +++ b/cdist/conf/type/__apt_mark/explorer/apt_version @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Ander Punnar (cdist at kvlt.ee) # @@ -26,6 +26,6 @@ apt_version_is=$(dpkg-query --show --showformat '${Version}' apt) apt_version_should=0.8.14.2 -dpkg --compare-versions $apt_version_should le $apt_version_is \ +dpkg --compare-versions "$apt_version_should" le "$apt_version_is" \ && echo 0 \ || echo 1 diff --git a/cdist/conf/type/__apt_mark/explorer/package_installed b/cdist/conf/type/__apt_mark/explorer/package_installed old mode 100644 new mode 100755 index c78ac3a9..0b072cbc --- a/cdist/conf/type/__apt_mark/explorer/package_installed +++ b/cdist/conf/type/__apt_mark/explorer/package_installed @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Ander Punnar (cdist at kvlt.ee) # @@ -24,7 +24,7 @@ else name="$__object_id" fi -dpkg-query --show --showformat '${Status}' $name 2>/dev/null \ - | grep -q 'ok installed' \ +dpkg-query --show --showformat '${Status}' "$name" 2>/dev/null \ + | grep -Fq 'ok installed' \ && echo 0 \ || echo 1 diff --git a/cdist/conf/type/__apt_mark/explorer/state b/cdist/conf/type/__apt_mark/explorer/state old mode 100644 new mode 100755 index 3b70003a..b7fe08fa --- a/cdist/conf/type/__apt_mark/explorer/state +++ b/cdist/conf/type/__apt_mark/explorer/state @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Ander Punnar (cdist at kvlt.ee) # @@ -24,4 +24,4 @@ else name="$__object_id" fi -apt-mark showhold | grep -q $name && echo hold || echo unhold +apt-mark showhold | grep -Fq "$name" && echo hold || echo unhold diff --git a/cdist/conf/type/__apt_mark/gencode-remote b/cdist/conf/type/__apt_mark/gencode-remote old mode 100644 new mode 100755 index c1ac58b3..bc995444 --- a/cdist/conf/type/__apt_mark/gencode-remote +++ b/cdist/conf/type/__apt_mark/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Ander Punnar (cdist at kvlt.ee) # diff --git a/cdist/conf/type/__apt_norecommends/manifest b/cdist/conf/type/__apt_norecommends/manifest index 9e633308..e737df89 100755 --- a/cdist/conf/type/__apt_norecommends/manifest +++ b/cdist/conf/type/__apt_norecommends/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__apt_ppa/explorer/state b/cdist/conf/type/__apt_ppa/explorer/state index 2bb4f65a..d47e7d20 100755 --- a/cdist/conf/type/__apt_ppa/explorer/state +++ b/cdist/conf/type/__apt_ppa/explorer/state @@ -23,10 +23,11 @@ name="$__object_id" +# shellcheck disable=SC1091 . /etc/lsb-release repo_name="${name#ppa:}" -repo_file_name="$(echo "$repo_name" | sed -e "s|[/:]|-|" -e "s|\.|_|")-${DISTRIB_CODENAME}.list" +repo_file_name="$(echo "$repo_name" | sed -e 's|[/:]|-|' -e 's|\.|_|')-${DISTRIB_CODENAME}.list" [ -s "/etc/apt/sources.list.d/${repo_file_name}" ] \ && echo present || echo absent diff --git a/cdist/conf/type/__apt_ppa/gencode-remote b/cdist/conf/type/__apt_ppa/gencode-remote index 300a0e1e..84ebebfe 100755 --- a/cdist/conf/type/__apt_ppa/gencode-remote +++ b/cdist/conf/type/__apt_ppa/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -29,9 +29,9 @@ fi case "$state_should" in present) - echo add-apt-repository \"$name\" + echo "add-apt-repository '$name'" ;; absent) - echo remove-apt-repository \"$name\" + echo "remove-apt-repository '$name'" ;; esac diff --git a/cdist/conf/type/__apt_ppa/manifest b/cdist/conf/type/__apt_ppa/manifest index a67c7613..c6f4e876 100755 --- a/cdist/conf/type/__apt_ppa/manifest +++ b/cdist/conf/type/__apt_ppa/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2016 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -18,8 +18,6 @@ # along with cdist. If not, see . # -name="$__object_id" - __package software-properties-common require="__package/software-properties-common" \ diff --git a/cdist/conf/type/__apt_source/gencode-remote b/cdist/conf/type/__apt_source/gencode-remote new file mode 100755 index 00000000..1e8592c6 --- /dev/null +++ b/cdist/conf/type/__apt_source/gencode-remote @@ -0,0 +1,28 @@ +#!/bin/sh -e +# +# 2018 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +name="$__object_id" +destination="/etc/apt/sources.list.d/${name}.list" + +if grep -q "^__file${destination}" "$__messages_in"; then + printf 'apt-get update || apt-get update\n' +fi + diff --git a/cdist/conf/type/__apt_source/man.rst b/cdist/conf/type/__apt_source/man.rst index 8aa6c144..d1acb388 100644 --- a/cdist/conf/type/__apt_source/man.rst +++ b/cdist/conf/type/__apt_source/man.rst @@ -8,7 +8,8 @@ cdist-type__apt_source - Manage apt sources DESCRIPTION ----------- -This cdist type allows you to manage apt sources. +This cdist type allows you to manage apt sources. It invokes index update +internally when needed so call of index updating type is not needed. REQUIRED PARAMETERS @@ -63,7 +64,7 @@ Steven Armstrong COPYING ------- -Copyright \(C) 2011-2014 Steven Armstrong. You can redistribute it +Copyright \(C) 2011-2018 Steven Armstrong. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__apt_source/manifest b/cdist/conf/type/__apt_source/manifest index 59c7c567..35f15909 100755 --- a/cdist/conf/type/__apt_source/manifest +++ b/cdist/conf/type/__apt_source/manifest @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2018 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -50,5 +50,3 @@ __file "/etc/apt/sources.list.d/${name}.list" \ --source "$__object/files/source.list" \ --owner root --group root --mode 0644 \ --state "$state" - -require="$__object_name" __apt_update_index diff --git a/cdist/conf/type/__apt_source/nonparallel b/cdist/conf/type/__apt_source/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__apt_update_index/gencode-remote b/cdist/conf/type/__apt_update_index/gencode-remote index 61ce11a9..70b59710 100755 --- a/cdist/conf/type/__apt_update_index/gencode-remote +++ b/cdist/conf/type/__apt_update_index/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__block/gencode-remote b/cdist/conf/type/__block/gencode-remote index 2e2147e5..1f5cc033 100755 --- a/cdist/conf/type/__block/gencode-remote +++ b/cdist/conf/type/__block/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -18,6 +18,11 @@ # along with cdist. If not, see . # +# quote function from http://www.etalabs.net/sh_tricks.html +quote() { + printf '%s\n' "$1" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/'/" +} + file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")" state_should=$(cat "$__object/parameter/state") prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id") @@ -46,7 +51,7 @@ tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) if [ -f "$file" ]; then cp -p "$file" "\$tmpfile" fi -awk -v prefix="^$prefix\$" -v suffix="^$suffix\$" ' +awk -v prefix=^$(quote "$prefix")\$ -v suffix=^$(quote "$suffix")\$ ' { if (match(\$0,prefix)) { triggered=1 diff --git a/cdist/conf/type/__block/manifest b/cdist/conf/type/__block/manifest index bf96181c..726950d3 100755 --- a/cdist/conf/type/__block/manifest +++ b/cdist/conf/type/__block/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013-2014 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -18,8 +18,6 @@ # along with cdist. If not, see . # - -file="$(cat "$__object/parameter/file" 2>/dev/null || echo "/$__object_id")" prefix=$(cat "$__object/parameter/prefix" 2>/dev/null || echo "#cdist:__block/$__object_id") suffix=$(cat "$__object/parameter/suffix" 2>/dev/null || echo "#/cdist:__block/$__object_id") text=$(cat "$__object/parameter/text") diff --git a/cdist/conf/type/__ccollect_source/gencode-remote b/cdist/conf/type/__ccollect_source/gencode-remote index c41b5179..57353c24 100755 --- a/cdist/conf/type/__ccollect_source/gencode-remote +++ b/cdist/conf/type/__ccollect_source/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Nico Schottelius (nico-cdist at schottelius.org) # @@ -42,21 +42,20 @@ get_current_value() { } set_group() { - echo chgrp \"$1\" \"$destination\" - echo chgrp $1 >> "$__messages_out" + echo "chgrp '$1' '$destination'" + echo "chgrp '$1'" >> "$__messages_out" } set_owner() { - echo chown \"$1\" \"$destination\" - echo chown $1 >> "$__messages_out" + echo "chown '$1' '$destination'" + echo "chown '$1'" >> "$__messages_out" } set_mode() { - echo chmod \"$1\" \"$destination\" - echo chmod $1 >> "$__messages_out" + echo "chmod '$1' '$destination'" + echo "chmod '$1'" >> "$__messages_out" } -set_attributes= case "$state_should" in present|exists) # Note: Mode - needs to happen last as a chown/chgrp can alter mode by @@ -67,11 +66,11 @@ case "$state_should" in # change 0xxx format to xxx format => same as stat returns if [ "$attribute" = mode ]; then - value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')" + value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" fi value_is="$(get_current_value "$attribute" "$value_should")" - if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then + if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then "set_$attribute" "$value_should" fi fi @@ -81,7 +80,7 @@ case "$state_should" in absent) if [ "$type" = "file" ]; then - echo rm -f \"$destination\" + echo "rm -f '$destination'" echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__ccollect_source/man.rst b/cdist/conf/type/__ccollect_source/man.rst index 617571bb..b0c23482 100644 --- a/cdist/conf/type/__ccollect_source/man.rst +++ b/cdist/conf/type/__ccollect_source/man.rst @@ -38,6 +38,8 @@ BOOLEAN PARAMETERS verbose Whether to report backup verbosely +create-destination + Create the directory specified in the destination parameter on the remote host EXAMPLES -------- @@ -50,6 +52,13 @@ EXAMPLES --exclude '/proc/*' --exclude '/sys/*' \ --verbose + __ccollect_source doc.ungleich.ch \ + --source doc.ungleich.ch:/ \ + --destination /backup/doc.ungleich.ch \ + --exclude '/proc/*' --exclude '/sys/*' \ + --verbose \ + --create-destination + SEE ALSO -------- diff --git a/cdist/conf/type/__ccollect_source/manifest b/cdist/conf/type/__ccollect_source/manifest index b95b75c3..727a4c97 100755 --- a/cdist/conf/type/__ccollect_source/manifest +++ b/cdist/conf/type/__ccollect_source/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Nico Schottelius (nico-cdist at schottelius.org) # @@ -22,7 +22,7 @@ name="$__object_id" state="$(cat "$__object/parameter/state")" source="$(cat "$__object/parameter/source")" destination="$(cat "$__object/parameter/destination")" -ccollectconf="$(cat "$__object/parameter/ccollectconf" | sed 's,/$,,')" +ccollectconf="$(sed 's,/$,,' "$__object/parameter/ccollectconf")" sourcedir="$ccollectconf/sources" basedir="$sourcedir/$name" @@ -53,3 +53,7 @@ if [ -f "$__object/parameter/exclude" ]; then __file "$exclude_file" --source - --state "$state" \ < "$__object/parameter/exclude" fi + +if [ -f "$__object/parameter/create-destination" ]; then + __directory "${destination}" --parents --state "${state}" +fi diff --git a/cdist/conf/type/__ccollect_source/parameter/boolean b/cdist/conf/type/__ccollect_source/parameter/boolean index c00ee94a..434c644f 100644 --- a/cdist/conf/type/__ccollect_source/parameter/boolean +++ b/cdist/conf/type/__ccollect_source/parameter/boolean @@ -1 +1,2 @@ verbose +create-destination diff --git a/cdist/conf/type/__cdist/man.rst b/cdist/conf/type/__cdist/man.rst index 9e1c72cb..be082781 100644 --- a/cdist/conf/type/__cdist/man.rst +++ b/cdist/conf/type/__cdist/man.rst @@ -30,7 +30,7 @@ username source Select the source from which to clone cdist from. - Defaults to "git://github.com/ungleich/cdist.git". + Defaults to "git@code.ungleich.ch:ungleich-public/cdist.git". branch @@ -47,7 +47,7 @@ EXAMPLES __cdist /home/cdist/cdist # Use alternative source - __cdist --source "git://github.com/ungleich/cdist" /home/cdist/cdist + __cdist --source "git@code.ungleich.ch:ungleich-public/cdist.git" /home/cdist/cdist AUTHORS diff --git a/cdist/conf/type/__cdist/manifest b/cdist/conf/type/__cdist/manifest index 7c0ae60e..a97cf288 100755 --- a/cdist/conf/type/__cdist/manifest +++ b/cdist/conf/type/__cdist/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__cdist/parameter/default/source b/cdist/conf/type/__cdist/parameter/default/source index 3f8e31ad..1ad3a250 100644 --- a/cdist/conf/type/__cdist/parameter/default/source +++ b/cdist/conf/type/__cdist/parameter/default/source @@ -1 +1 @@ -git://github.com/ungleich/cdist.git +git@code.ungleich.ch:ungleich-public/cdist.git diff --git a/cdist/conf/type/__cdistmarker/gencode-remote b/cdist/conf/type/__cdistmarker/gencode-remote index 5e889e52..e71955c4 100755 --- a/cdist/conf/type/__cdistmarker/gencode-remote +++ b/cdist/conf/type/__cdistmarker/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # Copyright (C) 2011 Daniel Maher (phrawzty+cdist at gmail.com) # diff --git a/cdist/conf/type/__check_messages/gencode-remote b/cdist/conf/type/__check_messages/gencode-remote new file mode 100755 index 00000000..ec36cecc --- /dev/null +++ b/cdist/conf/type/__check_messages/gencode-remote @@ -0,0 +1,26 @@ +#!/bin/sh -e +# +# 2019 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if grep -Eq \ + "$( cat "$__object/parameter/pattern" )" \ + "$__messages_in" +then + tee "$__messages_out" < "$__object/parameter/execute" +fi diff --git a/cdist/conf/type/__check_messages/man.rst b/cdist/conf/type/__check_messages/man.rst new file mode 100644 index 00000000..5c80a0ae --- /dev/null +++ b/cdist/conf/type/__check_messages/man.rst @@ -0,0 +1,52 @@ +cdist-type__check_messages(7) +============================= + +NAME +---- +cdist-type__check_messages - Check messages for pattern and execute command on match. + + +DESCRIPTION +----------- +Check messages for pattern and execute command on match. + +This type is useful if you chain together multiple related types using +dependencies and want to restart service if at least one type changes +something. + +For more information about messages see `cdist messaging `_. + +For more information about dependencies and execution order see +`cdist manifest `_ documentation. + + +REQUIRED PARAMETERS +------------------- +pattern + Extended regular expression pattern for search (passed to ``grep -E``). + +execute + Command to execute on pattern match. + + +EXAMPLES +-------- + +.. code-block:: sh + + __check_messages munin \ + --pattern '^__(file|link|line)/etc/munin/' \ + --execute 'service munin-node restart' + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2019 Ander Punnar. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__check_messages/parameter/required b/cdist/conf/type/__check_messages/parameter/required new file mode 100644 index 00000000..374363cb --- /dev/null +++ b/cdist/conf/type/__check_messages/parameter/required @@ -0,0 +1,2 @@ +pattern +execute diff --git a/cdist/conf/type/__chroot_mount/gencode-local b/cdist/conf/type/__chroot_mount/gencode-local new file mode 100755 index 00000000..b131346c --- /dev/null +++ b/cdist/conf/type/__chroot_mount/gencode-local @@ -0,0 +1,36 @@ +#!/bin/sh -e +# +# 2016 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +chroot="/$__object_id" + +if [ -f "$__object/parameter/manage-resolv-conf" ]; then + suffix="$(cat "$__object/parameter/manage-resolv-conf")" + resolv_conf="${chroot}/etc/resolv.conf" + original_resolv_conf="${resolv_conf}.${suffix}" + cat << DONE +$__remote_exec $__target_host << EOSSH +if [ -f "${resolv_conf}" ]; then + mv "${resolv_conf}" "${original_resolv_conf}" +fi +# copy hosts resolv.conf into chroot +cp /etc/resolv.conf "${resolv_conf}" +EOSSH +DONE +fi diff --git a/cdist/conf/type/__chroot_mount/gencode-remote b/cdist/conf/type/__chroot_mount/gencode-remote index 6d855f41..4fbb3ffc 100755 --- a/cdist/conf/type/__chroot_mount/gencode-remote +++ b/cdist/conf/type/__chroot_mount/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -41,8 +41,4 @@ mountpoint -q "${chroot}/dev/pts" \ [ -d "${chroot}/tmp" ] || mkdir -m 1777 "${chroot}/tmp" mountpoint -q "${chroot}/tmp" \ || mount -t tmpfs -o mode=1777,strictatime,nodev,nosuid tmpfs "${chroot}/tmp" - -if [ ! -f "${chroot}/etc/resolv.conf" ]; then - cp /etc/resolv.conf "${chroot}/etc/" -fi DONE diff --git a/cdist/conf/type/__chroot_mount/man.rst b/cdist/conf/type/__chroot_mount/man.rst index 0d7cdce3..41fd496b 100644 --- a/cdist/conf/type/__chroot_mount/man.rst +++ b/cdist/conf/type/__chroot_mount/man.rst @@ -1,5 +1,5 @@ cdist-type__chroot_mount(7) -=================================== +=========================== NAME ---- @@ -18,7 +18,17 @@ None OPTIONAL PARAMETERS ------------------- -None +manage-resolv-conf + manage /etc/resolv.conf inside the chroot. + Use the value of this parameter as the suffix to save a copy + of the current /etc/resolv.conf to /etc/resolv.conf.$suffix. + This is used by the __chroot_umount type to restore the initial + file content when unmounting the chroot. + + +BOOLEAN PARAMETERS +------------------ +None. EXAMPLES @@ -28,6 +38,9 @@ EXAMPLES __chroot_mount /path/to/chroot + __chroot_mount /path/to/chroot \ + --manage-resolv-conf "some-known-string" + AUTHORS ------- @@ -36,7 +49,7 @@ Steven Armstrong COPYING ------- -Copyright \(C) 2012 Steven Armstrong. You can redistribute it +Copyright \(C) 2012-2017 Steven Armstrong. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__chroot_mount/parameter/optional b/cdist/conf/type/__chroot_mount/parameter/optional new file mode 100644 index 00000000..27928f2c --- /dev/null +++ b/cdist/conf/type/__chroot_mount/parameter/optional @@ -0,0 +1 @@ +manage-resolv-conf diff --git a/cdist/conf/type/__chroot_umount/gencode-local b/cdist/conf/type/__chroot_umount/gencode-local new file mode 100755 index 00000000..b3cb69c6 --- /dev/null +++ b/cdist/conf/type/__chroot_umount/gencode-local @@ -0,0 +1,36 @@ +#!/bin/sh -e +# +# 2016 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +chroot="/$__object_id" + +if [ -f "$__object/parameter/manage-resolv-conf" ]; then + suffix="$(cat "$__object/parameter/manage-resolv-conf")" + resolv_conf="${chroot}/etc/resolv.conf" + original_resolv_conf="${resolv_conf}.${suffix}" +cat << DONE +$__remote_exec $__target_host << EOSSH +if [ -f "${original_resolv_conf}" ]; then + # restore original /etc/resolv.conf that we moved out of the way + # in __chroot_mount/gencode-local + mv -f "${original_resolv_conf}" "${resolv_conf}" +fi +EOSSH +DONE +fi diff --git a/cdist/conf/type/__chroot_umount/gencode-remote b/cdist/conf/type/__chroot_umount/gencode-remote index caf2c40c..ff669e1b 100755 --- a/cdist/conf/type/__chroot_umount/gencode-remote +++ b/cdist/conf/type/__chroot_umount/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -26,7 +26,6 @@ umount -l "${chroot}/dev/pts" umount -l "${chroot}/dev" umount -l "${chroot}/sys" umount -l "${chroot}/proc" -rm -f "${chroot}/etc/resolv.conf" if [ -d "${chroot}/etc/resolvconf/resolv.conf.d" ]; then # ensure /etc/resolvconf/resolv.conf.d/tail is not linked to \ # e.g. /etc/resolvconf/resolv.conf.d/original diff --git a/cdist/conf/type/__chroot_umount/man.rst b/cdist/conf/type/__chroot_umount/man.rst index ff116da5..2a15f362 100644 --- a/cdist/conf/type/__chroot_umount/man.rst +++ b/cdist/conf/type/__chroot_umount/man.rst @@ -18,7 +18,17 @@ None OPTIONAL PARAMETERS ------------------- -None +manage-resolv-conf + manage /etc/resolv.conf inside the chroot. + Use the value of this parameter as the suffix to find the backup file + that was saved by the __chroot_mount. + This is used by the to restore the initial file content when unmounting + the chroot. + + +BOOLEAN PARAMETERS +------------------ +None. EXAMPLES @@ -28,6 +38,9 @@ EXAMPLES __chroot_umount /path/to/chroot + __chroot_umount /path/to/chroot \ + --manage-resolv-conf "some-known-string" + SEE ALSO -------- @@ -41,7 +54,7 @@ Steven Armstrong COPYING ------- -Copyright \(C) 2012 Steven Armstrong. You can redistribute it +Copyright \(C) 2012-2017 Steven Armstrong. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__chroot_umount/manifest b/cdist/conf/type/__chroot_umount/manifest new file mode 100755 index 00000000..b3cb69c6 --- /dev/null +++ b/cdist/conf/type/__chroot_umount/manifest @@ -0,0 +1,36 @@ +#!/bin/sh -e +# +# 2016 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +chroot="/$__object_id" + +if [ -f "$__object/parameter/manage-resolv-conf" ]; then + suffix="$(cat "$__object/parameter/manage-resolv-conf")" + resolv_conf="${chroot}/etc/resolv.conf" + original_resolv_conf="${resolv_conf}.${suffix}" +cat << DONE +$__remote_exec $__target_host << EOSSH +if [ -f "${original_resolv_conf}" ]; then + # restore original /etc/resolv.conf that we moved out of the way + # in __chroot_mount/gencode-local + mv -f "${original_resolv_conf}" "${resolv_conf}" +fi +EOSSH +DONE +fi diff --git a/cdist/conf/type/__chroot_umount/parameter/optional b/cdist/conf/type/__chroot_umount/parameter/optional new file mode 100644 index 00000000..27928f2c --- /dev/null +++ b/cdist/conf/type/__chroot_umount/parameter/optional @@ -0,0 +1 @@ +manage-resolv-conf diff --git a/cdist/conf/type/__clean_path/explorer/list b/cdist/conf/type/__clean_path/explorer/list new file mode 100755 index 00000000..07d38127 --- /dev/null +++ b/cdist/conf/type/__clean_path/explorer/list @@ -0,0 +1,35 @@ +#!/bin/sh -e +# +# 2019 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +path="/$__object_id" + +[ ! -d "$path" ] && exit 0 + +pattern="$( cat "$__object/parameter/pattern" )" + +if [ -f "$__object/parameter/exclude" ] +then + exclude="$( cat "$__object/parameter/exclude" )" + + find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern" \ + -and -not -regex "$exclude" +else + find "$path" -mindepth 1 -maxdepth 1 -regex "$pattern" +fi diff --git a/cdist/conf/type/__clean_path/gencode-remote b/cdist/conf/type/__clean_path/gencode-remote new file mode 100755 index 00000000..998a70d8 --- /dev/null +++ b/cdist/conf/type/__clean_path/gencode-remote @@ -0,0 +1,48 @@ +#!/bin/sh -e +# +# 2019 Ander Punnar (ander-at-kvlt-dot-ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +[ ! -s "$__object/explorer/list" ] && exit 0 + +path="/$__object_id" + +pattern="$( cat "$__object/parameter/pattern" )" + +if [ -f "$__object/parameter/exclude" ] +then + exclude="$( cat "$__object/parameter/exclude" )" + + echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \ + "-and -not -regex '$exclude'" \ + '-exec rm -rf {} \;' +else + echo "find '$path' -mindepth 1 -maxdepth 1 -regex '$pattern'" \ + '-exec rm -rf {} \;' +fi + +while read -r f +do + echo "removed '$f'" >> "$__messages_out" +done \ +< "$__object/explorer/list" + +if [ -f "$__object/parameter/onchange" ] +then + cat "$__object/parameter/onchange" +fi diff --git a/cdist/conf/type/__clean_path/man.rst b/cdist/conf/type/__clean_path/man.rst new file mode 100644 index 00000000..826f4589 --- /dev/null +++ b/cdist/conf/type/__clean_path/man.rst @@ -0,0 +1,60 @@ +cdist-type__clean_path(7) +========================= + +NAME +---- +cdist-type__clean_path - Remove files and directories which match the pattern. + + +DESCRIPTION +----------- +Remove files and directories which match the pattern. + +Provided path (as __object_id) must be a directory. + +Patterns are passed to ``find``'s ``-regex`` - see ``find(1)`` for more details. + +Look up of files and directories is non-recursive (``-maxdepth 1``). + +Parent directory is excluded (``-mindepth 1``). + +This type is not POSIX compatible (sorry, Solaris users). + + +REQUIRED PARAMETERS +------------------- +pattern + Pattern of files which are removed from path. + + +OPTIONAL PARAMETERS +------------------- +exclude + Pattern of files which are excluded from removal. + +onchange + The code to run if files or directories were removed. + + +EXAMPLES +-------- + +.. code-block:: sh + + __clean_path /etc/apache2/conf-enabled \ + --pattern '.+' \ + --exclude '.+\(charset\.conf\|security\.conf\)' \ + --onchange 'service apache2 restart' + + +AUTHORS +------- +Ander Punnar + + +COPYING +------- +Copyright \(C) 2019 Ander Punnar. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__clean_path/parameter/optional b/cdist/conf/type/__clean_path/parameter/optional new file mode 100644 index 00000000..6f313474 --- /dev/null +++ b/cdist/conf/type/__clean_path/parameter/optional @@ -0,0 +1,2 @@ +exclude +onchange diff --git a/cdist/conf/type/__clean_path/parameter/required b/cdist/conf/type/__clean_path/parameter/required new file mode 100644 index 00000000..54774947 --- /dev/null +++ b/cdist/conf/type/__clean_path/parameter/required @@ -0,0 +1 @@ +pattern diff --git a/cdist/conf/type/__config_file/gencode-remote b/cdist/conf/type/__config_file/gencode-remote index e9b38c35..5f1626be 100755 --- a/cdist/conf/type/__config_file/gencode-remote +++ b/cdist/conf/type/__config_file/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -19,16 +19,9 @@ # destination="$__object_id" -state="$(cat "$__object/parameter/state")" - -if [ "$state" = "absent" ]; then - # nothing to do - exit 0 -fi if [ -f "$__object/parameter/onchange" ]; then if grep -q "^__file/${destination}" "$__messages_in"; then cat "$__object/parameter/onchange" fi fi - diff --git a/cdist/conf/type/__config_file/manifest b/cdist/conf/type/__config_file/manifest index 29add8b7..be8f9f67 100755 --- a/cdist/conf/type/__config_file/manifest +++ b/cdist/conf/type/__config_file/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -19,7 +19,8 @@ # set -- "/${__object_id}" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in source) source="$(cat "$__object/parameter/source")" diff --git a/cdist/conf/type/__consul/files/versions/1.0.6/cksum b/cdist/conf/type/__consul/files/versions/1.0.6/cksum new file mode 100644 index 00000000..b70b55f4 --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/1.0.6/cksum @@ -0,0 +1 @@ +4120550353 48801129 consul diff --git a/cdist/conf/type/__consul/files/versions/1.0.6/source b/cdist/conf/type/__consul/files/versions/1.0.6/source new file mode 100644 index 00000000..769d3134 --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/1.0.6/source @@ -0,0 +1 @@ +https://releases.hashicorp.com/consul/1.0.6/consul_1.0.6_linux_amd64.zip diff --git a/cdist/conf/type/__consul/files/versions/1.2.3/cksum b/cdist/conf/type/__consul/files/versions/1.2.3/cksum new file mode 100644 index 00000000..6352409e --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/1.2.3/cksum @@ -0,0 +1 @@ +191982 110369685 diff --git a/cdist/conf/type/__consul/files/versions/1.2.3/source b/cdist/conf/type/__consul/files/versions/1.2.3/source new file mode 100644 index 00000000..5e67bc37 --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/1.2.3/source @@ -0,0 +1 @@ +https://releases.hashicorp.com/consul/1.2.3/consul_1.2.3_linux_amd64.zip diff --git a/cdist/conf/type/__consul/files/versions/1.3.0/cksum b/cdist/conf/type/__consul/files/versions/1.3.0/cksum new file mode 100644 index 00000000..7a885378 --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/1.3.0/cksum @@ -0,0 +1 @@ +1714523667 98363467 consul diff --git a/cdist/conf/type/__consul/files/versions/1.3.0/source b/cdist/conf/type/__consul/files/versions/1.3.0/source new file mode 100644 index 00000000..18a1ba8e --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/1.3.0/source @@ -0,0 +1 @@ +https://releases.hashicorp.com/consul/1.3.0/consul_1.3.0_linux_amd64.zip diff --git a/cdist/conf/type/__consul/files/versions/1.5.0/cksum b/cdist/conf/type/__consul/files/versions/1.5.0/cksum new file mode 100644 index 00000000..efca9caa --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/1.5.0/cksum @@ -0,0 +1 @@ +886614099 103959898 consul diff --git a/cdist/conf/type/__consul/files/versions/1.5.0/source b/cdist/conf/type/__consul/files/versions/1.5.0/source new file mode 100644 index 00000000..cafa9248 --- /dev/null +++ b/cdist/conf/type/__consul/files/versions/1.5.0/source @@ -0,0 +1 @@ +https://releases.hashicorp.com/consul/1.5.0/consul_1.5.0_linux_amd64.zip diff --git a/cdist/conf/type/__consul/gencode-remote b/cdist/conf/type/__consul/gencode-remote new file mode 100755 index 00000000..2a21054f --- /dev/null +++ b/cdist/conf/type/__consul/gencode-remote @@ -0,0 +1,63 @@ +#!/bin/sh -e +# +# 2018 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +#set -x + +if [ ! -f "$__object/parameter/direct" ]; then + # Nothing here, staged file is used. + exit 0 +fi + +state=$(cat "$__object/parameter/state") +destination="/usr/local/bin/consul" + +if [ "$state" = "absent" ]; then + printf 'rm -f "%s"' "$destination" + exit 0 +fi + +versions_dir="$__type/files/versions" +version="$(cat "$__object/parameter/version")" +version_dir="$versions_dir/$version" + +source=$(cat "$version_dir/source") +source_file_name="${source##*/}" +cksum_should=$(cut -d' ' -f1,2 "$version_dir/cksum") + +cat << eof + tmpdir=\$(mktemp -d -p /tmp "${__type##*/}.XXXXXXXXXX") + curl -s -L "$source" > "\$tmpdir/$source_file_name" + unzip -p "\$tmpdir/$source_file_name" > "${destination}.tmp" + rm -rf "\$tmpdir" + + cksum_is=\$(cksum "${destination}.tmp" | cut -d' ' -f1,2) + if [ "\$cksum_is" = "$cksum_should" ]; then + rm -f "${destination}" + mv "${destination}.tmp" "${destination}" + chown root:root "$destination" + chmod 755 "$destination" + else + rm -f "${destination}.tmp" + echo "Failed to verify checksum for $__object_name" >&2 + exit 1 + fi +eof + +echo "/usr/local/bin/consul created" >> "$__messages_out" diff --git a/cdist/conf/type/__consul/man.rst b/cdist/conf/type/__consul/man.rst index 19ceb535..5b2db50a 100644 --- a/cdist/conf/type/__consul/man.rst +++ b/cdist/conf/type/__consul/man.rst @@ -10,7 +10,8 @@ DESCRIPTION ----------- Downloads and installs the consul binary from https://dl.bintray.com/mitchellh/consul. Note that the consul binary is downloaded on the server (the machine running -cdist) and then deployed to the target host using the __file type. +cdist) and then deployed to the target host using the __file type unless --direct +parameter is used. REQUIRED PARAMETERS @@ -28,6 +29,22 @@ version supported versions. Defaults to the latest known version. +BOOLEAN PARAMETERS +------------------ +direct + Download and deploy consul binary directly on the target machine. + + +MESSAGES +-------- +If consul binary is created using __staged_file then underlaying __file type messages are emitted. + +If consul binary is created by direct method then the following messages are emitted: + +/usr/local/bin/consul created + consul binary was created + + EXAMPLES -------- @@ -36,6 +53,9 @@ EXAMPLES # just install using defaults __consul + # install by downloading consul binary directly on the target machine + __consul --direct + # specific version __consul \ --version 0.4.1 @@ -43,7 +63,8 @@ EXAMPLES AUTHORS ------- -Steven Armstrong +| Steven Armstrong +| Darko Poljak COPYING diff --git a/cdist/conf/type/__consul/manifest b/cdist/conf/type/__consul/manifest index 7d0e73c5..156eb667 100755 --- a/cdist/conf/type/__consul/manifest +++ b/cdist/conf/type/__consul/manifest @@ -1,7 +1,8 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # 2016 Nico Schottelius (nico-cdist at schottelius.org) +# 2018 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -23,7 +24,7 @@ os=$(cat "$__global/explorer/os") case "$os" in - scientific|centos|redhat|ubuntu|debian|devuan|archlinux|gentoo) + alpine|scientific|centos|redhat|ubuntu|debian|devuan|archlinux|gentoo) # any linux should work : ;; @@ -44,12 +45,17 @@ if [ ! -d "$version_dir" ]; then exit 1 fi -__staged_file /usr/local/bin/consul \ - --source "$(cat "$version_dir/source")" \ - --cksum "$(cat "$version_dir/cksum")" \ - --fetch-command 'curl -s -L "%s"' \ - --prepare-command 'unzip -p "%s"' \ - --state "$(cat "$__object/parameter/state")" \ - --group root \ - --owner root \ - --mode 755 +if [ -f "$__object/parameter/direct" ]; then + __package unzip + __package curl +else + __staged_file /usr/local/bin/consul \ + --source "$(cat "$version_dir/source")" \ + --cksum "$(cat "$version_dir/cksum")" \ + --fetch-command 'curl -s -L "%s"' \ + --prepare-command 'unzip -p "%s"' \ + --state "$(cat "$__object/parameter/state")" \ + --group root \ + --owner root \ + --mode 755 +fi diff --git a/cdist/conf/type/__consul/parameter/boolean b/cdist/conf/type/__consul/parameter/boolean new file mode 100644 index 00000000..aa81b5e0 --- /dev/null +++ b/cdist/conf/type/__consul/parameter/boolean @@ -0,0 +1 @@ +direct diff --git a/cdist/conf/type/__consul/parameter/default/version b/cdist/conf/type/__consul/parameter/default/version index d2b13eb6..af0b7ddb 100644 --- a/cdist/conf/type/__consul/parameter/default/version +++ b/cdist/conf/type/__consul/parameter/default/version @@ -1 +1 @@ -0.6.4 +1.0.6 diff --git a/cdist/conf/type/__consul_agent/files/consul.sys-openrc b/cdist/conf/type/__consul_agent/files/consul.sys-openrc new file mode 100644 index 00000000..1dbe9375 --- /dev/null +++ b/cdist/conf/type/__consul_agent/files/consul.sys-openrc @@ -0,0 +1,38 @@ +#!/sbin/openrc-run +# 2019 Nico Schottelius (nico-cdist at schottelius.org) + +description="consul agent" + +pidfile="${CONSUL_PIDFILE:-"/var/run/$RC_SVCNAME/pidfile"}" +command="${CONSUL_BINARY:-"/usr/local/bin/consul"}" + + +checkconfig() { + if [ ! -d /var/run/consul ] ; then + mkdir -p /var/run/consul || return 1 + chown consul:consul /var/run/$NAME || return 1 + chmod 2770 /var/run/$NAME || return 1 + fi +} + +start() { + need net + + start-stop-daemon --start --quiet --oknodo \ + --pidfile "$pidfile" --background \ + --exec $command -- agent -pid-file="$pidfile" -config-dir /etc/consul/conf.d +} +start_pre() { + checkconfig +} + +stop() { + if [ "${RC_CMD}" = "restart" ] ; then + checkconfig || return 1 + fi + + ebegin "Stopping $RC_SVCNAME" + start-stop-daemon --stop --exec "$command" \ + --pidfile "$pidfile" --quiet + eend $? +} diff --git a/cdist/conf/type/__consul_agent/files/consul.sysv-debian b/cdist/conf/type/__consul_agent/files/consul.sysv-debian index a75c555d..4f43c000 100644 --- a/cdist/conf/type/__consul_agent/files/consul.sysv-debian +++ b/cdist/conf/type/__consul_agent/files/consul.sysv-debian @@ -1,6 +1,6 @@ #!/bin/sh # -# 2015 Nico Schottelius (nico-cdist at schottelius.org) +# 2015-2018 Nico Schottelius (nico-cdist at schottelius.org) # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. @@ -18,11 +18,24 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +### BEGIN INIT INFO +# Provides: consul +# Required-Start: $network $local_fs $remote_fs +# Required-Stop: $local_fs +# Should-Start: +# Should-Stop: +# Short-Description: consul +# Description: consul agent +# Default-Start: 2 3 4 5 +# Default-Stop: 0 1 6 +### END INIT INFO if [ -f "/etc/default/consul" ]; then + # shellcheck disable=SC1091 . /etc/default/consul fi +# shellcheck disable=SC1091 . /lib/lsb/init-functions NAME=consul diff --git a/cdist/conf/type/__consul_agent/files/consul.sysv-redhat b/cdist/conf/type/__consul_agent/files/consul.sysv-redhat index 13dafd2e..58fc9bd9 100644 --- a/cdist/conf/type/__consul_agent/files/consul.sysv-redhat +++ b/cdist/conf/type/__consul_agent/files/consul.sysv-redhat @@ -11,49 +11,52 @@ # pidfile: /var/run/consul/pidfile # Source function library. + +# shellcheck disable=SC1091 . /etc/init.d/functions NAME=consul CONSUL=/usr/local/bin/consul -CONFIG=/etc/$NAME/conf.d -PID_FILE=/var/run/$NAME/pidfile -LOG_FILE=/var/log/$NAME +CONFIG="/etc/$NAME/conf.d" +PID_FILE="/var/run/$NAME/pidfile" +LOG_FILE="/var/log/$NAME" -[ -e /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME -export GOMAXPROCS=${GOMAXPROCS:-2} +# shellcheck disable=SC1090 +[ -e "/etc/sysconfig/$NAME" ] && . "/etc/sysconfig/$NAME" +export GOMAXPROCS="${GOMAXPROCS:-2}" -mkdir -p /var/run/$NAME -chown consul:consul /var/run/$NAME -chmod 2770 /var/run/$NAME +mkdir -p "/var/run/$NAME" +chown consul:consul "/var/run/$NAME" +chmod 2770 "/var/run/$NAME" start() { - echo -n "Starting $NAME: " + printf "Starting %s: " "$NAME" daemon --user=consul \ --pidfile="$PID_FILE" \ "$CONSUL" agent -pid-file="$PID_FILE" -config-dir "$CONFIG" >> "$LOG_FILE" & retcode=$? - touch /var/lock/subsys/$NAME - return $retcode + touch "/var/lock/subsys/$NAME" + return "$retcode" } stop() { - echo -n "Shutting down $NAME: " - killproc -p "$PID_FILE" $NAME + printf "Shutting down %s: " "$NAME" + killproc -p "$PID_FILE" "$NAME" retcode=$? - rm -f /var/lock/subsys/$NAME - return $retcode + rm -f "/var/lock/subsys/$NAME" + return "$retcode" } case "$1" in start) - if $(status -p "$PID_FILE" $NAME >/dev/null); then + if status -p "$PID_FILE" "$NAME" >/dev/null; then echo "$NAME already running" else start fi ;; stop) - if $(status -p "$PID_FILE" $NAME >/dev/null); then + if status -p "$PID_FILE" "$NAME" >/dev/null; then stop else echo "$NAME not running" @@ -63,25 +66,25 @@ case "$1" in "$CONSUL" info ;; status) - status -p "$PID_FILE" $NAME + status -p "$PID_FILE" "$NAME" exit $? ;; restart) - if $(status -p "$PID_FILE" $NAME >/dev/null); then + if status -p "$PID_FILE" "$NAME" >/dev/null; then stop fi start ;; reload) - if $(status -p "$PID_FILE" $NAME >/dev/null); then - kill -HUP `cat $PID_FILE` + if status -p "$PID_FILE" "$NAME" >/dev/null; then + kill -HUP "$(cat "$PID_FILE")" else echo "$NAME not running" fi ;; condrestart) - if [ -f /var/lock/subsys/$NAME ]; then - if $(status -p "$PID_FILE" $NAME >/dev/null); then + if [ -f "/var/lock/subsys/$NAME" ]; then + if status -p "$PID_FILE" "$NAME" >/dev/null; then stop fi start diff --git a/cdist/conf/type/__consul_agent/gencode-remote b/cdist/conf/type/__consul_agent/gencode-remote index 04662967..997aa831 100755 --- a/cdist/conf/type/__consul_agent/gencode-remote +++ b/cdist/conf/type/__consul_agent/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__consul_agent/manifest b/cdist/conf/type/__consul_agent/manifest index 64efd366..a88d26ed 100755 --- a/cdist/conf/type/__consul_agent/manifest +++ b/cdist/conf/type/__consul_agent/manifest @@ -1,7 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) -# 2015 Nico Schottelius (nico-cdist at schottelius.org) +# 2015-2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -23,7 +23,7 @@ os=$(cat "$__global/explorer/os") case "$os" in - scientific|centos|debian|devuan|redhat|ubuntu) + alpine|scientific|centos|debian|devuan|redhat|ubuntu) # whitelist safeguard : ;; @@ -66,7 +66,7 @@ require="__directory/etc/consul" \ __directory "$conf_dir" \ --owner root --group "$group" --mode 750 --state "$state" -if [ -f "$__object/parameter/ca-file-source" -o -f "$__object/parameter/cert-file-source" -o -f "$__object/parameter/key-file-source" ]; then +if [ -f "$__object/parameter/ca-file-source" ] || [ -f "$__object/parameter/cert-file-source" ] || [ -f "$__object/parameter/key-file-source" ]; then # create directory for ssl certs require="__directory/etc/consul" \ __directory /etc/consul/ssl \ @@ -84,7 +84,8 @@ echo "{" # parameters we define ourself printf ' "data_dir": "%s"\n' "$data_dir" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state|user|group|json-config) continue ;; ca-file-source|cert-file-source|key-file-source) @@ -180,22 +181,25 @@ init_upstart() # Install init script to start on boot case "$os" in - centos|redhat) - os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")" - major_version="${os_version%%.*}" - case "$major_version" in - [456]) - init_sysvinit redhat - ;; - 7) - init_systemd - ;; - *) - echo "Unsupported CentOS/Redhat version: $os_version" >&2 - exit 1 - ;; - esac - ;; + alpine|devuan) + init_sysvinit debian + ;; + centos|redhat) + os_version="$(sed 's/[^0-9.]//g' "$__global/explorer/os_version")" + major_version="${os_version%%.*}" + case "$major_version" in + [456]) + init_sysvinit redhat + ;; + 7) + init_systemd + ;; + *) + echo "Unsupported CentOS/Redhat version: $os_version" >&2 + exit 1 + ;; + esac + ;; debian) os_version=$(cat "$__global/explorer/os_version") @@ -205,7 +209,7 @@ case "$os" in [567]) init_sysvinit debian ;; - 8) + [89]) init_systemd ;; *) @@ -213,13 +217,9 @@ case "$os" in exit 1 ;; esac - ;; - - devuan) - init_sysvinit debian - ;; + ;; ubuntu) init_upstart - ;; + ;; esac diff --git a/cdist/conf/type/__consul_check/manifest b/cdist/conf/type/__consul_check/manifest index 658e2598..c9f7add9 100755 --- a/cdist/conf/type/__consul_check/manifest +++ b/cdist/conf/type/__consul_check/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015-2016 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -40,7 +40,7 @@ if [ ! -f "$__object/parameter/interval" ]; then fi done fi -if [ -f "$__object/parameter/docker-container-id" -a ! -f "$__object/parameter/script" ]; then +if [ -f "$__object/parameter/docker-container-id" ] && [ ! -f "$__object/parameter/script" ]; then echo "When using --docker-container-id you must also define --script." >&2 exit 1 fi @@ -50,7 +50,8 @@ fi echo "{" printf ' "check": {\n' printf ' "name": "%s"\n' "$name" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state|name) continue ;; *) diff --git a/cdist/conf/type/__consul_reload/gencode-remote b/cdist/conf/type/__consul_reload/gencode-remote index 9369db73..839fd0c3 100755 --- a/cdist/conf/type/__consul_reload/gencode-remote +++ b/cdist/conf/type/__consul_reload/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__consul_service/manifest b/cdist/conf/type/__consul_service/manifest index 4f52d542..60397db7 100755 --- a/cdist/conf/type/__consul_service/manifest +++ b/cdist/conf/type/__consul_service/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -24,15 +24,15 @@ conf_file="service_${name}.json" state="$(cat "$__object/parameter/state")" # Sanity checks -if [ -f "$__object/parameter/check-script" -a -f "$__object/parameter/check-ttl" ]; then +if [ -f "$__object/parameter/check-script" ] && [ -f "$__object/parameter/check-ttl" ]; then echo "Use either --check-script together with --check-interval OR --check-ttl, but not both" >&2 exit 1 fi -if [ -f "$__object/parameter/check-script" -a ! -f "$__object/parameter/check-interval" ]; then +if [ -f "$__object/parameter/check-script" ] && [ ! -f "$__object/parameter/check-interval" ]; then echo "When using --check-script you must also define --check-interval" >&2 exit 1 fi -if [ -f "$__object/parameter/check-http" -a ! -f "$__object/parameter/check-interval" ]; then +if [ -f "$__object/parameter/check-http" ] && [ ! -f "$__object/parameter/check-interval" ]; then echo "When using --check-http you must also define --check-interval" >&2 exit 1 fi @@ -42,7 +42,8 @@ fi echo "{" printf ' "service": {\n' printf ' "name": "%s"\n' "$name" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state|name|check-interval) continue ;; check-script) diff --git a/cdist/conf/type/__consul_template/files/consul-template.sysv b/cdist/conf/type/__consul_template/files/consul-template.sysv index 0a463020..b263915a 100644 --- a/cdist/conf/type/__consul_template/files/consul-template.sysv +++ b/cdist/conf/type/__consul_template/files/consul-template.sysv @@ -10,72 +10,75 @@ # pidfile: /var/run/consul-template/pidfile # Source function library. + +# shellcheck disable=SC1091 . /etc/init.d/functions NAME=consul-template CONSUL_TEMPLATE=/usr/local/bin/consul-template -CONFIG=/etc/$NAME/conf.d -PID_FILE=/var/run/$NAME/pidfile -LOG_FILE=/var/log/$NAME +CONFIG="/etc/$NAME/conf.d" +PID_FILE="/var/run/$NAME/pidfile" +LOG_FILE="/var/log/$NAME" -[ -e /etc/sysconfig/$NAME ] && . /etc/sysconfig/$NAME -export CONSUL_TEMPLATE_LOG=${CONSUL_TEMPLATE_LOG:-info} -export GOMAXPROCS=${GOMAXPROCS:-2} +# shellcheck disable=SC1090 +[ -e "/etc/sysconfig/$NAME" ] && . "/etc/sysconfig/$NAME" +export CONSUL_TEMPLATE_LOG="${CONSUL_TEMPLATE_LOG:-info}" +export GOMAXPROCS="${GOMAXPROCS:-2}" -mkdir -p /var/run/$NAME +mkdir -p "/var/run/$NAME" start() { - echo -n "Starting $NAME: " + printf "Starting %s: " "$NAME" daemon --pidfile="$PID_FILE" \ "$CONSUL_TEMPLATE" -config "$CONFIG" >> "$LOG_FILE" 2>&1 & - echo $! > "$PID_FILE" + echo "$!" > "$PID_FILE" retcode=$? - touch /var/lock/subsys/$NAME - return $retcode + touch "/var/lock/subsys/$NAME" + return "$retcode" } stop() { - echo -n "Shutting down $NAME: " - killproc -p $PID_FILE $CONSUL_TEMPLATE + printf "Shutting down %s: " "$NAME" + killproc -p "$PID_FILE" "$CONSUL_TEMPLATE" retcode=$? - rm -f /var/lock/subsys/$NAME - return $retcode + rm -f "/var/lock/subsys/$NAME" + return "$retcode" } case "$1" in start) - if $(status -p "$PID_FILE" $NAME >/dev/null); then + if status -p "$PID_FILE" "$NAME" >/dev/null; then echo "$NAME already running" else start fi ;; stop) - if $(status -p "$PID_FILE" $NAME >/dev/null); then + if status -p "$PID_FILE" "$NAME" >/dev/null; then stop else echo "$NAME not running" fi ;; status) - status -p "$PID_FILE" $NAME + status -p "$PID_FILE" "$NAME" exit $? ;; restart) - if $(status -p "$PID_FILE" $NAME >/dev/null); then + if status -p "$PID_FILE" "$NAME" >/dev/null; then stop fi start ;; reload) - if $(status -p "$PID_FILE" $NAME >/dev/null); then - kill -HUP `cat $PID_FILE` + if status -p "$PID_FILE" "$NAME" >/dev/null; then + kill -HUP "$(cat "$PID_FILE")" else echo "$NAME not running" fi ;; condrestart) - if [ -f /var/lock/subsys/$NAME ]; then - if $(status -p "$PID_FILE" $NAME >/dev/null); then + if [ -f "/var/lock/subsys/$NAME" ]; then + if status -p "$PID_FILE" "$NAME" >/dev/null; then stop fi start diff --git a/cdist/conf/type/__consul_template/manifest b/cdist/conf/type/__consul_template/manifest index fd249185..b02fc332 100755 --- a/cdist/conf/type/__consul_template/manifest +++ b/cdist/conf/type/__consul_template/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -75,7 +75,8 @@ require="__directory/etc/consul-template" \ # Generate hcl config file ( -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in auth-password|state|ssl-*|syslog-*|version|vault-token|vault-ssl*) continue ;; auth-username) diff --git a/cdist/conf/type/__consul_template_template/manifest b/cdist/conf/type/__consul_template_template/manifest index b832075d..1eae1fad 100755 --- a/cdist/conf/type/__consul_template_template/manifest +++ b/cdist/conf/type/__consul_template_template/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -26,32 +26,36 @@ template_dir="/etc/consul-template/template" require="" # Sanity checks -if [ -f "$__object/parameter/source" -a -f "$__object/parameter/source-file" ]; then +if [ -f "$__object/parameter/source" ] && [ -f "$__object/parameter/source-file" ]; then echo "Use either --source OR --source-file, but not both." >&2 exit 1 fi -if [ ! -f "$__object/parameter/source" -a ! -f "$__object/parameter/source-file" ]; then +if [ ! -f "$__object/parameter/source" ] && [ ! -f "$__object/parameter/source-file" ]; then echo "Either --source OR --source-file must be given." >&2 exit 1 fi +if [ -f "$__object/parameter/source-file" ]; then + destination="${template_dir}/${name}" + require="__file${destination}" +fi + # Generate hcl config file -( +{ printf 'template {\n' -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in source-file) source="$(cat "$__object/parameter/$param")" if [ "$source" = "-" ]; then source="$__object/stdin" fi - destination="${template_dir}/${name}" require="__directory${template_dir}" \ __file "$destination" \ --owner root --group root --mode 640 \ --source "$source" \ --state "$state" - export require="__file${destination}" printf ' source = "%s"\n' "$destination" ;; @@ -65,7 +69,7 @@ for param in $(ls "$__object/parameter/"); do esac done printf '}\n' -) | \ +} | \ require="$require __directory${conf_dir}" \ __config_file "${conf_dir}/${conf_file}" \ --owner root --group root --mode 640 \ diff --git a/cdist/conf/type/__consul_watch_checks/manifest b/cdist/conf/type/__consul_watch_checks/manifest index c05ae9eb..5fdd7a74 100755 --- a/cdist/conf/type/__consul_watch_checks/manifest +++ b/cdist/conf/type/__consul_watch_checks/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -25,7 +25,7 @@ conf_file="watch_${watch_type}_${__object_id}.json" state="$(cat "$__object/parameter/state")" # Sanity checks -if [ -f "$__object/parameter/filter-service" -a -f "$__object/parameter/filter-state" ]; then +if [ -f "$__object/parameter/filter-service" ] && [ -f "$__object/parameter/filter-state" ]; then echo "Use either --filter-service or --filter-state but not both." >&2 exit 1 fi @@ -35,7 +35,8 @@ fi echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; filter-*) diff --git a/cdist/conf/type/__consul_watch_event/manifest b/cdist/conf/type/__consul_watch_event/manifest index 4e36a10d..61934656 100755 --- a/cdist/conf/type/__consul_watch_event/manifest +++ b/cdist/conf/type/__consul_watch_event/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__consul_watch_key/manifest b/cdist/conf/type/__consul_watch_key/manifest index 4e36a10d..61934656 100755 --- a/cdist/conf/type/__consul_watch_key/manifest +++ b/cdist/conf/type/__consul_watch_key/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__consul_watch_keyprefix/manifest b/cdist/conf/type/__consul_watch_keyprefix/manifest index 4e36a10d..61934656 100755 --- a/cdist/conf/type/__consul_watch_keyprefix/manifest +++ b/cdist/conf/type/__consul_watch_keyprefix/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__consul_watch_nodes/manifest b/cdist/conf/type/__consul_watch_nodes/manifest index 4e36a10d..61934656 100755 --- a/cdist/conf/type/__consul_watch_nodes/manifest +++ b/cdist/conf/type/__consul_watch_nodes/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__consul_watch_service/manifest b/cdist/conf/type/__consul_watch_service/manifest index 6011e288..db38eb18 100755 --- a/cdist/conf/type/__consul_watch_service/manifest +++ b/cdist/conf/type/__consul_watch_service/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; passingonly) diff --git a/cdist/conf/type/__consul_watch_services/manifest b/cdist/conf/type/__consul_watch_services/manifest index 4e36a10d..61934656 100755 --- a/cdist/conf/type/__consul_watch_services/manifest +++ b/cdist/conf/type/__consul_watch_services/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -29,7 +29,8 @@ state="$(cat "$__object/parameter/state")" echo "{" printf ' "watches": [{\n' printf ' "type": "%s"\n' "$watch_type" -for param in $(ls "$__object/parameter/"); do +cd "$__object/parameter/" +for param in *; do case "$param" in state) continue ;; *) diff --git a/cdist/conf/type/__cron/explorer/entry b/cdist/conf/type/__cron/explorer/entry index 2167e045..801861a3 100644 --- a/cdist/conf/type/__cron/explorer/entry +++ b/cdist/conf/type/__cron/explorer/entry @@ -24,7 +24,7 @@ user="$(cat "$__object/parameter/user")" if [ -f "$__object/parameter/raw_command" ]; then command="$(cat "$__object/parameter/command")" - crontab -u $user -l 2>/dev/null | grep "^$command\$" || true + crontab -u "$user" -l 2>/dev/null | grep "^$command\$" || true else - crontab -u $user -l 2>/dev/null | grep "# $name\$" || true + crontab -u "$user" -l 2>/dev/null | grep "# $name\$" || true fi diff --git a/cdist/conf/type/__cron/gencode-remote b/cdist/conf/type/__cron/gencode-remote old mode 100644 new mode 100755 index 3c3ed6b3..59398058 --- a/cdist/conf/type/__cron/gencode-remote +++ b/cdist/conf/type/__cron/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2013 Nico Schottelius (nico-cdist at schottelius.org) @@ -58,7 +58,7 @@ state_should="$(cat "$__object/parameter/state" 2>/dev/null || echo "present")" # These are the old markers prefix="#cdist:__cron/$__object_id" suffix="#/cdist:__cron/$__object_id" -filter="^# DO NOT EDIT THIS FILE|^# \(.* installed on |^# \(Cron version V|^# \(Cronie version .\..\)$" +filter='^# DO NOT EDIT THIS FILE|^# \(.* installed on |^# \(Cron version V|^# \(Cronie version .\..\)$' cat << DONE crontab -u $user -l 2>/dev/null | grep -v -E "$filter" | awk -v prefix="$prefix" -v suffix="$suffix" ' { diff --git a/cdist/conf/type/__cron/manifest b/cdist/conf/type/__cron/manifest old mode 100644 new mode 100755 index 9992df25..53973e07 --- a/cdist/conf/type/__cron/manifest +++ b/cdist/conf/type/__cron/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Thomas Oettli (otho at sfs.biz) # diff --git a/cdist/conf/type/__daemontools/files/init.d-svscan b/cdist/conf/type/__daemontools/files/init.d-svscan index 127dfdb3..996eb4e8 100644 --- a/cdist/conf/type/__daemontools/files/init.d-svscan +++ b/cdist/conf/type/__daemontools/files/init.d-svscan @@ -23,9 +23,9 @@ fi case "$1" in start) - echo -n "Starting daemontools: " - if [ ! `pidof svscan` ]; then - echo -n "svscan " + printf "Starting daemontools: " + if ! pidof svscan > /dev/null 2>&1; then + printf "svscan " env - PATH="$PATH" svscan /service 2>&1 | setuidgid daemon multilog t /var/log/svscan & echo "." else @@ -33,23 +33,28 @@ case "$1" in fi ;; stop) - echo -n "Stopping daemontools: " - if [ `pidof svscan` ]; then - echo -n "svscan" - while [ `pidof svscan` ]; do - kill `pidof svscan` - echo -n "." + printf "Stopping daemontools: " + pids="$(pidof svscan)" + if [ -n "${pids}" ] + then + printf "svscan" + while [ -n "${pids}" ] + do + # shellcheck disable=SC2086 + kill ${pids} + printf "." + pids="$(pidof svscan)" done fi - echo -n " services" - for i in `ls -d /service/*`; do - svc -dx $i - echo -n "." + printf " services" + for i in /service/*; do + svc -dx "$i" + printf "." done - echo -n " logging " - for i in `ls -d /service/*/log`; do - svc -dx $i - echo -n "." + printf " logging " + for i in /service/*/log; do + svc -dx "$i" + printf "." done echo "" ;; diff --git a/cdist/conf/type/__daemontools/man.rst b/cdist/conf/type/__daemontools/man.rst index a8e81e54..bc1b4d33 100644 --- a/cdist/conf/type/__daemontools/man.rst +++ b/cdist/conf/type/__daemontools/man.rst @@ -21,11 +21,16 @@ OPTIONAL PARAMETERS from-package Package to install. Must be compatible with the original daemontools. Example: daemontools-encore. Default: daemontools. +servicedir + Directory to scan for services. Default: `/service` + + BOOLEAN PARAMETERS ------------------ install-init-script Add an init script and set it to start on boot. + EXAMPLES -------- diff --git a/cdist/conf/type/__daemontools/manifest b/cdist/conf/type/__daemontools/manifest old mode 100644 new mode 100755 index 550994a7..b04c7e07 --- a/cdist/conf/type/__daemontools/manifest +++ b/cdist/conf/type/__daemontools/manifest @@ -1,20 +1,40 @@ -#!/bin/sh +#!/bin/sh -e pkg=$(cat "$__object/parameter/from-package") +servicedir=$(cat "$__object/parameter/servicedir") -__package $pkg +__package "$pkg" +__directory "$servicedir" --mode 700 -if [ -f "$__object/parameter/install-init-script" ]; then - init=$(cat "$__global/explorer/init") - case $init in - init) - __config_file /etc/init.d/svscan --mode 755 --source "$__type/files/init.d-svscan" - require="$require __config_file/etc/init.d/svscan" __start_on_boot svscan - require="$require __start_on_boot/svscan" __process svscan --start 'service svscan start' - ;; - *) - echo "Your init system ($init) is not supported by this type. Submit a patch at github.com/ungleich/cdist!" - exit 1 - ;; - esac -fi +os=$(cat "$__global/explorer/os") +init=$(cat "$__global/explorer/init") + +require="" +case $os in + freebsd) + # TODO change to __start_on_boot once it supports freebsd + __config_file /etc/rc.conf.d/svscan --source - <<-EOT + svscan_enable="YES" + svscan_servicedir="$servicedir" + EOT + require="$require __package/$pkg __directory/$servicedir __config_file/etc/rc.conf.d/svscan" \ + __process svscan --name ".*/svscan $servicedir" --start 'service svscan start' + ;; + *) + case $init in + init) + if [ -f "$__object/parameter/install-init-script" ]; then + __config_file /etc/init.d/svscan --mode 755 --source "$__type/files/init.d-svscan" + REQUIREEXTRA="__config_file/etc/init.d/svscan" + fi + require="$require $REQUIREEXTRA" __start_on_boot svscan + require="$require __package/$pkg __directory/$servicedir __start_on_boot/svscan" \ + __process svscan --name ".*/svscan $servicedir" --start 'service svscan start' + ;; + *) + echo "Your init system ($init) is not supported by this type. Submit a patch at github.com/ungleich/cdist!" + exit 1 + ;; + esac + ;; +esac diff --git a/cdist/conf/type/__daemontools/parameter/default/servicedir b/cdist/conf/type/__daemontools/parameter/default/servicedir new file mode 100644 index 00000000..b74e27f6 --- /dev/null +++ b/cdist/conf/type/__daemontools/parameter/default/servicedir @@ -0,0 +1 @@ +/service diff --git a/cdist/conf/type/__daemontools/parameter/optional b/cdist/conf/type/__daemontools/parameter/optional index 8eca305b..22c0805d 100644 --- a/cdist/conf/type/__daemontools/parameter/optional +++ b/cdist/conf/type/__daemontools/parameter/optional @@ -1 +1,2 @@ from-package +servicedir diff --git a/cdist/conf/type/__daemontools_service/explorer/svc b/cdist/conf/type/__daemontools_service/explorer/svc old mode 100644 new mode 100755 index d33fcea4..9ba462f2 --- a/cdist/conf/type/__daemontools_service/explorer/svc +++ b/cdist/conf/type/__daemontools_service/explorer/svc @@ -1 +1,2 @@ +#!/bin/sh command -v svc || true diff --git a/cdist/conf/type/__daemontools_service/manifest b/cdist/conf/type/__daemontools_service/manifest old mode 100644 new mode 100755 index 175066af..78bae285 --- a/cdist/conf/type/__daemontools_service/manifest +++ b/cdist/conf/type/__daemontools_service/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e RUN_PREFIX="#!/bin/sh exec 2>&1 @@ -25,14 +25,14 @@ badusage() { [ -z "$run$runfile" ] && badusage [ -n "$run" ] && [ -n "$runfile" ] && badusage -__directory $servicedir/$name/log/main --parents +__directory "$servicedir/$name/log/main" --parents echo "$RUN_PREFIX$run" | require="__directory/$servicedir/$name/log/main" __config_file "$servicedir/$name/run" \ --onchange "svc -t '$servicedir/$name' 2>/dev/null" \ --mode 755 \ --source "${runfile:--}" -echo "$RUN_PREFIX$logrun" | require="__directory/$servicedir/$name/log/main" __config_file $servicedir/$name/log/run \ +echo "$RUN_PREFIX$logrun" | require="__directory/$servicedir/$name/log/main" __config_file "$servicedir/$name/log/run" \ --onchange "svc -t '$servicedir/$name/log' 2>/dev/null" \ --mode 755 \ --source "-" diff --git a/cdist/conf/type/__debconf_set_selections/gencode-remote b/cdist/conf/type/__debconf_set_selections/gencode-remote index bb719c46..e99aef40 100755 --- a/cdist/conf/type/__debconf_set_selections/gencode-remote +++ b/cdist/conf/type/__debconf_set_selections/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__directory/explorer/stat b/cdist/conf/type/__directory/explorer/stat index 41bc8b04..03d466ba 100755 --- a/cdist/conf/type/__directory/explorer/stat +++ b/cdist/conf/type/__directory/explorer/stat @@ -25,23 +25,51 @@ destination="/$__object_id" os=$("$__explorer/os") case "$os" in - "freebsd"|"netbsd"|"openbsd") - # FIXME: should be something like this based on man page, but can not test - stat -f "type: %ST + "freebsd"|"netbsd"|"openbsd"|"macosx") + stat -f "type: %HT owner: %Du %Su group: %Dg %Sg -mode: %Op %Sp +mode: %Lp %Sp +" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' + ;; + alpine) + stat -c "type: %F +owner: %u %U +group: %g %G +mode: %a %A " "$destination" - ;; - "macosx") - stat -f "type: %HT - owner: %Du %Su - group: %Dg %Sg - mode: %Lp %Sp - " "$destination" + ;; + solaris) + ls1="$( ls -ld "$destination" )" + ls2="$( ls -ldn "$destination" )" + + if [ -f "$__object/parameter/mode" ] + then mode_should="$( cat "$__object/parameter/mode" )" + fi + + # yes, it is ugly hack, but if you know better way... + if [ -z "$( find "$destination" -perm "$mode_should" )" ] + then octets=888 + else octets="$( echo "$mode_should" | sed 's/^0//' )" + fi + + case "$( echo "$ls1" | cut -c1-1 )" in + -) echo 'type: regular file' ;; + d) echo 'type: directory' ;; + esac + + echo "owner: $( echo "$ls2" \ + | awk '{print $3}' ) $( echo "$ls1" \ + | awk '{print $3}' )" + + echo "group: $( echo "$ls2" \ + | awk '{print $4}' ) $( echo "$ls1" \ + | awk '{print $4}' )" + + echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" ;; *) - stat --printf="type: %F + stat --printf="type: %F owner: %u %U group: %g %G mode: %a %A diff --git a/cdist/conf/type/__directory/gencode-remote b/cdist/conf/type/__directory/gencode-remote index aba618ac..374db47a 100755 --- a/cdist/conf/type/__directory/gencode-remote +++ b/cdist/conf/type/__directory/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Steven Armstrong (steven-cdist armstrong.cc) @@ -57,18 +57,18 @@ get_current_value() { } set_group() { - echo chgrp $recursive \"$1\" \"$destination\" - echo chgrp $recursive $1 >> "$__messages_out" + echo "chgrp $recursive '$1' '$destination'" + echo "chgrp $recursive '$1'" >> "$__messages_out" } set_owner() { - echo chown $recursive \"$1\" \"$destination\" - echo chown $recursive $1 >> "$__messages_out" + echo "chown $recursive '$1' '$destination'" + echo "chown $recursive '$1'" >> "$__messages_out" } set_mode() { - echo chmod $recursive \"$1\" \"$destination\" - echo chmod $recursive $1 >> "$__messages_out" + echo "chmod $recursive '$1' '$destination'" + echo "chmod $recursive '$1'" >> "$__messages_out" } case "$state_should" in @@ -78,10 +78,10 @@ case "$state_should" in if [ "$type" != "none" ]; then # our destination is not a directory, remove whatever is there # and then create our directory and set all attributes - echo rm -f "\"$destination\"" + echo "rm -f '$destination'" echo "remove non directory" >> "$__messages_out" fi - echo "mkdir $mkdiropt \"$destination\"" + echo "mkdir $mkdiropt '$destination'" echo "create" >> "$__messages_out" fi @@ -94,7 +94,7 @@ case "$state_should" in # change 0xxx format to xxx format => same as stat returns if [ "$attribute" = mode ]; then - value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')" + value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" fi if [ "$set_attributes" = 1 ] || [ "$value_should" != "$value_is" ]; then @@ -105,7 +105,7 @@ case "$state_should" in ;; absent) if [ "$type" = "directory" ]; then - echo rm -rf \"$destination\" + echo "rm -rf '$destination'" echo remove >> "$__messages_out" fi ;; diff --git a/cdist/conf/type/__docker/man.rst b/cdist/conf/type/__docker/man.rst index 70b92cc7..718543a8 100644 --- a/cdist/conf/type/__docker/man.rst +++ b/cdist/conf/type/__docker/man.rst @@ -3,12 +3,12 @@ cdist-type__docker(7) NAME ---- -cdist-type__docker - install docker-engine +cdist-type__docker - install Docker CE DESCRIPTION ----------- -Installs latest docker-engine package from dockerproject.org. +Installs latest Docker Community Edition package. REQUIRED PARAMETERS @@ -18,16 +18,16 @@ None. OPTIONAL PARAMETERS ------------------- -None. +state + 'present' or 'absent', defaults to 'present' +version + The specific version to install. Defaults to the special value 'latest', + meaning the version the package manager will install by default. BOOLEAN PARAMETERS ------------------ -experimental - Install the experimental docker-engine package instead of the latest stable release. - -state - 'present' or 'absent', defaults to 'present' +None. EXAMPLES @@ -38,12 +38,11 @@ EXAMPLES # Install docker __docker - # Install experimental - __docker --experimental - # Remove docker __docker --state absent + # Install specific version + __docker --state present --version 18.03.0.ce AUTHORS ------- diff --git a/cdist/conf/type/__docker/manifest b/cdist/conf/type/__docker/manifest index 1f473afb..6a57d85a 100755 --- a/cdist/conf/type/__docker/manifest +++ b/cdist/conf/type/__docker/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -21,58 +21,84 @@ os=$(cat "$__global/explorer/os") state=$(cat "$__object/parameter/state") +version=$(cat "$__object/parameter/version") case "$os" in centos) - component="main" - if [ -f "$__object/parameter/experimental" ]; then - component="experimental" + # shellcheck source=/dev/null + if (. "$__global/explorer/os_release" && [ "${VERSION_ID}" = "7" ]); then + __yum_repo docker-ce-stable \ + --name 'Docker CE Stable' \ + --baseurl "https://download.docker.com/linux/centos/7/\$basearch/stable" \ + --enabled \ + --gpgcheck 1 \ + --gpgkey 'https://download.docker.com/linux/centos/gpg' \ + --state "${state}" + if [ "$version" != "latest" ]; then + require="__yum_repo/docker-ce-stable" __package docker-ce --version "${version}" --state "${state}" + else + require="__yum_repo/docker-ce-stable" __package docker-ce --state "${state}" + fi + else + echo "CentOS version 7 is required!" >&2 + exit 1 fi - __yum_repo docker \ - --name 'Docker Repository' \ - --baseurl "https://yum.dockerproject.org/repo/$component/centos/\$releasever/" \ - --enabled \ - --gpgcheck 1 \ - --gpgkey 'https://yum.dockerproject.org/gpg' \ - --state ${state} - require="__yum_repo/docker" __package docker-engine --state ${state} ;; - ubuntu) - component="main" - if [ -f "$__object/parameter/experimental" ]; then - component="experimental" - fi - __package apparmor --state ${state} - __package ca-certificates --state ${state} - __package apt-transport-https --state ${state} - __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D --state ${state} - export CDIST_ORDER_DEPENDENCY=on - __apt_source docker \ - --uri https://apt.dockerproject.org/repo \ - --distribution "ubuntu-$(cat "$__global/explorer/lsb_codename")" \ - --state ${state} \ - --component "$component" - __package docker-engine --state ${state} - unset CDIST_ORDER_DEPENDENCY - ;; - debian) - component="main" - if [ -f "$__object/parameter/experimental" ]; then - component="experimental" + ubuntu|debian) + if [ "${state}" = "present" ]; then + __package apt-transport-https + __package ca-certificates + __package gnupg2 fi + __apt_key_uri docker --name "Docker Release (CE deb) " \ + --uri "https://download.docker.com/linux/${os}/gpg" --state "${state}" - __package apt-transport-https --state ${state} - __package ca-certificates --state ${state} - __package gnupg2 --state ${state} - __apt_key docker --keyid 58118E89F3A912897C070ADBF76221572C52609D --state ${state} - export CDIST_ORDER_DEPENDENCY=on - __apt_source docker \ - --uri https://apt.dockerproject.org/repo \ - --distribution "debian-$(cat "$__global/explorer/lsb_codename")" \ - --state ${state} \ - --component "$component" - __package docker-engine --state ${state} - unset CDIST_ORDER_DEPENDENCY + require="__apt_key_uri/docker" __apt_source docker \ + --uri "https://download.docker.com/linux/${os}" \ + --distribution "$(cat "$__global/explorer/lsb_codename")" \ + --state "${state}" \ + --component "stable" + if [ "$version" != "latest" ]; then + require="__apt_source/docker" __package docker-ce --version "${version}" --state "${state}" + else + require="__apt_source/docker" __package docker-ce --state "${state}" + fi + ;; + devuan) + os_version="$(cat "$__global/explorer/os_version")" + + case "$os_version" in + ascii) + distribution="stretch" + ;; + jessie) + distribution="jessie" + ;; + *) + echo "Your devuan release ($os_version) is currently not supported by this type (${__type##*/}).">&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + ;; + esac + + if [ "${state}" = "present" ]; then + __package apt-transport-https + __package ca-certificates + __package gnupg2 + fi + __apt_key_uri docker --name "Docker Release (CE deb) " \ + --uri "https://download.docker.com/linux/${os}/gpg" --state "${state}" + + require="__apt_key_uri/docker" __apt_source docker \ + --uri "https://download.docker.com/linux/${os}" \ + --distribution "${distribution}" \ + --state "${state}" \ + --component "stable" + if [ "$version" != "latest" ]; then + require="__apt_source/docker" __package docker-ce --version "${version}" --state "${state}" + else + require="__apt_source/docker" __package docker-ce --state "${state}" + fi ;; *) diff --git a/cdist/conf/type/__docker/parameter/boolean b/cdist/conf/type/__docker/parameter/boolean deleted file mode 100644 index 9839eb20..00000000 --- a/cdist/conf/type/__docker/parameter/boolean +++ /dev/null @@ -1 +0,0 @@ -experimental diff --git a/cdist/conf/type/__docker/parameter/default/version b/cdist/conf/type/__docker/parameter/default/version new file mode 100644 index 00000000..a0f9a4b4 --- /dev/null +++ b/cdist/conf/type/__docker/parameter/default/version @@ -0,0 +1 @@ +latest diff --git a/cdist/conf/type/__docker/parameter/optional b/cdist/conf/type/__docker/parameter/optional index ff72b5c7..4d595ed7 100644 --- a/cdist/conf/type/__docker/parameter/optional +++ b/cdist/conf/type/__docker/parameter/optional @@ -1 +1,2 @@ state +version diff --git a/cdist/conf/type/__docker_compose/gencode-remote b/cdist/conf/type/__docker_compose/gencode-remote old mode 100644 new mode 100755 index bd1ad452..77fc2fdf --- a/cdist/conf/type/__docker_compose/gencode-remote +++ b/cdist/conf/type/__docker_compose/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Dominique Roux (dominique.roux at ungleich.ch) # @@ -22,10 +22,11 @@ version="$(cat "$__object/parameter/version")" state="$(cat "$__object/parameter/state")" -if [ ${state} = "present" ]; then +if [ "${state}" = "present" ]; then # Download docker-compose file - echo 'curl -L "https://github.com/docker/compose/releases/download/'${version}'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' - echo 'mv /tmp/docker-compose /usr/local/bin/docker-compose' + #shellcheck disable=SC2016 + echo 'curl -L "https://github.com/docker/compose/releases/download/'"${version}"'/docker-compose-$(uname -s)-$(uname -m)" -o /tmp/docker-compose' + echo 'mv /tmp/docker-compose /usr/local/bin/docker-compose' # Change permissions echo 'chmod +x /usr/local/bin/docker-compose' fi diff --git a/cdist/conf/type/__docker_compose/manifest b/cdist/conf/type/__docker_compose/manifest old mode 100644 new mode 100755 index 559375ef..f7de3a76 --- a/cdist/conf/type/__docker_compose/manifest +++ b/cdist/conf/type/__docker_compose/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Dominique Roux (dominique.roux at ungleich.ch) # @@ -22,10 +22,10 @@ state="$(cat "$__object/parameter/state")" # Needed packages -if [ ${state} = "present" ]; then +if [ "${state}" = "present" ]; then __docker __package curl -elif [ ${state} = "absent" ]; then +elif [ "${state}" = "absent" ]; then __file /usr/local/bin/docker-compose --state absent else echo "Unknown state: ${state}" >&2 diff --git a/cdist/conf/type/__docker_compose/parameter/default/version b/cdist/conf/type/__docker_compose/parameter/default/version index 0eed1a29..850e7424 100644 --- a/cdist/conf/type/__docker_compose/parameter/default/version +++ b/cdist/conf/type/__docker_compose/parameter/default/version @@ -1 +1 @@ -1.12.0 +1.14.0 diff --git a/cdist/conf/type/__docker_config/explorer/config-data b/cdist/conf/type/__docker_config/explorer/config-data new file mode 100755 index 00000000..b4bb0e11 --- /dev/null +++ b/cdist/conf/type/__docker_config/explorer/config-data @@ -0,0 +1,22 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +docker config inspect "${__object_id:?}" --format '{{json .Spec.Data}}' \ + 2>/dev/null | tr -d '"' | base64 -d diff --git a/cdist/conf/type/__docker_config/explorer/config-exists b/cdist/conf/type/__docker_config/explorer/config-exists new file mode 100755 index 00000000..58c207d4 --- /dev/null +++ b/cdist/conf/type/__docker_config/explorer/config-exists @@ -0,0 +1,25 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if docker config ls | grep -q " ${__object_id:?} "; then + echo yes +else + echo no +fi diff --git a/cdist/conf/type/__docker_config/gencode-remote b/cdist/conf/type/__docker_config/gencode-remote new file mode 100755 index 00000000..65497b7e --- /dev/null +++ b/cdist/conf/type/__docker_config/gencode-remote @@ -0,0 +1,69 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +config="${__object_id:?}" +config_exists=$(cat "${__object:?}/explorer/config-exists") +state=$(cat "${__object:?}/parameter/state") + +case "${state}" in + absent) + if [ "${config_exists}" != "yes" ]; then + exit 0 + fi + + echo "docker config rm \"${config}\"" + ;; + present) + source=$(cat "${__object}/parameter/source") + + if [ -z "${source}" ]; then + exit 0 + fi + + if [ "${source}" = "-" ]; then + source="${__object}/stdin" + fi + + if [ "${config_exists}" = "yes" ]; then + if cmp -s "${source}" "${__object}/explorer/config-data"; then + exit 0 + else + echo "docker config rm \"${config}\"" + fi + fi + + cat <<-EOF + source_file="\$(mktemp cdist.XXXXXXXXXX)" + + base64 -d > "\${source_file}" << eof + $(base64 "${source}") + eof + + docker config create "${config}" "\${source_file}" + + rm "\${source_file}" + EOF + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker_config/man.rst b/cdist/conf/type/__docker_config/man.rst new file mode 100644 index 00000000..7c74c8af --- /dev/null +++ b/cdist/conf/type/__docker_config/man.rst @@ -0,0 +1,55 @@ +cdist-type__docker_config(7) +============================ + +NAME +---- + +cdist-type__docker_config - Manage Docker configs + +DESCRIPTION +----------- + +This type manages Docker configs. + +OPTIONAL PARAMETERS +------------------- + +source + Path to the source file. If it is '-' (dash), read standard input. + +state + 'present' or 'absent', defaults to 'present' where: + + present + if the config does not exist, it is created + absent + the config is removed + +CAVEATS +------- + +Since Docker configs cannot be updated once created, this type tries removing +and recreating the config if it changes. If the config is used by a service at +the time of removing, then this type will fail. + +EXAMPLES +-------- + +.. code-block:: sh + + # Creates "foo" config from "bar" source file + __docker_config foo --source bar + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__docker_config/parameter/default/source b/cdist/conf/type/__docker_config/parameter/default/source new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__docker_config/parameter/default/state b/cdist/conf/type/__docker_config/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_config/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_config/parameter/optional b/cdist/conf/type/__docker_config/parameter/optional new file mode 100644 index 00000000..d77f3048 --- /dev/null +++ b/cdist/conf/type/__docker_config/parameter/optional @@ -0,0 +1,2 @@ +source +state diff --git a/cdist/conf/type/__docker_secret/explorer/secret-exists b/cdist/conf/type/__docker_secret/explorer/secret-exists new file mode 100755 index 00000000..1405f8bc --- /dev/null +++ b/cdist/conf/type/__docker_secret/explorer/secret-exists @@ -0,0 +1,25 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if docker secret ls | grep -q " ${__object_id:?} "; then + echo yes +else + echo no +fi diff --git a/cdist/conf/type/__docker_secret/gencode-remote b/cdist/conf/type/__docker_secret/gencode-remote new file mode 100755 index 00000000..c75e91d9 --- /dev/null +++ b/cdist/conf/type/__docker_secret/gencode-remote @@ -0,0 +1,65 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +secret="${__object_id:?}" +secret_exists=$(cat "${__object:?}/explorer/secret-exists") +state=$(cat "${__object:?}/parameter/state") + +case "${state}" in + absent) + if [ "${secret_exists}" != "yes" ]; then + exit 0 + fi + + echo "docker secret rm ${secret}" + ;; + present) + if [ "${secret_exists}" = "yes" ]; then + exit 0 + fi + + source=$(cat "${__object}/parameter/source") + + if [ -z "${source}" ]; then + exit 0 + fi + + if [ "${source}" = "-" ]; then + source="${__object}/stdin" + fi + + cat <<-EOF + source_file="\$(mktemp cdist.XXXXXXXXXX)" + + base64 -d > "\${source_file}" << eof + $(base64 "${source}") + eof + + docker secret create "${secret}" "\${source_file}" + + rm "\${source_file}" + EOF + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker_secret/man.rst b/cdist/conf/type/__docker_secret/man.rst new file mode 100644 index 00000000..7fe69623 --- /dev/null +++ b/cdist/conf/type/__docker_secret/man.rst @@ -0,0 +1,54 @@ +cdist-type__docker_secret(7) +============================ + +NAME +---- + +cdist-type__docker_secret - Manage Docker secrets + +DESCRIPTION +----------- + +This type manages Docker secrets. + +OPTIONAL PARAMETERS +------------------- + +source + Path to the source file. If it is '-' (dash), read standard input. + +state + 'present' or 'absent', defaults to 'present' where: + + present + if the secret does not exist, it is created + absent + the secret is removed + +CAVEATS +------- + +Since Docker secrets cannot be updated once created, this type takes no action +if the specified secret already exists. + +EXAMPLES +-------- + +.. code-block:: sh + + # Creates "foo" secret from "bar" source file + __docker_secret foo --source bar + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__docker_secret/parameter/default/source b/cdist/conf/type/__docker_secret/parameter/default/source new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__docker_secret/parameter/default/state b/cdist/conf/type/__docker_secret/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_secret/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_secret/parameter/optional b/cdist/conf/type/__docker_secret/parameter/optional new file mode 100644 index 00000000..d77f3048 --- /dev/null +++ b/cdist/conf/type/__docker_secret/parameter/optional @@ -0,0 +1,2 @@ +source +state diff --git a/cdist/conf/type/__docker_stack/explorer/stack-exists b/cdist/conf/type/__docker_stack/explorer/stack-exists new file mode 100755 index 00000000..4f511821 --- /dev/null +++ b/cdist/conf/type/__docker_stack/explorer/stack-exists @@ -0,0 +1,25 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +if docker stack ls | grep -q "^${__object_id:?} "; then + echo 1 +else + echo 0 +fi diff --git a/cdist/conf/type/__docker_stack/gencode-remote b/cdist/conf/type/__docker_stack/gencode-remote new file mode 100755 index 00000000..586271d0 --- /dev/null +++ b/cdist/conf/type/__docker_stack/gencode-remote @@ -0,0 +1,63 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +stack="${__object_id:?}" +state=$(cat "${__object:?}/parameter/state") + +case "${state}" in + absent) + stack_exists=$(cat "${__object:?}/explorer/stack-exists") + + if [ "${stack_exists}" -ne 1 ]; then + exit 0 + fi + + echo "docker stack rm ${stack}" + ;; + present) + compose_file=$(cat "${__object}/parameter/compose-file") + + if [ -z "${compose_file}" ]; then + exit 0 + fi + + if [ "${compose_file}" = "-" ]; then + compose_file="${__object}/stdin" + fi + + cat <<-EOF + compose_file="\$(mktemp cdist.XXXXXXXXXX)" + + base64 -d > "\${compose_file}" << eof + $(base64 "${compose_file}") + eof + + docker stack deploy --compose-file "\${compose_file}" \ + --prune --with-registry-auth ${stack} + + rm "\${compose_file}" + EOF + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker_stack/man.rst b/cdist/conf/type/__docker_stack/man.rst new file mode 100644 index 00000000..d0597c25 --- /dev/null +++ b/cdist/conf/type/__docker_stack/man.rst @@ -0,0 +1,54 @@ +cdist-type__docker_stack(7) +=========================== + +NAME +---- + +cdist-type__docker_stack - Manage Docker stacks + +DESCRIPTION +----------- + +This type manages service stacks. + +.. note:: + Since there is no easy way to tell whether a stack needs to be updated, + `docker stack deploy` is being run every time this type is invoked. + However, it does not mean this type is not idempotent. If Docker does not + detect changes, the existing stack will not be updated. + +OPTIONAL PARAMETERS +------------------- + +compose-file + Path to the compose file. If it is '-' (dash), read standard input. + +state + 'present' or 'absent', defaults to 'present' where: + + present + the stack is deployed + absent + the stack is removed + +EXAMPLES +-------- + +.. code-block:: sh + + # Deploys 'foo' stack defined in 'docker-compose.yml' compose file + __docker_stack foo --compose-file docker-compose.yml + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__docker_stack/parameter/default/compose-file b/cdist/conf/type/__docker_stack/parameter/default/compose-file new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__docker_stack/parameter/default/state b/cdist/conf/type/__docker_stack/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_stack/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_stack/parameter/optional b/cdist/conf/type/__docker_stack/parameter/optional new file mode 100644 index 00000000..b3457bd3 --- /dev/null +++ b/cdist/conf/type/__docker_stack/parameter/optional @@ -0,0 +1,2 @@ +compose-file +state diff --git a/cdist/conf/type/__docker_swarm/explorer/swarm-state b/cdist/conf/type/__docker_swarm/explorer/swarm-state new file mode 100755 index 00000000..2c9fd598 --- /dev/null +++ b/cdist/conf/type/__docker_swarm/explorer/swarm-state @@ -0,0 +1,21 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +docker info 2>/dev/null | grep '^ *Swarm: ' | awk '{print $2}' diff --git a/cdist/conf/type/__docker_swarm/gencode-remote b/cdist/conf/type/__docker_swarm/gencode-remote new file mode 100755 index 00000000..4b199a02 --- /dev/null +++ b/cdist/conf/type/__docker_swarm/gencode-remote @@ -0,0 +1,46 @@ +#!/bin/sh -e +# +# 2018 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +state=$(cat "${__object:?}/parameter/state") +swarm_state="$(cat "${__object}/explorer/swarm-state")" + +if [ -z "${swarm_state}" ]; then + echo "Unable to determine Swarm state. Is compatible version of Docker installed?" >&2 + + exit 1 +fi + +case "${state}" in + absent) + if [ "${swarm_state}" = "active" ]; then + echo "docker swarm leave --force" + fi + ;; + present) + if [ "${swarm_state}" = "inactive" ]; then + echo "docker swarm init" + fi + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__docker_swarm/man.rst b/cdist/conf/type/__docker_swarm/man.rst new file mode 100644 index 00000000..4dc408f0 --- /dev/null +++ b/cdist/conf/type/__docker_swarm/man.rst @@ -0,0 +1,49 @@ +cdist-type__docker_swarm(7) +=========================== + +NAME +---- + +cdist-type__docker_swarm - Manage Swarm + +DESCRIPTION +----------- + +This type can initialize Docker swarm mode. For more information about swarm +mode, see `Swarm mode overview `_. + +OPTIONAL PARAMETERS +------------------- + +state + 'present' or 'absent', defaults to 'present' where: + + present + Swarm is initialized + absent + Swarm is left + +EXAMPLES +-------- + +.. code-block:: sh + + # Initializes a swarm + __docker_swarm + + # Leaves a swarm + __docker_swarm --state absent + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__docker_swarm/parameter/default/state b/cdist/conf/type/__docker_swarm/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__docker_swarm/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__docker_swarm/parameter/optional b/cdist/conf/type/__docker_swarm/parameter/optional new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/cdist/conf/type/__docker_swarm/parameter/optional @@ -0,0 +1 @@ +state diff --git a/cdist/conf/type/__docker_swarm/singleton b/cdist/conf/type/__docker_swarm/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__dog_vdi/gencode-remote b/cdist/conf/type/__dog_vdi/gencode-remote old mode 100644 new mode 100755 index 56e4108a..9d49506c --- a/cdist/conf/type/__dog_vdi/gencode-remote +++ b/cdist/conf/type/__dog_vdi/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__dog_vdi/manifest b/cdist/conf/type/__dog_vdi/manifest old mode 100644 new mode 100755 index be327a3a..869bdede --- a/cdist/conf/type/__dog_vdi/manifest +++ b/cdist/conf/type/__dog_vdi/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__dot_file/explorer/home b/cdist/conf/type/__dot_file/explorer/home index 132cfc71..08d941bf 100755 --- a/cdist/conf/type/__dot_file/explorer/home +++ b/cdist/conf/type/__dot_file/explorer/home @@ -19,7 +19,7 @@ set -eu user="$(cat "${__object}/parameter/user")" -if which getent >/dev/null 2>&1; then +if command -v getent >/dev/null 2>&1; then line=$(getent passwd "${user}") else line=$(grep "^${user}:" /etc/passwd) diff --git a/cdist/conf/type/__dot_file/manifest b/cdist/conf/type/__dot_file/manifest index 4bc9f179..5e4957e5 100755 --- a/cdist/conf/type/__dot_file/manifest +++ b/cdist/conf/type/__dot_file/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # Copyright (C) 2016 Bogatov Dmitry # diff --git a/cdist/conf/type/__file/explorer/stat b/cdist/conf/type/__file/explorer/stat index 8a917556..13c1c208 100755 --- a/cdist/conf/type/__file/explorer/stat +++ b/cdist/conf/type/__file/explorer/stat @@ -1,6 +1,7 @@ #!/bin/sh # # 2013 Steven Armstrong (steven-cdist armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -25,25 +26,56 @@ destination="/$__object_id" os=$("$__explorer/os") case "$os" in - "freebsd"|"netbsd"|"openbsd") - # FIXME: should be something like this based on man page, but can not test - stat -f "type: %ST -owner: %Du %Su -group: %Dg %Sg -mode: %Op %Sp -size: %Dz -links: %Dl -" "$destination" - ;; - "macosx") - stat -f "type: %HT + "freebsd"|"netbsd"|"openbsd"|"macosx") + stat -f "type: %HT owner: %Du %Su group: %Dg %Sg mode: %Lp %Sp size: %Dz links: %Dl +" "$destination" | awk '/^type/ { print tolower($0); next; } { print; }' + ;; + alpine) + # busybox stat + stat -c "type: %F +owner: %u %U +group: %g %G +mode: %a %A +size: %s +links: %h " "$destination" - ;; + ;; + solaris) + ls1="$( ls -ld "$destination" )" + ls2="$( ls -ldn "$destination" )" + + if [ -f "$__object/parameter/mode" ] + then mode_should="$( cat "$__object/parameter/mode" )" + fi + + # yes, it is ugly hack, but if you know better way... + if [ -z "$( find "$destination" -perm "$mode_should" )" ] + then octets=888 + else octets="$( echo "$mode_should" | sed 's/^0//' )" + fi + + case "$( echo "$ls1" | cut -c1-1 )" in + -) echo 'type: regular file' ;; + d) echo 'type: directory' ;; + esac + + echo "owner: $( echo "$ls2" \ + | awk '{print $3}' ) $( echo "$ls1" \ + | awk '{print $3}' )" + + echo "group: $( echo "$ls2" \ + | awk '{print $4}' ) $( echo "$ls1" \ + | awk '{print $4}' )" + + echo "mode: $octets $( echo "$ls1" | awk '{print $1}' )" + echo "size: $( echo "$ls1" | awk '{print $5}' )" + echo "links: $( echo "$ls1" | awk '{print $2}' )" + ;; *) stat --printf="type: %F owner: %u %U @@ -52,5 +84,5 @@ mode: %a %A size: %s links: %h " "$destination" - ;; + ;; esac diff --git a/cdist/conf/type/__file/gencode-local b/cdist/conf/type/__file/gencode-local index 4caa6df6..fb9f9a92 100755 --- a/cdist/conf/type/__file/gencode-local +++ b/cdist/conf/type/__file/gencode-local @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Steven Armstrong (steven-cdist armstrong.cc) @@ -23,14 +23,31 @@ destination="/$__object_id" state_should="$(cat "$__object/parameter/state")" type="$(cat "$__object/explorer/type")" -[ "$state_should" = "exists" -a "$type" = "file" ] && exit 0 # nothing to do +[ "$state_should" = "exists" ] && [ "$type" = "file" ] && exit 0 # nothing to do + +if [ "$state_should" = "pre-exists" ]; then + if [ -f "$__object/parameter/source" ]; then + echo "--source cannot be used with --state pre-exists" + exit 1 + fi + + if [ "$type" = "file" ]; then + exit 0 # nothing to do + else + echo "File \"$destination\" does not exist" + exit 1 + fi +fi upload_file= create_file= -if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then +if [ "$state_should" = "present" ] || [ "$state_should" = "exists" ]; then if [ ! -f "$__object/parameter/source" ]; then - create_file=1 - echo create >> "$__messages_out" + remote_stat="$(cat "$__object/explorer/stat")" + if [ -z "$remote_stat" ]; then + create_file=1 + echo create >> "$__messages_out" + fi else source="$(cat "$__object/parameter/source")" if [ "$source" = "-" ]; then @@ -53,7 +70,7 @@ if [ "$state_should" = "present" -o "$state_should" = "exists" ]; then fi fi fi - if [ "$create_file" -o "$upload_file" ]; then + if [ "$create_file" ] || [ "$upload_file" ]; then # tell gencode-remote that we created or uploaded a file and that it must # set all attributes no matter what the explorer retreived mkdir "$__object/files" @@ -67,7 +84,7 @@ DONE if [ "$upload_file" ]; then echo upload >> "$__messages_out" # IPv6 fix - if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__file/gencode-remote b/cdist/conf/type/__file/gencode-remote index dcf3857b..b04c471e 100755 --- a/cdist/conf/type/__file/gencode-remote +++ b/cdist/conf/type/__file/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Steven Armstrong (steven-cdist armstrong.cc) @@ -23,7 +23,7 @@ destination="/$__object_id" state_should="$(cat "$__object/parameter/state")" type="$(cat "$__object/explorer/type")" stat_file="$__object/explorer/stat" - +fire_onchange='' get_current_value() { if [ -s "$stat_file" ]; then @@ -43,23 +43,25 @@ get_current_value() { } set_group() { - echo chgrp \"$1\" \"$destination\" - echo chgrp $1 >> "$__messages_out" + echo "chgrp '$1' '$destination'" + echo "chgrp '$1'" >> "$__messages_out" + fire_onchange=1 } set_owner() { - echo chown \"$1\" \"$destination\" - echo chown $1 >> "$__messages_out" + echo "chown '$1' '$destination'" + echo "chown '$1'" >> "$__messages_out" + fire_onchange=1 } set_mode() { - echo chmod \"$1\" \"$destination\" - echo chmod $1 >> "$__messages_out" + echo "chmod '$1' '$destination'" + echo "chmod '$1'" >> "$__messages_out" + fire_onchange=1 } -set_attributes= case "$state_should" in - present|exists) + present|exists|pre-exists) # Note: Mode - needs to happen last as a chown/chgrp can alter mode by # clearing S_ISUID and S_ISGID bits (see chown(2)) for attribute in group owner mode; do @@ -68,22 +70,27 @@ case "$state_should" in # change 0xxx format to xxx format => same as stat returns if [ "$attribute" = mode ]; then - value_should="$(echo $value_should | sed 's/^0\(...\)/\1/')" + value_should="$(echo "$value_should" | sed 's/^0\(...\)/\1/')" fi value_is="$(get_current_value "$attribute" "$value_should")" - if [ -f "$__object/files/set-attributes" -o "$value_should" != "$value_is" ]; then + if [ -f "$__object/files/set-attributes" ] || [ "$value_should" != "$value_is" ]; then "set_$attribute" "$value_should" fi fi done + if [ -f "$__object/files/set-attributes" ]; then + # set-attributes is created if file is created or uploaded in gencode-local + fire_onchange=1 + fi ;; absent) if [ "$type" = "file" ]; then - echo rm -f \"$destination\" + echo "rm -f '$destination'" echo remove >> "$__messages_out" + fire_onchange=1 fi ;; @@ -92,3 +99,9 @@ case "$state_should" in exit 1 ;; esac + +if [ -f "$__object/parameter/onchange" ]; then + if [ -n "$fire_onchange" ]; then + cat "$__object/parameter/onchange" + fi +fi diff --git a/cdist/conf/type/__file/man.rst b/cdist/conf/type/__file/man.rst index 7d9b413b..7a0603bb 100644 --- a/cdist/conf/type/__file/man.rst +++ b/cdist/conf/type/__file/man.rst @@ -23,6 +23,10 @@ symlink directory replace it with the source file +One exception is that when state is pre-exists, an error is raised if +the file would have been created otherwise (e.g. it is not present or +not a regular file). + In any case, make sure that the file attributes are as specified. @@ -33,7 +37,7 @@ None. OPTIONAL PARAMETERS ------------------- state - 'present', 'absent' or 'exists', defaults to 'present' where: + 'present', 'absent', 'exists' or 'pre-exists', defaults to 'present' where: present the file is exactly the one from source @@ -41,6 +45,9 @@ state the file does not exist exists the file from source but only if it doesn't already exist + pre-exists + check that the file exists and is a regular file, but do not + create or modify it group Group to chgrp to. @@ -56,6 +63,9 @@ source If not supplied, an empty file or directory will be created. If source is '-' (dash), take what was written to stdin as the file content. +onchange + The code to run if file is modified. + MESSAGES -------- chgrp @@ -93,6 +103,8 @@ EXAMPLES __file /home/frodo/.bashrc --source "/etc/skel/.bashrc" \ --state exists \ --owner frodo --mode 0600 + # Check that the file is present, show an error when it is not + __file /etc/somefile --state pre-exists # Take file content from stdin __file /tmp/whatever --owner root --group root --mode 644 --source - << DONE Here goes the content for /tmp/whatever diff --git a/cdist/conf/type/__file/parameter/optional b/cdist/conf/type/__file/parameter/optional index c696d592..9b98352c 100644 --- a/cdist/conf/type/__file/parameter/optional +++ b/cdist/conf/type/__file/parameter/optional @@ -3,3 +3,4 @@ group mode owner source +onchange diff --git a/cdist/conf/type/__filesystem/gencode-remote b/cdist/conf/type/__filesystem/gencode-remote old mode 100644 new mode 100755 index 3ca1c498..0bcdc13c --- a/cdist/conf/type/__filesystem/gencode-remote +++ b/cdist/conf/type/__filesystem/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 - 2016 Daniel Heule (hda at sfs.biz) # diff --git a/cdist/conf/type/__firewalld_rule/explorer/rule b/cdist/conf/type/__firewalld_rule/explorer/rule index 5a0e0265..0234e5b6 100644 --- a/cdist/conf/type/__firewalld_rule/explorer/rule +++ b/cdist/conf/type/__firewalld_rule/explorer/rule @@ -25,7 +25,7 @@ chain="$(cat "$__object/parameter/chain")" priority="$(cat "$__object/parameter/priority")" rule="$(cat "$__object/parameter/rule")" -if firewall-cmd --permanent --direct --query-rule "$protocol" "$table" "$chain" "$priority" $rule >/dev/null; then +if firewall-cmd --permanent --direct --query-rule "$protocol" "$table" "$chain" "$priority" "$rule" >/dev/null; then echo present else echo absent diff --git a/cdist/conf/type/__firewalld_rule/gencode-remote b/cdist/conf/type/__firewalld_rule/gencode-remote old mode 100644 new mode 100755 index 8f1ba28a..bd6d13e5 --- a/cdist/conf/type/__firewalld_rule/gencode-remote +++ b/cdist/conf/type/__firewalld_rule/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Nico Schottelius (nico-cdist at schottelius.org) # @@ -19,7 +19,6 @@ # # -name="$__object_id" state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/rule")" @@ -33,13 +32,13 @@ rule="$(cat "$__object/parameter/rule")" case "$state_should" in present) - echo firewall-cmd --quiet --permanent --direct --add-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule - echo firewall-cmd --quiet --direct --add-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule + echo "firewall-cmd --quiet --permanent --direct --add-rule '$protocol' '$table' '$chain' '$priority' $rule" + echo "firewall-cmd --quiet --direct --add-rule '$protocol' '$table' '$chain' '$priority' $rule" ;; absent) - echo firewall-cmd --quiet --permanent --direct --remove-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule - echo firewall-cmd --quiet --direct --remove-rule \"$protocol\" \"$table\" \"$chain\" \"$priority\" $rule + echo "firewall-cmd --quiet --permanent --direct --remove-rule '$protocol' '$table' '$chain' '$priority' $rule" + echo "firewall-cmd --quiet --direct --remove-rule '$protocol' '$table' '$chain' '$priority' $rule" ;; *) echo "Unknown state $state_should" >&2 diff --git a/cdist/conf/type/__firewalld_rule/manifest b/cdist/conf/type/__firewalld_rule/manifest old mode 100644 new mode 100755 index 5baf6da3..71156329 --- a/cdist/conf/type/__firewalld_rule/manifest +++ b/cdist/conf/type/__firewalld_rule/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 David Hürlimann (david at ungleich.ch) # diff --git a/cdist/conf/type/__firewalld_start/gencode-remote b/cdist/conf/type/__firewalld_start/gencode-remote old mode 100644 new mode 100755 index 7a3b6298..3e767f68 --- a/cdist/conf/type/__firewalld_start/gencode-remote +++ b/cdist/conf/type/__firewalld_start/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Darko Poljak(darko.poljak at ungleich.ch) # diff --git a/cdist/conf/type/__firewalld_start/manifest b/cdist/conf/type/__firewalld_start/manifest old mode 100644 new mode 100755 index 2c6a0219..98caaad9 --- a/cdist/conf/type/__firewalld_start/manifest +++ b/cdist/conf/type/__firewalld_start/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Darko Poljak (darko.poljak at ungleich.ch) # diff --git a/cdist/conf/type/__git/explorer/group b/cdist/conf/type/__git/explorer/group index 1308c710..3ddf9656 100644 --- a/cdist/conf/type/__git/explorer/group +++ b/cdist/conf/type/__git/explorer/group @@ -2,4 +2,4 @@ destination="/$__object_id/.git" -stat --print "%G" ${destination} 2>/dev/null || exit 0 +stat --print "%G" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__git/explorer/owner b/cdist/conf/type/__git/explorer/owner index 8c36b035..4c3cd431 100644 --- a/cdist/conf/type/__git/explorer/owner +++ b/cdist/conf/type/__git/explorer/owner @@ -2,4 +2,4 @@ destination="/$__object_id/.git" -stat --print "%U" ${destination} 2>/dev/null || exit 0 +stat --print "%U" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__git/gencode-remote b/cdist/conf/type/__git/gencode-remote old mode 100644 new mode 100755 index c4fc1ef2..5a9e23fc --- a/cdist/conf/type/__git/gencode-remote +++ b/cdist/conf/type/__git/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # @@ -35,10 +35,10 @@ owner="$(cat "$__object/parameter/owner")" group="$(cat "$__object/parameter/group")" mode="$(cat "$__object/parameter/mode")" -[ "$state_should" = "$state_is" -a \ - "$owner" = "$owner_is" -a \ - "$group" = "$group_is" -a \ - -n "$mode" ] && exit 0 +[ "$state_should" = "$state_is" ] && \ +[ "$owner" = "$owner_is" ] && \ +[ "$group" = "$group_is" ] && \ +[ -n "$mode" ] && exit 0 case $state_should in present) @@ -46,8 +46,8 @@ case $state_should in if [ "$state_should" != "$state_is" ]; then echo git clone --quiet --branch "$branch" "$source" "$destination" fi - if [ \( -n "$owner" -a "$owner_is" != "$owner" \) -o \ - \( -n "$group" -a "$group_is" != "$group" \) ]; then + if { [ -n "$owner" ] && [ "$owner_is" != "$owner" ]; } || \ + { [ -n "$group" ] && [ "$group_is" != "$group" ]; }; then echo chown -R "${owner}:${group}" "$destination" fi if [ -n "$mode" ]; then diff --git a/cdist/conf/type/__git/man.rst b/cdist/conf/type/__git/man.rst index 17e9c623..130925c8 100644 --- a/cdist/conf/type/__git/man.rst +++ b/cdist/conf/type/__git/man.rst @@ -44,7 +44,7 @@ EXAMPLES __git /home/services/dokuwiki --source git://github.com/splitbrain/dokuwiki.git # Checkout cdist, stay on branch 2.1 - __git /home/nico/cdist --source git://github.com/ungleich/cdist.git --branch 2.1 + __git /home/nico/cdist --source git@code.ungleich.ch:ungleich-public/cdist.git --branch 2.1 AUTHORS diff --git a/cdist/conf/type/__git/manifest b/cdist/conf/type/__git/manifest old mode 100644 new mode 100755 index b2b0feb0..6fb870f4 --- a/cdist/conf/type/__git/manifest +++ b/cdist/conf/type/__git/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__go_get/explorer/go-executable b/cdist/conf/type/__go_get/explorer/go-executable old mode 100644 new mode 100755 index 4c84ce07..87182282 --- a/cdist/conf/type/__go_get/explorer/go-executable +++ b/cdist/conf/type/__go_get/explorer/go-executable @@ -1,3 +1,6 @@ +#!/bin/sh +# shellcheck disable=SC1091 [ -f /etc/environment ] && . /etc/environment +# shellcheck disable=SC1091 [ -f /etc/profile ] && . /etc/profile go version 2>/dev/null || true diff --git a/cdist/conf/type/__go_get/gencode-remote b/cdist/conf/type/__go_get/gencode-remote old mode 100644 new mode 100755 index 5f1d3aae..4c47a70e --- a/cdist/conf/type/__go_get/gencode-remote +++ b/cdist/conf/type/__go_get/gencode-remote @@ -1,3 +1,5 @@ +#!/bin/sh -e + package=$__object_id cat<&2 && exit 1 diff --git a/cdist/conf/type/__golang_from_vendor/gencode-remote b/cdist/conf/type/__golang_from_vendor/gencode-remote old mode 100644 new mode 100755 index e372bf61..5200e9e3 --- a/cdist/conf/type/__golang_from_vendor/gencode-remote +++ b/cdist/conf/type/__golang_from_vendor/gencode-remote @@ -1,8 +1,8 @@ -#!/bin/sh +#!/bin/sh -e version=$(cat "$__object/parameter/version") -kernel_name=$(cat "$__global/explorer/kernel_name" | tr '[:upper:]' '[:lower:]') +kernel_name=$(tr '[:upper:]' '[:lower:]' < "$__global/explorer/kernel_name") machine=$(cat "$__global/explorer/machine") case $machine in x86_64|amd64) diff --git a/cdist/conf/type/__golang_from_vendor/manifest b/cdist/conf/type/__golang_from_vendor/manifest old mode 100644 new mode 100755 index 9d320830..ad39ddfb --- a/cdist/conf/type/__golang_from_vendor/manifest +++ b/cdist/conf/type/__golang_from_vendor/manifest @@ -1 +1,4 @@ +#!/bin/sh -e + +# shellcheck disable=SC2016 __line go_in_path --line 'export PATH=/usr/local/go/bin:$PATH' --file /etc/profile diff --git a/cdist/conf/type/__grafana_dashboard/manifest b/cdist/conf/type/__grafana_dashboard/manifest old mode 100644 new mode 100755 index b6e3020e..e652202b --- a/cdist/conf/type/__grafana_dashboard/manifest +++ b/cdist/conf/type/__grafana_dashboard/manifest @@ -1,32 +1,43 @@ -os=$(cat $__global/explorer/os) -os_version=$(cat $__global/explorer/os_version) +#!/bin/sh -e +os=$(cat "$__global/explorer/os") +os_version=$(cat "$__global/explorer/os_version") + +require="" case $os in debian|devuan) case $os_version in 8*|jessie) - __apt_key_uri grafana \ - --name 'Grafana Release Signing Key' \ - --uri https://packagecloud.io/gpg.key - - require="__apt_key_uri/grafana" __apt_source grafana \ - --uri https://packagecloud.io/grafana/stable/debian/ \ - --distribution jessie \ - --component main - - __package apt-transport-https - - require="__apt_source/grafana __package/apt-transport-https" __package grafana - require="__package/grafana" __start_on_boot grafana-server - ;; + # Differntation not needed anymore + apt_source_distribution=stable + ;; + 9*|ascii/ceres|ascii) + # Differntation not needed anymore + apt_source_distribution=stable + ;; *) - echo "Don't know how to install Grafana on $os $os_version. Send us a pull request!" + echo "Don't know how to install Grafana on $os $os_version. Send us a pull request!" >&2 exit 1 - ;; + ;; esac - ;; + + __apt_key_uri grafana \ + --name 'Grafana Release Signing Key' \ + --uri https://packages.grafana.com/gpg.key + + require="$require __apt_key_uri/grafana" __apt_source grafana \ + --uri https://packages.grafana.com/oss/deb \ + --distribution $apt_source_distribution \ + --component main + + __package apt-transport-https + + require="$require __apt_source/grafana __package/apt-transport-https" __package grafana + require="$require __package/grafana" __start_on_boot grafana-server + require="$require __start_on_boot/grafana-server" __process grafana-server --start "service grafana-server start" + ;; *) - echo "Don't know how to install Grafana on $os. Send us a pull request!" + echo "Don't know how to install Grafana on $os. Send us a pull request!" >&2 exit 1 - ;; + ;; esac diff --git a/cdist/conf/type/__group/explorer/gshadow b/cdist/conf/type/__group/explorer/gshadow index 2e2ab29d..ef40b7bc 100755 --- a/cdist/conf/type/__group/explorer/gshadow +++ b/cdist/conf/type/__group/explorer/gshadow @@ -22,7 +22,7 @@ # name=$__object_id -os="$($__explorer/os)" +os="$("$__explorer/os")" case "$os" in "freebsd"|"netbsd") diff --git a/cdist/conf/type/__group/gencode-remote b/cdist/conf/type/__group/gencode-remote index 2aaa83f3..6091c548 100755 --- a/cdist/conf/type/__group/gencode-remote +++ b/cdist/conf/type/__group/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2015 Steven Armstrong (steven-cdist at armstrong.cc) # 2011 Nico Schottelius (nico-cdist at schottelius.org) @@ -30,9 +30,9 @@ state="$(cat "$__object/parameter/state")" # Use short option names for portability shorten_property() { case "$1" in - gid) echo "-g";; - password) echo "-p";; - system) echo "-r";; + gid) echo " -g";; + password) echo " -p";; + system) echo " -r";; esac } @@ -40,11 +40,9 @@ shorten_property() { if [ "$state" = "present" ]; then case "$os" in freebsd) - supported_add_properties="gid" supported_change_properties="gid" ;; *) - supported_add_properties="gid password system" supported_change_properties="gid password" ;; esac @@ -63,8 +61,8 @@ if [ "$state" = "present" ]; then ;; esac if [ "$new_value" != "$current_value" ]; then - set -- "$@" "$(shorten_property $property)" \'$new_value\' - echo change $property $new_value $current_value >> "$__messages_out" + set -- "$@" "$(shorten_property "$property")" \'"$new_value"\' + echo "change $property $new_value $current_value" >> "$__messages_out" fi fi done @@ -83,9 +81,9 @@ if [ "$state" = "present" ]; then new_value="$(cat "$__object/parameter/$property")" if [ -z "$new_value" ]; then # Boolean parameters have no value - set -- "$@" "$(shorten_property $property)" + set -- "$@" "$(shorten_property "$property")" else - set -- "$@" "$(shorten_property $property)" \'$new_value\' + set -- "$@" "$(shorten_property "$property")" \'"$new_value"\' fi fi done diff --git a/cdist/conf/type/__hostname/gencode-remote b/cdist/conf/type/__hostname/gencode-remote index 4eb08723..8b5797dd 100755 --- a/cdist/conf/type/__hostname/gencode-remote +++ b/cdist/conf/type/__hostname/gencode-remote @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2014-2017 Steven Armstrong (steven-cdist at armstrong.cc) # 2014 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. @@ -22,7 +22,7 @@ if [ -f "$__object/parameter/name" ]; then name_should="$(cat "$__object/parameter/name")" else - name_should="$(echo "${__target_host%%.*}")" + name_should="${__target_host%%.*}" fi os=$(cat "$__global/explorer/os") @@ -35,13 +35,13 @@ has_hostnamectl=$(cat "$__object/explorer/has_hostnamectl") # If everything is ok -> exit # case "$os" in - archlinux|debian|suse|ubuntu|devuan) - if [ "$name_config" = "$name_should" -a "$name_running" = "$name_should" ]; then + archlinux|debian|suse|ubuntu|devuan|coreos|alpine) + if [ "$name_config" = "$name_should" ] && [ "$name_running" = "$name_should" ]; then exit 0 fi ;; - scientific|centos|openbsd) - if [ "$name_sysconfig" = "$name_should" -a "$name_running" = "$name_should" ]; then + scientific|centos|freebsd|openbsd) + if [ "$name_sysconfig" = "$name_should" ] && [ "$name_running" = "$name_should" ]; then exit 0 fi ;; @@ -56,20 +56,23 @@ esac # echo changed >> "$__messages_out" +# Use the good old way to set the hostname even on machines running systemd. +case "$os" in + archlinux|debian|ubuntu|devuan|centos|coreos|alpine) + printf "printf '%%s\\\\n' '$name_should' > /etc/hostname\\n" + echo "hostname -F /etc/hostname" + ;; + freebsd|openbsd) + echo "hostname '$name_should'" + ;; + suse) + echo "hostname '$name_should'" + printf "printf '%%s\\\\n' '$name_should' > /etc/HOSTNAME\\n" + ;; +esac + if [ "$has_hostnamectl" ]; then - echo "hostnamectl set-hostname '$name_should'" -else - case "$os" in - archlinux|debian|ubuntu|devuan) - echo "hostname '$name_should'" - echo "printf '%s\n' '$name_should' > /etc/hostname" - ;; - centos|openbsd) - echo "hostname '$name_should'" - ;; - suse) - echo "hostname '$name_should'" - echo "printf '%s\n' '$name_should' > /etc/HOSTNAME" - ;; - esac + # Allow hostnamectl set-hostname to fail silently. + # Who the fuck invented a tool that needs dbus to set the hostname anyway ... + echo "hostnamectl set-hostname '$name_should' || true" fi diff --git a/cdist/conf/type/__hostname/manifest b/cdist/conf/type/__hostname/manifest index 823d2f7e..8f1adf12 100755 --- a/cdist/conf/type/__hostname/manifest +++ b/cdist/conf/type/__hostname/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Steven Armstrong (steven-cdist at armstrong.cc) # 2014 Nico Schottelius (nico-cdist at schottelius.org) @@ -25,10 +25,10 @@ if [ -f "$__object/parameter/name" ]; then else case "$os" in openbsd) - name_should="$(echo "${__target_host}")" + name_should="${__target_host}" ;; *) - name_should="$(echo "${__target_host%%.*}")" + name_should="${__target_host%%.*}" ;; esac fi @@ -41,7 +41,7 @@ not_supported() { } case "$os" in - archlinux|debian|suse|ubuntu|devuan) + archlinux|debian|suse|ubuntu|devuan|coreos|alpine) # handled in gencode-remote : ;; @@ -52,6 +52,13 @@ case "$os" in --key HOSTNAME \ --value "$name_should" --exact_delimiter ;; + freebsd) + __key_value rcconf-hostname \ + --file /etc/rc.conf \ + --delimiter '=' \ + --key 'hostname' \ + --value "$name_should" + ;; openbsd) echo "$name_should" | __file /etc/myname --source - ;; diff --git a/cdist/conf/type/__hosts/manifest b/cdist/conf/type/__hosts/manifest index 6fa21608..c536b83b 100755 --- a/cdist/conf/type/__hosts/manifest +++ b/cdist/conf/type/__hosts/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # Copyright (C) 2015 Bogatov Dmitry # # This program is free software: you can redistribute it and/or modify diff --git a/cdist/conf/type/__install_bootloader_grub/explorer/target_os b/cdist/conf/type/__install_bootloader_grub/explorer/target_os new file mode 100755 index 00000000..f235710a --- /dev/null +++ b/cdist/conf/type/__install_bootloader_grub/explorer/target_os @@ -0,0 +1,100 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# 2014 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# All os variables are lower case. Keep this file in alphabetical +# order by os variable except in cases where order otherwise matters, +# in which case keep the primary os and its derivatives together in +# a block (see Debian and Redhat examples below). +# + +chroot="$(cat "$__object/parameter/chroot")" + +if grep -q ^Amazon "$chroot/etc/system-release" 2>/dev/null; then + echo amazon + exit 0 +fi + +if [ -f "$chroot/etc/arch-release" ]; then + echo archlinux + exit 0 +fi + +if [ -f "$chroot/etc/cdist-preos" ]; then + echo cdist-preos + exit 0 +fi + +### Debian and derivatives +if grep -q ^DISTRIB_ID=Ubuntu "$chroot/etc/lsb-release" 2>/dev/null; then + echo ubuntu + exit 0 +fi + +if [ -f "$chroot/etc/debian_version" ]; then + echo debian + exit 0 +fi +### + +if [ -f "$chroot/etc/gentoo-release" ]; then + echo gentoo + exit 0 +fi + +if [ -f "$chroot/etc/openwrt_version" ]; then + echo openwrt + exit 0 +fi + +if [ -f "$chroot/etc/owl-release" ]; then + echo owl + exit 0 +fi + +### Redhat and derivatives +if grep -q ^CentOS "$chroot/etc/redhat-release" 2>/dev/null; then + echo centos + exit 0 +fi + +if grep -q ^Fedora "$chroot/etc/redhat-release" 2>/dev/null; then + echo fedora + exit 0 +fi + +if [ -f "$chroot/etc/redhat-release" ]; then + echo redhat + exit 0 +fi +### + +if [ -f "$chroot/etc/SuSE-release" ]; then + echo suse + exit 0 +fi + +if [ -f "$chroot/etc/slackware-version" ]; then + echo slackware + exit 0 +fi + +echo "Unknown OS" >&2 +exit 1 diff --git a/cdist/conf/type/__install_bootloader_grub/gencode-remote b/cdist/conf/type/__install_bootloader_grub/gencode-remote index ed57331a..1caebbbf 100755 --- a/cdist/conf/type/__install_bootloader_grub/gencode-remote +++ b/cdist/conf/type/__install_bootloader_grub/gencode-remote @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2015 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -18,51 +18,74 @@ # along with cdist. If not, see . # -device="$(cat "$__object/parameter/device")" +device="$(cat "$__object/parameter/device" 2>/dev/null || echo "/$__object_id")" chroot="$(cat "$__object/parameter/chroot")" +target_os=$(cat "$__object/explorer/target_os") + +mkdir "$__object/files" +install_script="$__object/files/install_script" +# Link file descriptor #6 with stdout +exec 6>&1 +# Link stdout with $install_script +exec > "$install_script" + +# Generate script to install bootloader on distro +printf '#!/bin/sh -l\n' + +case "$target_os" in + ubuntu|debian) + if [ -s "$__global/explorer/efi" ]; then + # FIXME: untested. maybe also just run update-grub for EFI system? + printf 'grub-mkconfig --output=/boot/efi/EFI/%s/grub.cfg\n' "$target_os" + printf 'mkdir -p /boot/efi/EFI/BOOT\n' + printf 'cp /boot/efi/EFI/%s/grubx64.efi /boot/efi/EFI/BOOT/bootx64.efi' "$target_os" + else + printf 'grub-install "%s"\n' "$device" + printf 'update-grub\n' + fi + ;; + archlinux) + if [ -s "$__global/explorer/efi" ]; then + echo "EFI boot loader installation is on your operating system ($target_os) is currently not supported by this type (${__type##*/})." >&2 + echo "Please contribute an implementation for it if you can." >&2 + exit 1 + else + printf 'grub-install "%s"\n' "$device" + # bugfix/workarround: rebuild initramfs + # FIXME: doesn't belong here + printf 'mkinitcpio -p linux\n' + printf 'grub-mkconfig -o /boot/grub/grub.cfg\n' + fi + ;; + centos) + if [ -s "$__global/explorer/efi" ]; then + printf 'grub2-mkconfig --output=/boot/efi/EFI/%s/grub.cfg\n' "$target_os" + printf 'mkdir -p /boot/efi/EFI/BOOT\n' + printf 'cp /boot/efi/EFI/%s/grubx64.efi /boot/efi/EFI/BOOT/bootx64.efi' "$target_os" + else + printf 'grub2-install "%s"\n' "$device" + printf 'grub2-mkconfig --output=/boot/grub2/grub.cfg\n' + fi + ;; + *) + echo "Your operating system ($target_os) is currently not supported by this type (${__type##*/})." >&2 + echo "If you can, please contribute an implementation for it." >&2 + exit 1 + ;; +esac +# Restore stdout and close file descriptor #6. +exec 1>&6 6>&- + cat << DONE -os=\$( -if grep -q ^DISTRIB_ID=Ubuntu ${chroot}/etc/lsb-release 2>/dev/null; then - echo ubuntu - exit 0 -fi - -if [ -f ${chroot}/etc/arch-release ]; then - echo archlinux - exit 0 -fi - -if [ -f ${chroot}/etc/debian_version ]; then - echo debian - exit 0 -fi -) - # Ensure /tmp exists [ -d "${chroot}/tmp" ] || mkdir -m 1777 "${chroot}/tmp" # Generate script to run in chroot -script=\$(mktemp "${chroot}/tmp/__install_bootloader_grub.XXXXXXXXXX") -# Link file descriptor #6 with stdout -exec 6>&1 -# Link stdout with \$script -exec > \$script - -echo "#!/bin/sh -l" -echo "grub-install $device" -case \$os in - archlinux) - # bugfix/workarround: rebuild initramfs - # FIXME: doesn't belong here - echo "mkinitcpio -p linux" - echo "grub-mkconfig -o /boot/grub/grub.cfg" - ;; - ubuntu|debian) echo "update-grub" ;; -esac - -# Restore stdout and close file descriptor #6. -exec 1>&6 6>&- +script=\$(mktemp "${chroot}/tmp/${__type##*/}.XXXXXXXXXX") +cat > \$script << script_DONE +$(cat "$install_script") +script_DONE # Make script executable chmod +x "\$script" @@ -70,4 +93,5 @@ chmod +x "\$script" # Run script in chroot relative_script="\${script#$chroot}" chroot "$chroot" "\$relative_script" +rm -rf \$script DONE diff --git a/cdist/conf/type/__install_bootloader_grub/parameter/default/chroot b/cdist/conf/type/__install_bootloader_grub/parameter/default/chroot new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/cdist/conf/type/__install_bootloader_grub/parameter/default/chroot @@ -0,0 +1 @@ +/target diff --git a/cdist/conf/type/__install_chroot_mount/gencode-local b/cdist/conf/type/__install_chroot_mount/gencode-local new file mode 120000 index 00000000..68dcbd6a --- /dev/null +++ b/cdist/conf/type/__install_chroot_mount/gencode-local @@ -0,0 +1 @@ +../__chroot_mount/gencode-local \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_mount/parameter b/cdist/conf/type/__install_chroot_mount/parameter new file mode 120000 index 00000000..5b5c9e20 --- /dev/null +++ b/cdist/conf/type/__install_chroot_mount/parameter @@ -0,0 +1 @@ +../__chroot_mount/parameter \ No newline at end of file diff --git a/cdist/conf/type/__install_chroot_umount/parameter b/cdist/conf/type/__install_chroot_umount/parameter new file mode 120000 index 00000000..4148bcd0 --- /dev/null +++ b/cdist/conf/type/__install_chroot_umount/parameter @@ -0,0 +1 @@ +../__chroot_umount/parameter \ No newline at end of file diff --git a/cdist/conf/type/__install_config/files/remote/copy b/cdist/conf/type/__install_config/files/remote/copy index 5b6f555c..fa7fa9b7 100755 --- a/cdist/conf/type/__install_config/files/remote/copy +++ b/cdist/conf/type/__install_config/files/remote/copy @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -25,24 +25,24 @@ # log() { - echo "$@" | logger -t "__install_config copy" + #echo "$@" | logger -t "__install_config copy" : } chroot="$1"; shift target_host="$__target_host" -scp="scp -o User=root -q" - # postfix target_host with chroot location code="$(echo "$@" | sed "s|$target_host:|$target_host:$chroot|g")" log "target_host: $target_host" log "chroot: $chroot" -log "@: $@" +log "@: $*" log "code: $code" # copy files into chroot -$scp $code +# __default_remote_copy and code should be split +# shellcheck disable=SC2086 +$__default_remote_copy $code log "-----" diff --git a/cdist/conf/type/__install_config/files/remote/exec b/cdist/conf/type/__install_config/files/remote/exec index 58e6b162..c2057ebf 100755 --- a/cdist/conf/type/__install_config/files/remote/exec +++ b/cdist/conf/type/__install_config/files/remote/exec @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -25,7 +25,7 @@ # log() { - echo "$@" | logger -t "__install_config exec" + #echo "$@" | logger -t "__install_config exec" : } @@ -34,15 +34,19 @@ target_host="$__target_host" # In exec mode the first argument is the __target_host which we already got from env. Get rid of it. shift -ssh="ssh -o User=root -q $target_host" -code="$ssh chroot $chroot sh -c '$@'" +# escape ' with '"'"' +code="$(echo "$@" | sed -e "s/'/'\"'\"'/g")" +# shellcheck disable=SC2089 +code="chroot $chroot sh -e -c '$code'" log "target_host: $target_host" log "chroot: $chroot" -log "@: $@" +log "@: $*" log "code: $code" # Run the code -$code +# __default_remote_exec and code should be split +# shellcheck disable=SC2086,SC2090 +$__default_remote_exec "$target_host" $code log "-----" diff --git a/cdist/conf/type/__install_config/gencode-local b/cdist/conf/type/__install_config/gencode-local index 674dec25..dd4f2a78 100755 --- a/cdist/conf/type/__install_config/gencode-local +++ b/cdist/conf/type/__install_config/gencode-local @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2018 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -22,29 +22,14 @@ chroot="$(cat "$__object/parameter/chroot")" remote_exec="$__type/files/remote/exec" remote_copy="$__type/files/remote/copy" -cdist_args="-v" -[ "$__debug" = "yes" ] && cdist_args="$cdist_args -d" - cat << DONE -#echo "__apt_noautostart --state present" \ -# | cdist $cdist_args \ -# config \ -# --initial-manifest - \ -# --remote-exec="$remote_exec $chroot" \ -# --remote-copy="$remote_copy $chroot" \ -# $__target_host - -cdist $cdist_args \ - config \ +export __cdist_install_config=yes +export __cdist_log_level=$__cdist_log_level +export __default_remote_exec="$__remote_exec" +export __default_remote_copy="$__remote_copy" +cdist config \ --remote-exec="$remote_exec $chroot" \ --remote-copy="$remote_copy $chroot" \ $__target_host - -#echo "__apt_noautostart --state absent" \ -# | cdist $cdist_args \ -# config \ -# --initial-manifest - \ -# --remote-exec="$remote_exec $chroot" \ -# --remote-copy="$remote_copy $chroot" \ -# $__target_host DONE + diff --git a/cdist/conf/type/__install_config/parameter/default/chroot b/cdist/conf/type/__install_config/parameter/default/chroot new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/cdist/conf/type/__install_config/parameter/default/chroot @@ -0,0 +1 @@ +/target diff --git a/cdist/conf/type/__install_coreos/gencode-remote b/cdist/conf/type/__install_coreos/gencode-remote new file mode 100755 index 00000000..f550b5a5 --- /dev/null +++ b/cdist/conf/type/__install_coreos/gencode-remote @@ -0,0 +1,19 @@ +#!/bin/sh -e + +device=$(cat "${__object:?}/parameter/device") +ignition=$(cat "${__object}/parameter/ignition") + +cat < "\${ignition_file}" << eof +$(base64 "${ignition}") +eof + +coreos-install -d "${device}" \ + \$(if [ -s "\${ignition_file}" ]; then + printf -- "-i \${ignition_file}\\n" + fi) + +rm "\${ignition_file}" +EOF diff --git a/cdist/conf/type/__install_coreos/install b/cdist/conf/type/__install_coreos/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_coreos/man.rst b/cdist/conf/type/__install_coreos/man.rst new file mode 100644 index 00000000..314f9f2a --- /dev/null +++ b/cdist/conf/type/__install_coreos/man.rst @@ -0,0 +1,50 @@ +cdist-type__install_coreos(7) +============================= + +NAME +---- + +cdist-type__install_coreos - Install CoreOS + +DESCRIPTION +----------- + +This type installs CoreOS to a given device using coreos-install_, which is +present in CoreOS ISO by default. + +.. _coreos-install: https://raw.githubusercontent.com/coreos/init/master/bin/coreos-install + +REQUIRED PARAMETERS +------------------- + +device + A device CoreOS will be installed to. + +OPTIONAL PARAMETERS +------------------- + +ignition + Path to ignition config. + +EXAMPLES +-------- + +.. code-block:: sh + + __install_coreos \ + --device /dev/sda \ + --ignition ignition.json + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2018 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_coreos/parameter/default/ignition b/cdist/conf/type/__install_coreos/parameter/default/ignition new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_coreos/parameter/optional b/cdist/conf/type/__install_coreos/parameter/optional new file mode 100644 index 00000000..df284caa --- /dev/null +++ b/cdist/conf/type/__install_coreos/parameter/optional @@ -0,0 +1 @@ +ignition diff --git a/cdist/conf/type/__install_coreos/parameter/required b/cdist/conf/type/__install_coreos/parameter/required new file mode 100644 index 00000000..f89ee6a8 --- /dev/null +++ b/cdist/conf/type/__install_coreos/parameter/required @@ -0,0 +1 @@ +device diff --git a/cdist/conf/type/__install_coreos/singleton b/cdist/conf/type/__install_coreos/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_directory/explorer b/cdist/conf/type/__install_directory/explorer new file mode 120000 index 00000000..ba2591e1 --- /dev/null +++ b/cdist/conf/type/__install_directory/explorer @@ -0,0 +1 @@ +../__directory/explorer \ No newline at end of file diff --git a/cdist/conf/type/__install_directory/gencode-remote b/cdist/conf/type/__install_directory/gencode-remote new file mode 120000 index 00000000..c86d61c9 --- /dev/null +++ b/cdist/conf/type/__install_directory/gencode-remote @@ -0,0 +1 @@ +../__directory/gencode-remote \ No newline at end of file diff --git a/cdist/conf/type/__install_directory/install b/cdist/conf/type/__install_directory/install new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__install_directory/man.rst b/cdist/conf/type/__install_directory/man.rst new file mode 120000 index 00000000..1ad7fa84 --- /dev/null +++ b/cdist/conf/type/__install_directory/man.rst @@ -0,0 +1 @@ +../__directory/man.rst \ No newline at end of file diff --git a/cdist/conf/type/__install_directory/parameter b/cdist/conf/type/__install_directory/parameter new file mode 120000 index 00000000..e23d9672 --- /dev/null +++ b/cdist/conf/type/__install_directory/parameter @@ -0,0 +1 @@ +../__directory/parameter \ No newline at end of file diff --git a/cdist/conf/type/__install_fstab/manifest b/cdist/conf/type/__install_fstab/manifest index 74af53c0..c5d24f3c 100755 --- a/cdist/conf/type/__install_fstab/manifest +++ b/cdist/conf/type/__install_fstab/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__install_generate_fstab/gencode-local b/cdist/conf/type/__install_generate_fstab/gencode-local index d10e5b92..80455aaa 100755 --- a/cdist/conf/type/__install_generate_fstab/gencode-local +++ b/cdist/conf/type/__install_generate_fstab/gencode-local @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -23,12 +23,13 @@ cat "$__type/files/fstab.header" > "$destination" mkdir "$__object/files" # get current UUID's from target_host -$__remote_exec $__target_host blkid > "$__object/files/blkid" +$__remote_exec "$__target_host" blkid > "$__object/files/blkid" -for object in $(find "$__global/object/__install_mount" -path "*.cdist"); do +find "$__global/object/__install_mount" -type d -name "$__cdist_object_marker" | +while IFS= read -r object +do device="$(cat "$object/parameter/device")" dir="$(cat "$object/parameter/dir")" - prefix="$(cat "$object/parameter/prefix")" type="$(cat "$object/parameter/type")" if [ -f "$object/parameter/options" ]; then options="$(cat "$object/parameter/options")" @@ -44,12 +45,17 @@ for object in $(find "$__global/object/__install_mount" -path "*.cdist"); do tmpfs) pass=0 ;; + bind) + pass=0 + type=none + options="bind,$options" + ;; *) pass=1 ;; esac if [ -f "$__object/parameter/uuid" ]; then - uuid="$(grep -w $device "$__object/files/blkid" | awk '{print $2}')" + uuid="$(grep -w "$device" "$__object/files/blkid" | awk '{print $2}')" if [ -n "$uuid" ]; then echo "# $dir was on $device during installation" >> "$destination" device="$uuid" diff --git a/cdist/conf/type/__install_mkfs/gencode-remote b/cdist/conf/type/__install_mkfs/gencode-remote index da643cce..8fc2c98e 100755 --- a/cdist/conf/type/__install_mkfs/gencode-remote +++ b/cdist/conf/type/__install_mkfs/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # 2017 Nico Schottelius (nico-cdist at schottelius.org) @@ -24,7 +24,7 @@ type="$(cat "$__object/parameter/type")" case "$type" in swap) - echo "mkswap $device" + echo "mkswap -f $device" exit 0 ;; xfs) diff --git a/cdist/conf/type/__install_mkfs/manifest b/cdist/conf/type/__install_mkfs/manifest index e9d275a4..b0a21dae 100755 --- a/cdist/conf/type/__install_mkfs/manifest +++ b/cdist/conf/type/__install_mkfs/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -19,13 +19,7 @@ # # set defaults -if [ -f "$__object/parameter/device" ]; then - device="(cat "$__object/parameter/device")" -else +if [ ! -f "$__object/parameter/device" ]; then device="/$__object_id" echo "$device" > "$__object/parameter/device" fi - -type="(cat "$__object/parameter/type")" - -options="(cat "$__object/parameter/options")" diff --git a/cdist/conf/type/__install_mount/gencode-remote b/cdist/conf/type/__install_mount/gencode-remote index 3a35c139..4415f0ff 100755 --- a/cdist/conf/type/__install_mount/gencode-remote +++ b/cdist/conf/type/__install_mount/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -20,7 +20,9 @@ get_type_from_mkfs() { _device="$1" - for mkfs_object in $(find "$__global/object/__install_mkfs" -path "*.cdist"); do + find "$__global/object/__install_mkfs" -type d -name "$__cdist_object_marker" | + while IFS= read -r mkfs_object + do mkfs_device="$(cat "$mkfs_object/parameter/device")" if [ "$_device" = "$mkfs_device" ]; then cat "$mkfs_object/parameter/type" @@ -42,18 +44,25 @@ else # store for later use by others echo "$type" > "$__object/parameter/type" fi -[ -n "$type" ] || die "Can't determine type for $__object" +[ -n "$type" ] || { + echo "Can't determine type for $__object" >&2 + exit 1 +} if [ "$type" = "swap" ]; then - echo "swapon \"$device\"" + printf 'swapon "%s"\n' "$device" else - if [ -f "$__object/parameter/options" ]; then - options="$(cat "$__object/parameter/options")" - else - options="" - fi - [ -n "$options" ] && options="-o $options" mount_point="${prefix}${dir}" - - echo "[ -d \"$mount_point\" ] || mkdir -p \"$mount_point\"" - echo "mount -t \"$type\" $options \"$device\" \"$mount_point\"" + printf '[ -d "%s" ] || mkdir -p "%s"\n' "$mount_point" "$mount_point" + printf 'mount' + if [ "$type" = "bind" ]; then + printf ' --bind' + device="${prefix}${device}" + else + printf ' -t "%s"' "$type" + fi + if [ -f "$__object/parameter/options" ]; then + printf ' -o %s' "$(cat "$__object/parameter/options")" + fi + printf ' "%s"' "$device" + printf ' "%s"\n' "$mount_point" fi diff --git a/cdist/conf/type/__install_mount/manifest b/cdist/conf/type/__install_mount/manifest index 5afae7fc..72fc26e2 100755 --- a/cdist/conf/type/__install_mount/manifest +++ b/cdist/conf/type/__install_mount/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__install_partition_msdos/man.rst b/cdist/conf/type/__install_partition_msdos/man.rst index 5ebb9218..c408a614 100644 --- a/cdist/conf/type/__install_partition_msdos/man.rst +++ b/cdist/conf/type/__install_partition_msdos/man.rst @@ -19,6 +19,12 @@ type OPTIONAL PARAMETERS ------------------- +device + the device we're working on. Defaults to the string prefix of --partition + +minor + the partition number we're working on. Defaults to the numeric suffix of --partition + partition defaults to object_id @@ -49,6 +55,8 @@ EXAMPLES __install_partition_msdos /dev/sda6 --type 83 --size 50% # rest of the extended partition, linux __install_partition_msdos /dev/sda7 --type 83 --size + + # nvm device partition 2 + __install_partition_msdos /dev/nvme0n1p2 --device /dev/nvme0n1 --minor 2 --type 83 --size 128M --bootable true AUTHORS @@ -58,7 +66,7 @@ Steven Armstrong COPYING ------- -Copyright \(C) 2011 Steven Armstrong. You can redistribute it +Copyright \(C) 2011-2017 Steven Armstrong. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__install_partition_msdos/manifest b/cdist/conf/type/__install_partition_msdos/manifest index e55d3f24..b32605fa 100755 --- a/cdist/conf/type/__install_partition_msdos/manifest +++ b/cdist/conf/type/__install_partition_msdos/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -25,10 +25,15 @@ else partition="/$__object_id" echo "$partition" > "$__object/parameter/partition" fi -device="$(echo "$partition" | sed 's/[0-9]//g')" -echo "$device" > "$__object/parameter/device" -minor="$(echo "$partition" | sed 's/[^0-9]//g')" -echo "$minor" > "$__object/parameter/minor" + +if [ ! -f "$__object/parameter/device" ]; then + device="$(echo "$partition" | sed 's/[0-9]//g')" + echo "$device" > "$__object/parameter/device" +fi +if [ ! -f "$__object/parameter/minor" ]; then + minor="$(echo "$partition" | sed 's/[^0-9]//g')" + echo "$minor" > "$__object/parameter/minor" +fi if [ ! -f "$__object/parameter/bootable" ]; then echo "false" > "$__object/parameter/bootable" diff --git a/cdist/conf/type/__install_partition_msdos/parameter/optional b/cdist/conf/type/__install_partition_msdos/parameter/optional index b2b0a4c2..3b3f2083 100644 --- a/cdist/conf/type/__install_partition_msdos/parameter/optional +++ b/cdist/conf/type/__install_partition_msdos/parameter/optional @@ -1,3 +1,5 @@ +device +minor partition bootable size diff --git a/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh b/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh index cddc575d..2db9a441 100644 --- a/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh +++ b/cdist/conf/type/__install_partition_msdos_apply/files/lib.sh @@ -1,18 +1,20 @@ +#!/bin/sh + die() { - echo "[__install_partition_msdos_apply] $@" >&2 + echo "[__install_partition_msdos_apply] $*" >&2 exit 1 } debug() { - #echo "[__install_partition_msdos_apply] $@" >&2 + #echo "[__install_partition_msdos_apply] $*" >&2 : } fdisk_command() { - local device="$1" - local cmd="$2" + device="$1" + cmd="$2" debug fdisk_command "running fdisk command '${cmd}' on device ${device}" - printf "${cmd}\nw\n" | fdisk -c -u "$device" + printf '%s\nw\n' "${cmd}" | fdisk -c -u "$device" ret=$? # give disk some time sleep 1 @@ -20,49 +22,49 @@ fdisk_command() { } create_disklabel() { - local device=$1 + device=$1 debug create_disklabel "creating new msdos disklabel" - fdisk_command ${device} "o" + fdisk_command "${device}" "o" return $? } toggle_bootable() { - local device="$1" - local minor="$2" - fdisk_command ${device} "a\n${minor}\n" + device="$1" + minor="$2" + fdisk_command "${device}" "a\\n${minor}\\n" return $? } create_partition() { - local device="$1" - local minor="$2" - local size="$3" - local type="$4" - local primary_count="$5" + device="$1" + minor="$2" + size="$3" + type="$4" + primary_count="$5" - if [ "$type" = "extended" -o "$type" = "5" ]; then + if [ "$type" = "extended" ] || [ "$type" = "5" ]; then # Extended partition - primary_extended="e\n" - first_minor="${minor}\n" + primary_extended='e\n' + first_minor="${minor}\\n" [ "${minor}" = "4" ] && first_minor="" - type_minor="${minor}\n" + type_minor="${minor}\\n" [ "${minor}" = "1" ] && type_minor="" type="5" elif [ "${minor}" -lt "5" ]; then - primary_extended="p\n" - first_minor="${minor}\n" + primary_extended='p\n' + first_minor="${minor}\\n" [ "${minor}" = "4" ] && first_minor="" - type_minor="${minor}\n" + type_minor="${minor}\\n" [ "${minor}" = "1" ] && type_minor="" else # Logical partitions - first_minor="${minor}\n" - type_minor="${minor}\n" - primary_extended="l\n" + first_minor="${minor}\\n" + type_minor="${minor}\\n" + primary_extended='l\n' [ "$primary_count" -gt "3" ] && primary_extended="" fi [ -n "${size}" ] && size="+${size}M" - fdisk_command ${device} "n\n${primary_extended}${first_minor}\n${size}\nt\n${type_minor}${type}\n" + fdisk_command "${device}" "n\\n${primary_extended}${first_minor}\\n${size}\\nt\\n${type_minor}${type}\\n" return $? } diff --git a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote index a1547296..a0b46b2d 100755 --- a/cdist/conf/type/__install_partition_msdos_apply/gencode-remote +++ b/cdist/conf/type/__install_partition_msdos_apply/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -18,36 +18,38 @@ # along with cdist. If not, see . # +#set -x + die() { - echo "[__install_partition_msdos_apply] $@" >&2 + echo "[__install_partition_msdos_apply] $*" >&2 exit 1 } debug() { - #echo "[__install_partition_msdos_apply] $@" >&2 + #echo "[__install_partition_msdos_apply] $*" >&2 : } # Convert a size specifier 1G 100M or 50% into the corresponding numeric MB. size_to_mb() { - local size=$1 - local available_size="$2" + size=$1 + available_size="$2" - local number_suffix="$(echo ${size} | sed -e 's:\.[0-9]\+::' -e 's:\([0-9]\+\)\([KkMmGg%]\)[Bb]\?:\1|\2:')" - local number="$(echo ${number_suffix} | cut -d '|' -f1)" - local suffix="$(echo ${number_suffix} | cut -d '|' -f2)" + number_suffix="$(echo "${size}" | sed -e 's:\.[0-9]\+::' -e 's:\([0-9]\+\)\([KkMmGg%]\)[Bb]\?:\1|\2:')" + number="$(echo "${number_suffix}" | cut -d '|' -f1)" + suffix="$(echo "${number_suffix}" | cut -d '|' -f2)" case "$suffix" in K|k) - size="$(( $number / 1024 ))" + size="$(( number / 1024 ))" ;; M|m) size="$number" ;; G|g) - size="$(( $number * 1024 ))" + size="$(( number * 1024 ))" ;; %) - size="$(( $available_size * $number / 100 ))" + size="$(( available_size * number / 100 ))" ;; *) size="-1" @@ -57,13 +59,15 @@ size_to_mb() { get_objects() { objects_file=$(mktemp) - for object in $(find "$__global/object/__install_partition_msdos" -path "*.cdist"); do + find "$__global/object/__install_partition_msdos" -type d -name "$__cdist_object_marker" | + while IFS= read -r object + do object_device="$(cat "$object/parameter/device")" object_minor="$(cat "$object/parameter/minor")" - echo "$object_device $object_minor $object" >> $objects_file + echo "$object_device $object_minor $object" >> "$objects_file" done - sort -k 1,2 $objects_file | cut -d' ' -f 3 - rm $objects_file + sort -k 1,2 "$objects_file" | cut -d' ' -f 3 + rm "$objects_file" unset objects_file unset object unset object_device @@ -83,9 +87,9 @@ primary_count=0 for object in $objects; do device="$(cat "$object/parameter/device")" if [ "$current_device" != "$device" ]; then - echo "create_disklabel \"$device\" || die 'Failed to create disklabel for $device'" + echo "create_disklabel '$device' || die 'Failed to create disklabel for $device'" current_device="$device" - device_name=$(echo ${device} | sed -e 's:^/dev/::;s:/:\\/:g') + device_name=$(echo "${device}" | sed -e 's:^/dev/::;s:/:\\/:g') available_device_size=$(( $(awk "/${device_name}\$/ { print \$3; }" "$partitions") / 1024)) # make sure we don't go past the end of the drive available_device_size=$((available_device_size - 2)) @@ -106,7 +110,7 @@ for object in $objects; do if [ "${minor}" -lt "5" ]; then # Primary partitions - primary_count=$(( $primary_count + 1 )) + primary_count=$(( primary_count + 1 )) available_size=$available_device_size else # Logical partitions @@ -119,13 +123,13 @@ for object in $objects; do available_size=0 else partition_size=$(size_to_mb "$size" "$available_size") - available_size="$(( $available_size - $partition_size ))" + available_size="$(( available_size - partition_size ))" fi if [ "${minor}" -lt "5" ]; then # Primary partitions available_device_size=$available_size - if [ "$type" = "extended" -o "$type" = "5" ]; then + if [ "$type" = "extended" ] || [ "$type" = "5" ]; then # Extended partition available_extended_size=$partition_size fi diff --git a/cdist/conf/type/__install_reboot/gencode-remote b/cdist/conf/type/__install_reboot/gencode-remote index 4358347d..9a6322c1 100755 --- a/cdist/conf/type/__install_reboot/gencode-remote +++ b/cdist/conf/type/__install_reboot/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -18,6 +18,13 @@ # along with cdist. If not, see . # -options="$(cat "$__object/parameter/options")" +#echo "reboot $options" +cat << DONE +echo 1 > /proc/sys/kernel/sysrq +echo s > /proc/sysrq-trigger -echo "reboot $options" +# close file descriptors to detach from ssh +sh -c 'sleep 3; echo b > /proc/sysrq-trigger' > /dev/null 2>&1 /dev/null; then - vgchange -a n + +debug() { + echo "[DEBUG] \$@" >&2 +} + +find_md_device_names() { + local disk_name="\$1" + for slave in \$(find /sys/devices/virtual/block/*/slaves/ -name "\${disk_name}*"); do + debug "slave: \$slave" + for holder in \$slave/holders/*; do + debug "holder: \$holder" + if [ -d "\$holder/md" ]; then + debug "mdadm found at \$holder" + holder_name="\${holder##*/}" + echo "\$holder_name" + fi + done + done +} + +# disable any enabled volume group +if command -v vgchange >/dev/null; then + vgchange -a n +else + echo "WARNING: vgchange command not found" >&2 +fi + +# disable any running mdadm arrays related to $disk +for md_name in \$(find_md_device_names "$disk_name" | sort | uniq); do + echo "md_name: \$md_name" + if command -v mdadm >/dev/null; then + mdadm --stop "/dev/\$md_name" else - echo "WARNING: vgchange command not found" >&2 + echo "WARNING: mdadm command not found" >&2 + echo "WARNING: could not stop active mdadm raid for disk $disk" >&2 fi -fi +done -# stop mdadm raids if any -if [ -r /proc/mdstat ]; then - md_name="\$(awk "/$disk_name/ {print \$1}" /proc/mdstat)" - if [ -n "\$md_name" ]; then - if command -v mdadm >/dev/null; then - mdadm --stop "/dev/\$md_name" - else - echo "WARNING: mdadm command not found" >&2 - echo "WARNING: could not stop active mdadm raid for disk $disk" >&2 - fi - fi -fi - -if command -v pvremove >/dev/null; then - pvremove --force --force --yes "$disk" || true -else - echo "WARNING: pvremove command not found" >&2 -fi -if command -v mdadm >/dev/null; then - mdadm --zero-superblock --force "$disk" || true -else - echo "WARNING: mdadm command not found" >&2 -fi # clean disks from any legacy signatures if command -v wipefs >/dev/null; then wipefs -a "$disk" || true @@ -61,5 +67,5 @@ fi # erase partition table dd if=/dev/zero of=$disk bs=512 count=1 -printf 'w\n' | fdisk -u -c $disk || true +printf 'w\\n' | fdisk -u -c $disk || true DONE diff --git a/cdist/conf/type/__install_stage/gencode-remote b/cdist/conf/type/__install_stage/gencode-remote index 3b83ea61..776e9fd5 100755 --- a/cdist/conf/type/__install_stage/gencode-remote +++ b/cdist/conf/type/__install_stage/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -22,8 +22,14 @@ uri="$(cat "$__object/parameter/uri" 2>/dev/null \ || echo "$__object_id")" target="$(cat "$__object/parameter/target")" -[ "$__debug" = "yes" ] && curl="curl" || curl="curl -s" -[ "$__debug" = "yes" ] && tar="tar -xvzp" || tar="tar -xzp" +if [ "$__cdist_log_level" -le "10" ] +then + curl="curl" + tar="tar -xvzp" +else + curl="curl -s" + tar="tar -xzp" +fi if [ -f "$__object/parameter/insecure" ] ; then curl="$curl -k" diff --git a/cdist/conf/type/__install_stage/man.rst b/cdist/conf/type/__install_stage/man.rst index 6c68c543..fd764693 100644 --- a/cdist/conf/type/__install_stage/man.rst +++ b/cdist/conf/type/__install_stage/man.rst @@ -17,9 +17,9 @@ REQUIRED PARAMETERS uri The uri from which to fetch the tarball. Can be anything understood by curl, e.g: - | http://path/to/stage.tgz - | tftp:///path/to/stage.tgz - | file:///local/path/stage.tgz + | http://path/to/stage.tgz + | tftp:///path/to/stage.tgz + | file:///local/path/stage.tgz OPTIONAL PARAMETERS diff --git a/cdist/conf/type/__install_umount/gencode-remote b/cdist/conf/type/__install_umount/gencode-remote index c275fe5d..8dcfb253 100755 --- a/cdist/conf/type/__install_umount/gencode-remote +++ b/cdist/conf/type/__install_umount/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__install_umount/parameter/default/target b/cdist/conf/type/__install_umount/parameter/default/target new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/cdist/conf/type/__install_umount/parameter/default/target @@ -0,0 +1 @@ +/target diff --git a/cdist/conf/type/__install_umount/parameter/optional b/cdist/conf/type/__install_umount/parameter/optional new file mode 100644 index 00000000..eb5a316c --- /dev/null +++ b/cdist/conf/type/__install_umount/parameter/optional @@ -0,0 +1 @@ +target diff --git a/cdist/conf/type/__iptables_apply/files/init-script b/cdist/conf/type/__iptables_apply/files/init-script index 2247dcf5..d9c79ef7 100644 --- a/cdist/conf/type/__iptables_apply/files/init-script +++ b/cdist/conf/type/__iptables_apply/files/init-script @@ -24,13 +24,15 @@ case $1 in iptables-save > "$status" # Apply our ruleset - cd "$basedir" - count="$(ls -1 | wc -l)" + cd "$basedir" || exit + count="$(find . ! -name . -prune | wc -l)" # Only do something if there are rules if [ "$count" -ge 1 ]; then for rule in *; do echo "Applying iptables rule $rule ..." + # Rule should be split. + # shellcheck disable=SC2046 iptables $(cat "$rule") done fi diff --git a/cdist/conf/type/__iptables_apply/gencode-remote b/cdist/conf/type/__iptables_apply/gencode-remote old mode 100644 new mode 100755 index c15d4d7f..a80cb936 --- a/cdist/conf/type/__iptables_apply/gencode-remote +++ b/cdist/conf/type/__iptables_apply/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e if grep -q "^__file/etc/iptables.d/" "$__messages_in"; then echo /etc/init.d/iptables restart diff --git a/cdist/conf/type/__iptables_apply/manifest b/cdist/conf/type/__iptables_apply/manifest old mode 100644 new mode 100755 index 3bb2d976..0061d3de --- a/cdist/conf/type/__iptables_apply/manifest +++ b/cdist/conf/type/__iptables_apply/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__iptables_rule/manifest b/cdist/conf/type/__iptables_rule/manifest old mode 100644 new mode 100755 index 13cec523..ed78787f --- a/cdist/conf/type/__iptables_rule/manifest +++ b/cdist/conf/type/__iptables_rule/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__issue/manifest b/cdist/conf/type/__issue/manifest index d2720f2d..0f0b3d83 100755 --- a/cdist/conf/type/__issue/manifest +++ b/cdist/conf/type/__issue/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # @@ -25,6 +25,9 @@ os="$(cat "$__global/explorer/os")" if [ -f "$__object/parameter/source" ]; then source="$(cat "$__object/parameter/source")" + if [ "$source" = "-" ]; then + source="${__object}/stdin" + fi else case "$os" in archlinux|redhat) diff --git a/cdist/conf/type/__jail/manifest b/cdist/conf/type/__jail/manifest index 6df52c59..fad6a3a1 100755 --- a/cdist/conf/type/__jail/manifest +++ b/cdist/conf/type/__jail/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Jake Guffey (jake.guffey at eprotex.com) # @@ -35,16 +35,15 @@ fi jaildir="$(cat "$__object/parameter/jaildir")" -__directory ${jaildir} --parents +__directory "${jaildir}" --parents -set -- "$@" "$__object_id" "--state" "$state" +set -- "$@" "$__object_id" cd "$__object/parameter" -for property in $(ls .); do +for property in *; do set -- "$@" "--$property" "$(cat "$property")" done -ver="$(cat "$__global/explorer/os_version")" -if [ -n "$(echo "$ver" | grep '^10\.' )" ]; then # Version is 10.x +if grep -q '^10\.' "$(cat "$__global/explorer/os_version")" ; then # Version is 10.x __jail_freebsd10 "$@" else __jail_freebsd9 "$@" diff --git a/cdist/conf/type/__jail_freebsd10/explorer/status b/cdist/conf/type/__jail_freebsd10/explorer/status index 1ceba212..c8039f21 100755 --- a/cdist/conf/type/__jail_freebsd10/explorer/status +++ b/cdist/conf/type/__jail_freebsd10/explorer/status @@ -39,7 +39,7 @@ fi # backslash-escaped $jaildir sjaildir="$(echo ${jaildir} | sed 's#/#\\/#g')" -jls_output="$(jls | grep "[ ]${sjaildir}\/${name}\$")" || true +jls_output="$(jls | grep "[ ]${sjaildir}\\/${name}\$")" || true if [ -n "${jls_output}" ]; then echo "STARTED" diff --git a/cdist/conf/type/__jail_freebsd10/gencode-local b/cdist/conf/type/__jail_freebsd10/gencode-local index 8c1687a9..f163cad3 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-local +++ b/cdist/conf/type/__jail_freebsd10/gencode-local @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Jake Guffey (jake.guffey at eprotex.com) # @@ -44,7 +44,7 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__jail_freebsd10/gencode-remote b/cdist/conf/type/__jail_freebsd10/gencode-remote index ae68616d..4f376c25 100755 --- a/cdist/conf/type/__jail_freebsd10/gencode-remote +++ b/cdist/conf/type/__jail_freebsd10/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012,2014,2016 Jake Guffey (jake.guffey at jointheirstm.org) # @@ -36,7 +36,7 @@ state="$(cat "$__object/parameter/state")" started="true" # If the user wants the jail gone, it implies it shouldn't be started. -[ -f "$__object/parameter/stopped" -o "$state" = "absent" ] && started="false" +{ [ -f "$__object/parameter/stopped" ] || [ "$state" = "absent" ]; } && started="false" if [ -f "$__object/parameter/ip" ]; then ip="$(cat "$__object/parameter/ip")" @@ -45,7 +45,7 @@ else # when $state=present, it's required. Enforce this. if [ "$state" = "present" ]; then exec >&2 - echo "If --state is 'present,' --ip must be given\!" + printf 'If --state is "present", --ip must be given\!\n' exit 1 fi fi @@ -66,7 +66,7 @@ devfsruleset="$(cat "$__object/parameter/devfs-ruleset")" # devfs_ruleset being defined without devfs_enable being true # is pointless. Treat this as an error. -if [ -n "$devfsruleset" -a "$devfsenable" = "false" ]; then +if [ -n "$devfsruleset" ] && [ "$devfsenable" = "false" ]; then exec >&2 echo "Can't have --devfs-ruleset defined with --devfs-disable" exit 1 @@ -83,12 +83,12 @@ present="$(cat "$__object/explorer/present")" status="$(cat "$__object/explorer/status")" # Handle ip="addr, addr" format -if [ $(expr "${ip}" : ".*, .*") -gt "0" ]; then +if [ "$(expr "${ip}" : ".*, .*")" -gt "0" ]; then SAVE_IFS="$IFS" IFS=", " for cur_ip in ${ip}; do # Just get the last IP address for SSH to listen on - mgmt_ip=$(echo "${ip}" | cut '-d ' -f1) # In case using "ip netmask" format rather than CIDR + mgmt_ip=$(echo "${cur_ip}" | cut '-d ' -f1) # In case using "ip netmask" format rather than CIDR done IFS="$SAVE_IFS" else @@ -114,19 +114,19 @@ startJail() { deleteJail() { # Unmount the jail's mountpoints if necessary cat <=1 rw mount is mounted still - for DIR in "${output}"; do - umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print $3}')" + for DIR in "\${output}"; do + umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print \$3}')" done fi - output="\$(mount | grep "\/${name} (")" || true + output="\$(mount | grep "\\/${name} (")" || true if [ -n "\${output}" ]; then # ro mount is mounted still - umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print $3}')" + umount -F "/etc/fstab.${name}" "\$(echo "\${output}" | awk '{print \$3}')" fi EOF # Remove the jail's rw mountpoints @@ -275,9 +275,9 @@ cat <&1 >/dev/null # Close the FD==fail... @@ -290,7 +290,7 @@ add include \\\$devfsrules_unhide_basic add include \\\$devfsrules_unhide_login END fi - devfsruleset_num=\$(grep "\[${devfsruleset}=" /etc/devfs.rules | sed -n 's/\[.*=\([0-9]*\)\]/\1/pg') + devfsruleset_num=\$(grep "\\[${devfsruleset}=" /etc/devfs.rules | sed -n 's/\\[.*=\\([0-9]*\\)\\]/\\1/pg') if [ -n "\$devfsruleset_num" ]; then jaildata="\$jaildata devfs_ruleset=\"\${devfsruleset_num}\";" @@ -298,8 +298,8 @@ END fi EOF - - echo "printf \"%s\\n%s\n%s\n\" \"\$jailheader\" \"\$jaildata\" \"\$jailtrailer\" >>\"\$jailfile\"" + # shellcheck disable=SC2028 + echo "printf \"%s\\n%s\\n%s\\n\" \"\$jailheader\" \"\$jaildata\" \"\$jailtrailer\" >>\"\$jailfile\"" # Add $name to jail_list if $onboot=yes if [ "$onboot" = "yes" ]; then diff --git a/cdist/conf/type/__jail_freebsd9/explorer/status b/cdist/conf/type/__jail_freebsd9/explorer/status index 1ceba212..c8039f21 100755 --- a/cdist/conf/type/__jail_freebsd9/explorer/status +++ b/cdist/conf/type/__jail_freebsd9/explorer/status @@ -39,7 +39,7 @@ fi # backslash-escaped $jaildir sjaildir="$(echo ${jaildir} | sed 's#/#\\/#g')" -jls_output="$(jls | grep "[ ]${sjaildir}\/${name}\$")" || true +jls_output="$(jls | grep "[ ]${sjaildir}\\/${name}\$")" || true if [ -n "${jls_output}" ]; then echo "STARTED" diff --git a/cdist/conf/type/__jail_freebsd9/gencode-local b/cdist/conf/type/__jail_freebsd9/gencode-local index 67420a6f..bbdc9fcc 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-local +++ b/cdist/conf/type/__jail_freebsd9/gencode-local @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Jake Guffey (jake.guffey at eprotex.com) # @@ -40,7 +40,7 @@ basepresent="$(cat "$__object/explorer/basepresent")" if [ "$state" = "present" ]; then if [ "$basepresent" = "NONE" ]; then # IPv6 fix - if $(echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$') + if echo "${__target_host}" | grep -q -E '^[0-9a-fA-F:]+$' then my_target_host="[${__target_host}]" else diff --git a/cdist/conf/type/__jail_freebsd9/gencode-remote b/cdist/conf/type/__jail_freebsd9/gencode-remote index 6a4c64de..68229d3e 100755 --- a/cdist/conf/type/__jail_freebsd9/gencode-remote +++ b/cdist/conf/type/__jail_freebsd9/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012,2014,2016 Jake Guffey (jake.guffey at jointheirstm.org) # @@ -36,7 +36,7 @@ state="$(cat "$__object/parameter/state")" started="true" # If the user wants the jail gone, it implies it shouldn't be started. -[ -f "$__object/parameter/stopped" -o "$state" = "absent" ] && started="false" +{ [ -f "$__object/parameter/stopped" ] || [ "$state" = "absent" ]; } && started="false" if [ -f "$__object/parameter/ip" ]; then ip="$(cat "$__object/parameter/ip")" @@ -45,7 +45,7 @@ else # when $state=present, it's required. Enforce this. if [ "$state" = "present" ]; then exec >&2 - echo "If --state is 'present,' --ip must be given\!" + printf 'If --state is "present", --ip must be given\!\n' exit 1 fi fi @@ -70,7 +70,7 @@ devfsruleset="$(cat "$__object/parameter/devfs-ruleset")" # devfs_ruleset being defined without devfs_enable being true # is pointless. Treat this as an error. -if [ -n "$devfsruleset" -a "$devfsenable" = "false" ]; then +if [ -n "$devfsruleset" ] && [ "$devfsenable" = "false" ]; then exec >&2 echo "Can't have --devfs-ruleset defined with --devfs-disable" exit 1 @@ -86,14 +86,14 @@ present="$(cat "$__object/explorer/present")" status="$(cat "$__object/explorer/status")" # Handle ip="iface|addr, iface|addr" format -if [ $(expr "${ip}" : ".*|.*") -gt "0" ]; then +if [ "$(expr "${ip}" : ".*|.*")" -gt "0" ]; then # If we have multiple IPs defined, $interface doesn't make sense because ip="iface|addr, iface|addr" implies it interface="" SAVE_IFS="$IFS" IFS=", " for cur_ip in ${ip}; do # Just get the last IP address for SSH to listen on - mgmt_ip=$(echo "${ip}" | sed -E -e 's/^.*\|(.*)\/[0-9]+$/\1/') + mgmt_ip=$(echo "${cur_ip}" | sed -E -e 's/^.*\|(.*)\/[0-9]+$/\1/') done IFS="$SAVE_IFS" else @@ -119,19 +119,19 @@ startJail() { deleteJail() { # Unmount the jail's mountpoints if necessary cat <=1 rw mount is mounted still - for DIR in "${output}"; do - umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print $3}')" + for DIR in "\${output}"; do + umount -F "/etc/fstab.${name}" "\$(echo "${DIR}" | awk '{print \$3}')" done fi - output="\$(mount | grep "\/${name} (")" || true + output="\$(mount | grep "\\/${name} (")" || true if [ -n "\${output}" ]; then # ro mount is mounted still - umount -F "/etc/fstab.${name}" "\$(echo "${output}" | awk '{print $3}')" + umount -F "/etc/fstab.${name}" "\$(echo "\${output}" | awk '{print \$3}')" fi EOF # Remove the jail's rw mountpoints @@ -279,9 +279,9 @@ END if [ ! -f /etc/devfs.rules ]; then touch /etc/devfs.rules fi - if [ -z "\$(grep '\[jailrules=' /etc/devfs.rules)" ]; then # The default ruleset doesn't exist + if [ -z "\$(grep '\\[jailrules=' /etc/devfs.rules)" ]; then # The default ruleset doesn't exist # Get the highest-numbered ruleset - highest="\$(sed -n 's/\[.*=\([0-9]*\)\]/\1/pg' /etc/devfs.rules | sort -u | tail -n 1)" || true + highest="\$(sed -n 's/\\[.*=\\([0-9]*\\)\\]/\\1/pg' /etc/devfs.rules | sort -u | tail -n 1)" || true # increment by 1 let num="\${highest}+1" 2>&- >&- # add default ruleset diff --git a/cdist/conf/type/__key_value/explorer/state b/cdist/conf/type/__key_value/explorer/state index b990733d..7b2de1df 100755 --- a/cdist/conf/type/__key_value/explorer/state +++ b/cdist/conf/type/__key_value/explorer/state @@ -19,9 +19,9 @@ # along with cdist. If not, see . # -export key="$(cat "$__object/parameter/key" 2>/dev/null \ +key="$(cat "$__object/parameter/key" 2>/dev/null \ || echo "$__object_id")" -export state="$(cat "$__object/parameter/state")" +state="$(cat "$__object/parameter/state")" file="$(cat "$__object/parameter/file")" @@ -30,14 +30,15 @@ if [ ! -f "$file" ]; then exit fi -export delimiter="$(cat "$__object/parameter/delimiter")" -export value="$(cat "$__object/parameter/value" 2>/dev/null \ +delimiter="$(cat "$__object/parameter/delimiter")" +value="$(cat "$__object/parameter/value" 2>/dev/null \ || echo "__CDIST_NOTSET__")" if [ -f "$__object/parameter/exact_delimiter" ]; then - export exact_delimiter=1 + exact_delimiter=1 else - export exact_delimiter=0 + exact_delimiter=0 fi +export key state delimiter value exact_delimiter awk -f - "$file" <<"AWK_EOF" BEGIN { diff --git a/cdist/conf/type/__key_value/files/remote_script.sh b/cdist/conf/type/__key_value/files/remote_script.sh index 52b3f2de..f7a1add5 100644 --- a/cdist/conf/type/__key_value/files/remote_script.sh +++ b/cdist/conf/type/__key_value/files/remote_script.sh @@ -1,19 +1,21 @@ #!/bin/sh -export key="$(cat "$__object/parameter/key" 2>/dev/null \ +key="$(cat "$__object/parameter/key" 2>/dev/null \ || echo "$__object_id")" -export state="$(cat "$__object/parameter/state")" +state="$(cat "$__object/parameter/state")" file="$(cat "$__object/parameter/file")" -export delimiter="$(cat "$__object/parameter/delimiter")" -export value="$(cat "$__object/parameter/value" 2>/dev/null \ +delimiter="$(cat "$__object/parameter/delimiter")" +value="$(cat "$__object/parameter/value" 2>/dev/null \ || echo "__CDIST_NOTSET__")" +export key state delimiter value if [ -f "$__object/parameter/exact_delimiter" ]; then - export exact_delimiter=1 + exact_delimiter=1 else - export exact_delimiter=0 + exact_delimiter=0 fi +export exact_delimiter tmpfile=$(mktemp "${file}.cdist.XXXXXXXXXX") # preserve ownership and permissions by copying existing file over tmpfile diff --git a/cdist/conf/type/__key_value/gencode-remote b/cdist/conf/type/__key_value/gencode-remote index e6815cb6..13cc27c7 100755 --- a/cdist/conf/type/__key_value/gencode-remote +++ b/cdist/conf/type/__key_value/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2012-2014 Nico Schottelius (nico-cdist at schottelius.org) @@ -23,13 +23,14 @@ state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" +fire_onchange='' if [ "$state_is" = "$state_should" ]; then exit 0 fi # here we check only if the states are valid, -# emmit messages and +# emit messages and # let awk do the work ... case "$state_should" in absent) @@ -39,6 +40,7 @@ case "$state_should" in ;; wrongformat|wrongvalue|present) echo "remove" >> "$__messages_out" + fire_onchange=1 ;; *) echo "Unknown explorer state: $state_is" >&2 @@ -50,12 +52,15 @@ case "$state_should" in case "$state_is" in nosuchfile) echo "create" >> "$__messages_out" + fire_onchange=1 ;; absent) echo "insert" >> "$__messages_out" + fire_onchange=1 ;; wrongformated|wrongvalue) echo "change" >> "$__messages_out" + fire_onchange=1 ;; present) # nothing to do @@ -67,9 +72,13 @@ case "$state_should" in esac ;; *) - echo "Unknown state: $state_should" >&2 - exit 1 + echo "Unknown state: $state_should" >&2 + exit 1 ;; esac cat "$__type/files/remote_script.sh" + +if [ -n "$fire_onchange" ]; then + cat "$__object/parameter/onchange" +fi diff --git a/cdist/conf/type/__key_value/man.rst b/cdist/conf/type/__key_value/man.rst index f069d989..34e4aab2 100644 --- a/cdist/conf/type/__key_value/man.rst +++ b/cdist/conf/type/__key_value/man.rst @@ -34,6 +34,8 @@ comment but only if the key or value must be changed. You need to ensure yourself that the line is prefixed with the correct comment sign. (for example # or ; or wathever ..) +onchange + The code to run if the key or value changes (i.e. is inserted, removed or replaced). BOOLEAN PARAMETERS diff --git a/cdist/conf/type/__key_value/manifest b/cdist/conf/type/__key_value/manifest index 56f4c874..5a91f60c 100755 --- a/cdist/conf/type/__key_value/manifest +++ b/cdist/conf/type/__key_value/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2012 Nico Schottelius (nico-cdist at schottelius.org) @@ -21,7 +21,7 @@ state_should="$(cat "$__object/parameter/state")" -if [ "$state_should" = "present" -a ! -f "$__object/parameter/value" ]; then +if [ "$state_should" = "present" ] && [ ! -f "$__object/parameter/value" ]; then echo "Missing required parameter 'value'" >&2 exit 1 fi diff --git a/cdist/conf/type/__key_value/parameter/default/onchange b/cdist/conf/type/__key_value/parameter/default/onchange new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__key_value/parameter/optional b/cdist/conf/type/__key_value/parameter/optional index 666be2ae..d4b8cac0 100644 --- a/cdist/conf/type/__key_value/parameter/optional +++ b/cdist/conf/type/__key_value/parameter/optional @@ -2,3 +2,4 @@ key value state comment +onchange diff --git a/cdist/conf/type/__keyboard/manifest b/cdist/conf/type/__keyboard/manifest old mode 100644 new mode 100755 index 3bfddf0b..80cd4819 --- a/cdist/conf/type/__keyboard/manifest +++ b/cdist/conf/type/__keyboard/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # Carlos Ortigoza (carlos.ortigoza at ungleich.ch) # diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path new file mode 100755 index 00000000..3c6076df --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certbot-path @@ -0,0 +1,3 @@ +#!/bin/sh -e + +command -v certbot 2>/dev/null || true diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains new file mode 100755 index 00000000..db605b63 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-domains @@ -0,0 +1,8 @@ +#!/bin/sh -e + +certbot_path=$("${__type_explorer}/certbot-path") +if [ -n "${certbot_path}" ] +then + certbot certificates --cert-name "${__object_id:?}" | grep ' Domains: ' | \ + cut -d ' ' -f 6- | tr ' ' '\n' +fi diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists new file mode 100755 index 00000000..4e6f44db --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-exists @@ -0,0 +1,13 @@ +#!/bin/sh -e + +certbot_path=$("${__type_explorer}/certbot-path") +if [ -n "${certbot_path}" ] +then + if certbot certificates | grep -q " Certificate Name: ${__object_id:?}$"; then + echo yes + else + echo no + fi +else + echo no +fi diff --git a/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test new file mode 100755 index 00000000..9b445059 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/explorer/certificate-is-test @@ -0,0 +1,14 @@ +#!/bin/sh -e + +certbot_path=$("${__type_explorer}/certbot-path") +if [ -n "${certbot_path}" ] +then + if certbot certificates --cert-name "${__object_id:?}" | \ + grep -q 'INVALID: TEST_CERT'; then + echo yes + else + echo no + fi +else + echo no +fi diff --git a/cdist/conf/type/__letsencrypt_cert/gencode-remote b/cdist/conf/type/__letsencrypt_cert/gencode-remote new file mode 100755 index 00000000..375570a4 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/gencode-remote @@ -0,0 +1,82 @@ +#!/bin/sh -e + +certificate_exists=$(cat "${__object:?}/explorer/certificate-exists") +name="${__object_id:?}" +state=$(cat "${__object}/parameter/state") + +case "${state}" in + absent) + if [ "${certificate_exists}" = "no" ]; then + exit 0 + fi + + echo "certbot delete --cert-name '${name}' --quiet" + + echo remove >> "${__messages_out:?}" + ;; + present) + domain_param_file="${__object}/parameter/domain" + requested_domains=$(mktemp "${TMPDIR:-/tmp}/domain.cdist.XXXXXXXXXX") + if [ -f "${domain_param_file}" ]; then + cp "${domain_param_file}" "${requested_domains}" + else + echo "$__object_id" >> "${requested_domains}" + fi + + staging=no + if [ -f "${__object}/parameter/staging" ]; then + staging=yes + fi + + if [ "${certificate_exists}" = "yes" ]; then + existing_domains="${__object}/explorer/certificate-domains" + certificate_is_test=$(cat "${__object}/explorer/certificate-is-test") + + sort -uo "${requested_domains}" "${requested_domains}" + sort -uo "${existing_domains}" "${existing_domains}" + + if [ -z "$(comm -23 "${requested_domains}" "${existing_domains}")" ] && \ + [ "${certificate_is_test}" = "${staging}" ]; then + exit 0 + fi + fi + + admin_email="$(cat "$__object/parameter/admin-email")" + webroot="$(cat "$__object/parameter/webroot")" + + cat <<-EOF + certbot certonly \ + --agree-tos \ + --cert-name '${name}' \ + --email '${admin_email}' \ + --expand \ + --non-interactive \ + --quiet \ + $(if [ "${staging}" = "yes" ]; then + echo "--staging" + elif [ "${certificate_is_test}" != "${staging}" ]; then + echo "--force-renewal" + fi) \ + $(if [ -z "${webroot}" ]; then + echo "--standalone" + else + echo "--webroot --webroot-path '${webroot}'" + fi) \ + $(while read -r domain; do + echo "--domain '${domain}' \\" + done < "${requested_domains}") + EOF + rm -f "${requested_domains}" + + if [ "${certificate_exists}" = "no" ]; then + echo create >> "${__messages_out}" + else + echo change >> "${__messages_out}" + fi + ;; + *) + echo "Unsupported state: ${state}" >&2 + + exit 1 + ;; +esac diff --git a/cdist/conf/type/__letsencrypt_cert/man.rst b/cdist/conf/type/__letsencrypt_cert/man.rst new file mode 100644 index 00000000..c4ffc6bc --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/man.rst @@ -0,0 +1,109 @@ +cdist-type__letsencrypt_cert(7) +=============================== + +NAME +---- + +cdist-type__letsencrypt_cert - Get an SSL certificate from Let's Encrypt + +DESCRIPTION +----------- + +Automatically obtain a Let's Encrypt SSL certificate using Certbot. + +REQUIRED PARAMETERS +------------------- + +object id + A cert name. If domain parameter is not specified then it is used + as a domain to be included in the certificate. + +admin-email + Where to send Let's Encrypt emails like "certificate needs renewal". + +OPTIONAL PARAMETERS +------------------- + +state + 'present' or 'absent', defaults to 'present' where: + + present + if the certificate does not exist, it will be obtained + absent + the certificate will be removed + +webroot + The path to your webroot, as set up in your webserver config. If this + parameter is not present, Certbot will be run in standalone mode. + +OPTIONAL MULTIPLE PARAMETERS +---------------------------- + +renew-hook + Renew hook command directly passed to Certbot in cron job. + +domain + Domains to be included in the certificate. When specified then object id + is not used as a domain. + +BOOLEAN PARAMETERS +------------------ + +automatic-renewal + Install a cron job, which attempts to renew certificates daily. + +staging + Obtain a test certificate from a staging server. + +MESSAGES +-------- + +change + Certificte was changed. + +create + Certificte was created. + +remove + Certificte was removed. + +EXAMPLES +-------- + +.. code-block:: sh + + # use object id as domain + __letsencrypt_cert example.com \ + --admin-email root@example.com \ + --automatic-renewal \ + --renew-hook "service nginx reload" \ + --webroot /data/letsencrypt/root + +.. code-block:: sh + + # domain parameter is specified so object id is not used as domain + # and example.com needs to be included again with domain parameter + __letsencrypt_cert example.com \ + --admin-email root@example.com \ + --automatic-renewal \ + --domain example.com \ + --domain foo.example.com \ + --domain bar.example.com \ + --renew-hook "service nginx reload" \ + --webroot /data/letsencrypt/root + +AUTHORS +------- + +| Nico Schottelius +| Kamila Součková +| Darko Poljak +| Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2017-2018 Nico Schottelius, Kamila Součková, Darko Poljak and +Ľubomír Kučera. You can redistribute it and/or modify it under the terms of +the GNU General Public License as published by the Free Software Foundation, +either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__letsencrypt_cert/manifest b/cdist/conf/type/__letsencrypt_cert/manifest new file mode 100755 index 00000000..d598949e --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/manifest @@ -0,0 +1,105 @@ +#!/bin/sh + +certbot_fullpath="$(cat "${__object:?}/explorer/certbot-path")" + +if [ -z "${certbot_fullpath}" ]; then + os="$(cat "${__global:?}/explorer/os")" + os_version="$(cat "${__global}/explorer/os_version")" + + case "$os" in + debian) + case "$os_version" in + 8*) + __apt_source jessie-backports \ + --uri http://http.debian.net/debian \ + --distribution jessie-backports \ + --component main + + require="__apt_source/jessie-backports" __package_apt python-certbot \ + --target-release jessie-backports + require="__apt_source/jessie-backports" __package_apt certbot \ + --target-release jessie-backports + # Seems to be a missing dependency on debian 8 + __package python-ndg-httpsclient + ;; + 9*) + __apt_source stretch-backports \ + --uri http://http.debian.net/debian \ + --distribution stretch-backports \ + --component main + + require="__apt_source/stretch-backports" __package_apt python-certbot \ + --target-release stretch-backports + require="__apt_source/stretch-backports" __package_apt certbot \ + --target-release stretch-backports + ;; + *) + echo "Unsupported OS version: $os_version" >&2 + exit 1 + ;; + esac + + certbot_fullpath=/usr/bin/certbot + ;; + devuan) + case "$os_version" in + jessie) + __apt_source jessie-backports \ + --uri http://auto.mirror.devuan.org/merged \ + --distribution jessie-backports \ + --component main + + require="__apt_source/jessie-backports" __package_apt python-certbot \ + --target-release jessie-backports + require="__apt_source/jessie-backports" __package_apt certbot \ + --target-release jessie-backports + # Seems to be a missing dependency on debian 8 + __package python-ndg-httpsclient + ;; + ascii*) + __apt_source ascii-backports \ + --uri http://auto.mirror.devuan.org/merged \ + --distribution ascii-backports \ + --component main + + require="__apt_source/ascii-backports" __package_apt certbot \ + --target-release ascii-backports + ;; + bewoulf*) + __package_apt certbot + ;; + *) + echo "Unsupported OS version: $os_version" >&2 + exit 1 + ;; + esac + + certbot_fullpath=/usr/bin/certbot + ;; + freebsd) + __package py27-certbot + + certbot_fullpath=/usr/local/bin/certbot + ;; + *) + echo "Unsupported os: $os" >&2 + exit 1 + ;; + esac +fi + +if [ -f "${__object}/parameter/automatic-renewal" ]; then + renew_hook_param="${__object}/parameter/renew-hook" + renew_hook="" + if [ -f "${renew_hook_param}" ]; then + while read -r hook; do + renew_hook="${renew_hook} --renew-hook \"${hook}\"" + done < "${renew_hook_param}" + fi + + __cron letsencrypt-certbot \ + --user root \ + --command "${certbot_fullpath} renew -q ${renew_hook}" \ + --hour 0 \ + --minute 47 +fi diff --git a/cdist/conf/type/__letsencrypt_cert/nonparallel b/cdist/conf/type/__letsencrypt_cert/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/boolean b/cdist/conf/type/__letsencrypt_cert/parameter/boolean new file mode 100644 index 00000000..d5b8be99 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/boolean @@ -0,0 +1,2 @@ +automatic-renewal +staging diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/default/state b/cdist/conf/type/__letsencrypt_cert/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/default/webroot b/cdist/conf/type/__letsencrypt_cert/parameter/default/webroot new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/optional b/cdist/conf/type/__letsencrypt_cert/parameter/optional new file mode 100644 index 00000000..0a63b11e --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/optional @@ -0,0 +1,2 @@ +state +webroot diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple new file mode 100644 index 00000000..0e866d45 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/optional_multiple @@ -0,0 +1,2 @@ +domain +renew-hook diff --git a/cdist/conf/type/__letsencrypt_cert/parameter/required b/cdist/conf/type/__letsencrypt_cert/parameter/required new file mode 100644 index 00000000..bfe77226 --- /dev/null +++ b/cdist/conf/type/__letsencrypt_cert/parameter/required @@ -0,0 +1 @@ +admin-email diff --git a/cdist/conf/type/__line/explorer/state b/cdist/conf/type/__line/explorer/state index d04d5d09..2ef252c8 100755 --- a/cdist/conf/type/__line/explorer/state +++ b/cdist/conf/type/__line/explorer/state @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2018 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -17,26 +17,79 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -file="/$__object_id" -[ -f "$__object/parameter/file" ] && file=$(cat "$__object/parameter/file") +if [ -f "$__object/parameter/before" ]; then + position="before" +elif [ -f "$__object/parameter/after" ]; then + position="after" +else + # By default we append to the end of the file. + position="end" +fi if [ -f "$__object/parameter/regex" ]; then - regex=$(cat "$__object/parameter/regex") - greparg="" + needle="regex" else - if [ ! -f "$__object/parameter/line" ]; then - echo "Parameter line and regex missing - cannot explore" >&2 - exit 1 - fi - regex="$(cat "$__object/parameter/line")" - greparg="-F -x" + needle="line" fi -# Allow missing file - thus 2>/dev/null -if grep -q $greparg "$regex" "$file" 2>/dev/null; then - echo present +if [ -f "$__object/parameter/file" ]; then + file="$(cat "$__object/parameter/file")" else - echo absent + file="/$__object_id" fi + +if [ ! -f "$file" ]; then + echo "file_missing" + exit 0 +fi + +awk -v position="$position" -v needle="$needle" ' +function _find(_text, _pattern) { + if (needle == "regex") { + return match(_text, _pattern) + } else { + return index(_text, _pattern) + } +} +BEGIN { + getline anchor < (ENVIRON["__object"] "/parameter/" position) + getline pattern < (ENVIRON["__object"] "/parameter/" needle) + state = "absent" +} +{ + if (position == "after") { + if (match($0, anchor)) { + getline + if (_find($0, pattern)) { + state = "present" + } + else { + state = "wrongposition" + } + exit 0 + } + } + else if (position == "before") { + if (_find($0, pattern)) { + getline + if (match($0, anchor)) { + state = "present" + } + else { + state = "wrongposition" + } + exit 0 + } + } + else { + if (_find($0, pattern)) { + state = "present" + exit 0 + } + } +} +END { + print state +} +' "$file" diff --git a/cdist/conf/type/__line/gencode-remote b/cdist/conf/type/__line/gencode-remote index f73444e3..03e90c1b 100755 --- a/cdist/conf/type/__line/gencode-remote +++ b/cdist/conf/type/__line/gencode-remote @@ -1,7 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2012 Nico Schottelius (nico-cdist at schottelius.org) -# 2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2018 Steven Armstrong (steven-cdist at armstrong.cc) # # This file is part of cdist. # @@ -18,74 +17,112 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # -# -file="/$__object_id" -regex="" -state_should="present" -[ -f "$__object/parameter/file" ] && file=$(cat "$__object/parameter/file") -[ -f "$__object/parameter/regex" ] && regex=$(cat "$__object/parameter/regex") -[ -f "$__object/parameter/state" ] && state_should=$(cat "$__object/parameter/state") -[ -f "$__object/parameter/line" ] && line=$(cat "$__object/parameter/line") +if [ -f "$__object/parameter/before" ] && [ -f "$__object/parameter/after" ]; then + echo "Use either --before OR --after but not both." >&2 + exit 1 +fi +state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" -[ "$state_should" = "$state_is" ] && exit 0 +if [ "$state_should" = "$state_is" ]; then + # nothing to do + exit 0 +fi +if [ -f "$__object/parameter/before" ]; then + position="before" +elif [ -f "$__object/parameter/after" ]; then + position="after" +else + # By default we append to the end of the file. + position="end" +fi + +if [ -f "$__object/parameter/regex" ]; then + needle="regex" +else + needle="line" +fi + +if [ -f "$__object/parameter/file" ]; then + file="$(cat "$__object/parameter/file")" +else + file="/$__object_id" +fi + +add=0 +remove=0 case "$state_should" in - present) - if [ ! "$line" ]; then - echo "Required parameter \"line\" is missing" >&2 - exit 1 - fi + present) + if [ "$state_is" = "wrongposition" ]; then + echo updated >> "$__messages_out" + remove=1 + else + echo added >> "$__messages_out" + fi + add=1 + ;; + absent) + echo removed >> "$__messages_out" + remove=1 + ;; +esac - #echo "echo \"$line\" >> $file" - #line_sanitised=$(cat "$__object/parameter/line" | sed 's/"/\"/g') - # Idea: replace ' in the string: - # '"'"' - # |------> ': end the string - # |-|---> "'": create ' in the output string - # |--> ': continue the string - # - # Replace all \ so \t and other combinations are not interpreted - # - - - # line_sanitised=$(cat "$__object/parameter/line" | sed -e "s/'/'\"'\"'/g" -e 's/\\/\\\\/g') - # The one above does not work: - # --line "PS1='[\t] \[\033[1m\]\h\[\033[0m\]:\w\\$ '" - # becomes - # PS1='[\\t] \\[\\033[1m\\]\\h\\[\\033[0m\\]:\\w\\$ ' - - # Only replace ' with '"'"' and keep \ as they are - line_sanitised=$(cat "$__object/parameter/line" | sed -e "s/'/'\"'\"'/g") - printf '%s' "printf '%s\n' '$line_sanitised' >> $file" - - ;; - absent) - if [ "$regex" -a "$line" ]; then - echo "Mutally exclusive parameters regex and line given for state absent" >&2 - exit 1 - fi - - greparg="" - if [ "$line" ]; then - regex="$line" - greparg="-F -x" - fi - - cat << eof +cat << DONE tmpfile=\$(mktemp ${file}.cdist.XXXXXXXXXX) # preserve ownership and permissions of existing file if [ -f "$file" ]; then cp -p "$file" "\$tmpfile" fi -grep -v $greparg '$regex' '$file' > \$tmpfile || true + +awk -v position="$position" -v needle="$needle" -v remove=$remove -v add=$add ' +function _find(_text, _pattern) { + if (needle == "regex") { + return match(_text, _pattern) + } else { + return index(_text, _pattern) + } +} +BEGIN { + line_file = ENVIRON["__object"] "/parameter/line" + getline line < line_file + # Need to close line file as it may be re-read as pattern below. + close(line_file) + getline pattern < (ENVIRON["__object"] "/parameter/" needle) + getline anchor < (ENVIRON["__object"] "/parameter/" position) +} +{ + if (remove) { + if (_find(\$0, pattern)) { + # skip over this line -> remove it + next + } + } + if (add) { + if (anchor && match(\$0, anchor)) { + if (position == "before") { + print line + print + } else if (position == "after") { + print + print line + } + next + } + } + print +} +END { + if (add && position == "end") { + print line + } +} +' "$file" > "\$tmpfile" mv -f "\$tmpfile" "$file" -eof - ;; - *) - echo "Unknown state: $state_should" >&2 - exit 1 - ;; -esac +DONE + +if [ -f "$__object/parameter/onchange" ]; then + cat "$__object/parameter/onchange" +fi diff --git a/cdist/conf/type/__line/man.rst b/cdist/conf/type/__line/man.rst index e6adce9c..f76cab64 100644 --- a/cdist/conf/type/__line/man.rst +++ b/cdist/conf/type/__line/man.rst @@ -13,65 +13,104 @@ This cdist type allows you to add lines and remove lines from files. REQUIRED PARAMETERS ------------------- +None. + OPTIONAL PARAMETERS ------------------- -state - 'present' or 'absent', defaults to 'present' +after + Insert the given line after this pattern. -line - Specifies the line which should be absent or present - - Must be present, if state is present. - Must not be combined with regex, if state is absent. - -regex - If state is present, search for this pattern and add - given line, if the given regular expression does not match. - - In case of absent, ensure all lines matching the - regular expression are absent. - - The regular expression is interpreted by grep. - - Must not be combined with line, if state is absent. +before + Insert the given line before this pattern. file If supplied, use this as the destination file. Otherwise the object_id is used. +line + Specifies the line which should be absent or present. + + Must be present, if state is 'present'. + Ignored if regex is given and state is 'absent'. + +regex + If state is 'present', search for this pattern and if it matches add + the given line. + + If state is 'absent', ensure all lines matching the regular expression + are absent. + + The regular expression is interpreted by awk's match function. + +state + 'present' or 'absent', defaults to 'present' + +onchange + The code to run if line is added, removed or updated. + + +BOOLEAN PARAMETERS +------------------ +None. + + +MESSAGES +-------- +added + The line was added. + +updated + The line or its position was changed. + +removed + The line was removed. + EXAMPLES -------- .. code-block:: sh - # Manage the DAEMONS line in rc.conf - __line daemons --file /etc/rc.conf --line 'DAEMONS=(hwclock !network sshd crond postfix)' + # Manage a hosts entry for www.example.com. + __line /etc/hosts \ + --line '127.0.0.2 www.example.com' - # Ensure the home mount is present in /etc/fstab - explicitly make it present - __line home-fstab \ - --file /etc/fstab \ - --line 'filer.fs:/vol/home /home nfs defaults 0 0' \ - --state present + # Manage another hosts entry for test.example.com. + __line hosts:test.example.com \ + --file /etc/hosts \ + --line '127.0.0.3 test.example.com' - # Removes the line specifiend in "include_www" from the file "lighttpd.conf" - __line legacy_timezone --file /etc/rc.conf --regex 'TIMEZONE=.*' --state absent + # Remove the line starting with TIMEZONE from the /etc/rc.conf file. + __line legacy_timezone \ + --file /etc/rc.conf \ + --regex 'TIMEZONE=.*' \ + --state absent + + # Insert a line before another one. + __line password-auth-local:classify \ + --file /etc/pam.d/password-auth-local \ + --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \ + --before '^session[[:space:]]+include[[:space:]]+password-auth-ac$' + + # Insert a line after another one. + __line password-auth-local:classify \ + --file /etc/pam.d/password-auth-local \ + --line '-session required pam_exec.so debug log=/tmp/classify.log /usr/local/libexec/classify' \ + --after '^session[[:space:]]+include[[:space:]]+password-auth-ac$' SEE ALSO -------- -:strong:`grep`\ (1) +:strong:`cdist-type`\ (7) AUTHORS ------- -Nico Schottelius +Steven Armstrong COPYING ------- -Copyright \(C) 2012-2013 Nico Schottelius. You can redistribute it -and/or modify it under the terms of the GNU General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. +Copyright \(C) 2018 Steven Armstrong. Free use of this software is +granted under the terms of the GNU General Public License version 3 (GPLv3). diff --git a/cdist/conf/type/__line/parameter/default/state b/cdist/conf/type/__line/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__line/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__line/parameter/optional b/cdist/conf/type/__line/parameter/optional index 604a203e..1c34c699 100644 --- a/cdist/conf/type/__line/parameter/optional +++ b/cdist/conf/type/__line/parameter/optional @@ -1,4 +1,7 @@ -state -regex +after +before file line +regex +state +onchange diff --git a/cdist/conf/type/__link/explorer/state b/cdist/conf/type/__link/explorer/state index b8d8fc2b..7150df25 100755 --- a/cdist/conf/type/__link/explorer/state +++ b/cdist/conf/type/__link/explorer/state @@ -32,9 +32,9 @@ destination_dir="${destination%/*}" case "$type" in symbolic) - cd "$destination_dir" - source_is=$(ls -l "$destination" | sed 's/.*-> //g') + cd "$destination_dir" || exit 1 if [ -h "$destination" ]; then + source_is=$(readlink "$destination") # ignore trailing slashes for comparison if [ "${source_is%/}" = "${source%/}" ]; then echo present @@ -46,13 +46,19 @@ case "$type" in fi ;; hard) - cd "$destination_dir" + cd "$destination_dir" || exit 1 # check source relative to destination_dir if [ ! -e "$source" ]; then echo sourcemissing exit 0 fi + # Currently not worth the effor to change it, stat is not defined by POSIX + # and different OSes has different implementations for it. + # shellcheck disable=SC2012 destination_inode=$(ls -i "$destination" | awk '{print $1}') + # Currently not worth the effor to change it, stat is not defined by POSIX + # and different OSes has different implementations for it. + # shellcheck disable=SC2012 source_inode=$(ls -i "$source" | awk '{print $1}') if [ "$destination_inode" -eq "$source_inode" ]; then echo present diff --git a/cdist/conf/type/__link/explorer/type b/cdist/conf/type/__link/explorer/type index 579fd081..b322bf42 100755 --- a/cdist/conf/type/__link/explorer/type +++ b/cdist/conf/type/__link/explorer/type @@ -24,23 +24,26 @@ destination="/$__object_id" if [ ! -e "$destination" ]; then - echo none + echo none elif [ -h "$destination" ]; then - echo symlink + echo symlink elif [ -f "$destination" ]; then - type="$(cat "$__object/parameter/type")" - case "$type" in - hard) - link_count=$(ls -l "$destination" | awk '{ print $2 }') - if [ $link_count -gt 1 ]; then - echo hardlink - exit 0 - fi - ;; - esac - echo file + type="$(cat "$__object/parameter/type")" + case "$type" in + hard) + # Currently not worth the effor to change it, stat is not defined by POSIX + # and different OSes has different implementations for it. + # shellcheck disable=SC2012 + link_count=$(ls -l "$destination" | awk '{ print $2 }') + if [ "$link_count" -gt 1 ]; then + echo hardlink + exit 0 + fi + ;; + esac + echo file elif [ -d "$destination" ]; then - echo directory + echo directory else - echo unknown + echo unknown fi diff --git a/cdist/conf/type/__link/gencode-remote b/cdist/conf/type/__link/gencode-remote index 9e7831c7..45c22fcc 100755 --- a/cdist/conf/type/__link/gencode-remote +++ b/cdist/conf/type/__link/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # 2013-2014 Steven Armstrong (steven-cdist at armstrong.cc) @@ -48,21 +48,25 @@ case "$state_should" in if [ "$file_type" = "directory" ]; then # our destination is currently a directory, delete it printf 'rm -rf "%s" &&\n' "$destination" + echo "removed '$destination' (directory)" >> "$__messages_out" else if [ "$state_is" = "wrongsource" ]; then # our destination is a symlink but points to the wrong source, # delete it printf 'rm -f "%s" &&\n' "$destination" + echo "removed '$destination' (wrongsource)" >> "$__messages_out" fi fi # create our link printf 'ln %s -f "%s" "%s"\n' "$lnopt" "$source" "$destination" + echo "created '$destination'" >> "$__messages_out" ;; absent) # only delete if it is a sym/hard link - if [ "$file_type" = "symlink" -o "$file_type" = "hardlink" ]; then + if [ "$file_type" = "symlink" ] || [ "$file_type" = "hardlink" ]; then printf 'rm -f "%s"\n' "$destination" + echo "removed '$destination'" >> "$__messages_out" fi ;; *) diff --git a/cdist/conf/type/__link/man.rst b/cdist/conf/type/__link/man.rst index 9dc4665f..fe0ce425 100644 --- a/cdist/conf/type/__link/man.rst +++ b/cdist/conf/type/__link/man.rst @@ -27,6 +27,22 @@ state 'present' or 'absent', defaults to 'present' +MESSAGES +-------- + +created + Link to destination was created. + +removed + Link to destination was removed. + +removed (directory) + Destination was removed because state is ``present`` and destination was directory. + +removed (wrongsource) + Destination was removed because state is ``present`` and destination link source was wrong. + + EXAMPLES -------- diff --git a/cdist/conf/type/__locale/gencode-remote b/cdist/conf/type/__locale/gencode-remote old mode 100644 new mode 100755 index 538ce2cd..1feb9884 --- a/cdist/conf/type/__locale/gencode-remote +++ b/cdist/conf/type/__locale/gencode-remote @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2013 Nico Schottelius (nico-cdist at schottelius.org) +# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -37,6 +37,15 @@ locale_remove=$(echo "$locale" | sed 's/UTF-8/utf8/') state=$(cat "$__object/parameter/state") +os=$(cat "$__global/explorer/os") + +# Nothing to be done on alpine +case "$os" in + alpine) + exit 0 + ;; +esac + case "$state" in present) echo localedef -A "$alias" -f "$charmap" -i "$input" "$locale" diff --git a/cdist/conf/type/__locale/man.rst b/cdist/conf/type/__locale/man.rst index 60a4eacc..e36ab061 100644 --- a/cdist/conf/type/__locale/man.rst +++ b/cdist/conf/type/__locale/man.rst @@ -8,7 +8,8 @@ cdist-type__locale - Configure locales DESCRIPTION ----------- -This cdist type allows you to setup locales. +This cdist type allows you to setup locales. On systems that don't +support locale setting like alpine/musl libc, it is a no-op. OPTIONAL PARAMETERS @@ -44,6 +45,6 @@ Nico Schottelius COPYING ------- -Copyright \(C) 2013-2016 Nico Schottelius. Free use of this software is +Copyright \(C) 2013-2019 Nico Schottelius. Free use of this software is granted under the terms of the GNU General Public License version 3 or later (GPLv3+). diff --git a/cdist/conf/type/__locale/manifest b/cdist/conf/type/__locale/manifest old mode 100644 new mode 100755 index d360e9f3..9f1e17ac --- a/cdist/conf/type/__locale/manifest +++ b/cdist/conf/type/__locale/manifest @@ -1,6 +1,6 @@ -#!/bin/sh +#!/bin/sh -e # -# 2013-2015 Nico Schottelius (nico-cdist at schottelius.org) +# 2013-2019 Nico Schottelius (nico-cdist at schottelius.org) # 2015 David Hürlimann (david at ungleich.ch) # # This file is part of cdist. @@ -19,7 +19,7 @@ # along with cdist. If not, see . # # -# Install required packages +# Install required packages # os=$(cat "$__global/explorer/os") @@ -30,7 +30,7 @@ case "$os" in # Debian needs a seperate package __package locales --state present ;; - archlinux|suse|ubuntu|scientific|centos) + archlinux|suse|ubuntu|scientific|centos|alpine) : ;; *) diff --git a/cdist/conf/type/__locale_system/manifest b/cdist/conf/type/__locale_system/manifest old mode 100644 new mode 100755 index 02cf48df..80f7401b --- a/cdist/conf/type/__locale_system/manifest +++ b/cdist/conf/type/__locale_system/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2016 Steven Armstrong (steven-cdist at armstrong.cc) # 2016 Carlos Ortigoza (carlos.ortigoza at ungleich.ch) diff --git a/cdist/conf/type/__motd/gencode-remote b/cdist/conf/type/__motd/gencode-remote index 41fe3482..bc842cc8 100755 --- a/cdist/conf/type/__motd/gencode-remote +++ b/cdist/conf/type/__motd/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__motd/manifest b/cdist/conf/type/__motd/manifest index 4848a4c3..cd741cf4 100755 --- a/cdist/conf/type/__motd/manifest +++ b/cdist/conf/type/__motd/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Nico Schottelius (nico-cdist at schottelius.org) # @@ -22,6 +22,9 @@ # Select motd source if [ -f "$__object/parameter/source" ]; then source="$(cat "$__object/parameter/source")" + if [ "$source" = "-" ]; then + source="${__object}/stdin" + fi else source="$__type/files/motd" fi diff --git a/cdist/conf/type/__mount/gencode-remote b/cdist/conf/type/__mount/gencode-remote index 2626f3de..b2096764 100755 --- a/cdist/conf/type/__mount/gencode-remote +++ b/cdist/conf/type/__mount/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -39,7 +39,7 @@ case "$state_should" in printf ' -o %s' "$(cat "$__object/parameter/options")" fi printf ' %s' "$(cat "$__object/parameter/device")" - printf " %s\n" "$path" + printf ' %s\n' "$path" else # mount using existing fstab entry printf 'mount "%s"\n' "$path" diff --git a/cdist/conf/type/__mount/manifest b/cdist/conf/type/__mount/manifest index 472b6e2e..999d806c 100755 --- a/cdist/conf/type/__mount/manifest +++ b/cdist/conf/type/__mount/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -31,7 +31,7 @@ printf " %s" "$type" options="$(cat "$__object/parameter/options")" printf " %s" "$options" printf " %s" "$(cat "$__object/parameter/dump")" -printf " %s\n" "$(cat "$__object/parameter/pass")" +printf ' %s\n' "$(cat "$__object/parameter/pass")" ) | \ __block "$__object_name" \ --file "/etc/fstab" \ diff --git a/cdist/conf/type/__mysql_database/gencode-remote b/cdist/conf/type/__mysql_database/gencode-remote index b1c2e6a1..23e51b05 100755 --- a/cdist/conf/type/__mysql_database/gencode-remote +++ b/cdist/conf/type/__mysql_database/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Benedikt Koeppel (code@benediktkoeppel.ch) # diff --git a/cdist/conf/type/__package/explorer/pkgng_exists b/cdist/conf/type/__package/explorer/pkgng_exists index 355c5d65..6d69ba14 100755 --- a/cdist/conf/type/__package/explorer/pkgng_exists +++ b/cdist/conf/type/__package/explorer/pkgng_exists @@ -21,7 +21,7 @@ # Retrieve the status of a package - parsed dpkg output # -if [ "$($__explorer/os)" = "freebsd" ]; then +if [ "$("$__explorer/os")" = "freebsd" ]; then command -v pkg fi diff --git a/cdist/conf/type/__package/manifest b/cdist/conf/type/__package/manifest index 525691bb..a453c32b 100755 --- a/cdist/conf/type/__package/manifest +++ b/cdist/conf/type/__package/manifest @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -44,6 +45,7 @@ else suse) type="zypper" ;; openwrt) type="opkg" ;; openbsd) type="pkg_openbsd" ;; + alpine) type="apk" ;; *) echo "Don't know how to manage packages on: $os" >&2 exit 1 @@ -55,8 +57,8 @@ state="$(cat "$__object/parameter/state")" set -- "$@" "$__object_id" "--state" "$state" cd "$__object/parameter" -for property in $(ls .); do - if [ "$property" != "type" -a "$property" != "state" ]; then +for property in *; do + if [ "$property" != "type" ] && [ "$property" != "state" ]; then set -- "$@" "--$property" "$(cat "$property")" fi done diff --git a/cdist/conf/type/__package/nonparallel b/cdist/conf/type/__package/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkg_openbsd/explorer/pkg_version b/cdist/conf/type/__package_apk/explorer/state similarity index 70% rename from cdist/conf/type/__package_pkg_openbsd/explorer/pkg_version rename to cdist/conf/type/__package_apk/explorer/state index bc23a85d..b477ca7c 100755 --- a/cdist/conf/type/__package_pkg_openbsd/explorer/pkg_version +++ b/cdist/conf/type/__package_apk/explorer/state @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Andi Brönnimann (andi-cdist at v-net.ch) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -18,7 +18,7 @@ # along with cdist. If not, see . # # -# Retrieve the status of a package - parsed dpkg output +# Retrieve the status of a package - parsed apk output # if [ -f "$__object/parameter/name" ]; then @@ -27,5 +27,12 @@ else name="$__object_id" fi -#TODO: Is there a better way? -pkg_info | grep "$name" | sed 's .*\(-[0-9.][0-9.]*\).* \1 ' | sed 's/-//' +# Remove the @.. repo tag for finding out whether it is installed +# f.i. pass@testing => pass +name="$(echo "$name" | sed 's/@.*//')" + +if [ "$(apk list -I "$name")" ]; then + echo present +else + echo absent +fi diff --git a/cdist/conf/type/__package_apk/gencode-remote b/cdist/conf/type/__package_apk/gencode-remote new file mode 100755 index 00000000..79e3d2b6 --- /dev/null +++ b/cdist/conf/type/__package_apk/gencode-remote @@ -0,0 +1,49 @@ +#!/bin/sh -e +# +# 2019 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Manage packages on Debian and co. +# + +if [ -f "$__object/parameter/name" ]; then + name="$(cat "$__object/parameter/name")" +else + name="$__object_id" +fi + +state_should="$(cat "$__object/parameter/state")" +state_is="$(cat "$__object/explorer/state")" + +# Nothing to be done +[ "$state_is" = "$state_should" ] && exit 0 + +case "$state_should" in + present) + echo "apk add -q '$name'" + echo "installed" >> "$__messages_out" + ;; + absent) + echo "apk del -q '$name'" + echo "removed" >> "$__messages_out" + ;; + *) + echo "Unknown state: $state_should" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__package_apk/man.rst b/cdist/conf/type/__package_apk/man.rst new file mode 100644 index 00000000..bc2408b4 --- /dev/null +++ b/cdist/conf/type/__package_apk/man.rst @@ -0,0 +1,55 @@ +cdist-type__package_akp(7) +========================== + +NAME +---- +cdist-type__package_akp - Manage packages with akp + + +DESCRIPTION +----------- +apk is usually used on Alpine to manage packages. + + +REQUIRED PARAMETERS +------------------- +None + + +OPTIONAL PARAMETERS +------------------- +name + If supplied, use the name and not the object id as the package name. + +state + Either "present" or "absent", defaults to "present" + + +EXAMPLES +-------- + +.. code-block:: sh + + # Ensure zsh in installed + __package_apk zsh --state present + + # Remove package + __package_apk apache2 --state absent + + +SEE ALSO +-------- +:strong:`cdist-type__package`\ (7) + + +AUTHORS +------- +Nico Schottelius + + +COPYING +------- +Copyright \(C) 2019 Nico Schottelius. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__package_apk/nonparallel b/cdist/conf/type/__package_apk/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_apk/parameter/default/state b/cdist/conf/type/__package_apk/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__package_apk/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__package_apk/parameter/optional b/cdist/conf/type/__package_apk/parameter/optional new file mode 100644 index 00000000..1b423dc4 --- /dev/null +++ b/cdist/conf/type/__package_apk/parameter/optional @@ -0,0 +1,2 @@ +name +state diff --git a/cdist/conf/type/__package_apt/explorer/state b/cdist/conf/type/__package_apt/explorer/state index 04926b60..7ccd6fce 100755 --- a/cdist/conf/type/__package_apt/explorer/state +++ b/cdist/conf/type/__package_apt/explorer/state @@ -30,8 +30,9 @@ fi # Except dpkg failing, if package is not known / installed packages="$(apt-cache showpkg "$name" | sed -e "1,/Reverse Provides:/d" | cut -d ' ' -f 1) $name" for p in $packages; do - if [ -n "$(dpkg -s "$p" 2>/dev/null | grep "^Status: install ok installed$")" ]; then - echo "present $p" + if dpkg -s "$p" 2>/dev/null | grep --quiet "^Status: install ok installed$" ; then + version=$(dpkg -s "$p" 2>/dev/null | grep "^Version:" | cut -d ' ' -f 2) + echo "present $p $version" exit 0 fi done diff --git a/cdist/conf/type/__package_apt/gencode-remote b/cdist/conf/type/__package_apt/gencode-remote index ef313070..699eb0c9 100755 --- a/cdist/conf/type/__package_apt/gencode-remote +++ b/cdist/conf/type/__package_apt/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # @@ -29,12 +29,24 @@ fi state_should="$(cat "$__object/parameter/state")" +version_param="$__object/parameter/version" + +version="" +if [ -f "$version_param" ]; then + version="$(cat "$version_param")" +fi + if [ -f "$__object/parameter/target-release" ]; then target_release="--target-release $(cat "$__object/parameter/target-release")" else target_release="" fi +if [ -f "$__object/parameter/purge-if-absent" ]; then + purgeparam="--purge" +else + purgeparam="" +fi # FIXME: use grep directly, state is a list, not a line! @@ -42,22 +54,35 @@ state_is="$(cat "$__object/explorer/state")" case "$state_is" in present*) name="$(echo "$state_is" | cut -d ' ' -f 2)" + version_is="$(echo "$state_is" | cut -d ' ' -f 3)" state_is="present" ;; + *) + version_is="" + ;; esac # Hint if we need to avoid questions at some point: # DEBIAN_PRIORITY=critical can reduce the number of questions aptget="DEBIAN_FRONTEND=noninteractive apt-get --quiet --yes --no-install-recommends -o Dpkg::Options::=\"--force-confdef\" -o Dpkg::Options::=\"--force-confold\"" -[ "$state_is" = "$state_should" ] && exit 0 +if [ "$state_is" = "$state_should" ]; then + if [ -z "$version" ] || [ "$version" = "$version_is" ]; then + exit 0; + fi +fi case "$state_should" in present) - echo $aptget install $target_release \"$name\" + if [ -n "$version" ]; then + name="${name}=${version}" + fi + echo "$aptget install $target_release '$name'" + echo "installed" >> "$__messages_out" ;; absent) - echo $aptget remove \"$name\" + echo "$aptget remove $purgeparam '$name'" + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_apt/man.rst b/cdist/conf/type/__package_apt/man.rst index 0a7958d4..a3a70d91 100644 --- a/cdist/conf/type/__package_apt/man.rst +++ b/cdist/conf/type/__package_apt/man.rst @@ -29,6 +29,18 @@ target-release Passed on to apt-get install, see apt-get(8). Essentially allows you to retrieve packages from a different release +version + The version of the package to install. Default is to install the version + chosen by the local package manager. + + +BOOLEAN PARAMETERS +------------------ +purge-if-absent + If this parameter is given when state is `absent`, the package is + purged from the system (using `--purge`). + + EXAMPLES -------- diff --git a/cdist/conf/type/__package_apt/nonparallel b/cdist/conf/type/__package_apt/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_apt/parameter/boolean b/cdist/conf/type/__package_apt/parameter/boolean new file mode 100644 index 00000000..f9a0f6b0 --- /dev/null +++ b/cdist/conf/type/__package_apt/parameter/boolean @@ -0,0 +1 @@ +purge-if-absent diff --git a/cdist/conf/type/__package_dpkg/explorer/pkg_state b/cdist/conf/type/__package_dpkg/explorer/pkg_state new file mode 100644 index 00000000..d7487ed8 --- /dev/null +++ b/cdist/conf/type/__package_dpkg/explorer/pkg_state @@ -0,0 +1,11 @@ +#!/bin/sh -e + +package=$( basename "$__object_id" ) + +dpkg_status="$(dpkg-query --show --showformat='${db:Status-Abbrev} ${binary:Package}_${Version}_${Architecture}.deb\n' "${package%%_*}" 2>/dev/null || true)" + +if echo "$dpkg_status" | grep -q '^ii'; then + echo "${dpkg_status##* }" +fi + + diff --git a/cdist/conf/type/__package_dpkg/gencode-remote b/cdist/conf/type/__package_dpkg/gencode-remote index d4186e66..1c271748 100755 --- a/cdist/conf/type/__package_dpkg/gencode-remote +++ b/cdist/conf/type/__package_dpkg/gencode-remote @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Tomas Pospisek (tpo_deb sourcepole.ch) +# 2018 Thomas Eckert (tom at it-eckert.de) # # This file is based on cdist's __file/gencode-local and part of cdist. # @@ -26,5 +27,25 @@ # to conflict with dpkg's --force options). But currently we don't # do any checks or --force'ing. # +state=$( cat "$__object/parameter/state" ) +package=$( basename "$__object_id" ) +state_is="$(cat "$__object/explorer/pkg_state")" +state_should="" -echo "dpkg -i /var/cache/apt/archives/$__object_id" +[ "$state" = "absent" ] || state_should="$package" +[ "$state_is" = "$state_should" ] && exit 0 + +case "$state" in + present) + echo "dpkg --install /var/cache/apt/archives/$__object_id" + echo "installed" >> "$__messages_out" + ;; + absent) + [ -f "$__object/parameter/purge-if-absent" ] \ + && action="--purge" \ + || action="--remove" + echo "dpkg $action ${__object_id%%_*}" + echo "removed ($action)" >> "$__messages_out" + ;; + *) echo "ERROR: unknown state '$state'" >&2 ;; +esac diff --git a/cdist/conf/type/__package_dpkg/man.rst b/cdist/conf/type/__package_dpkg/man.rst index df2d86a7..828d8cdd 100644 --- a/cdist/conf/type/__package_dpkg/man.rst +++ b/cdist/conf/type/__package_dpkg/man.rst @@ -12,30 +12,77 @@ This type is used on Debian and variants (like Ubuntu) to install packages that are provided locally as \*.deb files. The object given to this type must be the name of the deb package. +The filename of the deb package has to follow Debian naming conventions, i.e. +`${binary:Package}_${Version}_${Architecture}.deb` (see `dpkg-query(1)` for +details). +OPTIONAL PARAMETERS +------------------- +state + `present` or `absent`, defaults to `present`. + REQUIRED PARAMETERS ------------------- source path to the \*.deb package + +BOOLEAN PARAMETERS +------------------ +purge-if-absent + If this parameter is given when state is `absent`, the package is + purged from the system (using `--purge`). + + +EXPLORER +-------- +pkg_state + Returns the full package name if package is installed, empty otherwise. + + +MESSAGES +-------- +installed + The deb-file was installed. + +removed (--remove) + The package was removed, keeping config. + +removed (--purge) + The package was removed including config (purged). + + EXAMPLES -------- .. code-block:: sh # Install foo and bar packages - __package_dpkg --source /tmp/foo_0.1_all.deb foo_0.1_all.deb - __package_dpkg --source $__type/files/bar_1.4.deb bar_1.4.deb + __package_dpkg foo_0.1_all.deb --source /tmp/foo_0.1_all.deb + __package_dpkg bar_1.4.deb --source $__type/files/bar_1.4.deb + + # uninstall baz: + __package_dpkg baz_1.4_amd64.deb \ + --source $__type/files/baz_1.4_amd64.deb \ + --state "absent" + # uninstall baz and also purge config-files: + __package_dpkg baz_1.4_amd64.deb \ + --source $__type/files/baz_1.4_amd64.deb \ + --purge-if-absent \ + --state "absent" SEE ALSO -------- -:strong:`cdist-type__package`\ (7) +:strong:`cdist-type__package`\ (7), :strong:`dpkg-query`\ (1) + AUTHORS ------- -Tomas Pospisek +| Tomas Pospisek +| Thomas Eckert + COPYING ------- diff --git a/cdist/conf/type/__package_dpkg/manifest b/cdist/conf/type/__package_dpkg/manifest old mode 100644 new mode 100755 index ff477c2d..6d228d8e --- a/cdist/conf/type/__package_dpkg/manifest +++ b/cdist/conf/type/__package_dpkg/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Tomas Pospisek (tpo_deb sourcepole.ch) # @@ -25,10 +25,16 @@ # do any checks or --force'ing. +state=$( cat "$__object/parameter/state" ) package_path=$( cat "$__object/parameter/source" ) package=$( basename "$__object_id" ) +state_is="$(cat "$__object/explorer/pkg_state")" +state_should="" + +[ "$state" = "absent" ] || state_should="$package" +[ "$state_is" = "$state_should" ] && exit 0 __file "/var/cache/apt/archives/$package" \ - --source "$package_path" \ - --state present + --source "$package_path" \ + --state "$state" diff --git a/cdist/conf/type/__package_dpkg/nonparallel b/cdist/conf/type/__package_dpkg/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_dpkg/parameter/boolean b/cdist/conf/type/__package_dpkg/parameter/boolean new file mode 100644 index 00000000..f9a0f6b0 --- /dev/null +++ b/cdist/conf/type/__package_dpkg/parameter/boolean @@ -0,0 +1 @@ +purge-if-absent diff --git a/cdist/conf/type/__package_dpkg/parameter/default/state b/cdist/conf/type/__package_dpkg/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__package_dpkg/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__package_dpkg/parameter/optional b/cdist/conf/type/__package_dpkg/parameter/optional new file mode 100644 index 00000000..ff72b5c7 --- /dev/null +++ b/cdist/conf/type/__package_dpkg/parameter/optional @@ -0,0 +1 @@ +state diff --git a/cdist/conf/type/__package_emerge/explorer/pkg_version b/cdist/conf/type/__package_emerge/explorer/pkg_version index 7053eaff..d02b9d6b 100644 --- a/cdist/conf/type/__package_emerge/explorer/pkg_version +++ b/cdist/conf/type/__package_emerge/explorer/pkg_version @@ -32,4 +32,5 @@ else name="$__object_id" fi +# shellcheck disable=SC2016 equery -q l -F '$cp $fullversion' "$name" || true diff --git a/cdist/conf/type/__package_emerge/gencode-remote b/cdist/conf/type/__package_emerge/gencode-remote old mode 100644 new mode 100755 index 1199fc72..e1b85ebb --- a/cdist/conf/type/__package_emerge/gencode-remote +++ b/cdist/conf/type/__package_emerge/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Thomas Oettli (otho at sfs.biz) # @@ -38,13 +38,13 @@ fi pkg_version="$(cat "$__object/explorer/pkg_version")" if [ -z "$pkg_version" ]; then state_is="absent" -elif [ -z "$version" -a $(echo "$pkg_version" | wc -l) -gt 1 ]; then - echo "Package name is not unique! The following packages are installed:" - echo "$pkg_version" +elif [ -z "$version" ] && [ "$(echo "$pkg_version" | wc -l)" -gt 1 ]; then + echo "Package name is not unique! The following packages are installed:" >&2 + echo "$pkg_version" >&2 exit 1 -elif [ -n "$version" -a $(echo "$pkg_version" | cut -d " " -f 1 | sort | uniq | wc -l) -gt 1 ]; then - echo "Package name is not unique! The following packages are installed:" - echo "$pkg_version" +elif [ -n "$version" ] && [ "$(echo "$pkg_version" | cut -d " " -f 1 | sort | uniq | wc -l)" -gt 1 ]; then + echo "Package name is not unique! The following packages are installed:" >&2 + echo "$pkg_version" >&2 exit 1 else state_is="present" @@ -57,16 +57,18 @@ fi # Exit if nothing is needed to be done -[ "$state_is" = "$state_should" ] && ( [ -z "$version" ] || [ "$installed_version" = "$version" ] ) && exit 0 -[ "$state_should" = "absent" ] && [ ! -z "$version" ] && [ "$installed_version" != "$version" ] && exit 0 +[ "$state_is" = "$state_should" ] && { [ -z "$version" ] || [ "$installed_version" = "$version" ]; } && exit 0 +[ "$state_should" = "absent" ] && [ -n "$version" ] && [ "$installed_version" != "$version" ] && exit 0 case "$state_should" in present) - echo "emerge \"$name\" &>/dev/null || exit 1" + echo "emerge '$name' &>/dev/null || exit 1" + echo "installed" >> "$__messages_out" ;; absent) - echo "emerge -C \"$name\" &>/dev/null || exit 1" + echo "emerge -C '$name' &>/dev/null || exit 1" + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_emerge/nonparallel b/cdist/conf/type/__package_emerge/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_emerge_dependencies/gencode-remote b/cdist/conf/type/__package_emerge_dependencies/gencode-remote old mode 100644 new mode 100755 index 0c84e53d..f3e6f76e --- a/cdist/conf/type/__package_emerge_dependencies/gencode-remote +++ b/cdist/conf/type/__package_emerge_dependencies/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e gentoolkit_installed="$(cat "$__object/explorer/gentoolkit_installed")" flaggie_installed="$(cat "$__object/explorer/flaggie_installed")" @@ -6,10 +6,11 @@ flaggie_installed="$(cat "$__object/explorer/flaggie_installed")" if [ "${gentoolkit_installed}" != "true" ]; then # emerge app-portage/gentoolkit echo "emerge app-portage/gentoolkit &> /dev/null || exit 1" + echo "installed app-portage/gentoolkit" >> "$__messages_out" fi if [ "${flaggie_installed}" != "true" ]; then # emerge app-portage/flaggie echo "emerge app-portage/flaggie &> /dev/null || exit 1" + echo "installed app-portage/flaggie" >> "$__messages_out" fi - diff --git a/cdist/conf/type/__package_emerge_dependencies/nonparallel b/cdist/conf/type/__package_emerge_dependencies/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_luarocks/explorer/pkg_status b/cdist/conf/type/__package_luarocks/explorer/pkg_status index 3eb73298..e83e8ce6 100755 --- a/cdist/conf/type/__package_luarocks/explorer/pkg_status +++ b/cdist/conf/type/__package_luarocks/explorer/pkg_status @@ -28,4 +28,4 @@ else fi # Accept luarocks failing if package is not known/installed -luarocks list "$name" | egrep -A1 "^$name$" || exit 0 +luarocks list "$name" | grep -E -A1 "^$name$" || exit 0 diff --git a/cdist/conf/type/__package_luarocks/gencode-remote b/cdist/conf/type/__package_luarocks/gencode-remote index 1046a936..d83b3c3a 100755 --- a/cdist/conf/type/__package_luarocks/gencode-remote +++ b/cdist/conf/type/__package_luarocks/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 SwellPath, Inc. # Christian G. Warden @@ -42,10 +42,12 @@ fi case "$state_should" in present) - echo luarocks install \"$name\" + echo "luarocks install '$name'" + echo "installed" >> "$__messages_out" ;; absent) - echo luarocks remove \"$name\" + echo "luarocks remove '$name'" + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_luarocks/manifest b/cdist/conf/type/__package_luarocks/manifest old mode 100644 new mode 100755 index 8e626714..7d8262ca --- a/cdist/conf/type/__package_luarocks/manifest +++ b/cdist/conf/type/__package_luarocks/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 SwellPath, Inc. # Christian G. Warden diff --git a/cdist/conf/type/__package_luarocks/nonparallel b/cdist/conf/type/__package_luarocks/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_opkg/gencode-remote b/cdist/conf/type/__package_opkg/gencode-remote index 2df31202..269d5f49 100755 --- a/cdist/conf/type/__package_opkg/gencode-remote +++ b/cdist/conf/type/__package_opkg/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011,2013 Nico Schottelius (nico-cdist at schottelius.org) # 2012 Giel van Schijndel (giel plus cdist at mortis dot eu) @@ -43,15 +43,17 @@ esac case "$state_should" in present) if [ "$present" = "notpresent" ]; then - echo opkg --verbosity=0 update + echo "opkg --verbosity=0 update" fi - echo opkg --verbosity=0 install \"$name\" + echo "opkg --verbosity=0 install '$name'" + echo "installed" >> "$__messages_out" ;; absent) - echo opkg --verbosity=0 remove \"$name\" + echo "opkg --verbosity=0 remove '$name'" + echo "removed" >> "$__messages_out" ;; *) - echo "Unknown state: $state" >&2 + echo "Unknown state: ${state_should}" >&2 exit 1 ;; esac diff --git a/cdist/conf/type/__package_opkg/nonparallel b/cdist/conf/type/__package_opkg/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pacman/gencode-remote b/cdist/conf/type/__package_pacman/gencode-remote index da1ac7c2..2e076ec3 100755 --- a/cdist/conf/type/__package_pacman/gencode-remote +++ b/cdist/conf/type/__package_pacman/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # @@ -45,10 +45,12 @@ fi case "$state_should" in present) - echo pacman --needed --noconfirm --noprogressbar -S \"$name\" + echo "pacman --needed --noconfirm --noprogressbar -S '$name'" + echo "installed" >> "$__messages_out" ;; absent) - echo pacman --noconfirm --noprogressbar -R \"$name\" + echo "pacman --noconfirm --noprogressbar -R '$name'" + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_pacman/nonparallel b/cdist/conf/type/__package_pacman/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pip/gencode-remote b/cdist/conf/type/__package_pip/gencode-remote old mode 100644 new mode 100755 index ccfdb92b..dcc4fdf9 --- a/cdist/conf/type/__package_pip/gencode-remote +++ b/cdist/conf/type/__package_pip/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # 2016 Darko Poljak (darko.poljak at gmail.com) @@ -53,18 +53,20 @@ case "$state_should" in present) if [ "$runas" ] then - echo "su -c \"$pip install -q $name\" $runas" + echo "su -c '$pip install -q $name' $runas" else echo $pip install -q "$name" fi + echo "installed" >> "$__messages_out" ;; absent) if [ "$runas" ] then - echo "su -c \"$pip uninstall -q -y $name\" $runas" + echo "su -c '$pip uninstall -q -y $name' $runas" else echo $pip uninstall -q -y "$name" fi + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_pip/nonparallel b/cdist/conf/type/__package_pip/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkg_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkg_freebsd/explorer/pkg_version index 1335ba79..0a1ab75c 100755 --- a/cdist/conf/type/__package_pkg_freebsd/explorer/pkg_version +++ b/cdist/conf/type/__package_pkg_freebsd/explorer/pkg_version @@ -30,7 +30,7 @@ fi # Don't produce "no pkgs installed" output -- breaks things PKG_OUTPUT=$(pkg_info 2>&1) if [ ! "$PKG_OUTPUT" = "pkg_info: no packages installed" ]; then - echo -n "$(echo "$PKG_OUTPUT" \ + printf "%s" "$(echo "$PKG_OUTPUT" \ | awk '{print $1}' \ | sed 's/^\(.*\)-\([^-]*\)$/name:\1 ver:\2/g' \ | grep "name:$name ver:" \ diff --git a/cdist/conf/type/__package_pkg_freebsd/gencode-remote b/cdist/conf/type/__package_pkg_freebsd/gencode-remote index 5866a0a8..3f88f6bc 100755 --- a/cdist/conf/type/__package_pkg_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_freebsd/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Jake Guffey (jake.guffey at eprotex.com) # @@ -33,12 +33,13 @@ assert () # If condition false, lineno=$2 - if [ ! $1 ] + if [ ! "$1" ] then echo "Assertion failed: \"$1\"" + # shellcheck disable=SC2039 echo "File \"$0\", line $lineno, called by $(caller 0)" exit $E_ASSERT_FAILED - fi + fi } # Debug @@ -66,7 +67,7 @@ cmd="" # FIXME: This is ugly. execcmd(){ # Set the PACKAGESITE if we're ADDing a new package - if [ "$1" = "add" -a -n "$pkgsite" ]; then + if [ "$1" = "add" ] && [ -n "$pkgsite" ]; then # Use http.../All/ if we know the exact version we want, use .../Latest/ otherwise pkgsite="export PACKAGESITE=${pkgsite}" [ -n "$version" ] && pkgsite="${pkgsite}/All/" || pkgsite="${pkgsite}/Latest/" @@ -88,6 +89,7 @@ if [ -n "$curr_version" ]; then # PKG *is* installed cmd="${rm_cmd} ${name}-${curr_version}" fi execcmd "remove" "${cmd}" + echo "removed" >> "$__messages_out" exit 0 else # Should be installed if [ -n "$version" ]; then # Want a specific version @@ -95,11 +97,13 @@ if [ -n "$curr_version" ]; then # PKG *is* installed exit 0 else # Current version is wrong, fix #updatepkg "$name" "$version" + # shellcheck disable=SC2039 assert "! ${version} = ${curr_version}" $LINENO cmd="${rm_cmd} ${name}-${curr_version}" execcmd "remove" "${cmd}" cmd="${add_cmd} -r ${name}-${version}" execcmd "add" "${cmd}" + echo "installed" >> "$__messages_out" fi else # Don't care what version to use exit 0 @@ -118,6 +122,7 @@ else # PKG *isn't* installed cmd="${cmd}-${version}" fi execcmd "add" "${cmd}" + echo "installed" >> "$__messages_out" exit 0 fi fi diff --git a/cdist/conf/type/__package_pkg_freebsd/nonparallel b/cdist/conf/type/__package_pkg_freebsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkg_openbsd/explorer/has_installurl b/cdist/conf/type/__package_pkg_openbsd/explorer/has_installurl new file mode 100755 index 00000000..68337cbb --- /dev/null +++ b/cdist/conf/type/__package_pkg_openbsd/explorer/has_installurl @@ -0,0 +1,36 @@ +#!/bin/sh +# +# Copyright 2017, Philippe Gregoire +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# +# Retrieve the installurl(5), as introduced in OpenBSD 6.1 +# +# As of 6.1, the file is supposed to contained a single line +# with the URL used to install from during install or upgrade. +# +# Allow for expansion and take the first non-commented (#) line. +# + +if [ -f /etc/installurl ]; then + printf 'yes' +else + printf 'no' +fi + +exit 0 diff --git a/cdist/conf/type/__package_pkg_openbsd/explorer/pkg_state b/cdist/conf/type/__package_pkg_openbsd/explorer/pkg_state new file mode 100755 index 00000000..9cd17787 --- /dev/null +++ b/cdist/conf/type/__package_pkg_openbsd/explorer/pkg_state @@ -0,0 +1,49 @@ +#!/bin/sh +# +# Copyright 2018, Takashi Yoshi +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Retrieve the status of a package - parsed pkg_info output +# + +if [ -f "${__object}/parameter/name" ] +then + pkgid="$(cat "${__object}/parameter/name")" +else + pkgid="${__object_id}" +fi + +if [ -f "${__object}/parameter/version" ] +then + pkgid="${pkgid}-$(cat "${__object}/parameter/version")" +fi + +if [ -f "${__object}/parameter/flavor" ] +then + # If a flavor but no version is given we need to add another -, + # otherwise pkg_info confuses the flavor with the version. + [ -f "${__object}/parameter/version" ] || pkgid="${pkgid}-" + + pkgid="${pkgid}-$(cat "${__object}/parameter/flavor")" +fi + + +pkg_info -q -I "inst:${pkgid}" >/dev/null 2>&1 \ + && echo 'present' || echo 'absent' + +exit 0 diff --git a/cdist/conf/type/__package_pkg_openbsd/gencode-remote b/cdist/conf/type/__package_pkg_openbsd/gencode-remote index 5ba5f7ef..5a21ce12 100755 --- a/cdist/conf/type/__package_pkg_openbsd/gencode-remote +++ b/cdist/conf/type/__package_pkg_openbsd/gencode-remote @@ -1,7 +1,8 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Andi Brönnimann (andi-cdist at v-net.ch) # 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2018 Takashi Yoshi # # This file is part of cdist. # @@ -22,97 +23,96 @@ # Manage packages with pkg on OpenBSD # -# Debug -# exec >&2 -# set -x +os_version=$(cat "${__global}/explorer/os_version") +machine=$(cat "${__global}/explorer/machine") -os_version="$(cat "$__global/explorer/os_version")" -machine="$(cat "$__global/explorer/machine")" - -if [ -f "$__object/parameter/version" ]; then - version="$(cat "$__object/parameter/version")" +if [ -f "${__object}/parameter/version" ]; then + version=$(cat "${__object}/parameter/version") fi -if [ -f "$__object/parameter/flavor" ]; then - flavor="$(cat "$__object/parameter/flavor")" +if [ -f "${__object}/parameter/flavor" ]; then + flavor=$(cat "${__object}/parameter/flavor") fi -# do not show progress bar -pkgopts="-x" +# Do not show progress bar +pkgopts='-x' -if [ -f "$__object/parameter/name" ]; then - name="$__object/parameter/name" +name="${__object_id}" +if [ -f "${__object}/parameter/name" ]; then + name=$(cat "${__object}/parameter/name") +fi + +if [ -n "${version}" ] && [ -n "${flavor}" ]; then + pkgid="${name}-${version}-${flavor}" +elif [ -n "${version}" ]; then + pkgid="${name}-${version}" +elif [ -f "${__object}/parameter/flavor" ]; then + pkgid="${name}--${flavor}" else - name="$__object_id" + pkgid="${name}" fi -if [ -n "$version" -a -n "$flavor" ]; then - pkgid="$name-$version-$flavor" -elif [ -n "$version" ]; then - pkgid="$name-$version" -elif [ -n "$flavor" ]; then - pkgid="$name--$flavor" +state_should=$(cat "${__object}/parameter/state") + +if [ -f "${__object}/parameter/pkg_path" ]; then + pkg_path=$(cat "${__object}/parameter/pkg_path") else - pkgid="$name" + has_installurl=$(cat "${__object}/explorer/has_installurl") + if [ 'yes' != "${has_installurl}" ]; then + # There is no default PKG_PATH, try to provide one + pkg_path="ftp://ftp.openbsd.org/pub/OpenBSD/${os_version}/packages/${machine}/" + fi fi -state_should="$(cat "$__object/parameter/state")" +state_is=$(cat "${__object}/explorer/pkg_state") +[ "${state_is}" = "${state_should}" ] && exit 0 -pkg_version="$(cat "$__object/explorer/pkg_version")" +case "${state_should}" in + present) + if [ -n "${pkg_path}" ]; then + echo "export PKG_PATH='${pkg_path}'" + fi -if [ -f "$__object/parameter/pkg_path" ]; then - pkg_path="$(cat "$__object/parameter/pkg_path")" -else - pkg_path="ftp://ftp.openbsd.org/pub/OpenBSD/$os_version/packages/$machine/" -fi + # Use this because pkg_add doesn't properly handle errors + cat <&1 || true) -if [ "$pkg_version" ]; then - state_is="present" -else - state_is="absent" +if ! pkg_info -q -I 'inst:${pkgid}' | grep -q '^${name}-${version}.*${flavor}$' 2>/dev/null +then + # We didn't find the package in the list of 'installed packages', so it failed. + # This is necessary because pkg_add doesn't return properly + + if [ -z "\${status}" ]; then + status='Failed to add package, uncaught exception.' + fi + echo "Error: \${status}" >&2 + exit 1 fi +EOF + echo 'installed' >> "${__messages_out}" + ;; -[ "$state_is" = "$state_should" ] && exit 0 + absent) + # Use this because pkg_delete doesn't properly handle errors + cat <&1 || true) -case "$state_should" in - present) - # use this because pkg_add doesn't properly handle errors - cat << eof -export PKG_PATH="$pkg_path" -status=\$(pkg_add "$pkgopts" "$pkgid" 2>&1) -pkg_info | grep "^${name}.*${version}.*${flavor}" > /dev/null 2>&1 +if pkg_info -q -I 'inst:${pkgid}' | grep -q '^${name}-${version}.*${flavor}' 2>/dev/null +then + # We found the package in the list of 'installed packages'. + # This would indicate that pkg_delete failed, send the output of pkg_delete -# We didn't find the package in the list of 'installed packages', so it failed -# This is necessary because pkg_add doesn't return properly -if [ \$? -ne 0 ]; then - if [ -z "\${status}" ]; then - status="Failed to add package, uncaught exception." - fi - echo "Error: \$status" - exit 1 + if [ -z "\${status}" ]; then + status='Failed to remove package, uncaught exception.' + fi + echo "Error: \${status}" >&2 + exit 1 fi -eof - ;; - - absent) - # use this because pkg_add doesn't properly handle errors - cat << eof -status=\$(pkg_delete "$pkgopts" "$pkgid") -pkg_info | grep "^${name}.*${version}.*${flavor}" > /dev/null 2>&1 - -# We found the package in the list of 'installed packages' -# This would indicate that pkg_delete failed, send the output of pkg_delete -if [ \$? -eq 0 ]; then - if [ -z "\${status}" ]; then - status="Failed to remove package, uncaught exception." - fi - echo "Error: \$status" - exit 1 -fi -eof - ;; - *) - echo "Unknown state: $state_should" >&2 +EOF + echo 'removed' >> "${__messages_out}" + ;; + *) + echo "Unknown state: ${state_should}" >&2 exit 1 - ;; + ;; esac diff --git a/cdist/conf/type/__package_pkg_openbsd/nonparallel b/cdist/conf/type/__package_pkg_openbsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version index 947857b9..92ce0623 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version +++ b/cdist/conf/type/__package_pkgng_freebsd/explorer/pkg_version @@ -29,7 +29,7 @@ fi # Don't produce "no pkgs installed" output -- breaks things PKG_OUTPUT=$(pkg info 2>&1) -echo -n "$(echo "$PKG_OUTPUT" \ +printf "%s" "$(echo "$PKG_OUTPUT" \ | awk '{print $1}' \ | sed 's/^\(.*\)-\([^-]*\)$/name:\1 ver:\2/g' \ | grep "name:$name ver:" \ diff --git a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote index 3c3e41e9..dd36efda 100755 --- a/cdist/conf/type/__package_pkgng_freebsd/gencode-remote +++ b/cdist/conf/type/__package_pkgng_freebsd/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Jake Guffey (jake.guffey at eprotex.com) # @@ -52,17 +52,20 @@ cmd="" # Parms: $1 -- mode, "rm", "add", or "upg" # $2 -- the command to be echoed execcmd(){ - local _cmd="" + _cmd="" case "$1" in add) _cmd="${add_cmd} $2" + echo "installed" >> "$__messages_out" ;; rm) _cmd="${rm_cmd} $2" + echo "removed" >> "$__messages_out" ;; upg) _cmd="${upg_cmd} $2" + echo "installed" >> "$__messages_out" ;; *) printf "Error. Don't understand command: %s" "$1" >&2 @@ -70,7 +73,7 @@ execcmd(){ ;; esac - echo "$_cmd 2>&- >&-" # Silence the output of the command + echo "$_cmd >/dev/null 2>&1" # Silence the output of the command echo "status=\$?" echo "if [ \"\$status\" -ne \"0\" ]; then" echo " echo \"Error: ${_cmd} exited nonzero with \$status\"'!' >&2" @@ -95,7 +98,7 @@ if [ -n "$curr_version" ]; then # PKG *is* installed if [ "$upgrade" = "true" ]; then execcmd "upg" "${cmd}" else - printf "Version %s is already installed and pkg-ng can't upgrade directly to version %s.\nTo upgrade to the latest version, use the --upgrade flag.\n" "$curr_version" "$version" >&2 + printf 'Version %s is already installed and pkg-ng cannot upgrade directly to version %s.\nTo upgrade to the latest version, use the --upgrade flag.\n' "$curr_version" "$version" >&2 exit 1 fi # PKG is supposed to be installed to the latest version diff --git a/cdist/conf/type/__package_pkgng_freebsd/nonparallel b/cdist/conf/type/__package_pkgng_freebsd/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_rubygem/gencode-remote b/cdist/conf/type/__package_rubygem/gencode-remote index dc755ad3..abb40653 100755 --- a/cdist/conf/type/__package_rubygem/gencode-remote +++ b/cdist/conf/type/__package_rubygem/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Chase Allen James # @@ -39,10 +39,12 @@ fi case "$state_should" in present) - echo gem install \"$name\" --no-ri --no-rdoc + echo "gem install '$name' --no-ri --no-rdoc" + echo "installed" >> "$__messages_out" ;; absent) - echo gem uninstall \"$name\" + echo "gem uninstall '$name'" + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_rubygem/nonparallel b/cdist/conf/type/__package_rubygem/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_update_index/explorer/currage b/cdist/conf/type/__package_update_index/explorer/currage new file mode 100644 index 00000000..3539b8e1 --- /dev/null +++ b/cdist/conf/type/__package_update_index/explorer/currage @@ -0,0 +1,40 @@ +#!/bin/sh +# +# 2018 Thomas Eckert (tom at it-eckert.de) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +type="$("$__type_explorer/type")" + +case "$type" in + apt) + if [ -f "/var/cache/apt/pkgcache.bin" ]; then + echo $(($(date +"%s")-$(stat --format '%Y' /var/cache/apt/pkgcache.bin))) + else + echo 0 + fi + ;; + pacman) + if [ -d "/var/lib/pacman/sync" ]; then + echo $(($(date +"%s")-$(stat --format '%Y' /var/lib/pacman/sync))) + else + echo 0 + fi + ;; + *) echo "Your specified type ($type) is currently not supported." >&2 + echo "Please contribute an implementation for it if you can." >&2 + ;; +esac diff --git a/cdist/conf/type/__package_update_index/explorer/type b/cdist/conf/type/__package_update_index/explorer/type new file mode 100644 index 00000000..35254c5f --- /dev/null +++ b/cdist/conf/type/__package_update_index/explorer/type @@ -0,0 +1,34 @@ +#!/bin/sh +# +# 2018 Stu Zhao (z12y12l12 at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . + +if [ -f "$__object/parameter/type" ]; then + cat "$__object/parameter/type" +else + # By default determine package manager based on operating system + os="$("$__explorer/os")" + case "$os" in + amazon|scientific|centos|fedora|redhat) echo "yum" ;; + debian|ubuntu|devuan) echo "apt" ;; + archlinux) echo "pacman" ;; + *) + echo "Don't know how to manage packages on: $os" >&2 + exit 1 + ;; + esac +fi diff --git a/cdist/conf/type/__package_update_index/gencode-remote b/cdist/conf/type/__package_update_index/gencode-remote index bf6a532d..738d38eb 100755 --- a/cdist/conf/type/__package_update_index/gencode-remote +++ b/cdist/conf/type/__package_update_index/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Ricardo Catalinas Jiménez (jimenezrick at gmail.com) # @@ -21,30 +21,34 @@ # Update the package index with the appropriate package manager # -type="$__object/parameter/type" - -if [ -f "$type" ]; then - type="$(cat "$type")" -else - # By default determine package manager based on operating system - os="$(cat "$__global/explorer/os")" - case "$os" in - amazon|scientific|centos|fedora|redhat) type="yum" ;; - debian|ubuntu|devuan) type="apt" ;; - archlinux) type="pacman" ;; - *) - echo "Don't know how to manage packages on: $os" >&2 - exit 1 - ;; - esac +type=$(cat "$__object/explorer/type") +currage="$(cat "$__object/explorer/currage")" +if [ -f "$__object/parameter/maxage" ]; then + maxage="$(cat "$__object/parameter/maxage")" fi +if [ -n "$maxage" ]; then + if [ "$type" != "apt" ] && [ "$type" != "pacman" ]; then + echo "ERROR: \"--maxage\" only supported for \"apt\" or \"pacman\" pkg-manager." >&2 + exit 1 + elif [ "$currage" -lt "$maxage" ]; then + exit 0 # no need to update + fi +fi + + case "$type" in yum) ;; - apt) echo "apt-get --quiet update" ;; - pacman) echo "pacman --noprogressbar --sync --refresh" ;; + apt) + echo "apt-get --quiet update" + echo "apt-cache updated (age was: $currage)" >> "$__messages_out" + ;; + pacman) + echo "pacman --noprogressbar --sync --refresh" + echo "pacman package database synced (age was: $currage)" >> "$__messages_out" + ;; *) - echo "Don't know how to manage packages on: $os" >&2 + echo "Don't know how to manage packages for type: $type" >&2 exit 1 ;; esac diff --git a/cdist/conf/type/__package_update_index/man.rst b/cdist/conf/type/__package_update_index/man.rst index 454aa05b..3cd787b9 100644 --- a/cdist/conf/type/__package_update_index/man.rst +++ b/cdist/conf/type/__package_update_index/man.rst @@ -27,6 +27,16 @@ type * yum for Red Hat * pacman for Arch Linux +maxage + Available for package manager apt and pacman, max time in seconds since + last update. Repo update is skipped if maxage is not reached yet. + +MESSAGES +-------- +apt-cache updated (age was: currage) + apt-cache was updated (run of `apt-get update`). `currage` is the time + in seconds since the previous run. + EXAMPLES -------- @@ -39,10 +49,17 @@ EXAMPLES # Force use of a specific package manager __package_update_index --type apt + # Only update every hour: + __package_update_index --maxage 3600 --type apt + + # same as above (on apt-type systems): + __package_update_index --maxage 3600 AUTHORS ------- -Ricardo Catalinas Jiménez +| Ricardo Catalinas Jiménez +| Thomas Eckert +| Stu Zhao COPYING diff --git a/cdist/conf/type/__package_update_index/nonparallel b/cdist/conf/type/__package_update_index/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_update_index/parameter/optional b/cdist/conf/type/__package_update_index/parameter/optional index aa80e646..7a0be716 100644 --- a/cdist/conf/type/__package_update_index/parameter/optional +++ b/cdist/conf/type/__package_update_index/parameter/optional @@ -1 +1,2 @@ type +maxage diff --git a/cdist/conf/type/__package_upgrade_all/gencode-remote b/cdist/conf/type/__package_upgrade_all/gencode-remote index 3e25f45f..38aa001e 100755 --- a/cdist/conf/type/__package_upgrade_all/gencode-remote +++ b/cdist/conf/type/__package_upgrade_all/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Ricardo Catalinas Jiménez (jimenezrick at gmail.com) # @@ -53,8 +53,8 @@ case "$type" in ;; apt) if [ -f "$apt_dist_upgrade" ] - then echo $aptget dist-upgrade - else echo $aptget upgrade + then echo "$aptget dist-upgrade" + else echo "$aptget upgrade" fi if [ -f "$apt_clean" ] diff --git a/cdist/conf/type/__package_upgrade_all/nonparallel b/cdist/conf/type/__package_upgrade_all/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_yum/explorer/pkg_version b/cdist/conf/type/__package_yum/explorer/pkg_version index fb3b7753..b81b0fe9 100755 --- a/cdist/conf/type/__package_yum/explorer/pkg_version +++ b/cdist/conf/type/__package_yum/explorer/pkg_version @@ -27,4 +27,4 @@ else name="$__object_id" fi -rpm -q --whatprovides "$name" 2>/dev/null || true +rpm -q "$name" 2>/dev/null || rpm -q --whatprovides "$name" 2>/dev/null || true diff --git a/cdist/conf/type/__package_yum/gencode-remote b/cdist/conf/type/__package_yum/gencode-remote index 08c5c2b5..b52953f6 100755 --- a/cdist/conf/type/__package_yum/gencode-remote +++ b/cdist/conf/type/__package_yum/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2014 Nico Schottelius (nico-cdist at schottelius.org) # @@ -43,10 +43,15 @@ else opts="--assumeyes --quiet" fi -not_installed="^no package provides" +not_provided="^no package provides" +not_installed='is not installed$' -if grep -q "$not_installed" "$__object/explorer/pkg_version"; then - state_is="absent" +if grep -q "$not_provided" "$__object/explorer/pkg_version"; then + if grep -q "$not_installed" "$__object/explorer/pkg_version"; then + state_is="absent" + else + state_is="present" + fi else state_is="present" fi @@ -55,10 +60,12 @@ fi case "$state_should" in present) - echo yum $opts install \"$install_name\" + echo "yum $opts install '$install_name'" + echo "installed" >> "$__messages_out" ;; absent) - echo yum $opts remove \"$name\" + echo "yum $opts remove '$name'" + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_yum/nonparallel b/cdist/conf/type/__package_yum/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__package_zypper/gencode-remote b/cdist/conf/type/__package_zypper/gencode-remote old mode 100644 new mode 100755 index d9f16f8d..e45dd9ff --- a/cdist/conf/type/__package_zypper/gencode-remote +++ b/cdist/conf/type/__package_zypper/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Daniel Heule (hda at sfs.biz) @@ -61,15 +61,18 @@ case "$state_should" in present) if [ -z "$version_should" ]; then [ "$state_is" = "present" ] && exit 0 # if state is present, we dont need to do anything - echo zypper $globalopts install --type \"$ptype\" --auto-agree-with-licenses \"$name\" ">/dev/null" + echo "zypper $globalopts install --type '$ptype' --auto-agree-with-licenses '$name' >/dev/null" + echo "removed" >> "$__messages_out" else [ "$state_is" = "present" ] && [ "$version_should" = "$version_is" ] && exit 0 # if state is present and version is correct, we dont need to do anything - echo zypper $globalopts install --oldpackage --type \"$ptype\" --auto-agree-with-licenses \"$name\" = \"$version_should\" ">/dev/null" + echo "zypper $globalopts install --oldpackage --type '$ptype' --auto-agree-with-licenses '$name' = '$version_should' >/dev/null" + echo "installed" >> "$__messages_out" fi ;; absent) [ "$state_is" = "absent" ] && exit 0 # if state is absent, we dont need to do anything - echo zypper $globalopts remove --type \"$ptype\" \"$name\" ">/dev/null" + echo "zypper $globalopts remove --type '$ptype' '$name' >/dev/null" + echo "removed" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__package_zypper/nonparallel b/cdist/conf/type/__package_zypper/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__pacman_conf/manifest b/cdist/conf/type/__pacman_conf/manifest old mode 100644 new mode 100755 index b9679577..a43f18a1 --- a/cdist/conf/type/__pacman_conf/manifest +++ b/cdist/conf/type/__pacman_conf/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Dominique Roux (dominique.roux4 at gmail.com) # @@ -59,13 +59,13 @@ if [ "${file}" ]; then if [ "${state}" = "present" ]; then - require="__file/${sec_path}/plain_file_${file}" __key_value ${file}_${key}\ - --file ${sec_path}/plain_file_${file} --key ${key} --value ${value} --delimiter ' = ' + require="__file/${sec_path}/plain_file_${file}" __key_value "${file}_${key}" \ + --file "${sec_path}/plain_file_${file}" --key "${key}" --value "${value}" --delimiter ' = ' exit 0 elif [ "${state}" = "absent" ]; then - require="__file/${sec_path}/plain_file_${file}" __key_value ${file}_${key}\ + require="__file/${sec_path}/plain_file_${file}" __key_value "${file}_${key}" \ --state absent exit 0 @@ -87,19 +87,19 @@ eof if [ "${MATCH}" -eq 1 ]; then if [ "${value}" = "on" ]; then - require="__file/${sec_path}/${section}" __line ${key}_${value}\ - --file ${sec_path}/${section} --line ${key} + require="__file/${sec_path}/${section}" __line "${key}_${value}" \ + --file "${sec_path}/${section}" --line "${key}" elif [ "${value}" = "off" ]; then - require="__file/${sec_path}/${section}" __line ${key}_${value}\ - --file ${sec_path}/${section} --line ${key} --state absent + require="__file/${sec_path}/${section}" __line "${key}_${value}" \ + --file "${sec_path}/${section}" --line "${key}" --state absent fi else contains_element "${key}" "${allowed_option_keys}" if [ "${MATCH}" -eq 1 ]; then - require="__file/${sec_path}/${section}" __key_value ${section}_${key}\ - --file ${sec_path}/${section} --key ${key} --value ${value} --delimiter ' = ' + require="__file/${sec_path}/${section}" __key_value "${section}_${key}" \ + --file "${sec_path}/${section}" --key "${key}" --value "${value}" --delimiter ' = ' else echo "Key: ${key} is not valid. Have a look at man pacman.conf" >&2 fi @@ -118,12 +118,12 @@ eof exit fi - require="__file/${sec_path}/repo_${section}" __key_value ${section}_${key}\ - --file ${sec_path}/repo_${section} --key ${key} --value ${value} --delimiter ' = ' + require="__file/${sec_path}/repo_${section}" __key_value "${section}_${key}" \ + --file "${sec_path}/repo_${section}" --key "${key}" --value "${value}" --delimiter ' = ' elif [ "${state}" = "absent" ]; then - require="__file/${sec_path}/repo_${section}" __key_value ${section}_${key}\ + require="__file/${sec_path}/repo_${section}" __key_value "${section}_${key}" \ --state absent else diff --git a/cdist/conf/type/__pacman_conf_integrate/manifest b/cdist/conf/type/__pacman_conf_integrate/manifest old mode 100644 new mode 100755 index 1d02f3b3..0ce0bee5 --- a/cdist/conf/type/__pacman_conf_integrate/manifest +++ b/cdist/conf/type/__pacman_conf_integrate/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Dominique Roux (dominique.roux4 at gmail.com # @@ -18,16 +18,14 @@ # along with cdist. If not, see . # -state=$(cat $__object/parameter/state 2>/dev/null) - -path="/etc/" +state=$(cat "$__object/parameter/state" 2>/dev/null) if [ "${state}" = "present" ]; then __file /etc/pacman.conf\ - --owner root --group root --mode 644 --source $__type/files/pacman.conf.cdist + --owner root --group root --mode 644 --source "$__type/files/pacman.conf.cdist" __file /etc/pacman.d/options\ - --owner root --group root --mode 644 --source $__type/files/options + --owner root --group root --mode 644 --source "$__type/files/options" __file /etc/pacman.d/repo_empty_placeholder\ --owner root --group root --mode 644 @@ -38,10 +36,10 @@ if [ "${state}" = "present" ]; then elif [ "${state}" = "absent" ]; then __file /etc/pacman.conf\ - --owner root --group root --mode 644 --source $__type/files/pacman.conf.pacman + --owner root --group root --mode 644 --source "$__type/files/pacman.conf.pacman" __file /etc/pacman.d/mirrorlist\ - --owner root --group root --mode 644 --source $__type/files/mirrorlist + --owner root --group root --mode 644 --source "$__type/files/mirrorlist" __file /etc/pacman.d/options\ --state absent diff --git a/cdist/conf/type/__pf_apply/explorer/rcvar b/cdist/conf/type/__pf_apply/explorer/rcvar index 20e9dfcc..7c8d535f 100755 --- a/cdist/conf/type/__pf_apply/explorer/rcvar +++ b/cdist/conf/type/__pf_apply/explorer/rcvar @@ -29,7 +29,7 @@ RC="/etc/rc.conf" PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')" -echo ${PFCONF:-"/etc/pf.conf"} +echo "${PFCONF:-"/etc/pf.conf"}" # Debug #set +x diff --git a/cdist/conf/type/__pf_apply/gencode-remote b/cdist/conf/type/__pf_apply/gencode-remote index f7c889b4..c8f7a25a 100755 --- a/cdist/conf/type/__pf_apply/gencode-remote +++ b/cdist/conf/type/__pf_apply/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Jake Guffey (jake.guffey at eprotex.com) # diff --git a/cdist/conf/type/__pf_ruleset/explorer/cksum b/cdist/conf/type/__pf_ruleset/explorer/cksum index f8679836..9be6c901 100755 --- a/cdist/conf/type/__pf_ruleset/explorer/cksum +++ b/cdist/conf/type/__pf_ruleset/explorer/cksum @@ -33,7 +33,7 @@ TMP="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')" PFCONF="${TMP:-"/etc/pf.conf"}" if [ -f "${PFCONF}" ]; then # The pf config file exists, find its cksum. - cksum -o 1 ${PFCONF} | cut -d= -f2 | awk '{print $1}' + cksum -o 1 "${PFCONF}" | cut -d= -f2 | awk '{print $1}' fi # Debug diff --git a/cdist/conf/type/__pf_ruleset/explorer/rcvar b/cdist/conf/type/__pf_ruleset/explorer/rcvar index 20e9dfcc..7c8d535f 100755 --- a/cdist/conf/type/__pf_ruleset/explorer/rcvar +++ b/cdist/conf/type/__pf_ruleset/explorer/rcvar @@ -29,7 +29,7 @@ RC="/etc/rc.conf" PFCONF="$(grep '^pf_rules=' ${RC} | cut -d= -f2 | sed 's/"//g')" -echo ${PFCONF:-"/etc/pf.conf"} +echo "${PFCONF:-"/etc/pf.conf"}" # Debug #set +x diff --git a/cdist/conf/type/__pf_ruleset/gencode-local b/cdist/conf/type/__pf_ruleset/gencode-local old mode 100644 new mode 100755 index 2db2ae06..11bfb0b1 --- a/cdist/conf/type/__pf_ruleset/gencode-local +++ b/cdist/conf/type/__pf_ruleset/gencode-local @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Jake Guffey (jake.guffey at eprotex.com) # @@ -54,7 +54,7 @@ case $uname in currentSum=\$(cksum -o 1 ${source} | cut -d= -f2 | sed 's/ //g') ;; *) - echo "Sorry, I do not know how to find a cksum on ${UNAME}." >&2 + echo "Sorry, I do not know how to find a cksum on ${uname}." >&2 exit 1 ;; esac @@ -69,10 +69,10 @@ fi if [ -n "${cksum}" ]; then if [ ! "\${currentSum}" = "${cksum}" ]; then - $__remote_copy "${source}" "${my_target_host}:${rcvar}.new" + $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new" fi else # File just doesn't exist yet - $__remote_copy "${source}" "${my_target_host}:${rcvar}.new" + $__remote_copy "${source}" "\${my_target_host}:${rcvar}.new" fi EOF diff --git a/cdist/conf/type/__pf_ruleset/gencode-remote b/cdist/conf/type/__pf_ruleset/gencode-remote old mode 100644 new mode 100755 index 6e9030ea..12760fdf --- a/cdist/conf/type/__pf_ruleset/gencode-remote +++ b/cdist/conf/type/__pf_ruleset/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Jake Guffey (jake.guffey at eprotex.com) # diff --git a/cdist/conf/type/__ping/gencode-remote b/cdist/conf/type/__ping/gencode-remote new file mode 100644 index 00000000..1341b954 --- /dev/null +++ b/cdist/conf/type/__ping/gencode-remote @@ -0,0 +1,12 @@ +#!/bin/sh -e +# +# Copyright (C) 2018 Olliver Schinagl +# +# SPDX-License-Identifier: GPL-3.0+ +# + +set -eu + +echo "echo 'pong'" + +exit 0 diff --git a/cdist/conf/type/__ping/man.rst b/cdist/conf/type/__ping/man.rst new file mode 100644 index 00000000..e08643dc --- /dev/null +++ b/cdist/conf/type/__ping/man.rst @@ -0,0 +1,43 @@ +cdist-type__ping(7) +================================== + +NAME +---- +cdist-type__ping - Try to connect to host and return 'pong' on success + + +DESCRIPTION +----------- +A simple type which tries to connect to a remote host and runs a simple command +to ensure everything is working. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + __ping + + +AUTHORS +------- +Olliver Schinagl + + +COPYING +------- +Copyright \(C) 2018 Schinagl. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__ping/singleton b/cdist/conf/type/__ping/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__postfix/manifest b/cdist/conf/type/__postfix/manifest index b425e072..f3616979 100755 --- a/cdist/conf/type/__postfix/manifest +++ b/cdist/conf/type/__postfix/manifest @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -22,7 +23,7 @@ os=$(cat "$__global/explorer/os") case "$os" in - ubuntu|debian|archlinux|suse|scientific|centos|devuan) + alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan) __package postfix --state present ;; *) diff --git a/cdist/conf/type/__postfix_master/gencode-remote b/cdist/conf/type/__postfix_master/gencode-remote index 51edc668..7c109a69 100755 --- a/cdist/conf/type/__postfix_master/gencode-remote +++ b/cdist/conf/type/__postfix_master/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__postfix_master/manifest b/cdist/conf/type/__postfix_master/manifest index af71b88e..0960ea41 100755 --- a/cdist/conf/type/__postfix_master/manifest +++ b/cdist/conf/type/__postfix_master/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -36,7 +36,6 @@ __postfix # Default to object_id service="$(cat "$__object/parameter/service" 2>/dev/null || echo "$__object_id")" -state="$(cat "$__object/parameter/state")" # NOTE: keep variables in sync in manifest,explorer,gencode-* prefix="#cdist:$__object_name" @@ -51,7 +50,6 @@ entry="$__object/files/entry" echo "# $(cat "$__object/parameter/comment")" fi printf "%s " "$service" - printf "%s " "$type" for parameter in type private unpriv chroot wakeup maxproc; do printf "%s " "$(cat "$__object/parameter/$parameter")" done diff --git a/cdist/conf/type/__postfix_postconf/explorer/value b/cdist/conf/type/__postfix_postconf/explorer/value index 17126c94..67dacad8 100755 --- a/cdist/conf/type/__postfix_postconf/explorer/value +++ b/cdist/conf/type/__postfix_postconf/explorer/value @@ -22,7 +22,7 @@ os=$("$__explorer/os") case "$os" in - ubuntu|debian|archlinux|suse|scientific|centos|devuan) + alpine|ubuntu|debian|archlinux|suse|scientific|centos|devuan) : ;; *) diff --git a/cdist/conf/type/__postfix_postconf/gencode-remote b/cdist/conf/type/__postfix_postconf/gencode-remote index f886499b..279dddd4 100755 --- a/cdist/conf/type/__postfix_postconf/gencode-remote +++ b/cdist/conf/type/__postfix_postconf/gencode-remote @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -21,7 +22,7 @@ os=$(cat "$__global/explorer/os") case "$os" in - ubuntu|debian|archlinux|suse|scientific|centos|devuan) + alpine|archlinux|centos|debian|devuan|suse|scientific|ubuntu) : ;; *) diff --git a/cdist/conf/type/__postfix_postconf/manifest b/cdist/conf/type/__postfix_postconf/manifest index dbce5364..a82e13d7 100755 --- a/cdist/conf/type/__postfix_postconf/manifest +++ b/cdist/conf/type/__postfix_postconf/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__postfix_postmap/gencode-remote b/cdist/conf/type/__postfix_postmap/gencode-remote index 1b370001..edb7711f 100755 --- a/cdist/conf/type/__postfix_postmap/gencode-remote +++ b/cdist/conf/type/__postfix_postmap/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__postfix_postmap/manifest b/cdist/conf/type/__postfix_postmap/manifest index dbce5364..a82e13d7 100755 --- a/cdist/conf/type/__postfix_postmap/manifest +++ b/cdist/conf/type/__postfix_postmap/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__postfix_reload/gencode-remote b/cdist/conf/type/__postfix_reload/gencode-remote index 0efd6022..7720dc49 100755 --- a/cdist/conf/type/__postfix_reload/gencode-remote +++ b/cdist/conf/type/__postfix_reload/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__postfix_reload/manifest b/cdist/conf/type/__postfix_reload/manifest index dbce5364..a82e13d7 100755 --- a/cdist/conf/type/__postfix_reload/manifest +++ b/cdist/conf/type/__postfix_reload/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__postgres_database/explorer/state b/cdist/conf/type/__postgres_database/explorer/state index dc9659e2..d68d4120 100755 --- a/cdist/conf/type/__postgres_database/explorer/state +++ b/cdist/conf/type/__postgres_database/explorer/state @@ -18,10 +18,25 @@ # along with cdist. If not, see . # +case "$("${__explorer}/os")" +in + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac + + name="$__object_id" -if su - postgres -c "echo '\q' | psql '$name'" 2>/dev/null; then - echo "present" +if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_database WHERE datname='$name'\"")" +then + echo 'present' else - echo "absent" + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_database/gencode-remote b/cdist/conf/type/__postgres_database/gencode-remote index c097efce..61cfa50d 100755 --- a/cdist/conf/type/__postgres_database/gencode-remote +++ b/cdist/conf/type/__postgres_database/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -18,6 +18,20 @@ # along with cdist. If not, see . # +case "$(cat "${__global}/explorer/os")" +in + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac + + name="$__object_id" state_should="$(cat "$__object/parameter/state")" state_is="$(cat "$__object/explorer/state")" @@ -29,10 +43,10 @@ if [ "$state_should" != "$state_is" ]; then if [ -f "$__object/parameter/owner" ]; then owner="-O '$(cat "$__object/parameter/owner")'" fi - echo "su - postgres -c \"createdb $owner '$name'\"" + echo "su - '$postgres_user' -c \"createdb $owner '$name'\"" ;; absent) - echo "su - postgres -c \"dropdb '$name'\"" + echo "su - '$postgres_user' -c \"dropdb '$name'\"" ;; esac fi diff --git a/cdist/conf/type/__postgres_extension/gencode-remote b/cdist/conf/type/__postgres_extension/gencode-remote index 3408df86..af9c97f1 100755 --- a/cdist/conf/type/__postgres_extension/gencode-remote +++ b/cdist/conf/type/__postgres_extension/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2013 Tomas Pospisek (tpo_deb at sourcepole.ch) @@ -22,6 +22,20 @@ # along with cdist. If not, see . # +case "$(cat "${__global}/explorer/os")" +in + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac + + dbname=$( echo "$__object_id" | cut -d":" -f1 ) extension=$( echo "$__object_id" | cut -d":" -f2 ) @@ -30,10 +44,10 @@ state_should=$( cat "$__object/parameter/state" ) case "$state_should" in present) cmd="CREATE EXTENSION IF NOT EXISTS $extension" - echo "su - postgres -c 'psql -c \"$cmd\" \"$dbname\"'" + echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'" ;; absent) - cmd="DROP EXTENSION IF EXISTS $extenstion" - echo "su - postgres -c 'psql -c \"$cmd\" \"$dbname\"'" + cmd="DROP EXTENSION IF EXISTS $extension" + echo "su - '$postgres_user' -c 'psql -c \"$cmd\" \"$dbname\"'" ;; esac diff --git a/cdist/conf/type/__postgres_role/explorer/state b/cdist/conf/type/__postgres_role/explorer/state index 8c102df9..c8e1fa9d 100755 --- a/cdist/conf/type/__postgres_role/explorer/state +++ b/cdist/conf/type/__postgres_role/explorer/state @@ -18,10 +18,25 @@ # along with cdist. If not, see . # +case "$("${__explorer}/os")" +in + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac + + name="$__object_id" -if su - postgres -c "psql -c '\du' | grep -q '^ *$name *|'"; then - echo "present" +if test -n "$(su - "$postgres_user" -c "psql postgres -twAc \"SELECT 1 FROM pg_roles WHERE rolname='$name'\"")" +then + echo 'present' else - echo "absent" + echo 'absent' fi diff --git a/cdist/conf/type/__postgres_role/gencode-remote b/cdist/conf/type/__postgres_role/gencode-remote index 0230e48e..fd56e85d 100755 --- a/cdist/conf/type/__postgres_role/gencode-remote +++ b/cdist/conf/type/__postgres_role/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -18,6 +18,20 @@ # along with cdist. If not, see . # +case "$(cat "${__global}/explorer/os")" +in + netbsd) + postgres_user='pgsql' + ;; + openbsd) + postgres_user='_postgresql' + ;; + *) + postgres_user='postgres' + ;; +esac + + name="$__object_id" state_is="$(cat "$__object/explorer/state")" state_should="$(cat "$__object/parameter/state")" @@ -34,16 +48,16 @@ case "$state_should" in if [ ! -f "$__object/parameter/$boolean" ]; then boolean="no${boolean}" fi - upper=$(echo $boolean | tr '[a-z]' '[A-Z]') + upper=$(echo $boolean | tr '[:lower:]' '[:upper:]') booleans="$booleans $upper" done - [ -n "$password" ] && password="PASSWORD '$password'" + [ -n "$password" ] && password="PASSWORD '$password'" - cmd="CREATE ROLE $name WITH $password $booleans" - echo "su - postgres -c \"psql -c \\\"$cmd\\\"\"" + cmd="CREATE ROLE $name WITH $password $booleans" + echo "su - '$postgres_user' -c \"psql postgres -wc \\\"$cmd\\\"\"" ;; absent) - echo "su - postgres -c \"dropuser \\\"$name\\\"\"" + echo "su - '$postgres_user' -c \"dropuser \\\"$name\\\"\"" ;; esac diff --git a/cdist/conf/type/__process/gencode-remote b/cdist/conf/type/__process/gencode-remote index 639940d9..ec9691b9 100755 --- a/cdist/conf/type/__process/gencode-remote +++ b/cdist/conf/type/__process/gencode-remote @@ -1,7 +1,8 @@ -#!/bin/sh +#!/bin/sh -e # # 2011-2012 Nico Schottelius (nico-cdist at schottelius.org) # 2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2017 Thomas Eckert (tom at it-eckert.de) # # This file is part of cdist. # @@ -45,13 +46,15 @@ case "$state_should" in else echo "$name" fi + echo "started" >> "$__messages_out" ;; absent) - if [ -f "$__object/parameter/stop" ]; then + if [ -f "$__object/parameter/stop" ]; then cat "$__object/parameter/stop" - else - echo kill "${runs}" - fi + else + echo kill "$(cat "$__object/parameter/runs")" + fi + echo "stopped" >> "$__messages_out" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__process/man.rst b/cdist/conf/type/__process/man.rst index e439f37c..e7303c55 100644 --- a/cdist/conf/type/__process/man.rst +++ b/cdist/conf/type/__process/man.rst @@ -30,6 +30,15 @@ start Executable to use for starting the process. +MESSAGES +-------- +started + The process was started. + +stopped + The process was stopped. + + EXAMPLES -------- @@ -63,7 +72,8 @@ SEE ALSO AUTHORS ------- -Nico Schottelius +| Nico Schottelius +| Thomas Eckert COPYING diff --git a/cdist/conf/type/__prometheus_alertmanager/man.rst b/cdist/conf/type/__prometheus_alertmanager/man.rst index ba99e7c8..67e97eaf 100644 --- a/cdist/conf/type/__prometheus_alertmanager/man.rst +++ b/cdist/conf/type/__prometheus_alertmanager/man.rst @@ -10,27 +10,27 @@ DESCRIPTION ----------- Install and configure Prometheus Alertmanager (https://prometheus.io/docs/alerting/alertmanager/). -This type create a daemontools-compatible service directory under /service/prometheus. -Daemontools (or something compatible) must be installed (in particular, the command `svc` must be executable). +Note that due to significant differences between Prometheus 1.x and 2.x, only 2.x is supported. It is your responsibility to make sure that your package manager installs 2.x. (On Devuan Ascii, the parameter `--install-from-backports` helps.) REQUIRED PARAMETERS ------------------- config Alertmanager configuration file. It will be saved as /etc/alertmanager/alertmanager.yml on the target. -listen-address - Passed as web.listen-address. OPTIONAL PARAMETERS ------------------- storage-path Where to put data. Default: /data/alertmanager. (Directory will be created if needed.) +retention-days + How long to retain data. Default: 90 days. BOOLEAN PARAMETERS ------------------ -None +install-from-backports + Valid on Devuan only. Will enable the backports apt source and install the package from there. Useful for getting a newer version. EXAMPLES @@ -38,21 +38,15 @@ EXAMPLES .. code-block:: sh - ALERTPORT=9093 - - __daemontools - __golang_from_vendor --version 1.8.1 # required for prometheus and many exporters - - require="__daemontools __golang_from_vendor" __prometheus_alertmanager \ - --with-daemontools \ - --config "$__manifest/files/alertmanager.yml" \ - --storage-path /data/alertmanager \ - --listen-address "[::]:$ALERTPORT" + __prometheus_alertmanager \ + --install-from-backports \ + --config "$__manifest/files/alertmanager.yml" \ + --storage-path /data/alertmanager SEE ALSO -------- -:strong:`cdist-type__prometheus_server`\ (7), :strong:`cdist-type__daemontools`\ (7), +:strong:`cdist-type__prometheus_server`\ (7), :strong:`cdist-type__grafana_dashboard`\ (7), Prometheus alerting documentation: https://prometheus.io/docs/alerting/overview/ AUTHORS @@ -61,7 +55,7 @@ Kamila Součková COPYING ------- -Copyright \(C) 2017 Kamila Součková. You can redistribute it +Copyright \(C) 2018 Kamila Součková. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__prometheus_alertmanager/manifest b/cdist/conf/type/__prometheus_alertmanager/manifest old mode 100644 new mode 100755 index d885f2ed..8ee818c3 --- a/cdist/conf/type/__prometheus_alertmanager/manifest +++ b/cdist/conf/type/__prometheus_alertmanager/manifest @@ -1,33 +1,64 @@ -#!/bin/sh +#!/bin/sh -e + +##### HARD-CODED CONFIG ##################################################### -GOBIN=/opt/gocode/bin # where to find go binaries CONF_DIR=/etc/prometheus -LOGLEVEL=info CONF=$CONF_DIR/alertmanager.yml -### Prometheus server ####################################################### +##### GET SETTINGS ########################################################## config="$(cat "$__object/parameter/config")" +retention_days="$(cat "$__object/parameter/retention-days")" storage_path="$(cat "$__object/parameter/storage-path")" -listen_address="$(cat "$__object/parameter/listen-address")" +# listen_address="$(cat "$__object/parameter/listen-address")" -FLAGS="config.file '$CONF' -storage.path '$storage_path' -web.listen-address '$listen_address' -log.level $LOGLEVEL" +##### INSTALL THE PACKAGE ################################################### -REAL_FLAGS="$(echo "$FLAGS" | sed -nE 's/^([^#]+).*/ --\1 \\/p')" +require_pkg="" # what to require if I want to require "the package" +require="" +if [ -f "$__object/parameter/install-from-backports" ]; then + os=$(cat "$__global/explorer/os") + os_version=$(cat "$__global/explorer/os_version") -__go_get github.com/prometheus/alertmanager/cmd/... + case $os in + devuan) + [ "$os_version" = "ascii/ceres" ] && os_version='ascii' # "ascii" used in the repo URLs + __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $os_version-backports --component main + require="$require __apt_source/backports" __package_apt prometheus-alertmanager --target-release $os_version-backports + require_pkg="__package_apt/prometheus-alertmanager" + ;; + *) + echo "--install-from-backports is only supported on Devuan -- ignoring." >&2 + echo "Send a pull request if you require it." >&2 + ;; + esac +else + __package prometheus-alertmanager + require_pkg="__package/prometheus-alertmanager" +fi -__user prometheus --system -require="__user/prometheus" __directory "$storage_path" --owner prometheus -require="__user/prometheus" __directory "$CONF_DIR" --owner prometheus +##### PREPARE PATHS AND SUCH ################################################ -__daemontools_service alertmanager --run "setuidgid prometheus $GOBIN/alertmanager $REAL_FLAGS" +require="$require $require_pkg" __directory "$storage_path" --owner prometheus --parents -require="$require __directory/$storage_path __user/prometheus" \ +# TODO this is a bug in the init script, patching it like this is awful and it should be reported +require="$require $require_pkg" \ +__key_value alertmanager_fix_init_script --file /etc/init.d/prometheus-alertmanager \ + --key "NAME" --value "prometheus-alertmanager" --delimiter "=" \ + --onchange "service prometheus-alertmanager restart" + +##### CONFIGURE ############################################################# + +FLAGS="--storage.path $storage_path --data.retention $((retention_days*24))h --web.listen-address [::]:9093 --cluster.advertise-address [::]:9093" + +require="$require $require_pkg" \ +__key_value alertmanager_args --file /etc/default/prometheus-alertmanager \ + --key "ARGS" --value "\"$FLAGS\"" --delimiter "=" \ + --onchange "service prometheus-alertmanager restart" + +require="$require __directory/$storage_path $require_pkg" \ __config_file $CONF \ - --source $config \ - --group prometheus --mode 640 \ - --onchange "$ONCHANGE" + --source "$config" \ + --group prometheus --mode 640 \ + --onchange "service prometheus-alertmanager reload" # TODO when a config-check tool is available, check config here + diff --git a/cdist/conf/type/__prometheus_alertmanager/parameter/boolean b/cdist/conf/type/__prometheus_alertmanager/parameter/boolean new file mode 100644 index 00000000..5d15e93d --- /dev/null +++ b/cdist/conf/type/__prometheus_alertmanager/parameter/boolean @@ -0,0 +1 @@ +install-from-backports diff --git a/cdist/conf/type/__prometheus_alertmanager/parameter/default/retention-days b/cdist/conf/type/__prometheus_alertmanager/parameter/default/retention-days new file mode 100644 index 00000000..d61f00d8 --- /dev/null +++ b/cdist/conf/type/__prometheus_alertmanager/parameter/default/retention-days @@ -0,0 +1 @@ +90 diff --git a/cdist/conf/type/__prometheus_alertmanager/parameter/optional b/cdist/conf/type/__prometheus_alertmanager/parameter/optional index f99d0d37..7fe79009 100644 --- a/cdist/conf/type/__prometheus_alertmanager/parameter/optional +++ b/cdist/conf/type/__prometheus_alertmanager/parameter/optional @@ -1 +1,2 @@ storage-path +retention-days diff --git a/cdist/conf/type/__prometheus_alertmanager/parameter/required b/cdist/conf/type/__prometheus_alertmanager/parameter/required index 02cb49d0..04204c7c 100644 --- a/cdist/conf/type/__prometheus_alertmanager/parameter/required +++ b/cdist/conf/type/__prometheus_alertmanager/parameter/required @@ -1,2 +1 @@ config -listen-address diff --git a/cdist/conf/type/__prometheus_exporter/files/blackbox.yml b/cdist/conf/type/__prometheus_exporter/files/blackbox.yml new file mode 100644 index 00000000..e567c127 --- /dev/null +++ b/cdist/conf/type/__prometheus_exporter/files/blackbox.yml @@ -0,0 +1,63 @@ +modules: + http_2xx: + prober: http + timeout: 3s + http: + method: GET + no_follow_redirects: false + fail_if_ssl: false + fail_if_not_ssl: false + # http_post_2xx: + # prober: http + # timeout: 5s + # http: + # method: POST + # headers: + # Content-Type: application/json + # body: '{}' + # tcp_connect_v4_example: + # prober: tcp + # timeout: 5s + # tcp: + # protocol: "tcp4" + # irc_banner_example: + # prober: tcp + # timeout: 5s + # tcp: + # query_response: + # - send: "NICK prober" + # - send: "USER prober prober prober :prober" + # - expect: "PING :([^ ]+)" + # send: "PONG ${1}" + # - expect: "^:[^ ]+ 001" + # icmp_example: + # prober: icmp + # timeout: 5s + # icmp: + # protocol: "icmp" + # preferred_ip_protocol: "ip4" + # dns_udp_example: + # prober: dns + # timeout: 5s + # dns: + # query_name: "www.prometheus.io" + # query_type: "A" + # valid_rcodes: + # - NOERROR + # validate_answer_rrs: + # fail_if_matches_regexp: + # - ".*127.0.0.1" + # fail_if_not_matches_regexp: + # - "www.prometheus.io.\t300\tIN\tA\t127.0.0.1" + # validate_authority_rrs: + # fail_if_matches_regexp: + # - ".*127.0.0.1" + # validate_additional_rrs: + # fail_if_matches_regexp: + # - ".*127.0.0.1" + # dns_tcp_example: + # prober: dns + # dns: + # protocol: "tcp" # accepts "tcp/tcp4/tcp6/udp/udp4/udp6", defaults to "udp" + # preferred_ip_protocol: "ip4" # used for "udp/tcp", defaults to "ip6" + # query_name: "www.prometheus.io" diff --git a/cdist/conf/type/__prometheus_exporter/man.rst b/cdist/conf/type/__prometheus_exporter/man.rst new file mode 100644 index 00000000..3b1ee4d7 --- /dev/null +++ b/cdist/conf/type/__prometheus_exporter/man.rst @@ -0,0 +1,70 @@ +cdist-type__prometheus_exporter(7) +================================== + +NAME +---- +cdist-type__prometheus_exporter - install some Prometheus exporters + + +DESCRIPTION +----------- +Install and configure some exporters to be used by the Prometheus monitoring system (https://prometheus.io/). + +This type creates a daemontools-compatible service directory under /service/$__object_id. +Daemontools (or something compatible) must be installed (in particular, the command `svc` must be executable). + +This type installs and builds the latest version from git, using go get. A recent version of golang as well +as build tools (make, g++, etc.) must be available. + +Currently supported exporters: + +- node +- blackbox +- ceph + + +REQUIRED PARAMETERS +------------------- +None + + +OPTIONAL PARAMETERS +------------------- +exporter + Which exporter to install and configure. Default: $__object_id. + Currently supported: node, blackbox, ceph. + + +BOOLEAN PARAMETERS +------------------ +add-consul-service + Add this exporter as a Consul service for automatic service discovery. + + +EXAMPLES +-------- + +.. code-block:: sh + + __daemontools + __golang_from_vendor --version 1.9 # required for prometheus and many exporters + + require="__daemontools __golang_from_vendor" __prometheus_exporter node + + +SEE ALSO +-------- +:strong:`cdist-type__daemontools`\ (7), :strong:`cdist-type__golang_from_vendor`\ (7), +:strong:`cdist-type__prometheus_server`\ (7), +Prometheus documentation: https://prometheus.io/docs/introduction/overview/ + +AUTHORS +------- +Kamila Součková + +COPYING +------- +Copyright \(C) 2017 Kamila Součková. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__prometheus_exporter/manifest b/cdist/conf/type/__prometheus_exporter/manifest new file mode 100644 index 00000000..b9e14531 --- /dev/null +++ b/cdist/conf/type/__prometheus_exporter/manifest @@ -0,0 +1,50 @@ +#!/bin/sh + +export GOBIN=/opt/gocode/bin # where to find go binaries + +exporter="$(cat "$__object/parameter/exporter")" +[ -z "$exporter" ] && exporter="$__object_id" + +__user prometheus --system + +require="" +case $exporter in + node) + TEXTFILES=/service/node-exporter/textfiles # path for the textfiles collector + __directory $TEXTFILES --parents --mode 777 + require="$require __golang_from_vendor" __go_get github.com/prometheus/node_exporter + + port=9100 + run="setuidgid prometheus $GOBIN/node_exporter -web.listen-address :$port -collector.textfile.directory=$TEXTFILES" + ;; + blackbox) + require="$require __daemontools_service/${exporter}-exporter __user/prometheus" __config_file "/service/${exporter}-exporter/blackbox.yml" \ + --source "$__type/files/blackbox.yml" \ + --group prometheus --mode 640 \ + --onchange "svc -h /service/${exporter}-exporter" + require="$require __golang_from_vendor" __go_get github.com/prometheus/blackbox_exporter + + port=9115 + run="setuidgid prometheus $GOBIN/blackbox_exporter -config.file=/service/${exporter}-exporter/blackbox.yml" + ;; + ceph) + __package librados-dev # dependency of ceph_exporter + require="$require __golang_from_vendor __package/librados-dev" __go_get github.com/digitalocean/ceph_exporter + + port=9128 + run="setuidgid ceph $GOBIN/ceph_exporter -ceph.config /etc/ceph/ceph.conf -telemetry.addr :$port" + ;; + *) + echo "Unsupported exporter: $exporter." >&2 + exit 1 + ;; +esac + +require="$require __daemontools" __daemontools_service "${exporter}-exporter" --run "$run" +if [ -f "$__object/parameter/add-consul-service" ]; then + __consul_service "${exporter}-exporter" --port "$port" --check-http "http://localhost:$port/metrics" --check-interval 10s +fi + +#__daemontools --install-init-script +__daemontools +__golang_from_vendor --version 1.9 # required for many exporters diff --git a/cdist/conf/type/__prometheus_exporter/parameter/boolean b/cdist/conf/type/__prometheus_exporter/parameter/boolean new file mode 100644 index 00000000..004af844 --- /dev/null +++ b/cdist/conf/type/__prometheus_exporter/parameter/boolean @@ -0,0 +1 @@ +add-consul-service diff --git a/cdist/conf/type/__prometheus_exporter/parameter/default/exporter b/cdist/conf/type/__prometheus_exporter/parameter/default/exporter new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__prometheus_exporter/parameter/optional b/cdist/conf/type/__prometheus_exporter/parameter/optional new file mode 100644 index 00000000..9cfaec5a --- /dev/null +++ b/cdist/conf/type/__prometheus_exporter/parameter/optional @@ -0,0 +1 @@ +exporter diff --git a/cdist/conf/type/__prometheus_server/man.rst b/cdist/conf/type/__prometheus_server/man.rst index fadebd3f..ab6a3c9b 100644 --- a/cdist/conf/type/__prometheus_server/man.rst +++ b/cdist/conf/type/__prometheus_server/man.rst @@ -10,18 +10,12 @@ DESCRIPTION ----------- Install and configure Prometheus (https://prometheus.io/). -This type creates a daemontools-compatible service directory under /service/prometheus. -Daemontools (or something compatible) must be installed (in particular, the command `svc` must be executable). - +Note that due to significant differences between Prometheus 1.x and 2.x, only 2.x is supported. It is your responsibility to make sure that your package manager installs 2.x. (On Devuan Ascii, the parameter `--install-from-backports` helps.) REQUIRED PARAMETERS ------------------- config Prometheus configuration file. It will be saved as /etc/prometheus/prometheus.yml on the target. -listen-address - Passed as web.listen-address. -alertmanager-url - Passed as alertmanager.url OPTIONAL PARAMETERS @@ -32,13 +26,12 @@ rule-files Path to rule files. They will be installed under /etc/prometheus/. You need to include `rule_files: [/etc/prometheus/]` in the config file if you use this. storage-path Where to put data. Default: /data/prometheus. (Directory will be created if needed.) -target-heap-size - Passed as storage.local.target-heap-size. Default: 1/2 of RAM. BOOLEAN PARAMETERS ------------------ -None +install-from-backports + Valid on Devuan only. Will enable the backports apt source and install the package from there. Useful for getting a newer version. EXAMPLES @@ -49,22 +42,17 @@ EXAMPLES PROMPORT=9090 ALERTPORT=9093 - __daemontools - __golang_from_vendor --version 1.8.1 # required for prometheus and many exporters - - require="__daemontools __golang_from_vendor" __prometheus_server \ - --with-daemontools \ + __prometheus_server \ + --install-from-backports \ --config "$__manifest/files/prometheus.yml" \ --retention-days 14 \ --storage-path /data/prometheus \ - --listen-address "[::]:$PROMPORT" \ - --rule-files "$__manifest/files/*.rules" \ - --alertmanager-url "http://monitoring1.node.consul:$ALERTPORT,http://monitoring2.node.consul:$ALERTPORT" + --rule-files "$__manifest/files/*.rules" SEE ALSO -------- -:strong:`cdist-type__prometheus_alertmanager`\ (7), :strong:`cdist-type__daemontools`\ (7), +:strong:`cdist-type__prometheus_alertmanager`\ (7), :strong:`cdist-type__grafana_dashboard`\ (7), Prometheus documentation: https://prometheus.io/docs/introduction/overview/ AUTHORS @@ -73,7 +61,7 @@ Kamila Součková COPYING ------- -Copyright \(C) 2017 Kamila Součková. You can redistribute it +Copyright \(C) 2018 Kamila Součková. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__prometheus_server/manifest b/cdist/conf/type/__prometheus_server/manifest old mode 100644 new mode 100755 index 3c5f16e3..8685130f --- a/cdist/conf/type/__prometheus_server/manifest +++ b/cdist/conf/type/__prometheus_server/manifest @@ -1,52 +1,71 @@ -#!/bin/sh +#!/bin/sh -e + +##### HARD-CODED CONFIG ##################################################### -GOBIN=/opt/gocode/bin # where to find go binaries CONF_DIR=/etc/prometheus CONF=$CONF_DIR/prometheus.yml -LOGLEVEL=info + +##### GET SETTINGS ########################################################## config="$(cat "$__object/parameter/config")" retention_days="$(cat "$__object/parameter/retention-days")" storage_path="$(cat "$__object/parameter/storage-path")" -listen_address="$(cat "$__object/parameter/listen-address")" -alertmanager_url="$(cat "$__object/parameter/alertmanager-url")" -target_heap_size="$(cat "$__object/parameter/target-heap-size")" rule_files="$(cat "$__object/parameter/rule-files")" # explorer in kB => convert; by default we go with 1/2 RAM -[ "$target_heap_size" = "auto" ] && target_heap_size=$(($(cat $__global/explorer/memory)*1024/2)) +[ "$target_heap_size" = "auto" ] && target_heap_size=$(($(cat "$__global/explorer/memory")*1024/2)) +##### INSTALL THE PACKAGE ################################################### -FLAGS="config.file '$CONF' -storage.local.path '$storage_path' -storage.local.target-heap-size $(($target_heap_size)) # in bytes; should be 2/3 of available memory because it may be hungry -storage.local.retention $(($retention_days*24))h # golang doesn't have days :D -web.listen-address '$listen_address' -alertmanager.url '$alertmanager_url' -log.level $LOGLEVEL" +require_pkg="" # what to require if I want to require "the package" +require="" +if [ -f "$__object/parameter/install-from-backports" ]; then + os=$(cat "$__global/explorer/os") + os_version=$(cat "$__global/explorer/os_version") -REAL_FLAGS="$(echo "$FLAGS" | sed -nE 's/^([^#]+).*/ --\1 \\/p')" + case $os in + devuan) + [ "$os_version" = "ascii/ceres" ] && os_version='ascii' # "ascii" used in the repo URLs + __apt_source backports --uri http://auto.mirror.devuan.org/merged --distribution $os_version-backports --component main + require="$require __apt_source/backports" __package_apt prometheus --target-release $os_version-backports + require_pkg="__package_apt/prometheus" + ;; + *) + echo "--install-from-backports is only supported on Devuan -- ignoring." >&2 + echo "Send a pull request if you require it." >&2 + ;; + esac +else + __package prometheus + require_pkg="__package/prometheus" +fi -__go_get github.com/prometheus/prometheus/cmd/... +##### PREPARE PATHS AND SUCH ################################################ -__user prometheus --system -require="__user/prometheus" __directory "$storage_path" --owner prometheus -require="__user/prometheus" __directory "$CONF_DIR" --owner prometheus +require="$require $require_pkg" __directory "$storage_path" --owner prometheus --parents -__daemontools_service prometheus --run "setuidgid prometheus $GOBIN/prometheus $REAL_FLAGS" +##### CONFIGURE ############################################################# -require="$require __directory/$storage_path __user/prometheus" \ +FLAGS="--storage.tsdb.path $storage_path --storage.tsdb.retention $((retention_days*24))h --web.listen-address [::]:9090" + +# TODO it would be neat to restart prometheus on change -- __key_value really should have an --onchange parameter +require="$require $require_pkg" \ +__key_value prometheus_args --file /etc/default/prometheus \ + --key "ARGS" --value "\"$FLAGS\"" --delimiter "=" \ + --onchange "service prometheus restart" + +require="$require __directory/$storage_path $require_pkg" \ __config_file $CONF \ - --source $config \ + --source "$config" \ --group prometheus --mode 640 \ - --onchange "$GOBIN/promtool check-config $CONF && svc -h /service/prometheus" + --onchange "promtool check config $CONF && service prometheus reload" for file in $rule_files; do - dest=$CONF_DIR/$(basename $file) - require="$require __directory/$CONF_DIR __user/prometheus" \ + dest=$CONF_DIR/$(basename "$file") + require="$require $require_pkg" \ __config_file "$dest" \ --source "$file" \ --owner prometheus \ - --onchange "$GOBIN/promtool check-rules '$dest' && svc -h /service/prometheus" + --onchange "promtool check rules '$dest' && service prometheus reload" done diff --git a/cdist/conf/type/__prometheus_server/parameter/boolean b/cdist/conf/type/__prometheus_server/parameter/boolean new file mode 100644 index 00000000..5d15e93d --- /dev/null +++ b/cdist/conf/type/__prometheus_server/parameter/boolean @@ -0,0 +1 @@ +install-from-backports diff --git a/cdist/conf/type/__prometheus_server/parameter/default/target-heap-size b/cdist/conf/type/__prometheus_server/parameter/default/target-heap-size deleted file mode 100644 index 865faf10..00000000 --- a/cdist/conf/type/__prometheus_server/parameter/default/target-heap-size +++ /dev/null @@ -1 +0,0 @@ -auto diff --git a/cdist/conf/type/__prometheus_server/parameter/optional b/cdist/conf/type/__prometheus_server/parameter/optional index 4d8d8f3e..cb437211 100644 --- a/cdist/conf/type/__prometheus_server/parameter/optional +++ b/cdist/conf/type/__prometheus_server/parameter/optional @@ -1,4 +1,3 @@ -target-heap-size retention-days rule-files storage-path diff --git a/cdist/conf/type/__prometheus_server/parameter/required b/cdist/conf/type/__prometheus_server/parameter/required index 49abf924..04204c7c 100644 --- a/cdist/conf/type/__prometheus_server/parameter/required +++ b/cdist/conf/type/__prometheus_server/parameter/required @@ -1,3 +1 @@ -alertmanager-url config -listen-address diff --git a/cdist/conf/type/__pyvenv/explorer/group b/cdist/conf/type/__pyvenv/explorer/group index ff072c5e..a655bda7 100755 --- a/cdist/conf/type/__pyvenv/explorer/group +++ b/cdist/conf/type/__pyvenv/explorer/group @@ -2,4 +2,4 @@ destination="/$__object_id" -stat --print "%G" ${destination} 2>/dev/null || exit 0 +stat --print "%G" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__pyvenv/explorer/owner b/cdist/conf/type/__pyvenv/explorer/owner index b77e3c6e..8b3c7f8e 100755 --- a/cdist/conf/type/__pyvenv/explorer/owner +++ b/cdist/conf/type/__pyvenv/explorer/owner @@ -2,4 +2,4 @@ destination="/$__object_id" -stat --print "%U" ${destination} 2>/dev/null || exit 0 +stat --print "%U" "${destination}" 2>/dev/null || exit 0 diff --git a/cdist/conf/type/__pyvenv/gencode-remote b/cdist/conf/type/__pyvenv/gencode-remote index 907e0ff6..04700683 100755 --- a/cdist/conf/type/__pyvenv/gencode-remote +++ b/cdist/conf/type/__pyvenv/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Darko Poljak (darko.poljak at gmail.com) # @@ -29,10 +29,10 @@ owner="$(cat "$__object/parameter/owner")" group="$(cat "$__object/parameter/group")" mode="$(cat "$__object/parameter/mode")" -[ "$state_should" = "$state_is" -a \ - "$owner" = "$owner_is" -a \ - "$group" = "$group_is" -a \ - -n "$mode" ] && exit 0 +[ "$state_should" = "$state_is" ] && \ +[ "$owner" = "$owner_is" ] && \ +[ "$group" = "$group_is" ] && \ +[ -n "$mode" ] && exit 0 destination="/$__object_id" venvparams="$(cat "$__object/parameter/venvparams")" @@ -47,10 +47,10 @@ fi case $state_should in present) if [ "$state_should" != "$state_is" ]; then - echo $pyvenv $venvparams "$destination" + echo "$pyvenv $venvparams $destination" fi - if [ \( -n "$owner" -a "$owner_is" != "$owner" \) -o \ - \( -n "$group" -a "$group_is" != "$group" \) ]; then + if { [ -n "$owner" ] && [ "$owner_is" != "$owner" ]; } || \ + { [ -n "$group" ] && [ "$group_is" != "$group" ]; }; then echo chown -R "${owner}:${group}" "$destination" fi if [ -n "$mode" ]; then diff --git a/cdist/conf/type/__pyvenv/manifest b/cdist/conf/type/__pyvenv/manifest index 3e41ad04..5d6a12e8 100755 --- a/cdist/conf/type/__pyvenv/manifest +++ b/cdist/conf/type/__pyvenv/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2016 Darko Poljak (darko.poljak at gmail.com) # diff --git a/cdist/conf/type/__qemu_img/gencode-remote b/cdist/conf/type/__qemu_img/gencode-remote old mode 100644 new mode 100755 index bffedd26..94816f58 --- a/cdist/conf/type/__qemu_img/gencode-remote +++ b/cdist/conf/type/__qemu_img/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # ################################################################################ # State: absent is handled by manifest - we need only to do stuff if image is @@ -18,4 +18,4 @@ format="$(cat "$__object/parameter/format")" size="$(cat "$__object/parameter/size")" diskimage="/$__object_id" -echo qemu-img create -f \"$format\" \"$diskimage\" \"$size\" +echo "qemu-img create -f '$format' '$diskimage' '$size'" diff --git a/cdist/conf/type/__qemu_img/manifest b/cdist/conf/type/__qemu_img/manifest old mode 100644 new mode 100755 index e0ff6e03..55f3bf16 --- a/cdist/conf/type/__qemu_img/manifest +++ b/cdist/conf/type/__qemu_img/manifest @@ -1,10 +1,9 @@ -#!/bin/sh +#!/bin/sh -e # ################################################################################ # Default settings # -format="$(cat "$__object/parameter/format")" state_should="$(cat "$__object/parameter/state")" diskimage="/$__object_id" diff --git a/cdist/conf/type/__rbenv/manifest b/cdist/conf/type/__rbenv/manifest old mode 100644 new mode 100755 index 767abdba..e5c3d2f8 --- a/cdist/conf/type/__rbenv/manifest +++ b/cdist/conf/type/__rbenv/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__rsync/gencode-local b/cdist/conf/type/__rsync/gencode-local old mode 100644 new mode 100755 index 8d268d7e..e36ded2f --- a/cdist/conf/type/__rsync/gencode-local +++ b/cdist/conf/type/__rsync/gencode-local @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Dominique Roux (dominique.roux4 at gmail.com) # @@ -29,9 +29,9 @@ fi set -- if [ -f "$__object/parameter/rsync-opts" ]; then - while read opts; do + while read -r opts; do set -- "$@" "--$opts" - done < $__object/parameter/rsync-opts + done < "$__object/parameter/rsync-opts" fi echo rsync -a \ diff --git a/cdist/conf/type/__rsync/gencode-remote b/cdist/conf/type/__rsync/gencode-remote old mode 100644 new mode 100755 index a1135ea6..074246af --- a/cdist/conf/type/__rsync/gencode-remote +++ b/cdist/conf/type/__rsync/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Dominique Roux (dominique.roux4 at gmail.com) # diff --git a/cdist/conf/type/__rsync/manifest b/cdist/conf/type/__rsync/manifest old mode 100644 new mode 100755 index 0e4cc1f4..9bd44c6d --- a/cdist/conf/type/__rsync/manifest +++ b/cdist/conf/type/__rsync/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Dominique Roux (dominique.roux4 at gmail.com) # diff --git a/cdist/conf/type/__rvm/explorer/state b/cdist/conf/type/__rvm/explorer/state index f43f5509..74d17048 100755 --- a/cdist/conf/type/__rvm/explorer/state +++ b/cdist/conf/type/__rvm/explorer/state @@ -28,7 +28,7 @@ if [ "$user" = "root" ]; then echo absent fi else - if su - $user -c "[ -d \"\$HOME/.rvm\" ]" ; then + if su - "$user" -c "[ -d \"\$HOME/.rvm\" ]" ; then echo "present" else echo "absent" diff --git a/cdist/conf/type/__rvm/gencode-remote b/cdist/conf/type/__rvm/gencode-remote index dbc6ba60..993191c1 100755 --- a/cdist/conf/type/__rvm/gencode-remote +++ b/cdist/conf/type/__rvm/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Evax Software # 2012 Nico Schottelius (nico-cdist at schottelius.org) @@ -34,7 +34,7 @@ DONE absent) cat << DONE su - $user -c "rm -Rf \"\\\$HOME/.rvm\"; -sed '/rvm\/scripts\/rvm/d' \"\\\$HOME/.bashrc\" > \"\\\$HOME/.bashrc.cdist-tmp\" +sed '/rvm\\/scripts\\/rvm/d' \"\\\$HOME/.bashrc\" > \"\\\$HOME/.bashrc.cdist-tmp\" mv \"\\\$HOME/.bashrc.cdist-tmp\" \"\\\$HOME/.bashrc\"" DONE ;; diff --git a/cdist/conf/type/__rvm/manifest b/cdist/conf/type/__rvm/manifest index 482c0d17..0230156b 100755 --- a/cdist/conf/type/__rvm/manifest +++ b/cdist/conf/type/__rvm/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Evax Software # diff --git a/cdist/conf/type/__rvm_gem/gencode-remote b/cdist/conf/type/__rvm_gem/gencode-remote index 1fe6e78e..9212de91 100755 --- a/cdist/conf/type/__rvm_gem/gencode-remote +++ b/cdist/conf/type/__rvm_gem/gencode-remote @@ -20,8 +20,6 @@ gem="$__object_id" gemset="$(cat "$__object/parameter/gemset")" -ruby="$(echo "$gemset" | cut -d '@' -f 1)" -gemsetname="$(echo "$gemset" | cut -d '@' -f 2)" state_is="$(cat "$__object/explorer/state")" user="$(cat "$__object/parameter/user")" state_should="$(cat "$__object/parameter/state")" diff --git a/cdist/conf/type/__rvm_gemset/explorer/state b/cdist/conf/type/__rvm_gemset/explorer/state index fa643a6e..e300453b 100755 --- a/cdist/conf/type/__rvm_gemset/explorer/state +++ b/cdist/conf/type/__rvm_gemset/explorer/state @@ -18,9 +18,6 @@ # along with cdist. If not, see . # -gemset="$__object_id" -ruby="$(echo "$gemset" | cut -d '@' -f 1)" -gemsetname="$(echo "$gemset" | cut -d '@' -f2)" user="$(cat "$__object/parameter/user")" if [ ! -e "~$user/.rvm/scripts/rvm" ] ; then @@ -28,7 +25,9 @@ if [ ! -e "~$user/.rvm/scripts/rvm" ] ; then exit 0 fi +# shellcheck disable=SC2016 if su - "$user" -c 'source ~/.rvm/scripts/rvm; rvm list strings | grep -q "^$ruby\$"'; then + # shellcheck disable=SC2016 if su - "$user" -c 'source ~/.rvm/scripts/rvm; rvm use "$ruby" > /dev/null; rvm gemset list strings | cut -f 1 -d " " | grep -q "^$gemsetname\$"'; then echo "present" exit 0 diff --git a/cdist/conf/type/__rvm_gemset/gencode-remote b/cdist/conf/type/__rvm_gemset/gencode-remote index f0c0052b..3cdc66a6 100755 --- a/cdist/conf/type/__rvm_gemset/gencode-remote +++ b/cdist/conf/type/__rvm_gemset/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Evax Software # 2012 Nico Schottelius (nico-cdist at schottelius.org) @@ -33,7 +33,7 @@ case "$state_should" in cat << DONE su - "$user" -c "source ~/.rvm/scripts/rvm; rvm $gemset --create" DONE - if -f "$__object/parameter/default"; then + if [ -f "$__object/parameter/default" ]; then cat << DONE su - "$user" -c "source ~/.rvm/scripts/rvm; rvm use --default $gemset" DONE diff --git a/cdist/conf/type/__rvm_ruby/gencode-remote b/cdist/conf/type/__rvm_ruby/gencode-remote index f1de3906..f2fd41ef 100755 --- a/cdist/conf/type/__rvm_ruby/gencode-remote +++ b/cdist/conf/type/__rvm_ruby/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Evax Software # @@ -21,7 +21,6 @@ ruby="$__object_id" state_is="$(cat "$__object/explorer/state")" user="$(cat "$__object/parameter/user")" -default="$(cat "$__object/parameter/default" 2>/dev/null || true)" state_should="$(cat "$__object/parameter/state")" [ "$state_is" = "$state_should" ] && exit 0 diff --git a/cdist/conf/type/__rvm_ruby/manifest b/cdist/conf/type/__rvm_ruby/manifest index db8fd830..3f63eb11 100755 --- a/cdist/conf/type/__rvm_ruby/manifest +++ b/cdist/conf/type/__rvm_ruby/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__ssh_authorized_key/explorer/entry b/cdist/conf/type/__ssh_authorized_key/explorer/entry index 78031ab5..ccab0afc 100755 --- a/cdist/conf/type/__ssh_authorized_key/explorer/entry +++ b/cdist/conf/type/__ssh_authorized_key/explorer/entry @@ -19,8 +19,16 @@ # # extract the keytype and base64 encoded key ignoring any options and comment -type_and_key="$(cat "$__object/parameter/key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" -file="$(cat $__object/parameter/file)" +type_and_key="$(tr ' ' '\n' < "$__object/parameter/key"| awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" +# If type_and_key is empty, which is the case with an invalid key, do not grep $file because it results +# in greping everything in file and all entries from file are removed. +if [ -n "${type_and_key}" ] +then + file="$(cat "$__object/parameter/file")" -# get any entries that match the type and key -grep ".*$type_and_key[ \n]" "$file" || true + # get any entries that match the type and key + + # NOTE: Do not match from the beginning of the line as there may be options + # preceeding the key. + grep "${type_and_key}\\([ \\n].*\\)*$" "$file" || true +fi diff --git a/cdist/conf/type/__ssh_authorized_key/gencode-remote b/cdist/conf/type/__ssh_authorized_key/gencode-remote index 6bbfa269..f37aa565 100755 --- a/cdist/conf/type/__ssh_authorized_key/gencode-remote +++ b/cdist/conf/type/__ssh_authorized_key/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -20,6 +20,15 @@ set -u +the_key="$(cat "$__object/parameter/key")" +# validate key +validated_key="$(echo "${the_key}" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" +if [ -z "${validated_key}" ] +then + echo "Key is invalid: \"${the_key}\"" >&2 + exit 1 +fi + remove_line() { file="$1" line="$2" @@ -39,7 +48,7 @@ add_line() { line="$2" # escape single quotes line_sanitised=$(echo "$line" | sed -e "s/'/'\"'\"'/g") - printf '%s' "printf '%s\n' '$line_sanitised' >> $file" + printf '%s' "printf '%s\\n' '$line_sanitised' >> $file" } @@ -50,16 +59,16 @@ mkdir "$__object/files" ( if [ -f "$__object/parameter/option" ]; then # comma seperated list of options - options="$(cat "$__object/parameter/option" | tr '\n' ',')" + options="$(tr '\n' ',' < "$__object/parameter/option")" printf '%s ' "${options%*,}" fi if [ -f "$__object/parameter/comment" ]; then # extract the keytype and base64 encoded key ignoring any options and comment - printf '%s ' "$(cat "$__object/parameter/key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" + printf '%s ' "$(echo "${the_key}" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" # override the comment with the one explicitly given printf '%s' "$(cat "$__object/parameter/comment")" else - printf '%s' "$(cat "$__object/parameter/key")" + printf '%s' "${the_key}" fi printf '\n' ) > "$__object/files/should" @@ -69,7 +78,7 @@ if [ -s "$__object/explorer/entry" ]; then # Note that the files have to be sorted for comparison with `comm`. sort "$__object/explorer/entry" > "$__object/files/is" comm -13 "$__object/files/should" "$__object/files/is" | { - while read entry; do + while read -r entry; do remove_line "$file" "$entry" done } @@ -79,7 +88,7 @@ fi entry="$(cat "$__object/files/should")" state_should="$(cat "$__object/parameter/state")" num_existing_entries=$(grep -c -F -x "$entry" "$__object/explorer/entry" || true) -if [ $num_existing_entries -eq 1 ]; then +if [ "$num_existing_entries" -eq 1 ]; then state_is="present" else # Posix grep does not define the -m option, so we can not remove a single @@ -102,8 +111,10 @@ fi case "$state_should" in present) add_line "$file" "$entry" + echo "added to $file ($entry)" >> "$__messages_out" ;; absent) remove_line "$file" "$entry" + echo "removed from $file ($entry)" >> "$__messages_out" ;; esac diff --git a/cdist/conf/type/__ssh_authorized_key/man.rst b/cdist/conf/type/__ssh_authorized_key/man.rst index b58ad879..087a3dae 100644 --- a/cdist/conf/type/__ssh_authorized_key/man.rst +++ b/cdist/conf/type/__ssh_authorized_key/man.rst @@ -36,6 +36,15 @@ state if the given keys should be 'present' or 'absent', defaults to 'present'. +MESSAGES +-------- +added to `file` (`entry`) + The key `entry` (with optional comment) was added to `file`. + +removed from `file` (`entry`) + The key `entry` (with optional comment) was removed from `file`. + + EXAMPLES -------- diff --git a/cdist/conf/type/__ssh_authorized_keys/manifest b/cdist/conf/type/__ssh_authorized_keys/manifest index 6a536e1b..b507c7ff 100755 --- a/cdist/conf/type/__ssh_authorized_keys/manifest +++ b/cdist/conf/type/__ssh_authorized_keys/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) # 2014 Nico Schottelius (nico-cdist at schottelius.org) @@ -23,7 +23,7 @@ owner="$(cat "$__object/parameter/owner" 2>/dev/null || echo "$__object_id")" state="$(cat "$__object/parameter/state" 2>/dev/null)" file="$(cat "$__object/explorer/file")" -if [ ! -f "$__object/parameter/noparent" -o ! -f "$__object/parameter/nofile" ]; then +if [ ! -f "$__object/parameter/noparent" ] || [ ! -f "$__object/parameter/nofile" ]; then group="$(cut -d':' -f 1 "$__object/explorer/group")" if [ -z "$group" ]; then echo "Failed to get owners group from explorer." >&2 @@ -45,23 +45,11 @@ if [ ! -f "$__object/parameter/noparent" -o ! -f "$__object/parameter/nofile" ]; fi fi -# Remove legacy blocks created by old versions of this type -# FIXME: remove me in 3.2+ -__block "$__object_name" \ - --file "$file" \ - --prefix "#cdist:$__object_name" \ - --suffix "#/cdist:$__object_name" \ - --state 'absent' \ - --text - << DONE -remove legacy block -DONE -export require="__block/$__object_name" - _cksum() { echo "$1" | cksum | cut -d' ' -f 1 } -while read key; do +while read -r key; do type_and_key="$(echo "$key" | tr ' ' '\n' | awk '/^(ssh|ecdsa)-[^ ]+/ { printf $1" "; getline; printf $1 }')" object_id="$(_cksum "$file")-$(_cksum "$type_and_key")" set -- "$object_id" @@ -69,7 +57,8 @@ while read key; do set -- "$@" --key "$key" set -- "$@" --state "$state" if [ -f "$__object/parameter/option" ]; then - set -- "$@" --option "$(cat "$__object/parameter/option")" + # shellcheck disable=SC2046 + set -- "$@" $(printf -- '--option %s ' $(cat "$__object/parameter/option")) fi if [ -f "$__object/parameter/comment" ]; then set -- "$@" --comment "$(cat "$__object/parameter/comment")" diff --git a/cdist/conf/type/__ssh_dot_ssh/manifest b/cdist/conf/type/__ssh_dot_ssh/manifest index 4b797afb..bc3a3952 100755 --- a/cdist/conf/type/__ssh_dot_ssh/manifest +++ b/cdist/conf/type/__ssh_dot_ssh/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2014 Steven Armstrong (steven-cdist at armstrong.cc) # 2014 Nico Schottelius (nico-cdist at schottelius.org) diff --git a/cdist/conf/type/__staged_file/gencode-local b/cdist/conf/type/__staged_file/gencode-local index 1a236789..ba9e8798 100755 --- a/cdist/conf/type/__staged_file/gencode-local +++ b/cdist/conf/type/__staged_file/gencode-local @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # 2015 Nico Schottelius (nico-cdist at schottelius.org) @@ -23,7 +23,6 @@ destination="$__object_id" source="$(cat "$__object/parameter/source")" -cksum="$(cat "$__object/parameter/cksum")" stage_dir="$(cat "$__object/parameter/stage-dir")" state="$(cat "$__object/parameter/state")" fetch_command="$(cat "$__object/parameter/fetch-command")" @@ -57,24 +56,30 @@ get_file() { } fetch_file() { + # shellcheck disable=SC2059 printf "$fetch_command" "$source" printf ' > "%s"\n' "$stage_file" } fetch_and_prepare_file() { - printf 'tmpdir="$(mktemp -d --tmpdir="/tmp" "%s")"\n' "${__type##*/}.XXXXXXXXXX" + # shellcheck disable=SC2016 + printf 'tmpdir="$(mktemp -d -p "/tmp" "%s")"\n' "${__type##*/}.XXXXXXXXXX" + # shellcheck disable=SC2016 printf 'cd "$tmpdir"\n' - printf "$fetch_command > \"%s\"\n" "$source" "$source_file_name" + # shellcheck disable=SC2059 + printf "$fetch_command > \"%s\"\\n" "$source" "$source_file_name" prepare_command="$(cat "$__object/parameter/prepare-command")" - printf "$prepare_command > \"%s\"\n" "$source_file_name" "$stage_file" + # shellcheck disable=SC2059 + printf "$prepare_command > \"%s\"\\n" "$source_file_name" "$stage_file" printf 'cd - >/dev/null\n' + # shellcheck disable=SC2016 printf 'rm -rf "$tmpdir"\n' } cat << DONE verify_cksum() { cksum_is="\$(cksum "$stage_file" | cut -d' ' -f1,2)" - cksum_should="$(cat "$__object/parameter/cksum" | cut -d' ' -f1,2)" + cksum_should="$(cut -d' ' -f1,2 "$__object/parameter/cksum")" if [ "\$cksum_is" = "\$cksum_should" ]; then return 0 else diff --git a/cdist/conf/type/__staged_file/manifest b/cdist/conf/type/__staged_file/manifest index 454948b4..c8e1fbbb 100755 --- a/cdist/conf/type/__staged_file/manifest +++ b/cdist/conf/type/__staged_file/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2015 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -19,11 +19,7 @@ # destination="$__object_id" -source="$(cat "$__object/parameter/source")" -cksum="$(cat "$__object/parameter/cksum")" stage_dir="$(cat "$__object/parameter/stage-dir")" -state="$(cat "$__object/parameter/state")" -fetch_command="$(cat "$__object/parameter/fetch-command")" stage_file="${stage_dir}/${destination}" set -- "/${destination}" diff --git a/cdist/conf/type/__start_on_boot/explorer/state b/cdist/conf/type/__start_on_boot/explorer/state index d49f01c7..b7a6cf0f 100644 --- a/cdist/conf/type/__start_on_boot/explorer/state +++ b/cdist/conf/type/__start_on_boot/explorer/state @@ -1,6 +1,6 @@ #!/bin/sh # -# 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2019 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Daniel Heule (hda at sfs.biz) # # This file is part of cdist. @@ -38,12 +38,27 @@ if [ "$init" = 'systemd' ]; then else case "$os" in debian|openwrt|devuan) - state="present" - [ -f "/etc/rc$runlevel.d/S"??"$name" ] || state="absent" + state="absent" + for file in "/etc/rc$runlevel.d/S"??"$name" + do + if [ -f "$file" ] + then + state="present" + break + fi + done ;; ubuntu) state="absent" - [ -f "/etc/rc$runlevel.d/S"??"$name" ] && state="present" + for file in "/etc/rc$runlevel.d/S"??"$name" + do + if [ -f "$file" ] + then + state="present" + break + fi + done + [ -f "/etc/init/${name}.conf" ] && state="present" ;; @@ -60,10 +75,25 @@ else state=$(chkconfig --check "$name" "$runlevel" || echo absent) [ "$state" ] || state="present" ;; - gentoo) - state="present" - [ -f "/etc/runlevels/${target_runlevel}/${name}" ] || state="absent" + gentoo|alpine) + state="absent" + for d in /etc/runlevels/*; do + if [ -f "/etc/runlevels/${d}/${name}" ];then + state="present" + break + fi + done ;; + freebsd) + state="absent" + service -e | grep "/$name$" && state="present" + ;; + openbsd) + state='absent' + # OpenBSD 5.7 and higher + rcctl ls on | grep "^${name}$" && state='present' + ;; + *) echo "Unsupported os: $os" >&2 exit 1 diff --git a/cdist/conf/type/__start_on_boot/gencode-remote b/cdist/conf/type/__start_on_boot/gencode-remote old mode 100644 new mode 100755 index 0ab67a1a..c900933f --- a/cdist/conf/type/__start_on_boot/gencode-remote +++ b/cdist/conf/type/__start_on_boot/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2016 Daniel Heule (hda at sfs.biz) @@ -37,16 +37,16 @@ case "$state_should" in if [ "$init" = 'systemd' ]; then # this handles ALL linux distros with systemd # e.g. archlinux, gentoo in some cases, new RHEL and SLES versions - echo "systemctl -q enable \"$name\"" + echo "systemctl -q enable '$name'" else case "$os" in debian) case "$os_version" in [1-7]*) - echo "update-rc.d \"$name\" defaults >/dev/null" + echo "update-rc.d '$name' defaults >/dev/null" ;; 8*) - echo "systemctl enable \"$name\"" + echo "systemctl enable '$name'" ;; *) echo "Unsupported version $os_version of $os" >&2 @@ -55,26 +55,35 @@ case "$state_should" in esac ;; devuan) - echo "update-rc.d \"$name\" defaults >/dev/null" + echo "update-rc.d '$name' defaults >/dev/null" ;; - gentoo) - echo rc-update add \"$name\" \"$target_runlevel\" + alpine|gentoo) + echo "rc-update add '$name' '$target_runlevel'" ;; amazon|scientific|centos|fedora|owl|redhat|suse) - echo chkconfig \"$name\" on + echo "chkconfig '$name' on" ;; openwrt) # 'enable' can be successful and still return a non-zero exit # code, deal with it by checking for success ourselves in that # case (the || ... part). - echo "/etc/init.d/\"$name\" enable || [ -f /etc/rc.d/S??\"$name\" ]" + echo "'/etc/init.d/$name' enable || [ -f /etc/rc.d/S??'$name' ]" ;; ubuntu) - echo "update-rc.d \"$name\" defaults >/dev/null" + echo "update-rc.d '$name' defaults >/dev/null" + ;; + + freebsd) + : # handled in manifest + ;; + + openbsd) + # OpenBSD 5.7 and higher + echo "rcctl enable '$name'" ;; *) @@ -89,24 +98,29 @@ case "$state_should" in if [ "$init" = 'systemd' ]; then # this handles ALL linux distros with systemd # e.g. archlinux, gentoo in some cases, new RHEL and SLES versions - echo "systemctl -q disable \"$name\"" + echo "systemctl -q disable '$name'" else case "$os" in debian|ubuntu|devuan) - echo update-rc.d -f \"$name\" remove + echo "update-rc.d -f '$name' remove" ;; - gentoo) - echo rc-update del \"$name\" \"$target_runlevel\" + alpine|gentoo) + echo "rc-update del '$name' '$target_runlevel'" ;; centos|fedora|owl|redhat|suse) - echo chkconfig \"$name\" off + echo "chkconfig '$name' off" ;; openwrt) - echo "\"/etc/init.d/$name\" disable" + echo "'/etc/init.d/$name' disable" + ;; + + openbsd) + # OpenBSD 5.7 and higher + echo "rcctl disable '$name'" ;; *) diff --git a/cdist/conf/type/__start_on_boot/man.rst b/cdist/conf/type/__start_on_boot/man.rst index 851d1a89..b7c73ab1 100644 --- a/cdist/conf/type/__start_on_boot/man.rst +++ b/cdist/conf/type/__start_on_boot/man.rst @@ -55,7 +55,7 @@ Nico Schottelius COPYING ------- -Copyright \(C) 2012 Nico Schottelius. You can redistribute it +Copyright \(C) 2012-2019 Nico Schottelius. You can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. diff --git a/cdist/conf/type/__start_on_boot/manifest b/cdist/conf/type/__start_on_boot/manifest new file mode 100644 index 00000000..c1c983ec --- /dev/null +++ b/cdist/conf/type/__start_on_boot/manifest @@ -0,0 +1,28 @@ +#!/bin/sh -e + +state_should="$(cat "$__object/parameter/state")" +state_is=$(cat "$__object/explorer/state") +name="$__object_id" + +# Short circuit if nothing is to be done +[ "$state_should" = "$state_is" ] && exit 0 + +os=$(cat "$__global/explorer/os") + +case "$os" in + freebsd) + if [ "$state_should" = 'present' ]; then + value='YES' + else + value='NO' + fi + __key_value "rcconf-$name-enable" \ + --file /etc/rc.conf \ + --key "${name}_enable" \ + --value "\"$value\"" \ + --delimiter '=' + ;; + *) + : # handled in gencode-remote + ;; +esac diff --git a/cdist/conf/type/__install_umount/manifest b/cdist/conf/type/__sysctl/explorer/conf-path similarity index 78% rename from cdist/conf/type/__install_umount/manifest rename to cdist/conf/type/__sysctl/explorer/conf-path index c547e167..ba35c4c6 100755 --- a/cdist/conf/type/__install_umount/manifest +++ b/cdist/conf/type/__sysctl/explorer/conf-path @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2018 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -18,6 +18,8 @@ # along with cdist. If not, see . # -# set defaults -target="$(cat "$__object/parameter/target" 2>/dev/null \ - || echo "/target" | tee "$__object/parameter/target")" +if [ -d "/etc/sysctl.d" ]; then + echo "/etc/sysctl.d/99-Z-sysctl-cdist.conf"; +else + echo "/etc/sysctl.conf"; +fi diff --git a/cdist/conf/type/__sysctl/gencode-remote b/cdist/conf/type/__sysctl/gencode-remote index 0f3b0b40..711d54e5 100755 --- a/cdist/conf/type/__sysctl/gencode-remote +++ b/cdist/conf/type/__sysctl/gencode-remote @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2018 Takashi Yoshi (takashi at yoshi.email) # # This file is part of cdist. # @@ -26,5 +27,29 @@ if [ "$value_should" = "$value_is" ]; then exit 0 fi +os=$(cat "$__global/explorer/os") +case "$os" in + # Linux + redhat|centos|ubuntu|debian|devuan|archlinux|gentoo|coreos) + flag='-w' + ;; + # BusyBox + alpine|openwrt) + flag='-w' + ;; + macosx) + # NOTE: Older versions of Mac OS X require the -w option. + # Even though the flag is not mentioned in new man pages anymore, + # it still works. + flag='-w' + ;; + netbsd) + flag='-w' + ;; + freebsd|openbsd) + flag='' + ;; +esac + # set the current runtime value -printf 'sysctl -w %s="%s"\n' "$__object_id" "$value_should" +printf 'sysctl %s %s="%s"\n' "$flag" "$__object_id" "$value_should" diff --git a/cdist/conf/type/__sysctl/manifest b/cdist/conf/type/__sysctl/manifest index ac9117c4..b4e2e902 100755 --- a/cdist/conf/type/__sysctl/manifest +++ b/cdist/conf/type/__sysctl/manifest @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Steven Armstrong (steven-cdist at armstrong.cc) +# 2018 Takashi Yoshi (takashi at yoshi.email) # # This file is part of cdist. # @@ -22,7 +23,12 @@ os=$(cat "$__global/explorer/os") case "$os" in - redhat|centos|ubuntu|debian|devuan|archlinux) + # Linux + redhat|centos|ubuntu|debian|devuan|archlinux|coreos) + : + ;; + # BSD + freebsd|macosx|netbsd|openbsd) : ;; *) @@ -32,8 +38,10 @@ case "$os" in ;; esac +conf_path=$(cat "$__object/explorer/conf-path") + __key_value "$__object_name" \ --key "$__object_id" \ - --file /etc/sysctl.conf \ + --file "${conf_path}" \ --value "$(cat "$__object/parameter/value")" \ --delimiter '=' diff --git a/cdist/conf/type/__install_config/manifest b/cdist/conf/type/__systemd_unit/explorer/enablement-state old mode 100755 new mode 100644 similarity index 78% rename from cdist/conf/type/__install_config/manifest rename to cdist/conf/type/__systemd_unit/explorer/enablement-state index f26297b4..5a5a4462 --- a/cdist/conf/type/__install_config/manifest +++ b/cdist/conf/type/__systemd_unit/explorer/enablement-state @@ -1,6 +1,6 @@ #!/bin/sh # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2017 Ľubomír Kučera # # This file is part of cdist. # @@ -18,6 +18,4 @@ # along with cdist. If not, see . # -# set defaults -chroot="$(cat "$__object/parameter/chroot" 2>/dev/null \ - || echo "/target" | tee "$__object/parameter/chroot")" +systemctl is-enabled "${__object_id}" 2>/dev/null || true diff --git a/cdist/conf/type/__systemd_unit/explorer/systemctl-present b/cdist/conf/type/__systemd_unit/explorer/systemctl-present new file mode 100644 index 00000000..7218affc --- /dev/null +++ b/cdist/conf/type/__systemd_unit/explorer/systemctl-present @@ -0,0 +1,21 @@ +#!/bin/sh +# +# 2017 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +command -v systemctl > /dev/null 2>&1 && echo 0 || echo 1 diff --git a/cdist/conf/type/__systemd_unit/explorer/unit-status b/cdist/conf/type/__systemd_unit/explorer/unit-status new file mode 100644 index 00000000..b68e5169 --- /dev/null +++ b/cdist/conf/type/__systemd_unit/explorer/unit-status @@ -0,0 +1,21 @@ +#!/bin/sh +# +# 2017 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +systemctl is-active "${__object_id}" || true diff --git a/cdist/conf/type/__systemd_unit/gencode-remote b/cdist/conf/type/__systemd_unit/gencode-remote new file mode 100644 index 00000000..967a6c87 --- /dev/null +++ b/cdist/conf/type/__systemd_unit/gencode-remote @@ -0,0 +1,76 @@ +#!/bin/sh -e +# +# 2017 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +name="${__object_id}" +state=$(cat "${__object}/parameter/state") +current_enablement_state=$(cat "${__object}/explorer/enablement-state") + +if [ "${state}" = "absent" ]; then + if [ -n "${current_enablement_state}" ]; then + echo "systemctl --now disable ${name}" + echo "rm -f /etc/systemd/system/${name}" + echo "systemctl daemon-reload" + fi + + exit 0 +fi + +unit_status=$(cat "${__object}/explorer/unit-status") +desired_enablement_state=$(cat "${__object}/parameter/enablement-state") + +if [ "${current_enablement_state}" = "masked" ] && \ + [ "${desired_enablement_state}" != "masked" ]; then + echo "systemctl unmask ${name}" +fi + +if [ -f "${__object}/parameter/restart" ]; then + if [ "${desired_enablement_state}" = "masked" ]; then + if [ "${unit_status}" = "active" ]; then + echo "systemctl stop ${name}" + fi + elif grep -q "^__file/etc/systemd/system/${name}" "${__messages_in}" || \ + [ "${unit_status}" != "active" ]; then + echo "systemctl restart ${name} || true" + fi +fi + +if [ "${current_enablement_state}" = "${desired_enablement_state}" ]; then + exit 0 +fi + +case "${desired_enablement_state}" in + "") + # Do nothing + : + ;; + enabled) + echo "systemctl enable ${name}" + ;; + disabled) + echo "systemctl disable ${name}" + ;; + masked) + echo "systemctl mask ${name}" + ;; + *) + echo "Unsupported unit status: ${desired_enablement_state}" >&2 + exit 1 + ;; +esac diff --git a/cdist/conf/type/__systemd_unit/man.rst b/cdist/conf/type/__systemd_unit/man.rst new file mode 100644 index 00000000..25a4e501 --- /dev/null +++ b/cdist/conf/type/__systemd_unit/man.rst @@ -0,0 +1,89 @@ +cdist-type__systemd_unit(7) +=========================== + +NAME +---- + +cdist-type__systemd_unit - Install a systemd unit + +DESCRIPTION +----------- + +This type manages systemd units in ``/etc/systemd/system/``. It can install, +enable and start a systemd unit. This is particularly useful on systems which +take advantage of systemd heavily (e.g., CoreOS). For more information about +systemd units, see SYSTEMD.UNIT(5). + +REQUIRED PARAMETERS +------------------- + +None. + +OPTIONAL PARAMETERS +------------------- + +enablement-state + 'enabled', 'disabled' or 'masked', where: + + enabled + enables the unit + disabled + disables the unit + masked + masks the unit + +source + Path to the config file. If source is '-' (dash), take what was written to + stdin as the config file content. + +state + 'present' or 'absent', defaults to 'present' where: + + present + the unit (or its mask) is installed + absent + The unit is stopped, disabled and uninstalled. If the unit was masked, + the mask is removed. + +BOOLEAN PARAMETERS +------------------ + +restart + Start the unit if it was inactive. Restart the unit if the unit file + changed. Stop the unit if new ``enablement-state`` is ``masked``. + +MESSAGES +-------- + +None. + +EXAMPLES +-------- + +.. code-block:: sh + + # Installs, enables and starts foobar.service + __systemd_unit foobar.service \ + --source "${__manifest}/files/foobar.service" \ + --enablement-state enabled \ + --restart + + # Disables the unit + __systemd_unit foobar.service --enablement-state disabled + + # Stops, disables and uninstalls foobar.service + __systemd_unit foobar.service --state absent + + +AUTHORS +------- + +Ľubomír Kučera + +COPYING +------- + +Copyright \(C) 2017 Ľubomír Kučera. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__systemd_unit/manifest b/cdist/conf/type/__systemd_unit/manifest new file mode 100644 index 00000000..688a00b1 --- /dev/null +++ b/cdist/conf/type/__systemd_unit/manifest @@ -0,0 +1,58 @@ +#!/bin/sh -e +# +# 2017 Ľubomír Kučera +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +systemctl_present=$(cat "${__object}/explorer/systemctl-present") + +if [ "${systemctl_present}" -ne 0 ]; then + echo "systemctl does not seem to be present on this system" >&2 + + exit 1 +fi + +name="${__object_id}" +source=$(cat "${__object}/parameter/source") +state=$(cat "${__object}/parameter/state") +enablement_state=$(cat "${__object}/parameter/enablement-state") + +# The unit must be disabled before removing its unit file. The unit file is +# therefore removed by gencode-remote of this type, not here. +if [ -z "${source}" ] || [ "${state}" = "absent" ]; then + exit 0 +fi + +# stdin is not propagated automatically to sub-objects +if [ "${source}" = "-" ]; then + source="${__object}/stdin" +fi + +unitfile_state="${state}" +if [ "${enablement_state}" = "masked" ]; then + # Masking creates a symlink from /etc/systemd/system/ to /dev/null. + # This process fails with "Failed to execute operation: Invalid argument" + # if file /etc/systemd/system/ already exists. We must therefore + # remove it. + unitfile_state="absent" +fi + +__config_file "/etc/systemd/system/${name}" \ + --mode 644 \ + --onchange "systemctl daemon-reload" \ + --source "${source}" \ + --state "${unitfile_state}" diff --git a/cdist/conf/type/__systemd_unit/parameter/boolean b/cdist/conf/type/__systemd_unit/parameter/boolean new file mode 100644 index 00000000..eea5a271 --- /dev/null +++ b/cdist/conf/type/__systemd_unit/parameter/boolean @@ -0,0 +1 @@ +restart diff --git a/cdist/conf/type/__systemd_unit/parameter/default/enablement-state b/cdist/conf/type/__systemd_unit/parameter/default/enablement-state new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__systemd_unit/parameter/default/source b/cdist/conf/type/__systemd_unit/parameter/default/source new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__systemd_unit/parameter/default/state b/cdist/conf/type/__systemd_unit/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__systemd_unit/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__systemd_unit/parameter/optional b/cdist/conf/type/__systemd_unit/parameter/optional new file mode 100644 index 00000000..e7cc7acf --- /dev/null +++ b/cdist/conf/type/__systemd_unit/parameter/optional @@ -0,0 +1,3 @@ +enablement-state +source +state diff --git a/cdist/conf/type/__timezone/explorer/timezone_is b/cdist/conf/type/__timezone/explorer/timezone_is new file mode 100755 index 00000000..a1aa813f --- /dev/null +++ b/cdist/conf/type/__timezone/explorer/timezone_is @@ -0,0 +1,23 @@ +#!/bin/sh -e +# +# 2017 Ander Punnar (cdist at kvlt.ee) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +[ -f /etc/timezone ] && cat /etc/timezone + +exit 0 diff --git a/cdist/conf/type/__timezone/gencode-remote b/cdist/conf/type/__timezone/gencode-remote index c07a61cb..5299f548 100755 --- a/cdist/conf/type/__timezone/gencode-remote +++ b/cdist/conf/type/__timezone/gencode-remote @@ -1,6 +1,7 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# 2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -20,11 +21,16 @@ # # This type allows to configure the desired localtime timezone. -timezone="$__object_id" +timezone_is=$(cat "$__object/explorer/timezone_is") +timezone_should="$__object_id" os=$(cat "$__global/explorer/os") +if [ "$timezone_is" = "$timezone_should" ]; then + exit 0 +fi + case "$os" in - ubuntu|debian|devuan) - echo "echo \"$timezone\" > /etc/timezone" + ubuntu|debian|devuan|coreos|alpine) + echo "echo \"$timezone_should\" > /etc/timezone" ;; esac diff --git a/cdist/conf/type/__timezone/manifest b/cdist/conf/type/__timezone/manifest index bcbe41c3..3d28ccba 100755 --- a/cdist/conf/type/__timezone/manifest +++ b/cdist/conf/type/__timezone/manifest @@ -1,8 +1,8 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Ramon Salvadó (rsalvado at gnuine dot com) # 2012-2015 Steven Armstrong (steven-cdist at armstrong.cc) -# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# 2012-2019 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. # @@ -26,7 +26,7 @@ timezone="$__object_id" os=$(cat "$__global/explorer/os") case "$os" in - archlinux|debian|ubuntu|devuan) + archlinux|debian|ubuntu|devuan|alpine) __package tzdata export require="__package/tzdata" ;; @@ -34,7 +34,11 @@ case "$os" in __package timezone export require="__package/timezone" ;; - freebsd|netbsd) + freebsd|netbsd|openbsd) + # whitelist + : + ;; + coreos) # whitelist : ;; diff --git a/cdist/conf/type/__ufw/gencode-remote b/cdist/conf/type/__ufw/gencode-remote new file mode 100644 index 00000000..fc62b591 --- /dev/null +++ b/cdist/conf/type/__ufw/gencode-remote @@ -0,0 +1,62 @@ +#!/bin/sh -e +# +# 2019 Mark Polyakov (mark--@--markasoftware.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +state="$(cat "$__object/parameter/state")" + +case "$state" in + enabled) + echo 'ufw --force enable' + ;; + + present) + echo 'ufw --force disable' + ;; + # absent will be uninstalled in manifest +esac + +if [ "$state" != absent ]; then + if [ -f "$__object/parameter/logging" ]; then + logging="$(cat "$__object/parameter/logging")" + case "$logging" in + off|low|medium|high|full) + echo "ufw --force logging $logging" + ;; + *) + echo 'Logging parameter must be off, low, medium, high, or full!' >&2 + exit 1 + ;; + esac + fi + + for direction in incoming outgoing routed; do + if [ -f "$__object/parameter/default_$direction" ]; then + treatment="$(cat "$__object/parameter/default_$direction")" + case "$treatment" in + allow|deny|reject) + echo "ufw --force default $treatment $direction" + ;; + *) + echo 'UFW default policies must be either "allow", "deny", or "reject".' >&2 + exit 1 + ;; + esac + fi + done +fi diff --git a/cdist/conf/type/__ufw/man.rst b/cdist/conf/type/__ufw/man.rst new file mode 100644 index 00000000..cc64fbb5 --- /dev/null +++ b/cdist/conf/type/__ufw/man.rst @@ -0,0 +1,59 @@ +cdist-type__ufw(7) +================== + +NAME +---- +cdist-type__ufw - Install the Uncomplicated FireWall + + +DESCRIPTION +----------- +Installs the Uncomplicated FireWall. Most modern distributions carry UFW in their main repositories, but on CentOS this type will automatically enable the EPEL repository. + +Some global configuration can also be set with this type. + +OPTIONAL PARAMETERS +------------------- +state + Either "enabled", "running", "present", or "absent". Defaults to "enabled", which registers UFW to start on boot. + +logging + Either "off", "low", "medium", "high", or "full". Will be passed to `ufw logging`. If not specified, logging level is not modified. + +default_incoming + Either "allow", "deny", or "reject". The default policy for dealing with ingress packets. + +default_outgoing + Either "allow", "deny", or "reject". The default policy for dealing with egress packets. + +default_routed + Either "allow", "deny", or "reject". The default policy for dealing with routed packets (passing through this machine). + + +EXAMPLES +-------- + +.. code-block:: sh + + # Install UFW + __ufw + # Setup UFW with maximum logging and no restrictions on routed packets. + __ufw --logging full --default_routed allow + + +SEE ALSO +-------- +:strong:`ufw`\ (8) + + +AUTHORS +------- +Mark Polyakov + + +COPYING +------- +Copyright \(C) 2019 Mark Polyakov. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__ufw/manifest b/cdist/conf/type/__ufw/manifest new file mode 100755 index 00000000..54309ff5 --- /dev/null +++ b/cdist/conf/type/__ufw/manifest @@ -0,0 +1,67 @@ +#!/bin/sh -e +# +# 2019 Mark Polyakov (mark--@--markasoftware.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +state="$(cat "$__object/parameter/state")" + +case "$state" in + present|enabled) + os="$(cat "$__global/explorer/os")" + + case "$os" in + centos) + # shellcheck source=/dev/null + if (. "$__global/explorer/os_release" && [ "${VERSION_ID}" = "7" ]); then + __package epel-release + require='__package/epel-release' __package ufw + else + echo 'CentOS version 7 is required!' + exit 1 + fi + ;; + *) + __package ufw + ;; + esac + + # ufw expects to always be enabled, then uses a switch in /etc to + # determine whether to "actually start" after the init system calls it. + # So, we have to both enable on bootup through init and run `ufw enable` + + # operators ae left-associative, so if !enabled it will never run + if [ "$(cat "$__global/explorer/os")" != ubuntu ] || \ + [ "$(cat "$__global/explorer/init")" != init ] && \ + [ "$state" = enabled ]; then + # Why don't we disable start_on_boot when state=present|absent? + # Because UFW should always be enabled at boot -- /etc/ufw/ufw.conf + # will stop it from "really" starting + require='__package/ufw' __start_on_boot ufw + fi + ;; + + absent) + __package ufw --state absent + ;; + + *) + echo 'State must be "enabled", "present", or "absent".' + exit 1 + ;; +esac + diff --git a/cdist/conf/type/__ufw/parameter/default/state b/cdist/conf/type/__ufw/parameter/default/state new file mode 100644 index 00000000..26ed6c9b --- /dev/null +++ b/cdist/conf/type/__ufw/parameter/default/state @@ -0,0 +1 @@ +enabled \ No newline at end of file diff --git a/cdist/conf/type/__ufw/parameter/optional b/cdist/conf/type/__ufw/parameter/optional new file mode 100644 index 00000000..0a4dec97 --- /dev/null +++ b/cdist/conf/type/__ufw/parameter/optional @@ -0,0 +1,5 @@ +state +logging +default_incoming +default_outgoing +default_routed \ No newline at end of file diff --git a/cdist/conf/type/__ufw/singleton b/cdist/conf/type/__ufw/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/conf/type/__ufw_rule/gencode-remote b/cdist/conf/type/__ufw_rule/gencode-remote new file mode 100755 index 00000000..4f1bf2c9 --- /dev/null +++ b/cdist/conf/type/__ufw_rule/gencode-remote @@ -0,0 +1,45 @@ +#!/bin/sh -e +# +# 2019 Mark Polyakov (mark@markasoftware.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# + +# This type does not bother with checking the current state of the rules. +# While it is possible to retrieve the list of rules in a consistent format from +# `ufw status`, it is a completely different format than the one used on the +# command line. I also do not suspect it is any faster. + +ufw='ufw --force rule' + +case "$(cat "$__object/parameter/state")" in + present) ;; + absent) + ufw="$ufw delete" + ;; + *) + echo 'State must be "present" or "absent".' >&2 + exit 1 + ;; +esac + +if [ -f "$__object/parameter/rule" ]; then + ufw="$ufw $(cat "$__object/parameter/rule")" +else + ufw="$ufw allow $__object_id" +fi + +echo "$ufw" diff --git a/cdist/conf/type/__ufw_rule/man.rst b/cdist/conf/type/__ufw_rule/man.rst new file mode 100644 index 00000000..996557f8 --- /dev/null +++ b/cdist/conf/type/__ufw_rule/man.rst @@ -0,0 +1,53 @@ +cdist-type__ufw_rule(7) +======================= + +NAME +---- +cdist-type__ufw_rule - A single UFW rule + + +DESCRIPTION +----------- +Adds or removes a single UFW rule. This type supports adding and deleting rules for port ranges or applications. + +Understanding what is "to" and what is "from" can be confusing. If the rule is ingress (default), then "from" is the remote machine and "to" is the local one. The opposite is true for egress traffic (--out). + +OPTIONAL PARAMETERS +------------------- +state + Either "present" or "absent". Defaults to "present". If "absent", only removes rules that exactly match the rule expected. + +rule + A firewall rule in UFW syntax. This is what you would usually write after `ufw` on the command line. Defaults to "allow" followed by the object ID. You can use either the short syntax (just allow|deny|reject|limit followed by a port or application name) or the full syntax. Do not include `delete` in your command. Set `--state absent` instead. + +EXAMPLES +-------- + +.. code-block:: sh + + # open port 80 (ufw allow 80) + __ufw_rule 80 + # Allow mosh application (if installed) + __ufw_rule mosh + # Allow all traffic from local network (ufw allow from 10.0.0.0/24) + __ufw_rule local --rule 'allow from 10.0.0.0/24' + # Block egress traffic from port 25 to 111.55.55.55 on interface eth0 + __ufw_rule block_smtp --rule 'deny out on eth0 from any port 25 to 111.55.55.55' + + +SEE ALSO +-------- +:strong:`ufw`\ (8) + + +AUTHORS +------- +Mark Polyakov + + +COPYING +------- +Copyright \(C) 2019 Mark Polyakov. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. diff --git a/cdist/conf/type/__ufw_rule/parameter/default/state b/cdist/conf/type/__ufw_rule/parameter/default/state new file mode 100644 index 00000000..e7f6134f --- /dev/null +++ b/cdist/conf/type/__ufw_rule/parameter/default/state @@ -0,0 +1 @@ +present diff --git a/cdist/conf/type/__ufw_rule/parameter/optional b/cdist/conf/type/__ufw_rule/parameter/optional new file mode 100644 index 00000000..0732d53d --- /dev/null +++ b/cdist/conf/type/__ufw_rule/parameter/optional @@ -0,0 +1,2 @@ +state +rule diff --git a/cdist/conf/type/__update_alternatives/gencode-remote b/cdist/conf/type/__update_alternatives/gencode-remote index 19ea9968..0e7b0d89 100755 --- a/cdist/conf/type/__update_alternatives/gencode-remote +++ b/cdist/conf/type/__update_alternatives/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Nico Schottelius (nico-cdist at schottelius.org) # diff --git a/cdist/conf/type/__user/explorer/group b/cdist/conf/type/__user/explorer/group index 98ce39c6..2aae2973 100755 --- a/cdist/conf/type/__user/explorer/group +++ b/cdist/conf/type/__user/explorer/group @@ -23,6 +23,11 @@ if [ -f "$__object/parameter/gid" ]; then gid=$(cat "$__object/parameter/gid") - getent group "$gid" || true + getent=$(command -v getent) + if [ X != X"${getent}" ]; then + "${getent}" group "$gid" || true + elif [ -f /etc/group ]; then + grep -E "^(${gid}|([^:]+:){2}${gid}):" /etc/group || true + fi fi diff --git a/cdist/conf/type/__user/explorer/passwd b/cdist/conf/type/__user/explorer/passwd index fdbfb193..677e3ff0 100755 --- a/cdist/conf/type/__user/explorer/passwd +++ b/cdist/conf/type/__user/explorer/passwd @@ -23,5 +23,9 @@ name=$__object_id -getent passwd "$name" || true - +getent=$(command -v getent) +if [ X != X"${getent}" ]; then + "${getent}" passwd "$name" || true +elif [ -f /etc/passwd ]; then + grep "^${name}:" /etc/passwd || true +fi diff --git a/cdist/conf/type/__user/explorer/shadow b/cdist/conf/type/__user/explorer/shadow index 1a8fd809..c49992d5 100755 --- a/cdist/conf/type/__user/explorer/shadow +++ b/cdist/conf/type/__user/explorer/shadow @@ -22,7 +22,7 @@ # name=$__object_id -os="$($__explorer/os)" +os="$("$__explorer/os")" # Default to using shadow passwords database="shadow" @@ -31,5 +31,9 @@ case "$os" in esac -getent "$database" "$name" || true - +getent=$(command -v getent) +if [ X != X"${getent}" ]; then + "${getent}" "$database" "$name" || true +elif [ -f /etc/shadow ]; then + grep "^${name}:" /etc/shadow || true +fi diff --git a/cdist/conf/type/__user/gencode-remote b/cdist/conf/type/__user/gencode-remote index 223d4d46..ee18c18f 100755 --- a/cdist/conf/type/__user/gencode-remote +++ b/cdist/conf/type/__user/gencode-remote @@ -1,8 +1,9 @@ -#!/bin/sh +#!/bin/sh -e # # 2011 Steven Armstrong (steven-cdist at armstrong.cc) # 2011 Nico Schottelius (nico-cdist at schottelius.org) # 2013 Daniel Heule (hda at sfs.biz) +# 2018 Thomas Eckert (tom at it-eckert.de) # # This file is part of cdist. # @@ -52,7 +53,7 @@ shorten_property() { if [ "$state" = "present" ]; then cd "$__object/parameter" if grep -q "^${name}:" "$__object/explorer/passwd"; then - for property in $(ls .); do + for property in *; do new_value="$(cat "$property")" unset current_value @@ -60,7 +61,7 @@ if [ "$state" = "present" ]; then case "$property" in gid) - if $(echo "$new_value" | grep -q '^[0-9][0-9]*$'); then + if echo "$new_value" | grep -q '^[0-9][0-9]*$'; then field=4 else # We were passed a group name. Compare the gid in @@ -97,7 +98,7 @@ if [ "$state" = "present" ]; then fi if [ "$new_value" != "$current_value" ]; then - set -- "$@" "$(shorten_property $property)" \'$new_value\' + set -- "$@" "$(shorten_property "$property")" \'"$new_value"\' fi done @@ -113,14 +114,14 @@ if [ "$state" = "present" ]; then fi else echo add >> "$__messages_out" - for property in $(ls .); do + for property in *; do [ "$property" = "state" ] && continue [ "$property" = "remove-home" ] && continue new_value="$(cat "$property")" if [ -z "$new_value" ];then # Boolean values have no value - set -- "$@" "$(shorten_property $property)" + set -- "$@" "$(shorten_property "$property")" else - set -- "$@" "$(shorten_property $property)" \'$new_value\' + set -- "$@" "$(shorten_property "$property")" \'"$new_value"\' fi done @@ -130,13 +131,17 @@ if [ "$state" = "present" ]; then echo useradd "$@" "$name" fi fi -else +elif [ "$state" = "absent" ]; then if grep -q "^${name}:" "$__object/explorer/passwd"; then #user exists, but state != present, so delete it if [ -f "$__object/parameter/remove-home" ]; then - echo userdel -r "${name}" + printf "userdel -r '%s' >/dev/null 2>&1\\n" "${name}" + echo "userdel -r" >> "$__messages_out" else - echo userdel "${name}" + printf "userdel '%s' >/dev/null 2>&1\\n" "${name}" + echo "userdel" >> "$__messages_out" fi fi +else + echo "Invalid state $state" >&2 fi diff --git a/cdist/conf/type/__user/man.rst b/cdist/conf/type/__user/man.rst index 5001bfa4..ef6b77af 100644 --- a/cdist/conf/type/__user/man.rst +++ b/cdist/conf/type/__user/man.rst @@ -60,6 +60,11 @@ mod add New user added +userdel -r + If user was deleted with homedir + +userdel + If user was deleted (keeping homedir) EXAMPLES -------- diff --git a/cdist/conf/type/__user/manifest b/cdist/conf/type/__user/manifest new file mode 100644 index 00000000..8f10b38c --- /dev/null +++ b/cdist/conf/type/__user/manifest @@ -0,0 +1,32 @@ +#!/bin/sh -e +# +# 2019 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Manage users. + +os=$(cat "$__global/explorer/os") + +case "$os" in + alpine) + __package shadow + ;; + *) + : + ;; +esac diff --git a/cdist/conf/type/__user_groups/gencode-remote b/cdist/conf/type/__user_groups/gencode-remote index 6728228c..8120761a 100755 --- a/cdist/conf/type/__user_groups/gencode-remote +++ b/cdist/conf/type/__user_groups/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2012 Steven Armstrong (steven-cdist at armstrong.cc) # @@ -42,10 +42,10 @@ if [ -z "$changed_groups" ]; then fi for group in $changed_groups; do - if [ "$os" = "netbsd" ]; then + if [ "$os" = "netbsd" ] || [ "$os" = "openbsd" ]; then case "$state_should" in present) echo "usermod -G \"$group\" \"$user\"" ;; - absent) echo 'NetBSD does not have a command to remove a user from a group' >&2 ; exit 1 ;; + absent) echo 'NetBSD and OpenBSD do not have a command to remove a user from a group' >&2 ; exit 1 ;; esac elif [ "$os" = "freebsd" ]; then case "$state_should" in @@ -59,8 +59,8 @@ for group in $changed_groups; do esac else case "$state_should" in - present) echo "gpasswd -a \"$group\" \"$user\"" ;; - absent) echo "gpasswd -d \"$group\" \"$user\"" ;; + present) echo "gpasswd -a \"$user\" \"$group\"" ;; + absent) echo "gpasswd -d \"$user\" \"$group\"" ;; esac fi done diff --git a/cdist/conf/type/__yum_repo/files/repo.template b/cdist/conf/type/__yum_repo/files/repo.template index 3e14c8b6..18ea9d2b 100755 --- a/cdist/conf/type/__yum_repo/files/repo.template +++ b/cdist/conf/type/__yum_repo/files/repo.template @@ -43,7 +43,7 @@ for key in baseurl gpgkey; do if [ -f "$__object/parameter/$key" ]; then printf '%s=' "$key" prefix='' - while read line; do + while read -r line; do printf '%s%s\n' "$prefix" "$line" prefix=' ' done < "$__object/parameter/$key" diff --git a/cdist/conf/type/__yum_repo/manifest b/cdist/conf/type/__yum_repo/manifest index 950c3b7a..5f60d32c 100755 --- a/cdist/conf/type/__yum_repo/manifest +++ b/cdist/conf/type/__yum_repo/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2014 Steven Armstrong (steven-cdist at armstrong.cc) # diff --git a/cdist/conf/type/__zypper_repo/explorer/all_repo_ids b/cdist/conf/type/__zypper_repo/explorer/all_repo_ids index b37d8ac5..7953158a 100644 --- a/cdist/conf/type/__zypper_repo/explorer/all_repo_ids +++ b/cdist/conf/type/__zypper_repo/explorer/all_repo_ids @@ -21,4 +21,5 @@ # Retrieve all repo id nummbers - parsed zypper output # # +# shellcheck disable=SC2005,SC2046 echo $(zypper lr | cut -d'|' -f 1 | grep -E '^[0-9]') diff --git a/cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids b/cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids index 2dfb946f..261d6073 100644 --- a/cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids +++ b/cdist/conf/type/__zypper_repo/explorer/enabled_repo_ids @@ -23,4 +23,6 @@ # # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper lr -E | cut -d'|' -f 1 | grep -E '^[0-9]') +# +# shellcheck disable=SC2005,SC2046 echo $(zypper lr | grep -E '^[0-9]([^|]+\|){3,3} Yes' | cut -d'|' -f 1) diff --git a/cdist/conf/type/__zypper_repo/explorer/repo_id b/cdist/conf/type/__zypper_repo/explorer/repo_id index 6a4791e6..d55a5cac 100644 --- a/cdist/conf/type/__zypper_repo/explorer/repo_id +++ b/cdist/conf/type/__zypper_repo/explorer/repo_id @@ -26,4 +26,5 @@ if [ -f "$__object/parameter/uri" ]; then else uri="$__object_id" fi -echo $(zypper lr -u | grep -F "$uri" | cut -d'|' -f 1 | grep -E '^[0-9]' ) +# shellcheck disable=SC2005,SC2046 +echo $(zypper lr -u | grep -F "$uri" | cut -d'|' -f 1 | grep -E '^[0-9]') diff --git a/cdist/conf/type/__zypper_repo/gencode-remote b/cdist/conf/type/__zypper_repo/gencode-remote old mode 100644 new mode 100755 index 26199c75..336488ae --- a/cdist/conf/type/__zypper_repo/gencode-remote +++ b/cdist/conf/type/__zypper_repo/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Daniel Heule (hda at sfs.biz) # @@ -70,25 +70,25 @@ case "$state" in fi if [ -z "$repo_id" ]; then # Repo not present, so we need to create it - echo zypper $zypper_def_opts addrepo "'$uri'" "'$desc'" + echo "zypper $zypper_def_opts addrepo '$uri' '$desc'" fi ;; absent) - if [ ! -z "$act_id" ]; then + if [ -n "$act_id" ]; then # Repo present (act_id not ""), so we ned to delete it - echo zypper $zypper_def_opts removerepo "$act_id" + echo "zypper $zypper_def_opts removerepo $act_id" fi ;; enabled) - if [ ! -z "$act_id" ] && [ "$repostate" = "disabled" ]; then + if [ -n "$act_id" ] && [ "$repostate" = "disabled" ]; then # Repo present (act_id not "") and repostate not enabled, so a enable call is needed - echo zypper $zypper_def_opts modifyrepo -e "$act_id" + echo "zypper $zypper_def_opts modifyrepo -e $act_id" fi ;; disabled) - if [ ! -z "$act_id" ] && [ "$repostate" = "enabled" ]; then + if [ -n "$act_id" ] && [ "$repostate" = "enabled" ]; then # Repo present (act_id not "") and repostate enabled, so a disable call is needed - echo zypper $zypper_def_opts modifyrepo -d "$act_id" + echo "zypper $zypper_def_opts modifyrepo -d $act_id" fi ;; *) diff --git a/cdist/conf/type/__zypper_service/explorer/repo_ids b/cdist/conf/type/__zypper_service/explorer/repo_ids index e831b76c..da506fea 100644 --- a/cdist/conf/type/__zypper_service/explorer/repo_ids +++ b/cdist/conf/type/__zypper_service/explorer/repo_ids @@ -24,4 +24,6 @@ # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper lr -u -E | cut -d'|' -f 1 | grep -E '^[0-9]') # on older systems, zypper doesn't know the parameter -E +# +# shellcheck disable=SC2005,SC2046 echo $(zypper lr -u | grep -E '^([^|]+\|){3,3} Yes' | cut -d'|' -f 1 | grep -E '^[0-9]') diff --git a/cdist/conf/type/__zypper_service/explorer/service_id b/cdist/conf/type/__zypper_service/explorer/service_id index bf5f0260..fbb983c8 100644 --- a/cdist/conf/type/__zypper_service/explorer/service_id +++ b/cdist/conf/type/__zypper_service/explorer/service_id @@ -27,4 +27,6 @@ else fi # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper ls -u -E | grep -E "\<$uri\>" | cut -d'|' -f 1 ) -echo $(zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | grep -E "\<$uri\>" | cut -d'|' -f 1 ) +# +# shellcheck disable=SC2005,SC2046 +echo $(zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | grep -E "\\<$uri\\>" | cut -d'|' -f 1) diff --git a/cdist/conf/type/__zypper_service/explorer/service_ids b/cdist/conf/type/__zypper_service/explorer/service_ids index 0f1f4186..5a26740e 100644 --- a/cdist/conf/type/__zypper_service/explorer/service_ids +++ b/cdist/conf/type/__zypper_service/explorer/service_ids @@ -22,4 +22,6 @@ # # simpler command which works only on SLES11 SP3 or newer: # echo $(zypper ls -u -E | cut -d'|' -f 1 | grep -E '^[0-9]') +# +# shellcheck disable=SC2005,SC2046 echo $(zypper ls -u | grep -E '^([^|]+\|){3,3} Yes' | cut -d'|' -f 1 | grep -E '^[0-9]') diff --git a/cdist/conf/type/__zypper_service/explorer/service_uri b/cdist/conf/type/__zypper_service/explorer/service_uri index 6eee47fb..2f3d0f94 100644 --- a/cdist/conf/type/__zypper_service/explorer/service_uri +++ b/cdist/conf/type/__zypper_service/explorer/service_uri @@ -25,4 +25,5 @@ if [ -f "$__object/parameter/uri" ]; then else uri="/$__object_id" fi -echo $(zypper ls -u | awk 'BEGIN { FS = "[ ]+\\|[ ]+" } ; $4 == "Yes" && $NF == "'$uri'" {print $NF}') +# shellcheck disable=SC2005,SC2046 +echo $(zypper ls -u | awk 'BEGIN { FS = "[ ]+\\|[ ]+" } ; $4 == "Yes" && $NF == "'"$uri"'" {print $NF}') diff --git a/cdist/conf/type/__zypper_service/gencode-remote b/cdist/conf/type/__zypper_service/gencode-remote old mode 100644 new mode 100755 index d16ba8ee..4ccfe301 --- a/cdist/conf/type/__zypper_service/gencode-remote +++ b/cdist/conf/type/__zypper_service/gencode-remote @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Daniel Heule (hda at sfs.biz) # @@ -46,7 +46,7 @@ exp_uri="$(cat "$__object/explorer/service_uri")" exp_id="$(cat "$__object/explorer/service_id")" # we need this list to remove ids, but we must do this in reverse order -exp_ids="$(cat "$__object/explorer/service_ids" | rev)" +exp_ids="$(rev "$__object/explorer/service_ids")" if [ "$uri" = "$exp_uri" ] ; then state_is="present" @@ -59,10 +59,10 @@ if [ -f "$__object/parameter/remove-all-other-services" ]; then # file exists -> True for i in $exp_ids; do if [ "$i" != "$exp_id" ] ; then - echo zypper $zypper_def_opts removeservice "$i" "&>/dev/null" + echo "zypper $zypper_def_opts removeservice $i &>/dev/null" fi done - echo zypper $zypper_def_opts refs "&>/dev/null" + echo "zypper $zypper_def_opts refs &>/dev/null" fi @@ -71,14 +71,14 @@ fi case "$state_should" in present) - echo zypper $zypper_def_opts addservice -t "$stype" "$uri" \"$desc\" - echo zypper $zypper_def_opts refs - echo zypper $zypper_def_opts ref + echo "zypper $zypper_def_opts addservice -t $stype $uri '$desc'" + echo "zypper $zypper_def_opts refs" + echo "zypper $zypper_def_opts ref" ;; absent) - echo zypper $zypper_def_opts removeservice "$service_id" - echo zypper $zypper_def_opts refs - echo zypper $zypper_def_opts ref + echo "zypper $zypper_def_opts removeservice $exp_id" + echo "zypper $zypper_def_opts refs" + echo "zypper $zypper_def_opts ref" ;; *) echo "Unknown state: $state_should" >&2 diff --git a/cdist/conf/type/__zypper_service/man.rst b/cdist/conf/type/__zypper_service/man.rst index ea48aebb..e082dc02 100644 --- a/cdist/conf/type/__zypper_service/man.rst +++ b/cdist/conf/type/__zypper_service/man.rst @@ -46,10 +46,10 @@ EXAMPLES # Ensure that internal SLES11 SP3 RIS is in installed and all other services and repos are discarded __zypper_service INTERNAL_SLES11_SP3 --service_desc "Internal SLES11 SP3 RIS" --uri "http://path/to/your/ris/dir" --remove-all-other-services --remove-all-repos - # Ensure that internal SLES11 SP3 RIS is in installed, no changes to ohter services or repos + # Ensure that internal SLES11 SP3 RIS is in installed, no changes to other services or repos __zypper_service INTERNAL_SLES11_SP3 --service_desc "Internal SLES11 SP3 RIS" --uri "http://path/to/your/ris/dir" - # Drop service by uri, no changes to ohter services or repos + # Drop service by uri, no changes to other services or repos __zypper_service INTERNAL_SLES11_SP3 --state absent --uri "http://path/to/your/ris/dir" diff --git a/cdist/conf/type/__zypper_service/manifest b/cdist/conf/type/__zypper_service/manifest old mode 100644 new mode 100755 index 7f853b3b..42a56830 --- a/cdist/conf/type/__zypper_service/manifest +++ b/cdist/conf/type/__zypper_service/manifest @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/sh -e # # 2013 Daniel Heule (hda at sfs.biz) # @@ -47,7 +47,7 @@ fi [ "$state_is" = "$state_should" ] && exit 0 # we need this list to remove ids, but we must do this in reverse order -exp_repos="$(cat "$__object/explorer/repo_ids" | rev)" +exp_repos="$(rev "$__object/explorer/repo_ids")" # boolean parameter if [ -f "$__object/parameter/remove-all-repos" ]; then diff --git a/cdist/config.py b/cdist/config.py index 03a2e6ee..26d07fc4 100644 --- a/cdist/config.py +++ b/cdist/config.py @@ -2,6 +2,8 @@ # -*- coding: utf-8 -*- # # 2010-2015 Nico Schottelius (nico-cdist at schottelius.org) +# 2013-2017 Steven Armstrong (steven-cdist at armstrong.cc) +# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -26,44 +28,103 @@ import sys import time import itertools import tempfile -import socket import multiprocessing -from cdist.mputil import mp_pool_run +from cdist.mputil import mp_pool_run, mp_sig_handler import atexit import shutil - +import socket import cdist import cdist.hostsource - import cdist.exec.local import cdist.exec.remote import cdist.util.ipaddr as ipaddr - -from cdist import core +import cdist.configuration +from cdist import core, inventory from cdist.util.remoteutil import inspect_ssh_mux_opts +def graph_check_cycle(graph): + # Start from each node in the graph and check for cycle starting from it. + for node in graph: + # Cycle path. + path = [node] + has_cycle = _graph_dfs_cycle(graph, node, path) + if has_cycle: + return has_cycle, path + return False, None + + +def _graph_dfs_cycle(graph, node, path): + for neighbour in graph.get(node, ()): + # If node is already in path then this is cycle. + if neighbour in path: + path.append(neighbour) + return True + path.append(neighbour) + rv = _graph_dfs_cycle(graph, neighbour, path) + if rv: + return True + # Remove last item from list - neighbour whose DFS path we have have + # just checked. + del path[-1] + return False + + class Config(object): """Cdist main class to hold arbitrary data""" - def __init__(self, local, remote, dry_run=False, jobs=None): + # list of paths (files and/or directories) that will be removed on finish + _paths_for_removal = [] + + @classmethod + def _register_path_for_removal(cls, path): + cls._paths_for_removal.append(path) + + @classmethod + def _remove_paths(cls): + while cls._paths_for_removal: + path = cls._paths_for_removal.pop() + if os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + + def __init__(self, local, remote, dry_run=False, jobs=None, + cleanup_cmds=None, remove_remote_files_dirs=False): self.local = local self.remote = remote self._open_logger() self.dry_run = dry_run self.jobs = jobs + if cleanup_cmds: + self.cleanup_cmds = cleanup_cmds + else: + self.cleanup_cmds = [] + self.remove_remote_files_dirs = remove_remote_files_dirs self.explorer = core.Explorer(self.local.target_host, self.local, - self.remote, jobs=self.jobs) - self.manifest = core.Manifest(self.local.target_host, self.local) - self.code = core.Code(self.local.target_host, self.local, self.remote) + self.remote, jobs=self.jobs, + dry_run=self.dry_run) + self.manifest = core.Manifest(self.local.target_host, self.local, + dry_run=self.dry_run) + self.code = core.Code(self.local.target_host, self.local, self.remote, + dry_run=self.dry_run) def _init_files_dirs(self): """Prepare files and directories for the run""" self.local.create_files_dirs() self.remote.create_files_dirs() + def _remove_remote_files_dirs(self): + """Remove remote files and directories for the run""" + self.remote.remove_files_dirs() + + def _remove_files_dirs(self): + """Remove files and directories for the run""" + if self.remove_remote_files_dirs: + self._remove_remote_files_dirs() + @staticmethod def hosts(source): try: @@ -73,6 +134,39 @@ class Config(object): "Error reading hosts from \'{}\': {}".format( source, e)) + @staticmethod + def construct_remote_exec_copy_patterns(args): + # default remote cmd patterns + args.remote_cmds_cleanup_pattern = "" + args.remote_exec_pattern = None + args.remote_copy_pattern = None + + # Determine forcing IPv4/IPv6 options if any, only for + # default remote commands. + if args.force_ipv: + force_addr_opt = " -{}".format(args.force_ipv) + else: + force_addr_opt = "" + + args_dict = vars(args) + # if remote-exec and/or remote-copy args are None then user + # didn't specify command line options nor env vars: + # inspect multiplexing options for default cdist.REMOTE_COPY/EXEC + if (args_dict['remote_copy'] is None or + args_dict['remote_exec'] is None): + mux_opts = inspect_ssh_mux_opts() + if args_dict['remote_exec'] is None: + args.remote_exec_pattern = (cdist.REMOTE_EXEC + + force_addr_opt + mux_opts) + if args_dict['remote_copy'] is None: + args.remote_copy_pattern = (cdist.REMOTE_COPY + + force_addr_opt + mux_opts) + if mux_opts: + cleanup_pattern = cdist.REMOTE_CMDS_CLEANUP_PATTERN + else: + cleanup_pattern = "" + args.remote_cmds_cleanup_pattern = cleanup_pattern + @classmethod def _check_and_prepare_args(cls, args): if args.manifest == '-' and args.hostfile == '-': @@ -83,7 +177,6 @@ class Config(object): if not (args.hostfile or args.host): args.hostfile = '-' - initial_manifest_tempfile = None if args.manifest == '-': # read initial manifest from stdin try: @@ -98,95 +191,149 @@ class Config(object): args.manifest = initial_manifest_temp_path atexit.register(lambda: os.remove(initial_manifest_temp_path)) - # default remote cmd patterns - args.remote_exec_pattern = None - args.remote_copy_pattern = None - - args_dict = vars(args) - # if remote-exec and/or remote-copy args are None then user - # didn't specify command line options nor env vars: - # inspect multiplexing options for default cdist.REMOTE_COPY/EXEC - if (args_dict['remote_copy'] is None or - args_dict['remote_exec'] is None): - mux_opts = inspect_ssh_mux_opts() - if args_dict['remote_exec'] is None: - args.remote_exec_pattern = cdist.REMOTE_EXEC + mux_opts - if args_dict['remote_copy'] is None: - args.remote_copy_pattern = cdist.REMOTE_COPY + mux_opts - - @classmethod - def _base_root_path(cls, args): - if args.out_path: - base_root_path = args.out_path - else: - base_root_path = tempfile.mkdtemp() - return base_root_path - @classmethod def commandline(cls, args): """Configure remote system""" - # FIXME: Refactor relict - remove later - log = logging.getLogger("cdist") + if (args.parallel and args.parallel != 1) or args.jobs: + if args.timestamp: + cdist.log.setupTimestampingParallelLogging() + else: + cdist.log.setupParallelLogging() + elif args.timestamp: + cdist.log.setupTimestampingLogging() + log = logging.getLogger("config") + + # No new child process if only one host at a time. + if args.parallel == 1: + log.debug("Only 1 parallel process, doing it sequentially") + args.parallel = 0 + + if args.parallel: + import signal + + signal.signal(signal.SIGTERM, mp_sig_handler) + signal.signal(signal.SIGHUP, mp_sig_handler) cls._check_and_prepare_args(args) - process = {} failed_hosts = [] time_start = time.time() - base_root_path = cls._base_root_path(args) + cls.construct_remote_exec_copy_patterns(args) + base_root_path = cls.create_base_root_path(args.out_path) hostcnt = 0 - for host in itertools.chain(cls.hosts(args.host), - cls.hosts(args.hostfile)): - hostdir = cdist.str_hash(host) - host_base_path = os.path.join(base_root_path, hostdir) + cfg = cdist.configuration.Configuration(args) + configuration = cfg.get_config(section='GLOBAL') + + if args.tag or args.all_tagged_hosts: + inventory.determine_default_inventory_dir(args, configuration) + if args.all_tagged_hosts: + inv_list = inventory.InventoryList( + hosts=None, istag=True, hostfile=None, + db_basedir=args.inventory_dir) + else: + inv_list = inventory.InventoryList( + hosts=args.host, istag=True, hostfile=args.hostfile, + db_basedir=args.inventory_dir, + has_all_tags=args.has_all_tags) + it = inv_list.entries() + else: + it = itertools.chain(cls.hosts(args.host), + cls.hosts(args.hostfile)) + + process_args = [] + if args.parallel: + log.trace("Processing hosts in parallel") + else: + log.trace("Processing hosts sequentially") + for entry in it: + if isinstance(entry, tuple): + # if configuring by specified tags + host = entry[0] + host_tags = entry[1] + else: + # if configuring by host then check inventory for tags + host = entry + inventory.determine_default_inventory_dir(args, configuration) + inv_list = inventory.InventoryList( + hosts=(host,), db_basedir=args.inventory_dir) + inv = tuple(inv_list.entries()) + if inv: + # host is present in inventory and has tags + host_tags = inv[0][1] + else: + # host is not present in inventory or has no tags + host_tags = None + host_base_path, hostdir = cls.create_host_base_dirs( + host, base_root_path) log.debug("Base root path for target host \"{}\" is \"{}\"".format( host, host_base_path)) hostcnt += 1 if args.parallel: - log.debug("Creating child process for %s", host) - process[host] = multiprocessing.Process( - target=cls.onehost, - args=(host, host_base_path, hostdir, args, True)) - process[host].start() + pargs = (host, host_tags, host_base_path, hostdir, args, True, + configuration) + log.trace(("Args for multiprocessing operation " + "for host {}: {}".format(host, pargs))) + process_args.append(pargs) else: try: - cls.onehost(host, host_base_path, hostdir, - args, parallel=False) - except cdist.Error as e: + cls.onehost(host, host_tags, host_base_path, hostdir, + args, parallel=False, + configuration=configuration) + except cdist.Error: failed_hosts.append(host) + if args.parallel and len(process_args) == 1: + log.debug("Only 1 host for parallel processing, doing it " + "sequentially") + try: + cls.onehost(*process_args[0]) + except cdist.Error: + failed_hosts.append(host) + elif args.parallel: + log.trace("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + log.trace(("Starting multiprocessing Pool for {} " + "parallel host operation".format(args.parallel))) - # Catch errors in parallel mode when joining - if args.parallel: - for host in process.keys(): - log.debug("Joining process %s", host) - process[host].join() + results = mp_pool_run(cls.onehost, + process_args, + jobs=args.parallel) + log.trace(("Multiprocessing for parallel host operation " + "finished")) + log.trace("Multiprocessing for parallel host operation " + "results: %s", results) - if not process[host].exitcode == 0: - failed_hosts.append(host) + failed_hosts = [host for host, result in results if not result] time_end = time.time() - log.info("Total processing time for %s host(s): %s", hostcnt, - (time_end - time_start)) + log.verbose("Total processing time for %s host(s): %s", hostcnt, + (time_end - time_start)) if len(failed_hosts) > 0: raise cdist.Error("Failed to configure the following hosts: " + " ".join(failed_hosts)) + elif not args.out_path: + # If tmp out path created then remove it, but only if no failed + # hosts. + shutil.rmtree(base_root_path) @classmethod def _resolve_ssh_control_path(cls): base_path = tempfile.mkdtemp() + cls._register_path_for_removal(base_path) control_path = os.path.join(base_path, "s") - atexit.register(lambda: shutil.rmtree(base_path)) return control_path @classmethod def _resolve_remote_cmds(cls, args): - control_path = cls._resolve_ssh_control_path() + if (args.remote_exec_pattern or + args.remote_copy_pattern or + args.remote_cmds_cleanup_pattern): + control_path = cls._resolve_ssh_control_path() # If we constructed patterns for remote commands then there is # placeholder for ssh ControlPath, format it and we have unique # ControlPath for each host. @@ -200,68 +347,158 @@ class Config(object): remote_copy = args.remote_copy_pattern.format(control_path) else: remote_copy = args.remote_copy - return (remote_exec, remote_copy, ) + if args.remote_cmds_cleanup_pattern: + remote_cmds_cleanup = args.remote_cmds_cleanup_pattern.format( + control_path) + else: + remote_cmds_cleanup = "" + return (remote_exec, remote_copy, remote_cmds_cleanup, ) + + @staticmethod + def _address_family(args): + if args.force_ipv == 4: + family = socket.AF_INET + elif args.force_ipv == 6: + family = socket.AF_INET6 + else: + family = 0 + return family + + @staticmethod + def resolve_target_addresses(host, family): + try: + return ipaddr.resolve_target_addresses(host, family) + except: + e = sys.exc_info()[1] + raise cdist.Error(("Error resolving target addresses for host '{}'" + ": {}").format(host, e)) @classmethod - def onehost(cls, host, host_base_path, host_dir_name, args, parallel): - """Configure ONE system""" + def onehost(cls, host, host_tags, host_base_path, host_dir_name, args, + parallel, configuration, remove_remote_files_dirs=False): + """Configure ONE system. + If operating in parallel then return tuple (host, True|False, ) + so that main process knows for which host function was successful. + """ log = logging.getLogger(host) try: - remote_exec, remote_copy = cls._resolve_remote_cmds(args) + remote_exec, remote_copy, cleanup_cmd = cls._resolve_remote_cmds( + args) log.debug("remote_exec for host \"{}\": {}".format( host, remote_exec)) log.debug("remote_copy for host \"{}\": {}".format( host, remote_copy)) - target_host = ipaddr.resolve_target_addresses(host) - log.debug("target_host: {}".format(target_host)) + family = cls._address_family(args) + log.debug("address family: {}".format(family)) + target_host = cls.resolve_target_addresses(host, family) + log.debug("target_host for host \"{}\": {}".format( + host, target_host)) local = cdist.exec.local.Local( target_host=target_host, + target_host_tags=host_tags, base_root_path=host_base_path, host_dir_name=host_dir_name, initial_manifest=args.manifest, - add_conf_dirs=args.conf_dir) + add_conf_dirs=args.conf_dir, + cache_path_pattern=args.cache_path_pattern, + quiet_mode=args.quiet, + configuration=configuration, + exec_path=sys.argv[0], + save_output_streams=args.save_output_streams) remote = cdist.exec.remote.Remote( target_host=target_host, remote_exec=remote_exec, - remote_copy=remote_copy) + remote_copy=remote_copy, + base_path=args.remote_out_path, + quiet_mode=args.quiet, + archiving_mode=args.use_archiving, + configuration=configuration, + stdout_base_path=local.stdout_base_path, + stderr_base_path=local.stderr_base_path, + save_output_streams=args.save_output_streams) - c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs) + cleanup_cmds = [] + if cleanup_cmd: + cleanup_cmds.append(cleanup_cmd) + c = cls(local, remote, dry_run=args.dry_run, jobs=args.jobs, + cleanup_cmds=cleanup_cmds, + remove_remote_files_dirs=remove_remote_files_dirs) c.run() + cls._remove_paths() except cdist.Error as e: log.error(e) if parallel: - # We are running in our own process here, need to sys.exit! - sys.exit(1) + return (host, False, ) else: raise - except KeyboardInterrupt: - # Ignore in parallel mode, we are existing anyway - if parallel: - sys.exit(0) - # Pass back to controlling code in sequential mode - else: - raise + if parallel: + return (host, True, ) + + @staticmethod + def create_base_root_path(out_path=None): + if out_path: + base_root_path = out_path + else: + base_root_path = tempfile.mkdtemp() + + return base_root_path + + @staticmethod + def create_host_base_dirs(host, base_root_path): + hostdir = cdist.str_hash(host) + host_base_path = os.path.join(base_root_path, hostdir) + + return (host_base_path, hostdir) def run(self): """Do what is most often done: deploy & cleanup""" start_time = time.time() + self.log.info("Starting {} run".format( + 'dry' if self.dry_run else 'configuration')) + self._init_files_dirs() self.explorer.run_global_explorers(self.local.global_explorer_out_path) - self.manifest.run_initial_manifest(self.local.initial_manifest) + try: + self.manifest.run_initial_manifest(self.local.initial_manifest) + except cdist.Error as e: + which = "init" + stdout_path = os.path.join(self.local.stdout_base_path, which) + stderr_path = os.path.join(self.local.stderr_base_path, which) + raise cdist.InitialManifestError(self.local.initial_manifest, + stdout_path, stderr_path, e) self.iterate_until_finished() + self.cleanup() + self._remove_files_dirs() - self.local.save_cache() - self.log.info("Finished successful run in %s seconds", - time.time() - start_time) + self.local.save_cache(start_time) + self.log.info("Finished {} run in {:.2f} seconds".format( + 'dry' if self.dry_run else 'successful', + time.time() - start_time)) + + def cleanup(self): + self.log.debug("Running cleanup commands") + for cleanup_cmd in self.cleanup_cmds: + cmd = cleanup_cmd.split() + cmd.append(self.local.target_host[0]) + try: + if self.log.getEffectiveLevel() <= logging.DEBUG: + quiet_mode = False + else: + quiet_mode = True + self.local.run(cmd, return_output=False, save_output=False, + quiet_mode=quiet_mode) + except cdist.Error as e: + # Log warning but continue. + self.log.warning("Cleanup command failed: %s", e) def object_list(self): """Short name for object list retrieval""" @@ -286,11 +523,12 @@ class Config(object): return objects_changed def _iterate_once_sequential(self): - self.log.info("Iteration in sequential mode") + self.log.debug("Iteration in sequential mode") objects_changed = False for cdist_object in self.object_list(): - if cdist_object.requirements_unfinished(cdist_object.requirements): + if cdist_object.requirements_unfinished( + cdist_object.requirements): """We cannot do anything for this poor object""" continue @@ -300,9 +538,10 @@ class Config(object): self.object_prepare(cdist_object) objects_changed = True - if cdist_object.requirements_unfinished(cdist_object.autorequire): + if cdist_object.requirements_unfinished( + cdist_object.autorequire): """The previous step created objects we depend on - - wait for them + wait for them """ continue @@ -313,7 +552,7 @@ class Config(object): return objects_changed def _iterate_once_parallel(self): - self.log.info("Iteration in parallel mode in {} jobs".format( + self.log.debug("Iteration in parallel mode in {} jobs".format( self.jobs)) objects_changed = False @@ -336,15 +575,39 @@ class Config(object): self.object_prepare(cargo[0]) objects_changed = True elif cargo: - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for {} parallel " + + self.log.trace("Multiprocessing cargo: %s", cargo) + + cargo_types = set() + for c in cargo: + cargo_types.add(c.cdist_type) + self.log.trace("Multiprocessing cargo_types: %s", cargo_types) + nt = len(cargo_types) + if nt == 1: + self.log.debug(("Only one type, transferring explorers " + "sequentially")) + self.explorer.transfer_type_explorers(cargo_types.pop()) + else: + self.log.trace(("Starting multiprocessing Pool for {} " + "parallel types explorers transferring".format( + nt))) + args = [ + (ct, ) for ct in cargo_types + ] + mp_pool_run(self.explorer.transfer_type_explorers, args, + jobs=self.jobs) + self.log.trace(("Multiprocessing for parallel transferring " + "types' explorers finished")) + + self.log.trace(("Starting multiprocessing Pool for {} parallel " "objects preparation".format(n))) args = [ - (c, ) for c in cargo + (c, False, ) for c in cargo ] mp_pool_run(self.object_prepare, args, jobs=self.jobs) - self.log.debug(("Multiprocessing for parallel object " + self.log.trace(("Multiprocessing for parallel object " "preparation finished")) objects_changed = True @@ -364,25 +627,44 @@ class Config(object): # self.object_run(cdist_object) # objects_changed = True - cargo.append(cdist_object) - n = len(cargo) - if n == 1: - self.log.debug("Only one object, running sequentially") - self.object_run(cargo[0]) - objects_changed = True - elif cargo: - self.log.debug("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for {} parallel " - "object run".format(n))) - args = [ - (c, ) for c in cargo - ] - mp_pool_run(self.object_run, args, jobs=self.jobs) - self.log.debug(("Multiprocessing for parallel object " - "run finished")) - objects_changed = True + # put objects in chuncks of distinct types + # so that there is no more than one object + # of the same type in one chunk because there is a + # possibility of object's process locking which + # prevents parallel execution at remote + # and do this only for nonparallel marked types + for chunk in cargo: + for obj in chunk: + if (obj.cdist_type == cdist_object.cdist_type and + cdist_object.cdist_type.is_nonparallel): + break + else: + chunk.append(cdist_object) + break + else: + chunk = [cdist_object, ] + cargo.append(chunk) + + for chunk in cargo: + self.log.trace("Running chunk: %s", chunk) + n = len(chunk) + if n == 1: + self.log.debug("Only one object, running sequentially") + self.object_run(chunk[0]) + objects_changed = True + elif chunk: + self.log.trace("Multiprocessing start method is {}".format( + multiprocessing.get_start_method())) + self.log.trace(("Starting multiprocessing Pool for {} " + "parallel object run".format(n))) + args = [ + (c, ) for c in chunk + ] + mp_pool_run(self.object_run, args, jobs=self.jobs) + self.log.trace(("Multiprocessing for parallel object " + "run finished")) + objects_changed = True return objects_changed @@ -401,6 +683,28 @@ class Config(object): self.__dict__.update(state) self._open_logger() + def _validate_dependencies(self): + ''' + Build dependency graph for unfinished objects and + check for cycles. + ''' + graph = {} + for cdist_object in self.object_list(): + obj_name = cdist_object.name + if obj_name not in graph: + graph[obj_name] = [] + if cdist_object.state == cdist_object.STATE_DONE: + continue + + for requirement in cdist_object.requirements_unfinished( + cdist_object.requirements): + graph[obj_name].append(requirement.name) + + for requirement in cdist_object.requirements_unfinished( + cdist_object.autorequire): + graph[obj_name].append(requirement.name) + return graph_check_cycle(graph) + def iterate_until_finished(self): """ Go through all objects and solve them @@ -410,6 +714,12 @@ class Config(object): objects_changed = True while objects_changed: + # Check for cycles as early as possible. + has_cycle, path = self._validate_dependencies() + if has_cycle: + raise cdist.UnresolvableRequirementsError( + "Cycle detected in object dependencies:\n{}!".format( + " -> ".join(path))) objects_changed = self.iterate_once() # Check whether all objects have been finished @@ -448,43 +758,71 @@ class Config(object): ("The requirements of the following objects could not be " "resolved:\n%s") % ("\n".join(info_string))) - def object_prepare(self, cdist_object): + def _handle_deprecation(self, cdist_object): + cdist_type = cdist_object.cdist_type + deprecated = cdist_type.deprecated + if deprecated is not None: + if deprecated: + self.log.warning("Type %s is deprecated: %s", cdist_type.name, + deprecated) + else: + self.log.warning("Type %s is deprecated.", cdist_type.name) + for param in cdist_object.parameters: + if param in cdist_type.deprecated_parameters: + msg = cdist_type.deprecated_parameters[param] + if msg: + format = "%s parameter of type %s is deprecated: %s" + args = [param, cdist_type.name, msg] + else: + format = "%s parameter of type %s is deprecated." + args = [param, cdist_type.name] + self.log.warning(format, *args) + + def object_prepare(self, cdist_object, transfer_type_explorers=True): """Prepare object: Run type explorer + manifest""" - self.log.info( - "Running manifest and explorers for " + cdist_object.name) - self.explorer.run_type_explorers(cdist_object) - self.manifest.run_type_manifest(cdist_object) - cdist_object.state = core.CdistObject.STATE_PREPARED + self._handle_deprecation(cdist_object) + self.log.verbose("Preparing object {}".format(cdist_object.name)) + self.log.verbose( + "Running manifest and explorers for " + cdist_object.name) + self.explorer.run_type_explorers(cdist_object, transfer_type_explorers) + try: + self.manifest.run_type_manifest(cdist_object) + cdist_object.state = core.CdistObject.STATE_PREPARED + except cdist.Error as e: + raise cdist.CdistObjectError(cdist_object, e) def object_run(self, cdist_object): """Run gencode and code for an object""" + try: + self.log.verbose("Running object " + cdist_object.name) + if cdist_object.state == core.CdistObject.STATE_DONE: + raise cdist.Error(("Attempting to run an already finished " + "object: %s"), cdist_object) - self.log.debug("Trying to run object %s" % (cdist_object.name)) - if cdist_object.state == core.CdistObject.STATE_DONE: - raise cdist.Error(("Attempting to run an already finished " - "object: %s"), cdist_object) - - cdist_type = cdist_object.cdist_type - - # Generate - self.log.info("Generating code for %s" % (cdist_object.name)) - cdist_object.code_local = self.code.run_gencode_local(cdist_object) - cdist_object.code_remote = self.code.run_gencode_remote(cdist_object) - if cdist_object.code_local or cdist_object.code_remote: - cdist_object.changed = True - - # Execute - if not self.dry_run: + # Generate + self.log.debug("Generating code for %s" % (cdist_object.name)) + cdist_object.code_local = self.code.run_gencode_local(cdist_object) + cdist_object.code_remote = self.code.run_gencode_remote( + cdist_object) if cdist_object.code_local or cdist_object.code_remote: - self.log.info("Executing code for %s" % (cdist_object.name)) - if cdist_object.code_local: - self.code.run_code_local(cdist_object) - if cdist_object.code_remote: - self.code.transfer_code_remote(cdist_object) - self.code.run_code_remote(cdist_object) - else: - self.log.info("Skipping code execution due to DRY RUN") + cdist_object.changed = True - # Mark this object as done - self.log.debug("Finishing run of " + cdist_object.name) - cdist_object.state = core.CdistObject.STATE_DONE + # Execute + if cdist_object.code_local or cdist_object.code_remote: + self.log.info("Processing %s" % (cdist_object.name)) + if not self.dry_run: + if cdist_object.code_local: + self.log.trace("Executing local code for %s" + % (cdist_object.name)) + self.code.run_code_local(cdist_object) + if cdist_object.code_remote: + self.log.trace("Executing remote code for %s" + % (cdist_object.name)) + self.code.transfer_code_remote(cdist_object) + self.code.run_code_remote(cdist_object) + + # Mark this object as done + self.log.trace("Finishing run of " + cdist_object.name) + cdist_object.state = core.CdistObject.STATE_DONE + except cdist.Error as e: + raise cdist.CdistObjectError(cdist_object, e) diff --git a/cdist/configuration.py b/cdist/configuration.py new file mode 100644 index 00000000..f05a5963 --- /dev/null +++ b/cdist/configuration.py @@ -0,0 +1,497 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + + +import configparser +import os +import cdist +import cdist.argparse +import re +import multiprocessing +import logging + + +class Singleton(type): + instance = None + + def __call__(cls, *args, **kwargs): + if 'singleton' in kwargs and not kwargs['singleton']: + return super(Singleton, cls).__call__(*args, **kwargs) + else: + if not cls.instance: + cls.instance = super(Singleton, cls).__call__(*args, **kwargs) + return cls.instance + + +_VERBOSITY_VALUES = ( + 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE', 'OFF', +) +_ARCHIVING_VALUES = ( + 'tar', 'tgz', 'tbz2', 'txz', 'none', +) + + +class OptionBase: + def __init__(self, name): + self.name = name + + def get_converter(self, *args, **kwargs): + raise NotImplementedError('Subclass should implement this method') + + def translate(self, val): + return val + + def update_value(self, currval, newval, update_appends=False): + '''Update current option value currval with new option value newval. + If update_appends is True and if currval and newval are lists then + resulting list contains all values in currval plus all values in + newval. Otherwise, newval is returned. + ''' + if (isinstance(currval, list) and isinstance(newval, list) and + update_appends): + rv = [] + if currval: + rv.extend(currval) + if newval: + rv.extend(newval) + if not rv: + rv = None + return rv + else: + return newval + + def should_override(self, currval, newval): + return True + + +class StringOption(OptionBase): + def __init__(self, name): + super().__init__(name) + + def get_converter(self): + def string_converter(val): + return self.translate(str(val)) + return string_converter + + def translate(self, val): + if val: + return val + else: + return None + + +class BooleanOption(OptionBase): + BOOLEAN_STATES = configparser.ConfigParser.BOOLEAN_STATES + + # If default_overrides is False then previous config value will not be + # overriden with default_value. + def __init__(self, name, default_overrides=True, default_value=True): + super().__init__(name) + self.default_overrides = default_overrides + self.default_value = default_value + + def get_converter(self): + def boolean_converter(val): + v = val.lower() + if v not in self.BOOLEAN_STATES: + raise ValueError('Invalid {} boolean value: {}'.format( + self.name, val)) + return self.translate(v) + return boolean_converter + + def translate(self, val): + return self.BOOLEAN_STATES[val] + + def should_override(self, currval, newval): + if not self.default_overrides: + return newval != self.default_value + return True + + +class IntOption(OptionBase): + def __init__(self, name): + super().__init__(name) + + def get_converter(self): + def int_converter(val): + return self.translate(int(val)) + return int_converter + + +class LowerBoundIntOption(IntOption): + def __init__(self, name, lower_bound): + super().__init__(name) + self.lower_bound = lower_bound + + def get_converter(self): + def lower_bound_converter(val): + converted = super(LowerBoundIntOption, self).get_converter()(val) + if converted < self.lower_bound: + raise ValueError("Invalid {} value: {} < {}".format( + self.name, val, self.lower_bound)) + return converted + return lower_bound_converter + + +class SpecialCasesLowerBoundIntOption(LowerBoundIntOption): + def __init__(self, name, lower_bound, special_cases_mapping): + super().__init__(name, lower_bound) + self.special_cases_mapping = special_cases_mapping + + def translate(self, val): + if val in self.special_cases_mapping: + return self.special_cases_mapping[val] + else: + return val + + +class JobsOption(SpecialCasesLowerBoundIntOption): + def __init__(self, name): + super().__init__(name, -1, {-1: multiprocessing.cpu_count()}) + + +class SelectOption(OptionBase): + def __init__(self, name, valid_values): + super().__init__(name) + self.valid_values = valid_values + + def get_converter(self): + def select_converter(val): + if val in self.valid_values: + return self.translate(val) + else: + raise ValueError("Invalid {} value: {}.".format( + self.name, val)) + return select_converter + + +class VerbosityOption(SelectOption): + def __init__(self): + super().__init__('verbosity', _VERBOSITY_VALUES) + + def translate(self, val): + name = 'VERBOSE_' + val + verbose = getattr(cdist.argparse, name) + return verbose + + +class DelimitedValuesOption(OptionBase): + def __init__(self, name, delimiter): + super().__init__(name) + self.delimiter = delimiter + + def get_converter(self): + def delimited_values_converter(val): + vals = re.split(r'(? ('__type_name', 'the/object_id') + """split_name('__type_name/the/object_id') + -> + ('__type_name', 'the/object_id') Split the given object name into it's type and object_id parts. @@ -122,7 +121,9 @@ class CdistObject(object): @staticmethod def join_name(type_name, object_id): - """join_name('__type_name', 'the/object_id') -> __type_name/the/object_id' + """join_name('__type_name', 'the/object_id') + -> + __type_name/the/object_id' Join the given type_name and object_id into an object name. @@ -145,9 +146,13 @@ class CdistObject(object): if '//' in self.object_id: raise IllegalObjectIdError( self.object_id, 'object_id may not contain //') - if self.object_id == '.': - raise IllegalObjectIdError( - self.object_id, 'object_id may not be a .') + + _invalid_object_ids = ('.', '/', ) + for ioid in _invalid_object_ids: + if self.object_id == ioid: + raise IllegalObjectIdError( + self.object_id, + 'object_id may not be a {}'.format(ioid)) # If no object_id and type is not singleton => error out if not self.object_id and not self.cdist_type.is_singleton: @@ -161,7 +166,8 @@ class CdistObject(object): # (parameters: %s)" % (self.cdist_type.name, self.parameters)) def object_from_name(self, object_name): - """Convenience method for creating an object instance from an object name. + """Convenience method for creating an object instance from an object + name. Mainly intended to create objects when resolving requirements. @@ -247,10 +253,11 @@ class CdistObject(object): """Create this cdist object on the filesystem. """ try: - os.makedirs(self.absolute_path, exist_ok=allow_overwrite) - absolute_parameter_path = os.path.join(self.base_path, - self.parameter_path) - os.makedirs(absolute_parameter_path, exist_ok=allow_overwrite) + for path in (self.absolute_path, + os.path.join(self.base_path, self.parameter_path), + self.stdout_path, + self.stderr_path): + os.makedirs(path, exist_ok=allow_overwrite) except EnvironmentError as error: raise cdist.Error(('Error creating directories for cdist object: ' '%s: %s') % (self, error)) diff --git a/cdist/core/cdist_type.py b/cdist/core/cdist_type.py index 14865386..4500f50d 100644 --- a/cdist/core/cdist_type.py +++ b/cdist/core/cdist_type.py @@ -21,19 +21,21 @@ # import os - import cdist +import cdist.core +import logging -class NoSuchTypeError(cdist.Error): +class InvalidTypeError(cdist.Error): def __init__(self, name, type_path, type_absolute_path): self.name = name self.type_path = type_path self.type_absolute_path = type_absolute_path + self.source_path = os.path.realpath(self.type_absolute_path) def __str__(self): - return "Type '%s' does not exist at %s" % ( - self.type_path, self.type_absolute_path) + return "Invalid type '%s' at '%s' defined at '%s'" % ( + self.type_path, self.type_absolute_path, self.source_path) class CdistType(object): @@ -45,13 +47,15 @@ class CdistType(object): """ + log = logging.getLogger("cdist-type") + def __init__(self, base_path, name): self.base_path = base_path self.name = name self.path = self.name self.absolute_path = os.path.join(self.base_path, self.path) if not os.path.isdir(self.absolute_path): - raise NoSuchTypeError(self.name, self.path, self.absolute_path) + raise InvalidTypeError(self.name, self.path, self.absolute_path) self.manifest_path = os.path.join(self.name, "manifest") self.explorer_path = os.path.join(self.name, "explorer") self.gencode_local_path = os.path.join(self.name, "gencode-local") @@ -65,17 +69,29 @@ class CdistType(object): self.__optional_multiple_parameters = None self.__boolean_parameters = None self.__parameter_defaults = None + self.__deprecated_parameters = None + + def __hash__(self): + return hash(self.name) @classmethod def list_types(cls, base_path): """Return a list of type instances""" for name in cls.list_type_names(base_path): - yield cls(base_path, name) + try: + yield cls(base_path, name) + except InvalidTypeError as e: + # ignore invalid type, log warning and continue + msg = "Ignoring invalid type '%s' at '%s' defined at '%s'" % ( + e.type_path, e.type_absolute_path, e.source_path) + cls.log.warning(msg) + # remove invalid from runtime conf dir + os.remove(e.type_absolute_path) @classmethod def list_type_names(cls, base_path): """Return a list of type names""" - return os.listdir(base_path) + return cdist.core.listdir(base_path) _instances = {} @@ -112,13 +128,30 @@ class CdistType(object): (if not: for configuration)""" return os.path.isfile(os.path.join(self.absolute_path, "install")) + @property + def is_nonparallel(self): + """Check whether a type is a non parallel, i.e. its objects + cannot run in parallel.""" + return os.path.isfile(os.path.join(self.absolute_path, "nonparallel")) + + @property + def deprecated(self): + """Get type deprecation message. If message is None then type + is not deprecated.""" + deprecated_path = os.path.join(self.absolute_path, "deprecated") + try: + with open(deprecated_path, 'r') as f: + return f.read() + except FileNotFoundError: + return None + @property def explorers(self): """Return a list of available explorers""" if not self.__explorers: try: - self.__explorers = os.listdir(os.path.join(self.absolute_path, - "explorer")) + self.__explorers = cdist.core.listdir( + os.path.join(self.absolute_path, "explorer")) except EnvironmentError: # error ignored self.__explorers = [] @@ -134,7 +167,9 @@ class CdistType(object): "parameter", "required")) as fd: for line in fd: - parameters.append(line.strip()) + line = line.strip() + if line: + parameters.append(line) except EnvironmentError: # error ignored pass @@ -152,7 +187,9 @@ class CdistType(object): "parameter", "required_multiple")) as fd: for line in fd: - parameters.append(line.strip()) + line = line.strip() + if line: + parameters.append(line) except EnvironmentError: # error ignored pass @@ -170,7 +207,9 @@ class CdistType(object): "parameter", "optional")) as fd: for line in fd: - parameters.append(line.strip()) + line = line.strip() + if line: + parameters.append(line) except EnvironmentError: # error ignored pass @@ -188,7 +227,9 @@ class CdistType(object): "parameter", "optional_multiple")) as fd: for line in fd: - parameters.append(line.strip()) + line = line.strip() + if line: + parameters.append(line) except EnvironmentError: # error ignored pass @@ -206,7 +247,9 @@ class CdistType(object): "parameter", "boolean")) as fd: for line in fd: - parameters.append(line.strip()) + line = line.strip() + if line: + parameters.append(line) except EnvironmentError: # error ignored pass @@ -222,7 +265,7 @@ class CdistType(object): defaults_dir = os.path.join(self.absolute_path, "parameter", "default") - for name in os.listdir(defaults_dir): + for name in cdist.core.listdir(defaults_dir): try: with open(os.path.join(defaults_dir, name)) as fd: defaults[name] = fd.read().strip() @@ -233,3 +276,23 @@ class CdistType(object): finally: self.__parameter_defaults = defaults return self.__parameter_defaults + + @property + def deprecated_parameters(self): + if not self.__deprecated_parameters: + deprecated = {} + try: + deprecated_dir = os.path.join(self.absolute_path, + "parameter", + "deprecated") + for name in cdist.core.listdir(deprecated_dir): + try: + with open(os.path.join(deprecated_dir, name)) as fd: + deprecated[name] = fd.read().strip() + except EnvironmentError: + pass # Swallow errors raised by open() or read() + except EnvironmentError: + pass # Swallow error raised by os.listdir() + finally: + self.__deprecated_parameters = deprecated + return self.__deprecated_parameters diff --git a/cdist/core/code.py b/cdist/core/code.py index e9e2edf0..1550880a 100644 --- a/cdist/core/code.py +++ b/cdist/core/code.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # 2014 Daniel Heule (hda at sfs.biz) # @@ -21,12 +21,8 @@ # # -import logging import os - -import cdist - -log = logging.getLogger(__name__) +from . import util ''' @@ -59,6 +55,7 @@ gencode-local __object_fq: full qualified object id, iow: $type.name + / + object_id __type: full qualified path to the type's dir __files: full qualified path to the files dir + __target_host_tags: comma spearated list of host tags returns: string containing the generated code or None @@ -77,6 +74,7 @@ gencode-remote __object_fq: full qualified object id, iow: $type.name + / + object_id __type: full qualified path to the type's dir __files: full qualified path to the files dir + __target_host_tags: comma spearated list of host tags returns: string containing the generated code or None @@ -99,7 +97,7 @@ class Code(object): """ # target_host is tuple (target_host, target_hostname, target_fqdn) - def __init__(self, target_host, local, remote): + def __init__(self, target_host, local, remote, dry_run=False): self.target_host = target_host self.local = local self.remote = remote @@ -109,8 +107,15 @@ class Code(object): '__target_fqdn': self.target_host[2], '__global': self.local.base_path, '__files': self.local.files_path, + '__target_host_tags': self.local.target_host_tags, + '__cdist_log_level': util.log_level_env_var_val(local.log), + '__cdist_log_level_name': util.log_level_name_env_var_val( + local.log), } + if dry_run: + self.env['__cdist_dry_run'] = '1' + def _run_gencode(self, cdist_object, which): cdist_type = cdist_object.cdist_type script = os.path.join(self.local.type_path, @@ -125,8 +130,18 @@ class Code(object): '__object_name': cdist_object.name, }) message_prefix = cdist_object.name - return self.local.run_script(script, env=env, return_output=True, - message_prefix=message_prefix) + if self.local.save_output_streams: + stderr_path = os.path.join(cdist_object.stderr_path, + 'gencode-' + which) + with open(stderr_path, 'ba+') as stderr: + return self.local.run_script(script, env=env, + return_output=True, + message_prefix=message_prefix, + stderr=stderr) + else: + return self.local.run_script(script, env=env, + return_output=True, + message_prefix=message_prefix) def run_gencode_local(self, cdist_object): """Run the gencode-local script for the given cdist object.""" @@ -143,15 +158,24 @@ class Code(object): cdist_object.code_remote_path) destination = os.path.join(self.remote.object_path, cdist_object.code_remote_path) - # FIXME: BUG: do not create destination, but top level of destination! - self.remote.mkdir(destination) + self.remote.mkdir(os.path.dirname(destination)) self.remote.transfer(source, destination) def _run_code(self, cdist_object, which, env=None): which_exec = getattr(self, which) script = os.path.join(which_exec.object_path, getattr(cdist_object, 'code_%s_path' % which)) - return which_exec.run_script(script, env=env) + if which_exec.save_output_streams: + stderr_path = os.path.join(cdist_object.stderr_path, + 'code-' + which) + stdout_path = os.path.join(cdist_object.stdout_path, + 'code-' + which) + with open(stderr_path, 'ba+') as stderr, \ + open(stdout_path, 'ba+') as stdout: + return which_exec.run_script(script, env=env, stdout=stdout, + stderr=stderr) + else: + return which_exec.run_script(script, env=env) def run_code_local(self, cdist_object): """Run the code-local script for the given cdist object.""" diff --git a/cdist/core/explorer.py b/cdist/core/explorer.py index 45afc5c0..353d7681 100644 --- a/cdist/core/explorer.py +++ b/cdist/core/explorer.py @@ -24,7 +24,9 @@ import logging import os import glob import multiprocessing +import cdist from cdist.mputil import mp_pool_run +from . import util ''' common: @@ -65,7 +67,7 @@ class Explorer(object): """Executes cdist explorers. """ - def __init__(self, target_host, local, remote, jobs=None): + def __init__(self, target_host, local, remote, jobs=None, dry_run=False): self.target_host = target_host self._open_logger() @@ -77,7 +79,15 @@ class Explorer(object): '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__explorer': self.remote.global_explorer_path, + '__target_host_tags': self.local.target_host_tags, + '__cdist_log_level': util.log_level_env_var_val(self.log), + '__cdist_log_level_name': util.log_level_name_env_var_val( + self.log), } + + if dry_run: + self.env['__cdist_dry_run'] = '1' + self._type_explorers_transferred = [] self.jobs = jobs @@ -95,7 +105,7 @@ class Explorer(object): out_path directory. """ - self.log.info("Running global explorers") + self.log.verbose("Running global explorers") self.transfer_global_explorers() if self.jobs is None: self._run_global_explorers_seq(out_path) @@ -103,28 +113,35 @@ class Explorer(object): self._run_global_explorers_parallel(out_path) def _run_global_explorer(self, explorer, out_path): - output = self.run_global_explorer(explorer) - path = os.path.join(out_path, explorer) - with open(path, 'w') as fd: - fd.write(output) + try: + path = os.path.join(out_path, explorer) + output = self.run_global_explorer(explorer) + with open(path, 'w') as fd: + fd.write(output) + except cdist.Error as e: + local_path = os.path.join(self.local.global_explorer_path, + explorer) + stderr_path = os.path.join(self.local.stderr_base_path, "remote") + raise cdist.GlobalExplorerError(explorer, local_path, stderr_path, + e) def _run_global_explorers_seq(self, out_path): - self.log.info("Running global explorers sequentially") + self.log.debug("Running global explorers sequentially") for explorer in self.list_global_explorer_names(): self._run_global_explorer(explorer, out_path) def _run_global_explorers_parallel(self, out_path): - self.log.info("Running global explorers in {} parallel jobs".format( + self.log.debug("Running global explorers in {} parallel jobs".format( self.jobs)) - self.log.debug("Multiprocessing start method is {}".format( + self.log.trace("Multiprocessing start method is {}".format( multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for global " + self.log.trace(("Starting multiprocessing Pool for global " "explorers run")) args = [ (e, out_path, ) for e in self.list_global_explorer_names() ] mp_pool_run(self._run_global_explorer, args, jobs=self.jobs) - self.log.debug(("Multiprocessing run for global explorers " + self.log.trace(("Multiprocessing run for global explorers " "finished")) # logger is not pickable, so remove it when we pickle @@ -141,7 +158,6 @@ class Explorer(object): def transfer_global_explorers(self): """Transfer the global explorers to the remote side.""" - self.remote.mkdir(self.remote.global_explorer_path) self.remote.transfer(self.local.global_explorer_path, self.remote.global_explorer_path, self.jobs) @@ -163,22 +179,39 @@ class Explorer(object): except EnvironmentError: return [] - def run_type_explorers(self, cdist_object): + def run_type_explorers(self, cdist_object, transfer_type_explorers=True): """Run the type explorers for the given object and save their output in the object. """ - self.log.debug("Transfering type explorers for type: %s", - cdist_object.cdist_type) - self.transfer_type_explorers(cdist_object.cdist_type) - self.log.debug("Transfering object parameters for object: %s", + self.log.verbose("Running type explorers for {}".format( + cdist_object.cdist_type)) + if transfer_type_explorers: + self.log.trace("Transferring type explorers for type: %s", + cdist_object.cdist_type) + self.transfer_type_explorers(cdist_object.cdist_type) + else: + self.log.trace(("No need for transferring type explorers for " + "type: %s"), + cdist_object.cdist_type) + self.log.trace("Transferring object parameters for object: %s", cdist_object.name) self.transfer_object_parameters(cdist_object) - for explorer in self.list_type_explorer_names(cdist_object.cdist_type): - output = self.run_type_explorer(explorer, cdist_object) - self.log.debug("Running type explorer '%s' for object '%s'", + cdist_type = cdist_object.cdist_type + for explorer in self.list_type_explorer_names(cdist_type): + self.log.trace("Running type explorer '%s' for object '%s'", explorer, cdist_object.name) - cdist_object.explorers[explorer] = output + try: + output = self.run_type_explorer(explorer, cdist_object) + cdist_object.explorers[explorer] = output + except cdist.Error as e: + path = os.path.join(self.local.type_path, + cdist_type.explorer_path, + explorer) + stderr_path = os.path.join(self.local.stderr_base_path, + "remote") + raise cdist.CdistObjectExplorerError( + cdist_object, explorer, path, stderr_path, e) def run_type_explorer(self, explorer, cdist_object): """Run the given type explorer for the given object and return @@ -203,14 +236,13 @@ class Explorer(object): remote side.""" if cdist_type.explorers: if cdist_type.name in self._type_explorers_transferred: - self.log.debug("Skipping retransfer of type explorers for: %s", - cdist_type) + self.log.trace(("Skipping retransfer of type explorers " + "for: %s"), cdist_type) else: source = os.path.join(self.local.type_path, cdist_type.explorer_path) destination = os.path.join(self.remote.type_path, cdist_type.explorer_path) - self.remote.mkdir(destination) self.remote.transfer(source, destination) self.remote.run(["chmod", "0700", "%s/*" % (destination)]) self._type_explorers_transferred.append(cdist_type.name) @@ -222,5 +254,4 @@ class Explorer(object): cdist_object.parameter_path) destination = os.path.join(self.remote.object_path, cdist_object.parameter_path) - self.remote.mkdir(destination) self.remote.transfer(source, destination) diff --git a/cdist/core/manifest.py b/cdist/core/manifest.py index 29f96c4f..07af0ef8 100644 --- a/cdist/core/manifest.py +++ b/cdist/core/manifest.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2013 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. @@ -24,6 +24,7 @@ import logging import os import cdist +from . import util ''' common: @@ -42,6 +43,7 @@ common: types are defined for use in type emulator == local.type_path __files: full qualified path to the files dir + __target_host_tags: comma spearated list of host tags initial manifest is: script: full qualified path to the initial manifest @@ -94,7 +96,7 @@ class Manifest(object): """Executes cdist manifests. """ - def __init__(self, target_host, local): + def __init__(self, target_host, local, dry_run=False): self.target_host = target_host self.local = local @@ -109,10 +111,14 @@ class Manifest(object): '__target_hostname': self.target_host[1], '__target_fqdn': self.target_host[2], '__files': self.local.files_path, + '__target_host_tags': self.local.target_host_tags, + '__cdist_log_level': util.log_level_env_var_val(self.log), + '__cdist_log_level_name': util.log_level_name_env_var_val( + self.log), } - if self.log.getEffectiveLevel() == logging.DEBUG: - self.env.update({'__cdist_debug': "yes"}) + if dry_run: + self.env['__cdist_dry_run'] = '1' def _open_logger(self): self.log = logging.getLogger(self.target_host[0]) @@ -145,16 +151,27 @@ class Manifest(object): else: user_supplied = True - self.log.info("Running initial manifest " + initial_manifest) - if not os.path.isfile(initial_manifest): raise NoInitialManifestError(initial_manifest, user_supplied) message_prefix = "initialmanifest" - self.local.run_script(initial_manifest, - env=self.env_initial_manifest(initial_manifest), - message_prefix=message_prefix, - save_output=False) + self.log.verbose("Running initial manifest " + initial_manifest) + which = "init" + if self.local.save_output_streams: + stderr_path = os.path.join(self.local.stderr_base_path, which) + stdout_path = os.path.join(self.local.stdout_base_path, which) + with open(stderr_path, 'ba+') as stderr, \ + open(stdout_path, 'ba+') as stdout: + self.local.run_script( + initial_manifest, + env=self.env_initial_manifest(initial_manifest), + message_prefix=message_prefix, + stdout=stdout, stderr=stderr) + else: + self.local.run_script( + initial_manifest, + env=self.env_initial_manifest(initial_manifest), + message_prefix=message_prefix) def env_type_manifest(self, cdist_object): type_manifest = os.path.join(self.local.type_path, @@ -176,8 +193,22 @@ class Manifest(object): type_manifest = os.path.join(self.local.type_path, cdist_object.cdist_type.manifest_path) message_prefix = cdist_object.name + which = 'manifest' if os.path.isfile(type_manifest): - self.local.run_script(type_manifest, - env=self.env_type_manifest(cdist_object), - message_prefix=message_prefix, - save_output=False) + self.log.verbose("Running type manifest %s for object %s", + type_manifest, cdist_object.name) + if self.local.save_output_streams: + stderr_path = os.path.join(cdist_object.stderr_path, which) + stdout_path = os.path.join(cdist_object.stdout_path, which) + with open(stderr_path, 'ba+') as stderr, \ + open(stdout_path, 'ba+') as stdout: + self.local.run_script( + type_manifest, + env=self.env_type_manifest(cdist_object), + message_prefix=message_prefix, + stdout=stdout, stderr=stderr) + else: + self.local.run_script( + type_manifest, + env=self.env_type_manifest(cdist_object), + message_prefix=message_prefix) diff --git a/cdist/core/util.py b/cdist/core/util.py new file mode 100644 index 00000000..64570d34 --- /dev/null +++ b/cdist/core/util.py @@ -0,0 +1,45 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import logging + + +def listdir(path='.', include_dot=False): + """os.listdir but do not include entries whose names begin with a dot('.') + if include_dot is False. + """ + if include_dot: + return os.listdir(path) + else: + return [x for x in os.listdir(path) if not _ishidden(x)] + + +def _ishidden(path): + return path[0] in ('.', b'.'[0]) + + +def log_level_env_var_val(log): + return str(log.getEffectiveLevel()) + + +def log_level_name_env_var_val(log): + return logging.getLevelName(log.getEffectiveLevel()) diff --git a/cdist/emulator.py b/cdist/emulator.py index cdbe5b08..3cf82f84 100644 --- a/cdist/emulator.py +++ b/cdist/emulator.py @@ -28,6 +28,7 @@ import sys import cdist from cdist import core +from cdist import flock class MissingRequiredEnvironmentVariableError(cdist.Error): @@ -94,20 +95,33 @@ class Emulator(object): """Emulate type commands (i.e. __file and co)""" self.commandline() - self.setup_object() - self.save_stdin() - self.record_requirements() - self.record_auto_requirements() - self.log.debug("Finished %s %s" % ( - self.cdist_object.path, self.parameters)) + self.init_object() + + # locking for parallel execution + with flock.Flock(self.flock_path): + self.setup_object() + self.save_stdin() + self.record_requirements() + self.record_auto_requirements() + self.log.trace("Finished %s %s" % ( + self.cdist_object.path, self.parameters)) def __init_log(self): """Setup logging facility""" - if '__cdist_debug' in self.env: - logging.root.setLevel(logging.DEBUG) + if '__cdist_log_level' in self.env: + try: + loglevel = self.env['__cdist_log_level'] + level = int(loglevel) + except ValueError: + level = logging.WARNING else: - logging.root.setLevel(logging.INFO) + level = logging.WARNING + try: + logging.root.setLevel(level) + except (ValueError, TypeError): + # if invalid __cdist_log_level value + logging.root.setLevel(logging.WARNING) self.log = logging.getLogger(self.target_host[0]) @@ -148,10 +162,10 @@ class Emulator(object): # And finally parse/verify parameter self.args = parser.parse_args(self.argv[1:]) - self.log.debug('Args: %s' % self.args) + self.log.trace('Args: %s' % self.args) - def setup_object(self): - # Setup object - and ensure it is not in args + def init_object(self): + # Initialize object - and ensure it is not in args if self.cdist_type.is_singleton: self.object_id = '' else: @@ -162,7 +176,36 @@ class Emulator(object): self.cdist_object = core.CdistObject( self.cdist_type, self.object_base_path, self.object_marker, self.object_id) + lockfname = ('.' + self.cdist_type.name + + self.object_id + '_' + + self.object_marker + '.lock') + lockfname = lockfname.replace(os.sep, '_') + self.flock_path = os.path.join(self.object_base_path, lockfname) + def _object_params_in_context(self): + ''' Get cdist_object parameters dict adopted by context. + Context consists of cdist_type boolean, optional, required, + optional_multiple and required_multiple parameters. If parameter + is multiple parameter then its value is a list. + This adaptation works on cdist_object.parameters which are read from + directory based dict where it is unknown what kind of data is in + file. If there is only one line in the file it is unknown if this + is a value of required/optional parameter or if it is one value of + multiple values parameter. + ''' + params = {} + if self.cdist_object.exists: + for param in self.cdist_object.parameters: + value = ('' if param in self.cdist_type.boolean_parameters + else self.cdist_object.parameters[param]) + if ((param in self.cdist_type.required_multiple_parameters or + param in self.cdist_type.optional_multiple_parameters) and + not isinstance(value, list)): + value = [value] + params[param] = value + return params + + def setup_object(self): # Create object with given parameters self.parameters = {} for key, value in vars(self.args).items(): @@ -173,15 +216,15 @@ class Emulator(object): # Make existing requirements a set so that we can compare it # later with new requirements. self._existing_reqs = set(self.cdist_object.requirements) - if self.cdist_object.parameters != self.parameters: + obj_params = self._object_params_in_context() + if obj_params != self.parameters: errmsg = ("Object %s already exists with conflicting " "parameters:\n%s: %s\n%s: %s" % ( self.cdist_object.name, " ".join(self.cdist_object.source), - self.cdist_object.parameters, + obj_params, self.object_source, self.parameters)) - self.log.error(errmsg) raise cdist.Error(errmsg) else: if self.cdist_object.exists: @@ -227,20 +270,21 @@ class Emulator(object): # Raises an error, if object cannot be created try: cdist_object = self.cdist_object.object_from_name(requirement) - except core.cdist_type.NoSuchTypeError as e: + except core.cdist_type.InvalidTypeError as e: self.log.error(("%s requires object %s, but type %s does not" " exist. Defined at %s" % ( self.cdist_object.name, requirement, e.name, self.object_source))) raise - except core.cdist_object.MissingObjectIdError as e: + except core.cdist_object.MissingObjectIdError: self.log.error(("%s requires object %s without object id." " Defined at %s" % (self.cdist_object.name, requirement, self.object_source))) raise - self.log.debug("Recording requirement: %s", requirement) + self.log.debug("Recording requirement %s for %s", + requirement, self.cdist_object.name) # Save the sanitised version, not the user supplied one # (__file//bar => __file/bar) @@ -256,19 +300,34 @@ class Emulator(object): # (this would leed to an circular dependency) if ("CDIST_ORDER_DEPENDENCY" in self.env and 'CDIST_OVERRIDE' not in self.env): - # load object name created bevor this one from typeorder file ... + # load object name created befor this one from typeorder file ... with open(self.typeorder_path, 'r') as typecreationfile: typecreationorder = typecreationfile.readlines() - # get the type created bevore this one ... + # get the type created before this one ... try: lastcreatedtype = typecreationorder[-2].strip() - if 'require' in self.env: - self.env['require'] += " " + lastcreatedtype + # __object_name is the name of the object whose type + # manifest is currently executed + __object_name = self.env.get('__object_name', None) + if lastcreatedtype == __object_name: + self.log.debug(("Not injecting require for " + "CDIST_ORDER_DEPENDENCY: %s for %s," + " %s's type manifest is currently" + " being executed"), + lastcreatedtype, + self.cdist_object.name, + lastcreatedtype) else: - self.env['require'] = lastcreatedtype - self.log.debug(("Injecting require for " - "CDIST_ORDER_DEPENDENCY: %s for %s"), - lastcreatedtype, self.cdist_object.name) + if 'require' in self.env: + appendix = " " + lastcreatedtype + if appendix not in self.env['require']: + self.env['require'] += appendix + else: + self.env['require'] = lastcreatedtype + self.log.debug(("Injecting require for " + "CDIST_ORDER_DEPENDENCY: %s for %s"), + lastcreatedtype, + self.cdist_object.name) except IndexError: # if no second last line, we are on the first type, # so do not set a requirement @@ -317,4 +376,6 @@ class Emulator(object): # But only if the user hasn't said otherwise. # Must prevent circular dependencies. if parent.name not in current_object.requirements: + self.log.debug("Recording autorequirement %s for %s", + current_object.name, parent.name) parent.autorequire.append(current_object.name) diff --git a/cdist/exec/local.py b/cdist/exec/local.py index e078dbd2..f83c85df 100644 --- a/cdist/exec/local.py +++ b/cdist/exec/local.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc) # 2011-2015 Nico Schottelius (nico-cdist at schottelius.org) -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -21,7 +21,6 @@ # # -import io import os import sys import re @@ -29,11 +28,13 @@ import subprocess import shutil import logging import tempfile +import time +import datetime import cdist import cdist.message from cdist import core -import cdist.exec.util as exec_util +import cdist.exec.util as util CONF_SUBDIRS_LINKED = ["explorer", "files", "manifest", "type", ] @@ -47,24 +48,39 @@ class Local(object): """ def __init__(self, target_host, + target_host_tags, base_root_path, host_dir_name, exec_path=sys.argv[0], initial_manifest=None, - add_conf_dirs=None): + add_conf_dirs=None, + cache_path_pattern=None, + quiet_mode=False, + configuration=None, + save_output_streams=True): self.target_host = target_host + if target_host_tags is None: + self.target_host_tags = "" + else: + self.target_host_tags = ",".join(target_host_tags) self.hostdir = host_dir_name self.base_path = os.path.join(base_root_path, "data") self.exec_path = exec_path self.custom_initial_manifest = initial_manifest self._add_conf_dirs = add_conf_dirs + self.cache_path_pattern = cache_path_pattern + self.quiet_mode = quiet_mode + if configuration: + self.configuration = configuration + else: + self.configuration = {} + self.save_output_streams = save_output_streams self._init_log() self._init_permissions() self.mkdir(self.base_path) - # FIXME: create dir that does not require moving later self._init_cache_dir(None) self._init_paths() self._init_object_marker() @@ -77,10 +93,7 @@ class Local(object): @property def home_dir(self): - if 'HOME' in os.environ: - return os.path.join(os.environ['HOME'], ".cdist") - else: - return None + return cdist.home_dir() def _init_log(self): self.log = logging.getLogger(self.target_host[0]) @@ -109,9 +122,11 @@ class Local(object): "explorer") self.object_path = os.path.join(self.base_path, "object") self.messages_path = os.path.join(self.base_path, "messages") - self.files_path = os.path.join(self.conf_path, "files") + self.stdout_base_path = os.path.join(self.base_path, "stdout") + self.stderr_base_path = os.path.join(self.base_path, "stderr") # Depending on conf_path + self.files_path = os.path.join(self.conf_path, "files") self.global_explorer_path = os.path.join(self.conf_path, "explorer") self.manifest_path = os.path.join(self.conf_path, "manifest") self.initial_manifest = (self.custom_initial_manifest or @@ -135,10 +150,14 @@ class Local(object): self.conf_dirs.append(self.home_dir) # Add directories defined in the CDIST_PATH environment variable - if 'CDIST_PATH' in os.environ: - cdist_path_dirs = re.split(r'(? %s", source, destination) - self.rmdir(destination) + self.log.trace("Remote transfer: %s -> %s", source, destination) + # self.rmdir(destination) if os.path.isdir(source): self.mkdir(destination) - if jobs: - self._transfer_dir_parallel(source, destination, jobs) - else: - self._transfer_dir_sequential(source, destination) + used_archiving = False + if self.archiving_mode: + self.log.trace("Remote transfer in archiving mode") + import cdist.autil as autil + + # create archive + tarpath, fcnt = autil.tar(source, self.archiving_mode) + if tarpath is None: + self.log.trace(("Files count {} is lower than {} limit, " + "skipping archiving").format( + fcnt, autil.FILES_LIMIT)) + else: + self.log.trace(("Archiving mode, tarpath: %s, file count: " + "%s"), tarpath, fcnt) + # get archive name + tarname = os.path.basename(tarpath) + self.log.trace("Archiving mode tarname: %s", tarname) + # archive path at the remote + desttarpath = os.path.join(destination, tarname) + self.log.trace( + "Archiving mode desttarpath: %s", desttarpath) + # transfer archive to the remote side + self.log.trace("Archiving mode: transferring") + self._transfer_file(tarpath, desttarpath) + # extract archive at the remote + self.log.trace("Archiving mode: extracting") + self.extract_archive(desttarpath, self.archiving_mode) + # remove remote archive + self.log.trace("Archiving mode: removing remote archive") + self.rmfile(desttarpath) + # remove local archive + self.log.trace("Archiving mode: removing local archive") + os.remove(tarpath) + used_archiving = True + if not used_archiving: + self._transfer_dir(source, destination) elif jobs: raise cdist.Error("Source {} is not a directory".format(source)) else: - command = self._copy.split() - command.extend([source, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - self._run_command(command) + self._transfer_file(source, destination) - def _transfer_dir_sequential(self, source, destination): + def _transfer_dir(self, source, destination): + command = self._copy.split() for f in glob.glob1(source, '*'): - command = self._copy.split() path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - self._run_command(command) + command.extend([path]) + command.extend(['{0}:{1}'.format( + _wrap_addr(self.target_host[0]), destination)]) + self._run_command(command) - def _transfer_dir_parallel(self, source, destination, jobs): - """Transfer a directory to the remote side in parallel mode.""" - self.log.info("Remote transfer in {} parallel jobs".format( - jobs)) - self.log.debug("Multiprocessing start method is {}".format( - multiprocessing.get_start_method())) - self.log.debug(("Starting multiprocessing Pool for parallel " - "remote transfer")) - args = [] - for f in glob.glob1(source, '*'): - command = self._copy.split() - path = os.path.join(source, f) - command.extend([path, '{0}:{1}'.format( - _wrap_addr(self.target_host[0]), destination)]) - args.append((command, )) - mp_pool_run(self._run_command, args, jobs=jobs) - self.log.debug(("Multiprocessing for parallel transfer " - "finished")) - - def run_script(self, script, env=None, return_output=False): + def run_script(self, script, env=None, return_output=False, stdout=None, + stderr=None): """Run the given script with the given environment on the remote side. Return the output as a string. """ - command = [os.environ.get('CDIST_REMOTE_SHELL', "/bin/sh"), "-e"] + command = [ + self.configuration.get('remote_shell', "/bin/sh"), + "-e" + ] command.append(script) - return self.run(command, env, return_output) + return self.run(command, env=env, return_output=return_output, + stdout=stdout, stderr=stderr) - def run(self, command, env=None, return_output=False): + def run(self, command, env=None, return_output=False, stdout=None, + stderr=None): """Run the given command with the given environment on the remote side. Return the output as a string. @@ -184,20 +244,19 @@ class Remote(object): cmd = self._exec.split() cmd.append(self.target_host[0]) - # FIXME: replace this by -o SendEnv name -o SendEnv name ... to ssh? # can't pass environment to remote side, so prepend command with # variable declarations # cdist command prepended with variable assignments expects - # posix shell (bourne, bash) at the remote as user default shell. - # If remote user shell isn't poxis shell, but for e.g. csh/tcsh + # POSIX shell (bourne, bash) at the remote as user default shell. + # If remote user shell isn't POSIX shell, but for e.g. csh/tcsh # then these var assignments are not var assignments for this # remote shell, it tries to execute it as a command and fails. # So really do this by default: # /bin/sh -c 'export ; command' # so that constructed remote command isn't dependent on remote # shell. Do this only if env is not None. env breaks this. - # Explicitly use /bin/sh, because var assignments assume poxis + # Explicitly use /bin/sh, because var assignments assume POSIX # shell already. # This leaves the posibility to write script that needs to be run # remotely in e.g. csh and setting up CDIST_REMOTE_SHELL to e.g. @@ -209,9 +268,11 @@ class Remote(object): cmd.append(string_cmd) else: cmd.extend(command) - return self._run_command(cmd, env=env, return_output=return_output) + return self._run_command(cmd, env=env, return_output=return_output, + stdout=stdout, stderr=stderr) - def _run_command(self, command, env=None, return_output=False): + def _run_command(self, command, env=None, return_output=False, stdout=None, + stderr=None): """Run the given command with the given environment. Return the output as a string. @@ -219,6 +280,19 @@ class Remote(object): assert isinstance(command, (list, tuple)), ( "list or tuple argument expected, got: %s" % command) + if return_output and stdout is not subprocess.PIPE: + self.log.debug("return_output is True, ignoring stdout") + + close_stdout = False + close_stderr = False + if self.save_output_streams: + if not return_output and stdout is None: + stdout = util.get_std_fd(self.stdout_base_path, 'remote') + close_stdout = True + if stderr is None: + stderr = util.get_std_fd(self.stderr_base_path, 'remote') + close_stderr = True + # export target_host, target_hostname, target_fqdn # for use in __remote_{exec,copy} scripts os_environ = os.environ.copy() @@ -226,17 +300,30 @@ class Remote(object): os_environ['__target_hostname'] = self.target_host[1] os_environ['__target_fqdn'] = self.target_host[2] - self.log.debug("Remote run: %s", command) + self.log.trace("Remote run: %s", command) try: - output, errout = exec_util.call_get_output(command, env=os_environ) - self.log.debug("Remote stdout: {}".format(output)) - # Currently, stderr is not captured. - # self.log.debug("Remote stderr: {}".format(errout)) + if self.quiet_mode: + stderr = subprocess.DEVNULL + close_stderr = False if return_output: - return output.decode() - except subprocess.CalledProcessError as e: - exec_util.handle_called_process_error(e, command) - except OSError as error: - raise cdist.Error(" ".join(command) + ": " + error.args[1]) + output = subprocess.check_output(command, env=os_environ, + stderr=stderr).decode() + else: + subprocess.check_call(command, env=os_environ, stdout=stdout, + stderr=stderr) + output = None + + if self.save_output_streams: + util.log_std_fd(self.log, command, stderr, 'Remote stderr') + util.log_std_fd(self.log, command, stdout, 'Remote stdout') + + return output + except (OSError, subprocess.CalledProcessError) as error: + raise cdist.Error(" ".join(command) + ": " + str(error.args[1])) except UnicodeDecodeError: raise DecodeError(command) + finally: + if close_stdout: + stdout.close() + if close_stderr: + stderr.close() diff --git a/cdist/exec/util.py b/cdist/exec/util.py index 864a73a3..c96f757b 100644 --- a/cdist/exec/util.py +++ b/cdist/exec/util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -20,7 +20,7 @@ # import subprocess -import sys +import os from tempfile import TemporaryFile import cdist @@ -116,16 +116,18 @@ import cdist # return (result.stdout, result.stderr) -def call_get_output(command, env=None): +# Currently not used. +def call_get_output(command, env=None, stderr=None): """Run the given command with the given environment. Return the tuple of stdout and stderr output as a byte strings. """ assert isinstance(command, (list, tuple)), ( "list or tuple argument expected, got: {}".format(command)) - return (_call_get_stdout(command, env), None) + return (_call_get_stdout(command, env, stderr), None) +# Currently not used. def handle_called_process_error(err, command): # Currently, stderr is not captured. # errout = None @@ -134,13 +136,20 @@ def handle_called_process_error(err, command): # "stdout: {}\n" # "stderr: {}").format( # err.returncode, err.output, errout)) - raise cdist.Error("Command failed: " + " ".join(command) + - (" with returncode: {}\n" - "stdout: {}").format( - err.returncode, err.output)) + if err.output: + output = err.output + else: + output = '' + raise cdist.Error(("Command failed: '{}'\n" + "return code: {}\n" + "---- BEGIN stdout ----\n" + "{}" + ("\n" if output else "") + + "---- END stdout ----").format( + " ".join(command), err.returncode, output)) -def _call_get_stdout(command, env=None): +# Currently not used. +def _call_get_stdout(command, env=None, stderr=None): """Run the given command with the given environment. Return the stdout output as a byte string, stderr is ignored. """ @@ -148,8 +157,21 @@ def _call_get_stdout(command, env=None): "list or tuple argument expected, got: {}".format(command)) with TemporaryFile() as fout: - subprocess.check_call(command, env=env, stdout=fout) + subprocess.check_call(command, env=env, stdout=fout, stderr=stderr) fout.seek(0) output = fout.read() return output + + +def get_std_fd(base_path, name): + path = os.path.join(base_path, name) + stdfd = open(path, 'ba+') + return stdfd + + +def log_std_fd(log, command, stdfd, prefix): + if stdfd is not None and stdfd != subprocess.DEVNULL: + stdfd.seek(0, 0) + log.trace("Command: {}; {}: {}".format( + command, prefix, stdfd.read().decode())) diff --git a/cdist/flock.py b/cdist/flock.py new file mode 100644 index 00000000..d8bac916 --- /dev/null +++ b/cdist/flock.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import fcntl +import logging +import os + + +log = logging.getLogger('cdist-flock') + + +class Flock(): + def __init__(self, path): + self.path = path + self.lockfd = None + + def flock(self): + log.debug('Acquiring lock on %s', self.path) + self.lockfd = open(self.path, 'w+') + fcntl.flock(self.lockfd, fcntl.LOCK_EX) + log.debug('Acquired lock on %s', self.path) + + def funlock(self): + log.debug('Releasing lock on %s', self.path) + fcntl.flock(self.lockfd, fcntl.LOCK_UN) + self.lockfd.close() + self.lockfd = None + try: + os.remove(self.path) + except FileNotFoundError: + pass + log.debug('Released lock on %s', self.path) + + def __enter__(self): + self.flock() + return self + + def __exit__(self, *args): + self.funlock() + return False diff --git a/cdist/hostsource.py b/cdist/hostsource.py index 9c2c0616..a7b8f0b4 100644 --- a/cdist/hostsource.py +++ b/cdist/hostsource.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -22,6 +22,25 @@ import fileinput +def hostfile_process_line(line, strip_func=str.strip): + """Return entry from read line or None if no entry present.""" + if not line: + return None + # remove comment if present + comment_index = line.find('#') + if comment_index >= 0: + foo = line[:comment_index] + else: + foo = line + # remove leading and trailing whitespaces + foo = strip_func(foo) + # skip empty lines + if foo: + return foo + else: + return None + + class HostSource(object): """ Host source object. @@ -32,22 +51,7 @@ class HostSource(object): self.source = source def _process_file_line(self, line): - """Return host from read line or None if no host present.""" - if not line: - return None - # remove comment if present - comment_index = line.find('#') - if comment_index >= 0: - host = line[:comment_index] - else: - host = line - # remove leading and trailing whitespaces - host = host.strip() - # skip empty lines - if host: - return host - else: - return None + return hostfile_process_line(line) def _hosts_from_sequence(self): for host in self.source: diff --git a/cdist/integration.py b/cdist/integration.py new file mode 100644 index 00000000..ee742cc5 --- /dev/null +++ b/cdist/integration.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import cdist +# needed for cdist.argparse +import cdist.banner +import cdist.config +import cdist.install +import cdist.shell +import cdist.inventory +import cdist.argparse +import cdist.log +import cdist.config +import cdist.install +import sys +import os +import os.path +import collections +import uuid +import shutil + + +def find_cdist_exec_in_path(): + """Search cdist executable in os.get_exec_path() entries. + """ + for path in os.get_exec_path(): + cdist_path = os.path.join(path, 'cdist') + if os.access(cdist_path, os.X_OK): + return cdist_path + return None + + +_mydir = os.path.dirname(__file__) + + +def find_cdist_exec(): + """Search cdist executable starting from local lib directory. + + Detect if ../scripts/cdist (from local lib direcotry) exists and + if it is executable. If not then try to find cdist exec path in + os.get_exec_path() entries. If no cdist path is found rasie + cdist.Error. + """ + cdist_path = os.path.abspath(os.path.join(_mydir, '..', 'scripts', + 'cdist')) + if os.access(cdist_path, os.X_OK): + return cdist_path + cdist_path = find_cdist_exec_in_path() + if not cdist_path: + raise cdist.Error('Cannot find cdist executable from local lib ' + 'directory: {}, nor in PATH: {}.'.format( + _mydir, os.environ.get('PATH'))) + return cdist_path + + +ACTION_CONFIG = 'config' +ACTION_INSTALL = 'install' + + +def _process_hosts_simple(action, host, manifest, verbose, + cdist_path=None): + """Perform cdist action ('config' or 'install') on hosts with specified + manifest using default other cdist options. host parameter can be a + string or iterbale of hosts. verbose is a desired verbosity level + which defaults to VERBOSE_INFO. cdist_path is path to cdist executable, + if it is None then integration lib tries to find it. + """ + if isinstance(host, str): + hosts = [host, ] + elif isinstance(host, collections.Iterable): + hosts = host + else: + raise cdist.Error('Invalid host argument: {}'.format(host)) + + # Setup sys.argv[0] since cdist relies on command line invocation. + if not cdist_path: + cdist_path = find_cdist_exec() + sys.argv[0] = cdist_path + + cname = action.title() + module = getattr(cdist, action) + theclass = getattr(module, cname) + + # Build argv for cdist and use argparse for argument parsing. + remote_out_dir_base = os.path.join('/', 'var', 'lib', 'cdist') + uid = str(uuid.uuid1()) + out_dir = remote_out_dir_base + uid + cache_path_pattern = '%h-' + uid + argv = [action, '-i', manifest, '-r', out_dir, '-C', cache_path_pattern, ] + for i in range(verbose): + argv.append('-v') + for x in hosts: + argv.append(x) + + parser, cfg = cdist.argparse.parse_and_configure(argv, singleton=False) + args = cfg.get_args() + configuration = cfg.get_config(section='GLOBAL') + + theclass.construct_remote_exec_copy_patterns(args) + base_root_path = theclass.create_base_root_path(None) + + for target_host in args.host: + host_base_path, hostdir = theclass.create_host_base_dirs( + target_host, base_root_path) + theclass.onehost(target_host, None, host_base_path, hostdir, args, + parallel=False, configuration=configuration, + remove_remote_files_dirs=True) + shutil.rmtree(base_root_path) + + +def configure_hosts_simple(host, manifest, + verbose=cdist.argparse.VERBOSE_INFO, + cdist_path=None): + """Configure hosts with specified manifest using default other cdist + options. host parameter can be a string or iterbale of hosts. verbose + is a desired verbosity level which defaults to VERBOSE_INFO. + cdist_path is path to cdist executable, if it is None then integration + lib tries to find it. + """ + _process_hosts_simple(action=ACTION_CONFIG, host=host, + manifest=manifest, verbose=verbose, + cdist_path=cdist_path) + + +def install_hosts_simple(host, manifest, + verbose=cdist.argparse.VERBOSE_INFO, + cdist_path=None): + """Install hosts with specified manifest using default other cdist + options. host parameter can be a string or iterbale of hosts. verbose + is a desired verbosity level which defaults to VERBOSE_INFO. + cdist_path is path to cdist executable, if it is None then integration + lib tries to find it. + """ + _process_hosts_simple(action=ACTION_INSTALL, host=host, + manifest=manifest, verbose=verbose, + cdist_path=cdist_path) diff --git a/cdist/inventory.py b/cdist/inventory.py new file mode 100644 index 00000000..138a2034 --- /dev/null +++ b/cdist/inventory.py @@ -0,0 +1,405 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import cdist +import logging +import os +import os.path +import itertools +import sys +import cdist.configuration +from cdist.hostsource import hostfile_process_line + +DIST_INVENTORY_DB_NAME = "inventory" + +dist_inventory_db = os.path.abspath(os.path.join( + os.path.dirname(cdist.__file__), DIST_INVENTORY_DB_NAME)) + + +def determine_default_inventory_dir(args, configuration): + # The order of inventory dir setting by decreasing priority + # 1. inventory_dir from configuration + # 2. ~/.cdist/inventory if HOME env var is set + # 3. distribution inventory directory + inventory_dir_set = False + if 'inventory_dir' in configuration: + val = configuration['inventory_dir'] + if val: + args.inventory_dir = val + inventory_dir_set = True + if not inventory_dir_set: + home = cdist.home_dir() + if home: + args.inventory_dir = os.path.join(home, DIST_INVENTORY_DB_NAME) + else: + args.inventory_dir = dist_inventory_db + + +def contains_all(big, little): + """Return True if big contains all elements from little, + False otherwise. + """ + return set(little).issubset(set(big)) + + +def contains_any(big, little): + """Return True if big contains any element from little, + False otherwise. + """ + for x in little: + if x in big: + return True + return False + + +def check_always_true(x, y): + return True + + +def rstrip_nl(s): + '''str.rstrip "\n" from s''' + return str.rstrip(s, "\n") + + +class Inventory(object): + """Inventory main class""" + + def __init__(self, db_basedir=dist_inventory_db, configuration=None): + self.db_basedir = db_basedir + if configuration: + self.configuration = configuration + else: + self.configuration = {} + self.log = logging.getLogger("inventory") + self.init_db() + + def init_db(self): + self.log.trace("Init db: {}".format(self.db_basedir)) + if not os.path.exists(self.db_basedir): + os.makedirs(self.db_basedir, exist_ok=True) + elif not os.path.isdir(self.db_basedir): + raise cdist.Error(("Invalid inventory db basedir \'{}\'," + " must be a directory").format(self.db_basedir)) + + @staticmethod + def strlist_to_list(slist): + if slist: + result = [x for x in slist.split(',') if x] + else: + result = [] + return result + + def _input_values(self, source): + """Yield input values from source. + Source can be a sequence or filename (stdin if '-'). + In case of filename each line represents one input value. + """ + if isinstance(source, str): + import fileinput + try: + with fileinput.FileInput(files=(source)) as f: + for x in f: + result = hostfile_process_line(x, strip_func=rstrip_nl) + if result: + yield result + except (IOError, OSError) as e: + raise cdist.Error("Error reading from \'{}\'".format( + source)) + else: + if source: + for x in source: + if x: + yield x + + def _host_path(self, host): + hostpath = os.path.join(self.db_basedir, host) + return hostpath + + def _all_hosts(self): + return os.listdir(self.db_basedir) + + def _check_host(self, hostpath): + if not os.path.exists(hostpath): + return False + else: + if not os.path.isfile(hostpath): + raise cdist.Error(("Host path \'{}\' exists, but is not" + " a valid file").format(hostpath)) + return True + + def _read_host_tags(self, hostpath): + result = set() + with open(hostpath, "rt") as f: + for tag in f: + tag = tag.rstrip("\n") + if tag: + result.add(tag) + return result + + def _get_host_tags(self, host): + hostpath = self._host_path(host) + if self._check_host(hostpath): + return self._read_host_tags(hostpath) + else: + return None + + def _write_host_tags(self, host, tags): + hostpath = self._host_path(host) + if self._check_host(hostpath): + with open(hostpath, "wt") as f: + for tag in tags: + f.write("{}\n".format(tag)) + return True + else: + return False + + @classmethod + def commandline(cls, args): + """Manipulate inventory db""" + log = logging.getLogger("inventory") + if 'taglist' in args: + args.taglist = cls.strlist_to_list(args.taglist) + + cfg = cdist.configuration.Configuration(args) + configuration = cfg.get_config(section='GLOBAL') + determine_default_inventory_dir(args, configuration) + + log.debug("Using inventory: {}".format(args.inventory_dir)) + log.trace("Inventory args: {}".format(vars(args))) + log.trace("Inventory command: {}".format(args.subcommand)) + + if args.subcommand == "list": + c = InventoryList(hosts=args.host, istag=args.tag, + hostfile=args.hostfile, + db_basedir=args.inventory_dir, + list_only_host=args.list_only_host, + has_all_tags=args.has_all_tags, + configuration=configuration) + elif args.subcommand == "add-host": + c = InventoryHost(hosts=args.host, hostfile=args.hostfile, + db_basedir=args.inventory_dir, + configuration=configuration) + elif args.subcommand == "del-host": + c = InventoryHost(hosts=args.host, hostfile=args.hostfile, + all=args.all, db_basedir=args.inventory_dir, + action="del", configuration=configuration) + elif args.subcommand == "add-tag": + c = InventoryTag(hosts=args.host, tags=args.taglist, + hostfile=args.hostfile, tagfile=args.tagfile, + db_basedir=args.inventory_dir, + configuration=configuration) + elif args.subcommand == "del-tag": + c = InventoryTag(hosts=args.host, tags=args.taglist, + hostfile=args.hostfile, tagfile=args.tagfile, + all=args.all, db_basedir=args.inventory_dir, + action="del", configuration=configuration) + else: + raise cdist.Error("Unknown inventory command \'{}\'".format( + args.subcommand)) + c.run() + + +class InventoryList(Inventory): + def __init__(self, hosts=None, istag=False, hostfile=None, + list_only_host=False, has_all_tags=False, + db_basedir=dist_inventory_db, configuration=None): + super().__init__(db_basedir, configuration) + self.hosts = hosts + self.istag = istag + self.hostfile = hostfile + self.list_only_host = list_only_host + self.has_all_tags = has_all_tags + + def _print(self, host, tags): + if self.list_only_host: + print("{}".format(host)) + else: + print("{} {}".format(host, ",".join(sorted(tags)))) + + def _do_list(self, it_tags, it_hosts, check_func): + if (it_tags is not None): + param_tags = set(it_tags) + self.log.trace("param_tags: {}".format(param_tags)) + else: + param_tags = set() + for host in it_hosts: + self.log.trace("host: {}".format(host)) + tags = self._get_host_tags(host) + if tags is None: + self.log.debug("Host \'{}\' not found, skipped".format(host)) + continue + self.log.trace("tags: {}".format(tags)) + if check_func(tags, param_tags): + yield host, tags + + def entries(self): + if not self.hosts and not self.hostfile: + self.log.trace("Listing all hosts") + it_hosts = self._all_hosts() + it_tags = None + check_func = check_always_true + else: + it = itertools.chain(self._input_values(self.hosts), + self._input_values(self.hostfile)) + if self.istag: + self.log.trace("Listing by tag(s)") + it_hosts = self._all_hosts() + it_tags = it + if self.has_all_tags: + check_func = contains_all + else: + check_func = contains_any + else: + self.log.trace("Listing by host(s)") + it_hosts = it + it_tags = None + check_func = check_always_true + for host, tags in self._do_list(it_tags, it_hosts, check_func): + yield host, tags + + def host_entries(self): + for host, tags in self.entries(): + yield host + + def run(self): + for host, tags in self.entries(): + self._print(host, tags) + + +class InventoryHost(Inventory): + def __init__(self, hosts=None, hostfile=None, + db_basedir=dist_inventory_db, all=False, action="add", + configuration=None): + super().__init__(db_basedir, configuration) + self.actions = ("add", "del") + if action not in self.actions: + raise cdist.Error("Invalid action \'{}\', valid actions are:" + " {}\n".format(action, self.actions.keys())) + self.action = action + self.hosts = hosts + self.hostfile = hostfile + self.all = all + + if not self.hosts and not self.hostfile: + self.hostfile = "-" + + def _new_hostpath(self, hostpath): + # create empty file + with open(hostpath, "w"): + pass + + def _action(self, host): + if self.action == "add": + self.log.debug("Adding host \'{}\'".format(host)) + elif self.action == "del": + self.log.debug("Deleting host \'{}\'".format(host)) + hostpath = self._host_path(host) + self.log.trace("hostpath: {}".format(hostpath)) + if self.action == "add" and not os.path.exists(hostpath): + self._new_hostpath(hostpath) + else: + if not os.path.isfile(hostpath): + raise cdist.Error(("Host path \'{}\' is" + " not a valid file").format(hostpath)) + if self.action == "del": + os.remove(hostpath) + + def run(self): + if self.action == "del" and self.all: + self.log.trace("Doing for all hosts") + it = self._all_hosts() + else: + self.log.trace("Doing for specified hosts") + it = itertools.chain(self._input_values(self.hosts), + self._input_values(self.hostfile)) + for host in it: + self._action(host) + + +class InventoryTag(Inventory): + def __init__(self, hosts=None, tags=None, hostfile=None, tagfile=None, + db_basedir=dist_inventory_db, all=False, action="add", + configuration=None): + super().__init__(db_basedir, configuration) + self.actions = ("add", "del") + if action not in self.actions: + raise cdist.Error("Invalid action \'{}\', valid actions are:" + " {}\n".format(action, self.actions.keys())) + self.action = action + self.hosts = hosts + self.tags = tags + self.hostfile = hostfile + self.tagfile = tagfile + self.all = all + + if not self.hosts and not self.hostfile: + self.allhosts = True + else: + self.allhosts = False + if not self.tags and not self.tagfile: + self.tagfile = "-" + + if self.hostfile == "-" and self.tagfile == "-": + raise cdist.Error("Cannot read both, hosts and tags, from stdin") + + def _read_input_tags(self): + self.input_tags = set() + for tag in itertools.chain(self._input_values(self.tags), + self._input_values(self.tagfile)): + self.input_tags.add(tag) + + def _action(self, host): + host_tags = self._get_host_tags(host) + if host_tags is None: + print("Host \'{}\' does not exist, skipping".format(host), + file=sys.stderr) + return + self.log.trace("existing host_tags: {}".format(host_tags)) + if self.action == "del" and self.all: + host_tags = set() + else: + for tag in self.input_tags: + if self.action == "add": + self.log.debug("Adding tag \'{}\' for host \'{}\'".format( + tag, host)) + host_tags.add(tag) + elif self.action == "del": + self.log.debug("Deleting tag \'{}\' for host " + "\'{}\'".format(tag, host)) + if tag in host_tags: + host_tags.remove(tag) + self.log.trace("new host tags: {}".format(host_tags)) + if not self._write_host_tags(host, host_tags): + self.log.trace("{} does not exist, skipped".format(host)) + + def run(self): + if self.allhosts: + self.log.trace("Doing for all hosts") + it = self._all_hosts() + else: + self.log.trace("Doing for specified hosts") + it = itertools.chain(self._input_values(self.hosts), + self._input_values(self.hostfile)) + if not(self.action == "del" and self.all): + self._read_input_tags() + for host in it: + self._action(host) diff --git a/cdist/log.py b/cdist/log.py index 2341c282..5d431130 100644 --- a/cdist/log.py +++ b/cdist/log.py @@ -21,19 +21,120 @@ # import logging +import sys +import datetime -class Log(logging.Logger): +# Define additional cdist logging levels. +logging.OFF = logging.CRITICAL + 10 # disable logging +logging.addLevelName(logging.OFF, 'OFF') + +logging.VERBOSE = logging.INFO - 5 +logging.addLevelName(logging.VERBOSE, 'VERBOSE') + + +def _verbose(msg, *args, **kwargs): + logging.log(logging.VERBOSE, msg, *args, **kwargs) + + +logging.verbose = _verbose + +logging.TRACE = logging.DEBUG - 5 +logging.addLevelName(logging.TRACE, 'TRACE') + + +def _trace(msg, *args, **kwargs): + logging.log(logging.TRACE, msg, *args, **kwargs) + + +logging.trace = _trace + + +class DefaultLog(logging.Logger): + + FORMAT = '%(levelname)s: %(message)s' + + class StdoutFilter(logging.Filter): + def filter(self, rec): + return rec.levelno != logging.ERROR + + class StderrFilter(logging.Filter): + def filter(self, rec): + return rec.levelno == logging.ERROR def __init__(self, name): - - self.name = name super().__init__(name) + + formatter = logging.Formatter(self.FORMAT) + self.addFilter(self) + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.addFilter(self.StdoutFilter()) + stdout_handler.setLevel(logging.TRACE) + stdout_handler.setFormatter(formatter) + + stderr_handler = logging.StreamHandler(sys.stderr) + stderr_handler.addFilter(self.StderrFilter()) + stderr_handler.setLevel(logging.ERROR) + stderr_handler.setFormatter(formatter) + + self.addHandler(stdout_handler) + self.addHandler(stderr_handler) + def filter(self, record): """Prefix messages with logger name""" record.msg = self.name + ": " + str(record.msg) return True + + def verbose(self, msg, *args, **kwargs): + self.log(logging.VERBOSE, msg, *args, **kwargs) + + def trace(self, msg, *args, **kwargs): + self.log(logging.TRACE, msg, *args, **kwargs) + + +class TimestampingLog(DefaultLog): + + def filter(self, record): + """Add timestamp to messages""" + + super().filter(record) + now = datetime.datetime.now() + timestamp = now.strftime("%Y%m%d%H%M%S.%f") + record.msg = "[" + timestamp + "] " + str(record.msg) + + return True + + +class ParallelLog(DefaultLog): + FORMAT = '%(levelname)s: [%(process)d]: %(message)s' + + +class TimestampingParallelLog(TimestampingLog, ParallelLog): + pass + + +def setupDefaultLogging(): + del logging.getLogger().handlers[:] + logging.setLoggerClass(DefaultLog) + + +def setupTimestampingLogging(): + del logging.getLogger().handlers[:] + logging.setLoggerClass(TimestampingLog) + + +def setupTimestampingParallelLogging(): + del logging.getLogger().handlers[:] + logging.setLoggerClass(TimestampingParallelLog) + + +def setupParallelLogging(): + del logging.getLogger().handlers[:] + logging.setLoggerClass(ParallelLog) + + +setupDefaultLogging() diff --git a/cdist/message.py b/cdist/message.py index 98a6e8cf..450fc3c3 100644 --- a/cdist/message.py +++ b/cdist/message.py @@ -24,8 +24,6 @@ import os import shutil import tempfile -import cdist - log = logging.getLogger(__name__) diff --git a/cdist/mputil.py b/cdist/mputil.py index e564d749..56fcfe39 100644 --- a/cdist/mputil.py +++ b/cdist/mputil.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2017 Darko Poljak (darko.poljak at gmail.com) # # This file is part of cdist. # @@ -21,14 +21,26 @@ import multiprocessing +import concurrent.futures as cf import itertools +import os +import signal +import logging + + +log = logging.getLogger("cdist-mputil") + + +def mp_sig_handler(signum, frame): + log.trace("signal %s, SIGKILL whole process group", signum) + os.killpg(os.getpgrp(), signal.SIGKILL) def mp_pool_run(func, args=None, kwds=None, jobs=multiprocessing.cpu_count()): - """ Run func using multiprocessing.Pool with jobs jobs and supplied - iterable of args and kwds with one entry for each parallel func - instance. - Return list of results. + """Run func using concurrent.futures.ProcessPoolExecutor with jobs jobs + and supplied iterables of args and kwds with one entry for each + parallel func instance. + Return list of results. """ if args and kwds: fargs = zip(args, kwds) @@ -39,10 +51,15 @@ def mp_pool_run(func, args=None, kwds=None, jobs=multiprocessing.cpu_count()): else: return [func(), ] - with multiprocessing.Pool(jobs) as pool: - results = [ - pool.apply_async(func, a, k) - for a, k in fargs - ] - retval = [r.get() for r in results] - return retval + retval = [] + with cf.ProcessPoolExecutor(jobs) as executor: + try: + results = [ + executor.submit(func, *a, **k) for a, k in fargs + ] + for f in cf.as_completed(results): + retval.append(f.result()) + return retval + except KeyboardInterrupt: + mp_sig_handler(signal.SIGINT, None) + raise diff --git a/cdist/shell.py b/cdist/shell.py index 9378efc3..60b6a9f0 100644 --- a/cdist/shell.py +++ b/cdist/shell.py @@ -21,7 +21,6 @@ import logging import os -import subprocess import tempfile # initialise cdist @@ -44,6 +43,7 @@ class Shell(object): "cdist-shell-no-target-host", "cdist-shell-no-target-host", ) + self.target_host_tags = "" host_dir_name = cdist.str_hash(self.target_host[0]) base_root_path = tempfile.mkdtemp() @@ -51,6 +51,7 @@ class Shell(object): self.local = cdist.exec.local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=host_dir_name) @@ -77,6 +78,7 @@ class Shell(object): '__manifest': self.local.manifest_path, '__explorer': self.local.global_explorer_path, '__files': self.local.files_path, + '__target_host_tags': self.local.target_host_tags, } self.env.update(additional_env) @@ -86,10 +88,9 @@ class Shell(object): self._init_files_dirs() self._init_environment() - log.info("Starting shell...") - # save_output=False -> do not catch stdout and stderr + log.trace("Starting shell...") self.local.run([self.shell], self.env, save_output=False) - log.info("Finished shell.") + log.trace("Finished shell.") @classmethod def commandline(cls, args): diff --git a/cdist/sphinxext/manpage.py b/cdist/sphinxext/manpage.py index 97b41f03..135fe22e 100644 --- a/cdist/sphinxext/manpage.py +++ b/cdist/sphinxext/manpage.py @@ -28,12 +28,13 @@ class ManualPageWriter(sphinx.writers.manpage.ManualPageWriter): def __init__(self, builder): super().__init__(builder) self.translator_class = ( - self.builder.translator_class or ManualPageTranslator) + self.builder.translator_class or ManualPageTranslator) class ManualPageBuilder(sphinx.builders.manpage.ManualPageBuilder): name = 'cman' + default_translator_class = ManualPageTranslator def write(self, *ignored): docwriter = ManualPageWriter(self) diff --git a/cdist/test/__init__.py b/cdist/test/__init__.py index 83b0c618..faa3686a 100644 --- a/cdist/test/__init__.py +++ b/cdist/test/__init__.py @@ -42,6 +42,7 @@ class CdistTestCase(unittest.TestCase): 'cdisttesthost', 'cdisttesthost', ) + target_host_tags = "tag1,tag2,tag3" def mkdtemp(self, **kwargs): return tempfile.mkdtemp(prefix='tmp.cdist.test.', **kwargs) diff --git a/cdist/test/__main__.py b/cdist/test/__main__.py index 08e839d1..c8c7df3b 100644 --- a/cdist/test/__main__.py +++ b/cdist/test/__main__.py @@ -45,4 +45,5 @@ for test_module in test_modules: suites.append(suite) all_suites = unittest.TestSuite(suites) -unittest.TextTestRunner(verbosity=2).run(all_suites) +rv = unittest.TextTestRunner(verbosity=2).run(all_suites).wasSuccessful() +sys.exit(not rv) diff --git a/cdist/test/autil/__init__.py b/cdist/test/autil/__init__.py new file mode 100644 index 00000000..a78feaad --- /dev/null +++ b/cdist/test/autil/__init__.py @@ -0,0 +1,57 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +from cdist import test +import cdist.autil as autil +import os +import os.path as op +import tarfile + + +my_dir = op.abspath(op.dirname(__file__)) +fixtures = op.join(my_dir, 'fixtures') +explorers_path = op.join(fixtures, 'explorer') + + +class AUtilTestCase(test.CdistTestCase): + def test_tar(self): + test_modes = { + 'tar': 'r:', + 'tgz': 'r:gz', + 'tbz2': 'r:bz2', + 'txz': 'r:xz', + } + source = explorers_path + for mode in test_modes: + tarpath, fcnt = autil.tar(source, mode) + self.assertIsNotNone(tarpath) + fcnt = 0 + with tarfile.open(tarpath, test_modes[mode]) as tar: + for tarinfo in tar: + fcnt += 1 + os.remove(tarpath) + self.assertGreater(fcnt, 0) + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/cdist/test/autil/fixtures/explorer/cpu_cores b/cdist/test/autil/fixtures/explorer/cpu_cores new file mode 100755 index 00000000..7f7a955e --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/cpu_cores @@ -0,0 +1,40 @@ +#!/bin/sh +# +# 2014 Daniel Heule (hda at sfs.biz) +# 2014 Thomas Oettli (otho at sfs.biz) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +# FIXME: other system types (not linux ...) + +os=$("$__explorer/os") +case "$os" in + "macosx") + echo "$(sysctl -n hw.physicalcpu)" + ;; + + *) + if [ -r /proc/cpuinfo ]; then + cores="$(grep "core id" /proc/cpuinfo | sort | uniq | wc -l)" + if [ ${cores} -eq 0 ]; then + cores="1" + fi + echo "$cores" + fi + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/cpu_sockets b/cdist/test/autil/fixtures/explorer/cpu_sockets new file mode 100755 index 00000000..8a8194df --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/cpu_sockets @@ -0,0 +1,40 @@ +#!/bin/sh +# +# 2014 Daniel Heule (hda at sfs.biz) +# 2014 Thomas Oettli (otho at sfs.biz) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +# FIXME: other system types (not linux ...) + +os=$("$__explorer/os") +case "$os" in + "macosx") + echo "$(system_profiler SPHardwareDataType | grep "Number of Processors" | awk -F': ' '{print $2}')" + ;; + + *) + if [ -r /proc/cpuinfo ]; then + sockets="$(grep "physical id" /proc/cpuinfo | sort | uniq | wc -l)" + if [ ${sockets} -eq 0 ]; then + sockets="$(cat /proc/cpuinfo | grep "processor" | wc -l)" + fi + echo "${sockets}" + fi + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/disks b/cdist/test/autil/fixtures/explorer/disks new file mode 100644 index 00000000..52fef81e --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/disks @@ -0,0 +1,2 @@ +cd /dev +echo sd? hd? vd? diff --git a/cdist/test/autil/fixtures/explorer/hostname b/cdist/test/autil/fixtures/explorer/hostname new file mode 100755 index 00000000..7715c6b0 --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/hostname @@ -0,0 +1,25 @@ +#!/bin/sh +# +# 2010-2014 Nico Schottelius (nico-cdist at schottelius.org) +# 2012 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +if command -v uname >/dev/null; then + uname -n +fi diff --git a/cdist/test/autil/fixtures/explorer/init b/cdist/test/autil/fixtures/explorer/init new file mode 100755 index 00000000..2693a0d3 --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/init @@ -0,0 +1,35 @@ +#!/bin/sh +# +# 2016 Daniel Heule (hda at sfs.biz) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# Returns the process name of pid 1 ( normaly the init system ) +# for example at linux this value is "init" or "systemd" in most cases +# + +uname_s="$(uname -s)" + +case "$uname_s" in + Linux|FreeBSD) + ps -o comm= -p 1 || true + ;; + *) + # return a empty string as unknown value + echo "" + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/interfaces b/cdist/test/autil/fixtures/explorer/interfaces new file mode 100755 index 00000000..c1f2a57a --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/interfaces @@ -0,0 +1,51 @@ +#!/bin/sh +# +# 2012 Sébastien Gross +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# List all network interfaces in explorer/ifaces. One interface per line. +# +# If your OS is not supported please provide a ifconfig output +# + +# Use ip, if available +if command -v ip >/dev/null; then + ip -o link show | sed -n 's/^[0-9]\+: \(.\+\): <.*/\1/p' + exit 0 +fi + +if ! command -v ifconfig >/dev/null; then + # no ifconfig, nothing we could do + exit 0 +fi + +uname_s="$(uname -s)" +REGEXP='s/^(.*)(:[[:space:]]*flags=|Link encap).*/\1/p' + +case "$uname_s" in + Darwin) + ifconfig -a | sed -n -E "$REGEXP" + ;; + Linux|*BSD) + ifconfig -a | sed -n -r "$REGEXP" + ;; + *) + echo "Unsupported ifconfig output for $uname_s" >&2 + exit 1 + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/kernel_name b/cdist/test/autil/fixtures/explorer/kernel_name new file mode 100644 index 00000000..98ebac2a --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/kernel_name @@ -0,0 +1 @@ +uname -s diff --git a/cdist/test/autil/fixtures/explorer/lsb_codename b/cdist/test/autil/fixtures/explorer/lsb_codename new file mode 100755 index 00000000..eebd3e0f --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/lsb_codename @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +set +e +case "$($__explorer/os)" in + openwrt) + (. /etc/openwrt_release && echo "$DISTRIB_CODENAME") + ;; + *) + lsb_release=$(command -v lsb_release) + if [ -x "$lsb_release" ]; then + $lsb_release --short --codename + fi + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/lsb_description b/cdist/test/autil/fixtures/explorer/lsb_description new file mode 100755 index 00000000..23f45421 --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/lsb_description @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +set +e +case "$($__explorer/os)" in + openwrt) + (. /etc/openwrt_release && echo "$DISTRIB_DESCRIPTION") + ;; + *) + lsb_release=$(command -v lsb_release) + if [ -x "$lsb_release" ]; then + $lsb_release --short --description + fi + ;; +esac diff --git a/cdist/conf/type/__install_bootloader_grub/manifest b/cdist/test/autil/fixtures/explorer/lsb_id similarity index 74% rename from cdist/conf/type/__install_bootloader_grub/manifest rename to cdist/test/autil/fixtures/explorer/lsb_id index 4c7c4955..9754eb63 100755 --- a/cdist/conf/type/__install_bootloader_grub/manifest +++ b/cdist/test/autil/fixtures/explorer/lsb_id @@ -17,9 +17,17 @@ # You should have received a copy of the GNU General Public License # along with cdist. If not, see . # +# -# set defaults -device="$(cat "$__object/parameter/device" 2>/dev/null \ - || echo "/$__object_id" | tee "$__object/parameter/device")" -chroot="$(cat "$__object/parameter/chroot" 2>/dev/null \ - || echo "/target" | tee "$__object/parameter/chroot")" +set +e +case "$($__explorer/os)" in + openwrt) + (. /etc/openwrt_release && echo "$DISTRIB_ID") + ;; + *) + lsb_release=$(command -v lsb_release) + if [ -x "$lsb_release" ]; then + $lsb_release --short --id + fi + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/lsb_release b/cdist/test/autil/fixtures/explorer/lsb_release new file mode 100755 index 00000000..35b5547c --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/lsb_release @@ -0,0 +1,33 @@ +#!/bin/sh +# +# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +set +e +case "$($__explorer/os)" in + openwrt) + (. /etc/openwrt_release && echo "$DISTRIB_RELEASE") + ;; + *) + lsb_release=$(command -v lsb_release) + if [ -x "$lsb_release" ]; then + $lsb_release --short --release + fi + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/machine b/cdist/test/autil/fixtures/explorer/machine new file mode 100755 index 00000000..d4a0e106 --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/machine @@ -0,0 +1,27 @@ +#!/bin/sh +# +# 2010-2011 Andi Brönnimann (andi-cdist at v-net.ch) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# All os variables are lower case +# +# + +if command -v uname 2>&1 >/dev/null; then + uname -m +fi diff --git a/cdist/test/autil/fixtures/explorer/machine_type b/cdist/test/autil/fixtures/explorer/machine_type new file mode 100755 index 00000000..eb3c9d36 --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/machine_type @@ -0,0 +1,66 @@ +#!/bin/sh +# +# 2014 Daniel Heule (hda at sfs.biz) +# 2014 Thomas Oettli (otho at sfs.biz) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +# FIXME: other system types (not linux ...) + +if [ -d "/proc/vz" -a ! -d "/proc/bc" ]; then + echo openvz + exit +fi + +if [ -e "/proc/1/environ" ] && + cat "/proc/1/environ" | tr '\000' '\n' | grep -Eiq '^container='; then + echo lxc + exit +fi + +if [ -r /proc/cpuinfo ]; then + # this should only exist on virtual guest machines, + # tested on vmware, xen, kvm + if grep -q "hypervisor" /proc/cpuinfo; then + # this file is aviable in xen guest systems + if [ -r /sys/hypervisor/type ]; then + if grep -q -i "xen" /sys/hypervisor/type; then + echo virtual_by_xen + exit + fi + else + if [ -r /sys/class/dmi/id/product_name ]; then + if grep -q -i 'vmware' /sys/class/dmi/id/product_name; then + echo "virtual_by_vmware" + exit + elif grep -q -i 'bochs' /sys/class/dmi/id/product_name; then + echo "virtual_by_kvm" + exit + elif grep -q -i 'virtualbox' /sys/class/dmi/id/product_name; then + echo "virtual_by_virtualbox" + exit + fi + fi + fi + echo "virtual_by_unknown" + else + echo "physical" + fi +else + echo "unknown" +fi diff --git a/cdist/test/autil/fixtures/explorer/memory b/cdist/test/autil/fixtures/explorer/memory new file mode 100755 index 00000000..05db865f --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/memory @@ -0,0 +1,36 @@ +#!/bin/sh +# +# 2014 Daniel Heule (hda at sfs.biz) +# 2014 Thomas Oettli (otho at sfs.biz) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +# FIXME: other system types (not linux ...) + +os=$("$__explorer/os") +case "$os" in + "macosx") + echo "$(sysctl -n hw.memsize)/1024" | bc + ;; + + *) + if [ -r /proc/meminfo ]; then + grep "MemTotal:" /proc/meminfo | awk '{print $2}' + fi + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/os b/cdist/test/autil/fixtures/explorer/os new file mode 100755 index 00000000..094685ea --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/os @@ -0,0 +1,143 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# All os variables are lower case. Keep this file in alphabetical +# order by os variable except in cases where order otherwise matters, +# in which case keep the primary os and its derivatives together in +# a block (see Debian and Redhat examples below). +# + +if grep -q ^Amazon /etc/system-release 2>/dev/null; then + echo amazon + exit 0 +fi + +if [ -f /etc/arch-release ]; then + echo archlinux + exit 0 +fi + +if [ -f /etc/cdist-preos ]; then + echo cdist-preos + exit 0 +fi + +if [ -d /gnu/store ]; then + echo guixsd + exit 0 +fi + +### Debian and derivatives +if grep -q ^DISTRIB_ID=Ubuntu /etc/lsb-release 2>/dev/null; then + echo ubuntu + exit 0 +fi + +if [ -f /etc/debian_version ]; then + echo debian + exit 0 +fi + +if [ -f /etc/devuan_version ]; then + echo devuan + exit 0 +fi +### + +if [ -f /etc/gentoo-release ]; then + echo gentoo + exit 0 +fi + +if [ -f /etc/openwrt_version ]; then + echo openwrt + exit 0 +fi + +if [ -f /etc/owl-release ]; then + echo owl + exit 0 +fi + +### Redhat and derivatives +if grep -q ^Scientific /etc/redhat-release 2>/dev/null; then + echo scientific + exit 0 +fi + +if grep -q ^CentOS /etc/redhat-release 2>/dev/null; then + echo centos + exit 0 +fi + +if grep -q ^Fedora /etc/redhat-release 2>/dev/null; then + echo fedora + exit 0 +fi + +if grep -q ^Mitel /etc/redhat-release 2>/dev/null; then + echo mitel + exit 0 +fi + +if [ -f /etc/redhat-release ]; then + echo redhat + exit 0 +fi +### + +if [ -f /etc/SuSE-release ]; then + echo suse + exit 0 +fi + +if [ -f /etc/slackware-version ]; then + echo slackware + exit 0 +fi + +uname_s="$(uname -s)" + +# Assume there is no tr on the client -> do lower case ourselves +case "$uname_s" in + Darwin) + echo macosx + exit 0 + ;; + NetBSD) + echo netbsd + exit 0 + ;; + FreeBSD) + echo freebsd + exit 0 + ;; + OpenBSD) + echo openbsd + exit 0 + ;; + SunOS) + echo solaris + exit 0 + ;; +esac + +echo "Unknown OS" >&2 +exit 1 diff --git a/cdist/test/autil/fixtures/explorer/os_version b/cdist/test/autil/fixtures/explorer/os_version new file mode 100755 index 00000000..380782cc --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/os_version @@ -0,0 +1,73 @@ +#!/bin/sh +# +# 2010-2011 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# +# All os variables are lower case +# +# + +case "$($__explorer/os)" in + amazon) + cat /etc/system-release + ;; + archlinux) + # empty, but well... + cat /etc/arch-release + ;; + debian) + cat /etc/debian_version + ;; + devuan) + cat /etc/devuan_version + ;; + fedora) + cat /etc/fedora-release + ;; + gentoo) + cat /etc/gentoo-release + ;; + macosx) + sw_vers -productVersion + ;; + *bsd|solaris) + uname -r + ;; + openwrt) + cat /etc/openwrt_version + ;; + owl) + cat /etc/owl-release + ;; + redhat|centos|mitel|scientific) + cat /etc/redhat-release + ;; + slackware) + cat /etc/slackware-version + ;; + suse) + if [ -f /etc/os-release ]; then + cat /etc/os-release + else + cat /etc/SuSE-release + fi + ;; + ubuntu) + lsb_release -sr + ;; +esac diff --git a/cdist/test/autil/fixtures/explorer/runlevel b/cdist/test/autil/fixtures/explorer/runlevel new file mode 100755 index 00000000..02d3a245 --- /dev/null +++ b/cdist/test/autil/fixtures/explorer/runlevel @@ -0,0 +1,26 @@ +#!/bin/sh +# +# 2012 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +set +e +executable=$(command -v runlevel) +if [ -x "$executable" ]; then + "$executable" | awk '{ print $2 }' +fi diff --git a/cdist/test/capture_output/__init__.py b/cdist/test/capture_output/__init__.py new file mode 100644 index 00000000..229cbf70 --- /dev/null +++ b/cdist/test/capture_output/__init__.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# +# 2011-2013 Steven Armstrong (steven-cdist at armstrong.cc) +# 2012-2013 Nico Schottelius (nico-cdist at schottelius.org) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import shutil + +import cdist +from cdist import core +from cdist import test +from cdist.exec import local +from cdist.exec import remote +from cdist.core import code +from cdist.core import manifest + +import os.path as op +my_dir = op.abspath(op.dirname(__file__)) +fixtures = op.join(my_dir, 'fixtures') +conf_dir = op.join(fixtures, 'conf') + + +class CaptureOutputTestCase(test.CdistTestCase): + + def setUp(self): + # logging.root.setLevel(logging.TRACE) + self.temp_dir = self.mkdtemp() + + self.local_dir = os.path.join(self.temp_dir, "local") + self.hostdir = cdist.str_hash(self.target_host[0]) + self.host_base_path = os.path.join(self.local_dir, self.hostdir) + os.makedirs(self.host_base_path) + self.local = local.Local( + target_host=self.target_host, + target_host_tags=None, + base_root_path=self.host_base_path, + host_dir_name=self.hostdir, + exec_path=cdist.test.cdist_exec_path, + add_conf_dirs=[conf_dir]) + self.local.create_files_dirs() + + self.remote_dir = self.mkdtemp() + remote_exec = self.remote_exec + remote_copy = self.remote_copy + self.remote = remote.Remote( + target_host=self.target_host, + remote_exec=remote_exec, + remote_copy=remote_copy, + base_path=self.remote_dir, + stdout_base_path=self.local.stdout_base_path, + stderr_base_path=self.local.stderr_base_path) + self.remote.create_files_dirs() + + self.code = code.Code(self.target_host, self.local, self.remote) + + self.manifest = manifest.Manifest(self.target_host, self.local) + + self.cdist_type = core.CdistType(self.local.type_path, + '__write_to_stdout_and_stderr') + self.cdist_object = core.CdistObject(self.cdist_type, + self.local.object_path, + self.local.object_marker_name, + '') + self.cdist_object.create() + self.output_dirs = { + 'object': { + 'stdout': os.path.join(self.cdist_object.absolute_path, + 'stdout'), + 'stderr': os.path.join(self.cdist_object.absolute_path, + 'stderr'), + }, + 'init': { + 'stdout': os.path.join(self.local.base_path, 'stdout'), + 'stderr': os.path.join(self.local.base_path, 'stderr'), + }, + } + + def tearDown(self): + shutil.rmtree(self.local_dir) + shutil.rmtree(self.remote_dir) + shutil.rmtree(self.temp_dir) + + def _test_output(self, which, target, streams=('stdout', 'stderr')): + for stream in streams: + _should = '{0}: {1}\n'.format(which, stream) + stream_path = os.path.join(self.output_dirs[target][stream], which) + with open(stream_path, 'r') as fd: + _is = fd.read() + self.assertEqual(_should, _is) + + def test_capture_code_output(self): + self.cdist_object.code_local = self.code.run_gencode_local( + self.cdist_object) + self._test_output('gencode-local', 'object', ('stderr',)) + + self.code.run_code_local(self.cdist_object) + self._test_output('code-local', 'object') + + self.cdist_object.code_remote = self.code.run_gencode_remote( + self.cdist_object) + self._test_output('gencode-remote', 'object', ('stderr',)) + + self.code.transfer_code_remote(self.cdist_object) + self.code.run_code_remote(self.cdist_object) + self._test_output('code-remote', 'object') + + def test_capture_manifest_output(self): + self.manifest.run_type_manifest(self.cdist_object) + self._test_output('manifest', 'object') + + def test_capture_init_manifest_output(self): + initial_manifest = os.path.join(conf_dir, 'manifest', 'init') + self.manifest.run_initial_manifest(initial_manifest) + self._test_output('init', 'init') + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/cdist/test/capture_output/fixtures/conf/manifest/init b/cdist/test/capture_output/fixtures/conf/manifest/init new file mode 100755 index 00000000..68d7da97 --- /dev/null +++ b/cdist/test/capture_output/fixtures/conf/manifest/init @@ -0,0 +1,4 @@ +#!/bin/sh + +echo "init: stdout" +echo "init: stderr" >&2 diff --git a/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-local b/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-local new file mode 100755 index 00000000..1946dbd3 --- /dev/null +++ b/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-local @@ -0,0 +1,6 @@ +#!/bin/sh + +echo "gencode-local: stderr" >&2 + +echo "echo \"code-local: stdout\"" +echo "echo \"code-local: stderr\" >&2" diff --git a/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-remote b/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-remote new file mode 100755 index 00000000..f713b932 --- /dev/null +++ b/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-remote @@ -0,0 +1,6 @@ +#!/bin/sh + +echo "gencode-remote: stderr" >&2 + +echo "echo \"code-remote: stdout\"" +echo "echo \"code-remote: stderr\" >&2" diff --git a/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/manifest b/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/manifest new file mode 100755 index 00000000..4f122f25 --- /dev/null +++ b/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/manifest @@ -0,0 +1,4 @@ +#!/bin/sh + +echo "manifest: stdout" +echo "manifest: stderr" >&2 diff --git a/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/singleton b/cdist/test/capture_output/fixtures/conf/type/__write_to_stdout_and_stderr/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/capture_output_disabled/__init__.py b/cdist/test/capture_output_disabled/__init__.py new file mode 100644 index 00000000..828e80f1 --- /dev/null +++ b/cdist/test/capture_output_disabled/__init__.py @@ -0,0 +1,140 @@ +# -*- coding: utf-8 -*- +# +# 2018 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import shutil + +import cdist +from cdist import core +from cdist import test +from cdist.exec import local +from cdist.exec import remote +from cdist.core import code +from cdist.core import manifest + +import os.path as op +my_dir = op.abspath(op.dirname(__file__)) +fixtures = op.join(my_dir, 'fixtures') +conf_dir = op.join(fixtures, 'conf') + + +class CaptureOutputDisabledTestCase(test.CdistTestCase): + + def setUp(self): + # logging.root.setLevel(logging.TRACE) + save_output_streams = False + self.temp_dir = self.mkdtemp() + + self.local_dir = os.path.join(self.temp_dir, "local") + self.hostdir = cdist.str_hash(self.target_host[0]) + self.host_base_path = os.path.join(self.local_dir, self.hostdir) + os.makedirs(self.host_base_path) + self.local = local.Local( + target_host=self.target_host, + target_host_tags=None, + base_root_path=self.host_base_path, + host_dir_name=self.hostdir, + exec_path=cdist.test.cdist_exec_path, + add_conf_dirs=[conf_dir], + save_output_streams=save_output_streams) + self.local.create_files_dirs() + + self.remote_dir = self.mkdtemp() + remote_exec = self.remote_exec + remote_copy = self.remote_copy + self.remote = remote.Remote( + target_host=self.target_host, + remote_exec=remote_exec, + remote_copy=remote_copy, + base_path=self.remote_dir, + stdout_base_path=self.local.stdout_base_path, + stderr_base_path=self.local.stderr_base_path, + save_output_streams=save_output_streams) + self.remote.create_files_dirs() + + self.code = code.Code(self.target_host, self.local, self.remote) + + self.manifest = manifest.Manifest(self.target_host, self.local) + + self.cdist_type = core.CdistType(self.local.type_path, + '__write_to_stdout_and_stderr') + self.cdist_object = core.CdistObject(self.cdist_type, + self.local.object_path, + self.local.object_marker_name, + '') + self.cdist_object.create() + self.output_dirs = { + 'object': { + 'stdout': os.path.join(self.cdist_object.absolute_path, + 'stdout'), + 'stderr': os.path.join(self.cdist_object.absolute_path, + 'stderr'), + }, + 'init': { + 'stdout': os.path.join(self.local.base_path, 'stdout'), + 'stderr': os.path.join(self.local.base_path, 'stderr'), + }, + } + + def tearDown(self): + shutil.rmtree(self.local_dir) + shutil.rmtree(self.remote_dir) + shutil.rmtree(self.temp_dir) + + def _test_output(self, which, target, streams=('stdout', 'stderr')): + for stream in streams: + stream_path = os.path.join(self.output_dirs[target][stream], which) + if os.path.exists(stream_path): + with open(stream_path, 'r') as fd: + _is = fd.read() + self.assertEqual("", _is) + # else ok when not exists + + def test_capture_code_output_disabled(self): + self.cdist_object.code_local = self.code.run_gencode_local( + self.cdist_object) + self._test_output('gencode-local', 'object', ('stderr',)) + + self.code.run_code_local(self.cdist_object) + self._test_output('code-local', 'object') + + self.cdist_object.code_remote = self.code.run_gencode_remote( + self.cdist_object) + self._test_output('gencode-remote', 'object', ('stderr',)) + + self.code.transfer_code_remote(self.cdist_object) + self.code.run_code_remote(self.cdist_object) + self._test_output('code-remote', 'object') + + def test_capture_manifest_output_disabled(self): + self.manifest.run_type_manifest(self.cdist_object) + self._test_output('manifest', 'object') + + def test_capture_init_manifest_output_disabled(self): + initial_manifest = os.path.join(conf_dir, 'manifest', 'init') + self.manifest.run_initial_manifest(initial_manifest) + self._test_output('init', 'init') + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/cdist/test/capture_output_disabled/fixtures/conf/manifest/init b/cdist/test/capture_output_disabled/fixtures/conf/manifest/init new file mode 100755 index 00000000..68d7da97 --- /dev/null +++ b/cdist/test/capture_output_disabled/fixtures/conf/manifest/init @@ -0,0 +1,4 @@ +#!/bin/sh + +echo "init: stdout" +echo "init: stderr" >&2 diff --git a/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-local b/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-local new file mode 100755 index 00000000..1946dbd3 --- /dev/null +++ b/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-local @@ -0,0 +1,6 @@ +#!/bin/sh + +echo "gencode-local: stderr" >&2 + +echo "echo \"code-local: stdout\"" +echo "echo \"code-local: stderr\" >&2" diff --git a/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-remote b/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-remote new file mode 100755 index 00000000..f713b932 --- /dev/null +++ b/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/gencode-remote @@ -0,0 +1,6 @@ +#!/bin/sh + +echo "gencode-remote: stderr" >&2 + +echo "echo \"code-remote: stdout\"" +echo "echo \"code-remote: stderr\" >&2" diff --git a/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/manifest b/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/manifest new file mode 100755 index 00000000..4f122f25 --- /dev/null +++ b/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/manifest @@ -0,0 +1,4 @@ +#!/bin/sh + +echo "manifest: stdout" +echo "manifest: stderr" >&2 diff --git a/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/singleton b/cdist/test/capture_output_disabled/fixtures/conf/type/__write_to_stdout_and_stderr/singleton new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/cdist_object/__init__.py b/cdist/test/cdist_object/__init__.py index d03c0642..a9c20cd3 100644 --- a/cdist/test/cdist_object/__init__.py +++ b/cdist/test/cdist_object/__init__.py @@ -147,6 +147,13 @@ class ObjectIdTestCase(test.CdistTestCase): core.CdistObject(cdist_type, self.object_base_path, OBJECT_MARKER_NAME, illegal_object_id) + def test_object_id_equals_slash(self): + cdist_type = core.CdistType(type_base_path, '__third') + illegal_object_id = '/' + with self.assertRaises(core.IllegalObjectIdError): + core.CdistObject(cdist_type, self.object_base_path, + OBJECT_MARKER_NAME, illegal_object_id) + def test_object_id_on_singleton_type(self): cdist_type = core.CdistType(type_base_path, '__test_singleton') illegal_object_id = 'object_id' diff --git a/cdist/test/cdist_type/__init__.py b/cdist/test/cdist_type/__init__.py index 6ed3f87c..a51a1e6f 100644 --- a/cdist/test/cdist_type/__init__.py +++ b/cdist/test/cdist_type/__init__.py @@ -55,7 +55,7 @@ class TypeTestCase(test.CdistTestCase): def test_nonexistent_type(self): base_path = fixtures - self.assertRaises(core.NoSuchTypeError, core.CdistType, base_path, + self.assertRaises(core.InvalidTypeError, core.CdistType, base_path, '__i-dont-exist') def test_name(self): @@ -113,6 +113,26 @@ class TypeTestCase(test.CdistTestCase): cdist_type = core.CdistType(base_path, '__not_singleton') self.assertFalse(cdist_type.is_singleton) + def test_nonparallel_is_nonparallel(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__nonparallel') + self.assertTrue(cdist_type.is_nonparallel) + + def test_not_nonparallel_is_nonparallel(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__not_nonparallel') + self.assertFalse(cdist_type.is_nonparallel) + + def test_deprecated(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__deprecated') + self.assertIsNotNone(cdist_type.deprecated) + + def test_not_deprecated(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__not_deprecated') + self.assertIsNone(cdist_type.deprecated) + def test_install_is_install(self): base_path = fixtures cdist_type = core.CdistType(base_path, '__install') @@ -180,3 +200,18 @@ class TypeTestCase(test.CdistTestCase): self.assertEqual( list(sorted(cdist_type.parameter_defaults.keys())), ['bar', 'foo']) + + def test_without_deprecated_parameters(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, + '__without_deprecated_parameters') + self.assertEqual(cdist_type.deprecated_parameters, {}) + + def test_with_deprecated_parameters(self): + base_path = fixtures + cdist_type = core.CdistType(base_path, '__with_deprecated_parameters') + self.assertTrue('eggs' in cdist_type.deprecated_parameters) + self.assertTrue('spam' in cdist_type.deprecated_parameters) + self.assertEqual(cdist_type.deprecated_parameters['eggs'], + 'Deprecated') + self.assertEqual(cdist_type.deprecated_parameters['spam'], '') diff --git a/cdist/test/cdist_type/fixtures/__deprecated/deprecated b/cdist/test/cdist_type/fixtures/__deprecated/deprecated new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/cdist_type/fixtures/__nonparallel/nonparallel b/cdist/test/cdist_type/fixtures/__nonparallel/nonparallel new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/cdist_type/fixtures/__not_nonparallel/.keep b/cdist/test/cdist_type/fixtures/__not_nonparallel/.keep new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs new file mode 100644 index 00000000..69d9f456 --- /dev/null +++ b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/eggs @@ -0,0 +1 @@ +Deprecated diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/spam b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/deprecated/spam new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional new file mode 100644 index 00000000..bfe09199 --- /dev/null +++ b/cdist/test/cdist_type/fixtures/__with_deprecated_parameters/parameter/optional @@ -0,0 +1,3 @@ +spam +eggs +sausage diff --git a/cdist/test/code/__init__.py b/cdist/test/code/__init__.py index 83c93f8b..bf80110d 100644 --- a/cdist/test/code/__init__.py +++ b/cdist/test/code/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2011-2017 Steven Armstrong (steven-cdist at armstrong.cc) # 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # # This file is part of cdist. @@ -23,6 +23,7 @@ import getpass import os import shutil +import logging import cdist from cdist import core @@ -46,6 +47,7 @@ class CodeTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=cdist.test.cdist_exec_path, @@ -59,7 +61,9 @@ class CodeTestCase(test.CdistTestCase): target_host=self.target_host, remote_exec=remote_exec, remote_copy=remote_copy, - base_path=self.remote_dir) + base_path=self.remote_dir, + stdout_base_path=self.local.stdout_base_path, + stderr_base_path=self.local.stderr_base_path) self.remote.create_files_dirs() self.code = code.Code(self.target_host, self.local, self.remote) @@ -97,6 +101,11 @@ class CodeTestCase(test.CdistTestCase): self.cdist_object.object_id) self.assertEqual(output_dict['__object_name'], self.cdist_object.name) self.assertEqual(output_dict['__files'], self.local.files_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) + self.assertEqual(output_dict['__cdist_log_level'], + str(logging.WARNING)) + self.assertEqual(output_dict['__cdist_log_level_name'], 'WARNING') def test_run_gencode_remote_environment(self): output_string = self.code.run_gencode_remote(self.cdist_object) @@ -120,6 +129,11 @@ class CodeTestCase(test.CdistTestCase): self.cdist_object.object_id) self.assertEqual(output_dict['__object_name'], self.cdist_object.name) self.assertEqual(output_dict['__files'], self.local.files_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) + self.assertEqual(output_dict['__cdist_log_level'], + str(logging.WARNING)) + self.assertEqual(output_dict['__cdist_log_level_name'], 'WARNING') def test_transfer_code_remote(self): self.cdist_object.code_remote = self.code.run_gencode_remote( @@ -140,6 +154,7 @@ class CodeTestCase(test.CdistTestCase): self.code.transfer_code_remote(self.cdist_object) self.code.run_code_remote(self.cdist_object) + if __name__ == '__main__': import unittest unittest.main() diff --git a/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local b/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local index 7fa70342..2829d633 100755 --- a/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local +++ b/cdist/test/code/fixtures/conf/type/__dump_environment/gencode-local @@ -9,3 +9,6 @@ echo "echo __object: $__object" echo "echo __object_id: $__object_id" echo "echo __object_name: $__object_name" echo "echo __files: $__files" +echo "echo __target_host_tags: $__target_host_tags" +echo "echo __cdist_log_level: $__cdist_log_level" +echo "echo __cdist_log_level_name: $__cdist_log_level_name" diff --git a/cdist/test/config/__init__.py b/cdist/test/config/__init__.py index af1aa38f..499593e3 100644 --- a/cdist/test/config/__init__.py +++ b/cdist/test/config/__init__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# 2010-2011 Steven Armstrong (steven-cdist at armstrong.cc) +# 2010-2017 Steven Armstrong (steven-cdist at armstrong.cc) # 2012-2015 Nico Schottelius (nico-cdist at schottelius.org) # 2014 Daniel Heule (hda at sfs.biz) # @@ -23,7 +23,6 @@ import os import shutil -import tempfile from cdist import test from cdist import core @@ -45,6 +44,19 @@ expected_object_names = sorted([ '__third/moon']) +class CdistObjectErrorContext(object): + def __init__(self, original_error): + self.original_error = original_error + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, tb): + if exc_type is not None: + if exc_value.original_error: + raise exc_value.original_error + + class ConfigRunTestCase(test.CdistTestCase): def setUp(self): @@ -60,6 +72,7 @@ class ConfigRunTestCase(test.CdistTestCase): os.makedirs(self.host_base_path) self.local = cdist.exec.local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=self.host_base_path, host_dir_name=self.hostdir) @@ -86,7 +99,9 @@ class ConfigRunTestCase(test.CdistTestCase): target_host=self.target_host, remote_copy=self.remote_copy, remote_exec=self.remote_exec, - base_path=self.remote_dir) + base_path=self.remote_dir, + stdout_base_path=self.local.stdout_base_path, + stderr_base_path=self.local.stderr_base_path) self.local.object_path = self.object_base_path self.local.type_path = type_base_path @@ -101,6 +116,20 @@ class ConfigRunTestCase(test.CdistTestCase): os.environ = self.orig_environ shutil.rmtree(self.temp_dir) + def assertRaisesCdistObjectError(self, original_error, callable_obj): + """ + Test if a raised CdistObjectError was caused by the given + original_error. + """ + with self.assertRaises(original_error): + try: + callable_obj() + except cdist.CdistObjectError as e: + if e.original_error: + raise e.original_error + else: + raise + def test_dependency_resolution(self): first = self.object_index['__first/man'] second = self.object_index['__second/on-the'] @@ -136,34 +165,39 @@ class ConfigRunTestCase(test.CdistTestCase): first.requirements = [second.name] second.requirements = [first.name] - with self.assertRaises(cdist.UnresolvableRequirementsError): - self.config.iterate_until_finished() + self.assertRaisesCdistObjectError( + cdist.UnresolvableRequirementsError, + self.config.iterate_until_finished) def test_missing_requirements(self): """Throw an error if requiring something non-existing""" first = self.object_index['__first/man'] first.requirements = ['__first/not/exist'] - with self.assertRaises(cdist.UnresolvableRequirementsError): - self.config.iterate_until_finished() + self.assertRaisesCdistObjectError( + cdist.UnresolvableRequirementsError, + self.config.iterate_until_finished) def test_requirement_broken_type(self): """Unknown type should be detected in the resolving process""" first = self.object_index['__first/man'] first.requirements = ['__nosuchtype/not/exist'] - with self.assertRaises(cdist.core.cdist_type.NoSuchTypeError): - self.config.iterate_until_finished() + self.assertRaisesCdistObjectError( + cdist.core.cdist_type.InvalidTypeError, + self.config.iterate_until_finished) def test_requirement_singleton_where_no_singleton(self): """Missing object id should be detected in the resolving process""" first = self.object_index['__first/man'] first.requirements = ['__first'] - with self.assertRaises(cdist.core.cdist_object.MissingObjectIdError): - self.config.iterate_until_finished() + self.assertRaisesCdistObjectError( + cdist.core.cdist_object.MissingObjectIdError, + self.config.iterate_until_finished) def test_dryrun(self): """Test if the dryrun option is working like expected""" drylocal = cdist.exec.local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=self.host_base_path, host_dir_name=self.hostdir, # exec_path can not derivated from sys.argv in case of unittest @@ -177,10 +211,11 @@ class ConfigRunTestCase(test.CdistTestCase): dryrun.run() # if we are here, dryrun works like expected - def test_desp_resolver(self): + def test_deps_resolver(self): """Test to show dependency resolver warning message.""" local = cdist.exec.local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=os.path.abspath(os.path.join( @@ -193,6 +228,71 @@ class ConfigRunTestCase(test.CdistTestCase): config = cdist.config.Config(local, self.remote, dry_run=True) config.run() + def test_graph_check_cycle_empty(self): + graph = {} + has_cycle, path = cdist.config.graph_check_cycle(graph) + self.assertFalse(has_cycle) + + def test_graph_check_cycle_1(self): + # + # a -> b -> c + # | + # +--> d -> e + graph = { + 'a': ['b', ], + 'b': ['c', 'd', ], + 'd': ['e', ], + } + has_cycle, path = cdist.config.graph_check_cycle(graph) + self.assertFalse(has_cycle) + + def test_graph_check_cycle_2(self): + # + # a -> b -> c + # /\ | + # \ | + # +-------+ + graph = { + 'a': ['b', ], + 'b': ['c', ], + 'c': ['a', ], + } + has_cycle, path = cdist.config.graph_check_cycle(graph) + self.assertTrue(has_cycle) + self.assertGreater(path.count(path[-1]), 1) + + def test_graph_check_cycle_3(self): + # + # a -> b -> c + # \ \ + # \ +--> g + # \ /\ + # \ /| + # +-> d -> e | + # \ | + # + --> f + # + # h -> i --> j + # | /\ | + # \/ | \/ + # n m <- k + graph = { + 'a': ['b', 'd', ], + 'b': ['c', ], + 'c': ['g', ], + 'd': ['e', 'f', ], + 'e': ['g', ], + 'f': ['g', ], + 'h': ['i', 'n', ], + 'i': ['j', ], + 'j': ['k', ], + 'k': ['m', ], + 'm': ['i', ], + } + has_cycle, path = cdist.config.graph_check_cycle(graph) + self.assertTrue(has_cycle) + self.assertGreater(path.count(path[-1]), 1) + # Currently the resolving code will simply detect that this object does # not exist. It should probably check if the type is a singleton as well diff --git a/cdist/test/configuration/__init__.py b/cdist/test/configuration/__init__.py new file mode 100644 index 00000000..182868a6 --- /dev/null +++ b/cdist/test/configuration/__init__.py @@ -0,0 +1,1415 @@ +# -*- coding: utf-8 -*- +# +# 2017 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import configparser +import os +import multiprocessing +import cdist.configuration as cc +import os.path as op +import argparse +from cdist import test +import cdist.argparse as cap +import logging + +my_dir = op.abspath(op.dirname(__file__)) +fixtures = op.join(my_dir, 'fixtures') +interpolation_config_file = op.join(fixtures, "interpolation-test.cfg") + + +def newConfigParser(): + return configparser.ConfigParser(interpolation=None) + + +class ConfigurationOptionsTestCase(test.CdistTestCase): + + def test_OptionBase(self): + option = cc.OptionBase('test') + test_cases = ( + ([], [], True, None, ), + (['spam', 'eggs', ], [], True, ['spam', 'eggs', ], ), + ([], ['spam', 'eggs', ], True, ['spam', 'eggs', ], ), + ( + ['spam', 'eggs', ], + ['ham', 'spamspam', ], + True, + ['spam', 'eggs', 'ham', 'spamspam', ], + ), + (['spam', 'eggs', ], 'spam:eggs', True, 'spam:eggs', ), + ('spam:eggs', ['spam', 'eggs', ], True, ['spam', 'eggs', ], ), + ('spam', 'eggs', True, 'eggs', ), + + (['spam', 'eggs', ], 'spam:eggs', True, 'spam:eggs', ), + + ('spam:eggs', ['spam', 'eggs', ], False, ['spam', 'eggs', ], ), + ('spam', 'eggs', False, 'eggs', ), + ( + ['spam', 'eggs', ], + ['ham', 'spamspam', ], + False, + ['ham', 'spamspam', ], + ), + ) + for currval, newval, update_appends, expected in test_cases: + self.assertEqual( + option.update_value(currval, newval, + update_appends=update_appends), + expected) + + def test_StringOption(self): + option = cc.StringOption('test') + self.assertIsNone(option.translate('')) + self.assertEqual(option.translate('spam'), 'spam') + converter = option.get_converter() + self.assertEqual(converter('spam'), 'spam') + self.assertIsNone(converter('')) + + def test_BooleanOption(self): + option = cc.BooleanOption('test') + for x in cc.BooleanOption.BOOLEAN_STATES: + self.assertEqual(option.translate(x), + cc.BooleanOption.BOOLEAN_STATES[x]) + converter = option.get_converter() + self.assertRaises(ValueError, converter, 'of') + for x in cc.BooleanOption.BOOLEAN_STATES: + self.assertEqual(converter(x), cc.BooleanOption.BOOLEAN_STATES[x]) + + def test_IntOption(self): + option = cc.IntOption('test') + converter = option.get_converter() + self.assertRaises(ValueError, converter, 'x') + for x in range(-5, 10): + self.assertEqual(converter(str(x)), x) + + def test_LowerBoundIntOption(self): + option = cc.LowerBoundIntOption('test', -1) + converter = option.get_converter() + self.assertRaises(ValueError, converter, -2) + for x in range(-1, 10): + self.assertEqual(converter(str(x)), x) + + def test_SpecialCasesLowerBoundIntOption(self): + special_cases = { + -1: 8, + -2: 10, + } + option = cc.SpecialCasesLowerBoundIntOption('test', -1, special_cases) + for x in special_cases: + self.assertEqual(option.translate(x), special_cases[x]) + + def test_SelectOption(self): + valid_values = ('spam', 'eggs', 'ham', ) + option = cc.SelectOption('test', valid_values) + converter = option.get_converter() + self.assertRaises(ValueError, converter, 'spamspam') + for x in valid_values: + self.assertEqual(converter(x), x) + + def test_DelimitedValuesOption(self): + option = cc.DelimitedValuesOption('test', ':') + converter = option.get_converter() + value = 'spam:eggs::ham' + self.assertEqual(converter(value), ['spam', 'eggs', 'ham', ]) + self.assertIsNone(converter('')) + + def test_LogLevelOption(self): + option = cc.LogLevelOption() + converter = option.get_converter() + value = str(logging.DEBUG) + conv_val = converter(value) + self.assertEqual(conv_val, cap.VERBOSE_DEBUG) + value = str(logging.INFO) + conv_val = converter(value) + self.assertEqual(conv_val, cap.VERBOSE_INFO) + for value in ('11', '80', 'a'): + self.assertRaises(ValueError, converter, value) + + +class ConfigurationTestCase(test.CdistTestCase): + + def setUp(self): + # Create test config file. + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + config_custom = newConfigParser() + config_custom['GLOBAL'] = { + 'parallel': '4', + 'archiving': 'txz', + } + + config_custom2 = newConfigParser() + config_custom2['GLOBAL'] = { + 'parallel': '16', + 'archiving': 'tbz2', + 'remote_copy': 'myscp', + } + + self.expected_config_dict = { + 'GLOBAL': { + 'beta': False, + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': None, + 'cache_path_pattern': None, + 'conf_dir': None, + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': None, + 'remote_exec': None, + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': None, + }, + } + + self.config_file = os.path.join(fixtures, 'cdist.cfg') + with open(self.config_file, 'w') as f: + config.write(f) + + self.custom_config_file = os.path.join(fixtures, 'cdist_custom.cfg') + with open(self.custom_config_file, 'w') as f: + config_custom.write(f) + + self.custom_config_file2 = os.path.join(fixtures, 'cdist_custom2.cfg') + with open(self.custom_config_file2, 'w') as f: + config_custom2.write(f) + + config['TEST'] = {} + self.invalid_config_file1 = os.path.join(fixtures, + 'cdist_invalid1.cfg') + with open(self.invalid_config_file1, 'w') as f: + config.write(f) + + del config['TEST'] + config['GLOBAL']['test'] = 'test' + self.invalid_config_file2 = os.path.join(fixtures, + 'cdist_invalid2.cfg') + with open(self.invalid_config_file2, 'w') as f: + config.write(f) + + del config['GLOBAL']['test'] + config['GLOBAL']['archiving'] = 'zip' + self.invalid_config_file3 = os.path.join(fixtures, + 'cdist_invalid3.cfg') + with open(self.invalid_config_file3, 'w') as f: + config.write(f) + + self.maxDiff = None + + def tearDown(self): + os.remove(self.config_file) + os.remove(self.custom_config_file) + os.remove(self.custom_config_file2) + os.remove(self.invalid_config_file1) + os.remove(self.invalid_config_file2) + os.remove(self.invalid_config_file3) + + # remove files from tests + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + custom_config_file = os.path.join(fixtures, 'cdist-custom.cfg') + if os.path.exists(global_config_file): + os.remove(global_config_file) + if os.path.exists(local_config_file): + os.remove(local_config_file) + if os.path.exists(custom_config_file): + os.remove(custom_config_file) + + def test_singleton(self): + x = cc.Configuration(None, env={}, config_files=()) + args = argparse.Namespace() + args.a = 'a' + y = cc.Configuration(args, env={}, config_files=()) + self.assertIs(x, y) + + def test_non_singleton(self): + x = cc.Configuration(None, env={}, config_files=(), singleton=False) + args = argparse.Namespace() + args.a = 'a' + y = cc.Configuration(args, env={}, config_files=(), singleton=False) + self.assertIsNot(x, y) + + def test_read_config_file(self): + config = cc.Configuration(None, env={}, config_files=()) + d = config._read_config_file(self.config_file) + self.assertEqual(d, self.expected_config_dict) + + for x in range(1, 4): + config_file = getattr(self, 'invalid_config_file' + str(x)) + with self.assertRaises(ValueError): + config._read_config_file(config_file) + + def test_read_env_var_config(self): + config = cc.Configuration(None, env={}, config_files=()) + env = { + 'a': 'a', + 'CDIST_BETA': '1', + 'CDIST_PATH': '/usr/local/cdist:~/.cdist', + } + expected = { + 'beta': True, + 'conf_dir': ['/usr/local/cdist', '~/.cdist', ], + } + section = 'GLOBAL' + d = config._read_env_var_config(env, section) + self.assertEqual(d, expected) + + del env['CDIST_BETA'] + del expected['beta'] + d = config._read_env_var_config(env, section) + self.assertEqual(d, expected) + + def test_read_args_config(self): + config = cc.Configuration(None, env={}, config_files=()) + args = argparse.Namespace() + args.beta = False + args.conf_dir = ['/usr/local/cdist1', ] + args.verbose = 3 + args.tag = 'test' + + expected = { + 'conf_dir': ['/usr/local/cdist1', ], + 'verbosity': 3, + 'beta': False, + } + args_dict = vars(args) + d = config._read_args_config(args_dict) + self.assertEqual(d, expected) + self.assertNotEqual(d, args_dict) + + def test_update_config_dict(self): + config = { + 'GLOBAL': { + 'conf_dir': ['/usr/local/cdist', ], + 'parallel': -1, + }, + } + newconfig = { + 'GLOBAL': { + 'conf_dir': ['~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + expected = { + 'GLOBAL': { + 'conf_dir': ['/usr/local/cdist', '~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + configuration = cc.Configuration(None, env={}, config_files=()) + configuration._update_config_dict(config, newconfig, + update_appends=True) + self.assertEqual(config, expected) + expected = { + 'GLOBAL': { + 'conf_dir': ['~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + configuration._update_config_dict(config, newconfig, + update_appends=False) + self.assertEqual(config, expected) + + def test_update_config_dict_section(self): + config = { + 'GLOBAL': { + 'conf_dir': ['/usr/local/cdist', ], + 'parallel': -1, + }, + } + newconfig = { + 'conf_dir': ['~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + } + expected = { + 'GLOBAL': { + 'conf_dir': ['/usr/local/cdist', '~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + configuration = cc.Configuration(None, env={}, config_files=()) + configuration._update_config_dict_section('GLOBAL', config, newconfig, + update_appends=True) + self.assertEqual(config, expected) + expected = { + 'GLOBAL': { + 'conf_dir': ['~/.cdist', ], + 'parallel': 2, + 'local_shell': '/usr/local/bin/sh', + }, + } + configuration._update_config_dict_section('GLOBAL', config, newconfig, + update_appends=False) + self.assertEqual(config, expected) + + def test_configuration1(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + } + args = argparse.Namespace() + expected_config_dict = { + 'GLOBAL': { + 'verbosity': 0, + }, + } + + # bypass singleton so we can test further + cc.Configuration.instance = None + configuration = cc.Configuration(args, env=env, + config_files=('cdist.cfg')) + self.assertIsNotNone(configuration.args) + self.assertIsNotNone(configuration.env) + self.assertIsNotNone(configuration.config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration2(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + } + args = argparse.Namespace() + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': False, + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': None, + 'cache_path_pattern': None, + 'conf_dir': None, + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': None, + 'remote_exec': None, + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': None, + }, + } + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration3(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + } + args = argparse.Namespace() + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': ['/opt/cdist', ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'tar', + }, + } + config_files = (global_config_file, local_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration4(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': None, + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/cdist/conf', + '/usr/local/share/cdist/conf', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': None, + 'remote_exec': None, + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': None, + }, + } + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration5(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/cdist/conf', + '/usr/local/share/cdist/conf', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'jobs': 0, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'tar', + }, + } + config_files = (global_config_file, local_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_update_defaults_for_unset(self): + config = { + 'GLOBAL': { + }, + } + expected_config = { + 'GLOBAL': { + 'verbosity': 0, + }, + } + cfg = cc.Configuration(None, env={}, config_files=()) + cfg._update_defaults_for_unset(config) + self.assertEqual(config, expected_config) + + def test_configuration6(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + args.inventory_dir = '/opt/sysadmin/cdist/inventory' + args.conf_dir = ['/opt/sysadmin/cdist/conf', ] + args.manifest = '/opt/sysadmin/cdist/conf/manifest/init' + args.jobs = 10 + args.verbose = None + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/opt/sysadmin/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/cdist/conf', + '/usr/local/share/cdist/conf', + '/opt/sysadmin/cdist/conf', + ], + 'init_manifest': '/opt/sysadmin/cdist/conf/manifest/init', + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'jobs': 10, + 'parallel': multiprocessing.cpu_count(), + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'tar', + }, + } + config_files = (global_config_file, local_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration7(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'conf_dir': '/opt/conf/cdist', + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'parallel': '15', + 'archiving': 'txz', + } + + custom_config_file = os.path.join(fixtures, 'cdist-custom.cfg') + with open(custom_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/conf/cdist', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'jobs': 0, + 'parallel': 15, + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'txz', + }, + } + + config_files = (global_config_file, local_config_file, ) + + args.config_file = custom_config_file + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration8(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'conf_dir': '/opt/conf/cdist', + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'parallel': '15', + 'archiving': 'txz', + } + + custom_config_file = os.path.join(fixtures, 'cdist-custom.cfg') + with open(custom_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/conf/cdist', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'jobs': 0, + 'parallel': 15, + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'txz', + }, + } + + config_files = (global_config_file, local_config_file, ) + + os.environ['CDIST_CONFIG_FILE'] = custom_config_file + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_get_args(self): + env = { + 'PATH': '/usr/local/bin:/usr/bin:/bin', + 'TEST': 'test', + 'CDIST_PATH': '/opt/cdist/conf:/usr/local/share/cdist/conf', + 'REMOTE_COPY': 'scp', + 'REMOTE_EXEC': 'ssh', + 'CDIST_BETA': '1', + 'CDIST_LOCAL_SHELL': '/usr/bin/sh', + 'CDIST_REMOTE_SHELL': '/usr/bin/sh', + } + args = argparse.Namespace() + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'off', + 'local_shell': '/bin/sh', + 'remote_shell': '/bin/sh', + 'inventory_dir': '', + 'cache_path_pattern': '', + 'conf_dir': '', + 'init_manifest': '', + 'out_path': '', + 'remote_out_path': '', + 'remote_copy': '', + 'remote_exec': '', + 'jobs': '0', + 'parallel': '-1', + 'verbosity': 'INFO', + 'archiving': 'none', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'beta': 'on', + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'conf_dir': '/opt/cdist', + 'remote_copy': 'myscp', + 'remote_exec': 'myexec', + 'parallel': '-1', + 'archiving': 'tar', + } + + local_config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(local_config_file, 'w') as f: + config.write(f) + + config = newConfigParser() + config['GLOBAL'] = { + 'conf_dir': '/opt/conf/cdist', + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'parallel': '15', + 'archiving': 'txz', + } + + custom_config_file = os.path.join(fixtures, 'cdist-custom.cfg') + with open(custom_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'beta': True, + 'local_shell': '/usr/bin/sh', + 'remote_shell': '/usr/bin/sh', + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/conf/cdist', + ], + 'init_manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'jobs': 0, + 'parallel': 15, + 'verbosity': cap.VERBOSE_INFO, + 'archiving': 'txz', + }, + } + + config_files = (global_config_file, local_config_file, ) + + os.environ['CDIST_CONFIG_FILE'] = custom_config_file + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + args = configuration.get_args() + dargs = vars(args) + expected_args = { + 'beta': True, + 'inventory_dir': '/var/db/cdist/inventory', + 'cache_path_pattern': None, + 'conf_dir': [ + '/opt/conf/cdist', + ], + 'manifest': None, + 'out_path': None, + 'remote_out_path': None, + 'remote_copy': 'scpcustom', + 'remote_exec': 'sshcustom', + 'jobs': 0, + 'parallel': 15, + 'verbose': cap.VERBOSE_INFO, + 'use_archiving': 'txz', + } + + self.assertEqual(dargs, expected_args) + + def test_configuration_empty_value_in_file(self): + config = newConfigParser() + config['GLOBAL'] = { + 'inventory_dir': '', + 'conf_dir': '', + } + + config_file = os.path.join(fixtures, 'cdist-local.cfg') + with open(config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'inventory_dir': None, + 'conf_dir': None, + 'verbosity': 0, + }, + } + + config_files = (config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + configuration = cc.Configuration(args, env={}, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_cdist_log_level_env_var(self): + env = { + '__cdist_log_level': str(logging.DEBUG), + } + args = argparse.Namespace() + + expected_config_dict = { + 'GLOBAL': { + 'verbosity': cap.VERBOSE_DEBUG, + }, + } + + # bypass singleton so we can test further + cc.Configuration.instance = None + + configuration = cc.Configuration(args, env=env, + config_files=()) + self.assertEqual(configuration.config, expected_config_dict) + + # bypass singleton so we can test further + cc.Configuration.instance = None + env['__cdist_log_level'] = '80' + with self.assertRaises(ValueError): + configuration = cc.Configuration(args, env=env, + config_files=()) + + # bypass singleton so we can test further + cc.Configuration.instance = None + env['__cdist_log_level'] = 'x' + with self.assertRaises(ValueError): + configuration = cc.Configuration(args, env=env, + config_files=()) + + def test_configuration_disable_saving_output_streams1(self): + config = newConfigParser() + config['GLOBAL'] = { + 'save_output_streams': 'True', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'save_output_streams': True, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.save_output_streams = True + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_disable_saving_output_streams2(self): + config = newConfigParser() + config['GLOBAL'] = { + 'save_output_streams': 'False', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'save_output_streams': False, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.save_output_streams = True + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_disable_saving_output_streams3(self): + config = newConfigParser() + config['GLOBAL'] = { + 'save_output_streams': 'False', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'save_output_streams': False, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.save_output_streams = False + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_disable_saving_output_streams4(self): + config = newConfigParser() + config['GLOBAL'] = { + 'save_output_streams': 'True', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'save_output_streams': False, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.save_output_streams = False + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_read_config_file_with_interpolation(self): + try: + config = cc.Configuration(None, env={}, config_files=()) + d = config._read_config_file(interpolation_config_file) + val = d['GLOBAL']['cache_path_pattern'] + self.assertIsNotNone(val) + self.assertEqual(val, '%N') + except configparser.InterpolationSyntaxError as e: + self.fail("Exception should not have been raised: {}".format( + e)) + + def test_configuration_timestamping_log_1(self): + config = newConfigParser() + config['GLOBAL'] = { + 'timestamp': 'True', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'timestamp': True, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.timestamp = True + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_timestamping_log_2(self): + config = newConfigParser() + config['GLOBAL'] = { + 'timestamp': 'False', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'timestamp': True, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.timestamp = True + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_timestamping_log_3(self): + config = newConfigParser() + config['GLOBAL'] = { + 'timestamp': 'False', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'timestamp': False, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.timestamp = False + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + def test_configuration_timestamping_log_4(self): + config = newConfigParser() + config['GLOBAL'] = { + 'timestamp': 'True', + } + + global_config_file = os.path.join(fixtures, 'cdist-global.cfg') + with open(global_config_file, 'w') as f: + config.write(f) + + expected_config_dict = { + 'GLOBAL': { + 'timestamp': False, + 'verbosity': 0, + }, + } + + config_files = (global_config_file, ) + + # bypass singleton so we can test further + cc.Configuration.instance = None + + args = argparse.Namespace() + args.timestamp = False + configuration = cc.Configuration(args, env=None, + config_files=config_files) + self.assertEqual(configuration.config, expected_config_dict) + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/cdist/test/configuration/fixtures/.nonempty b/cdist/test/configuration/fixtures/.nonempty new file mode 100644 index 00000000..e69de29b diff --git a/cdist/test/configuration/fixtures/interpolation-test.cfg b/cdist/test/configuration/fixtures/interpolation-test.cfg new file mode 100644 index 00000000..df723121 --- /dev/null +++ b/cdist/test/configuration/fixtures/interpolation-test.cfg @@ -0,0 +1,2 @@ +[GLOBAL] +cache_path_pattern = %N diff --git a/cdist/test/emulator/__init__.py b/cdist/test/emulator/__init__.py index 51de3180..5691093c 100644 --- a/cdist/test/emulator/__init__.py +++ b/cdist/test/emulator/__init__.py @@ -27,6 +27,7 @@ import shutil import string import filecmp import random +import logging import cdist from cdist import test @@ -53,6 +54,7 @@ class EmulatorTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -62,6 +64,8 @@ class EmulatorTestCase(test.CdistTestCase): self.manifest = core.Manifest(self.target_host, self.local) self.env = self.manifest.env_initial_manifest(self.script) self.env['__cdist_object_marker'] = self.local.object_marker_name + if '__cdist_log_level' in self.env: + del self.env['__cdist_log_level'] def tearDown(self): shutil.rmtree(self.temp_dir) @@ -72,14 +76,14 @@ class EmulatorTestCase(test.CdistTestCase): def test_nonexistent_type_exec(self): argv = ['__does-not-exist'] - self.assertRaises(core.cdist_type.NoSuchTypeError, emulator.Emulator, + self.assertRaises(core.cdist_type.InvalidTypeError, emulator.Emulator, argv, env=self.env) def test_nonexistent_type_requirement(self): argv = ['__file', '/tmp/foobar'] self.env['require'] = '__does-not-exist/some-id' emu = emulator.Emulator(argv, env=self.env) - self.assertRaises(core.cdist_type.NoSuchTypeError, emu.run) + self.assertRaises(core.cdist_type.InvalidTypeError, emu.run) def test_illegal_object_id_requirement(self): argv = ['__file', '/tmp/foobar'] @@ -114,6 +118,31 @@ class EmulatorTestCase(test.CdistTestCase): emu = emulator.Emulator(argv, env=self.env) # if we get here all is fine + def test_loglevel(self): + argv = ['__file', '/tmp/foobar'] + self.env['require'] = '__file/etc/*' + emu = emulator.Emulator(argv, env=self.env) + emu_loglevel = emu.log.getEffectiveLevel() + self.assertEqual(emu_loglevel, logging.WARNING) + self.env['__cdist_log_level'] = str(logging.DEBUG) + emu = emulator.Emulator(argv, env=self.env) + emu_loglevel = emu.log.getEffectiveLevel() + self.assertEqual(emu_loglevel, logging.DEBUG) + del self.env['__cdist_log_level'] + + def test_invalid_loglevel_value(self): + argv = ['__file', '/tmp/foobar'] + self.env['require'] = '__file/etc/*' + emu = emulator.Emulator(argv, env=self.env) + emu_loglevel = emu.log.getEffectiveLevel() + self.assertEqual(emu_loglevel, logging.WARNING) + # lowercase is invalid + self.env['__cdist_log_level'] = 'debug' + emu = emulator.Emulator(argv, env=self.env) + emu_loglevel = emu.log.getEffectiveLevel() + self.assertEqual(emu_loglevel, logging.WARNING) + del self.env['__cdist_log_level'] + def test_requirement_via_order_dependency(self): self.env['CDIST_ORDER_DEPENDENCY'] = 'on' argv = ['__planet', 'erde'] @@ -156,6 +185,7 @@ class EmulatorConflictingRequirementsTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -246,6 +276,7 @@ class AutoRequireEmulatorTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -279,6 +310,7 @@ class OverrideTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -322,6 +354,7 @@ class ArgumentsTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -387,6 +420,27 @@ class ArgumentsTestCase(test.CdistTestCase): self.assertEqual(cdist_object.parameters['required1'], value) self.assertEqual(cdist_object.parameters['required2'], value) + def test_required_multiple_arguments(self): + """check whether assigning required multiple parameter works""" + + type_name = '__arguments_required_multiple' + object_id = 'some-id' + value1 = 'value1' + value2 = 'value2' + argv = [type_name, object_id, '--required1', value1, + '--required1', value2] + os.environ.update(self.env) + emu = emulator.Emulator(argv) + emu.run() + + cdist_type = core.CdistType(self.local.type_path, type_name) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + object_id) + self.assertTrue('required1' in cdist_object.parameters) + self.assertTrue(value1 in cdist_object.parameters['required1']) + self.assertTrue(value2 in cdist_object.parameters['required1']) + # def test_required_missing(self): # type_name = '__arguments_required' # object_id = 'some-id' @@ -414,6 +468,25 @@ class ArgumentsTestCase(test.CdistTestCase): self.assertFalse('optional2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['optional1'], value) + def test_optional_multiple(self): + type_name = '__arguments_optional_multiple' + object_id = 'some-id' + value1 = 'value1' + value2 = 'value2' + argv = [type_name, object_id, '--optional1', value1, '--optional1', + value2] + os.environ.update(self.env) + emu = emulator.Emulator(argv) + emu.run() + + cdist_type = core.CdistType(self.local.type_path, type_name) + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + object_id) + self.assertTrue('optional1' in cdist_object.parameters) + self.assertTrue(value1 in cdist_object.parameters['optional1']) + self.assertTrue(value2 in cdist_object.parameters['optional1']) + def test_argument_defaults(self): type_name = '__argument_defaults' object_id = 'some-id' @@ -431,6 +504,29 @@ class ArgumentsTestCase(test.CdistTestCase): self.assertFalse('optional2' in cdist_object.parameters) self.assertEqual(cdist_object.parameters['optional1'], value) + def test_object_params_in_context(self): + type_name = '__arguments_all' + object_id = 'some-id' + argv = [type_name, object_id, '--opt', 'opt', '--req', 'req', + '--bool', '--optmul', 'val1', '--optmul', 'val2', + '--reqmul', 'val3', '--reqmul', 'val4', + '--optmul1', 'val5', '--reqmul1', 'val6'] + os.environ.update(self.env) + emu = emulator.Emulator(argv) + emu.run() + + obj_params = emu._object_params_in_context() + obj_params_expected = { + 'bool': '', + 'opt': 'opt', + 'optmul1': ['val5', ], + 'optmul': ['val1', 'val2', ], + 'req': 'req', + 'reqmul1': ['val6', ], + 'reqmul': ['val3', 'val4', ], + } + self.assertEqual(obj_params, obj_params_expected) + class StdinTestCase(test.CdistTestCase): @@ -445,6 +541,7 @@ class StdinTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -511,6 +608,7 @@ class EmulatorAlreadyExistingRequirementsWarnTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=host_base_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, diff --git a/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/boolean b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/boolean new file mode 100644 index 00000000..46a27912 --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/boolean @@ -0,0 +1 @@ +bool diff --git a/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/optional b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/optional new file mode 100644 index 00000000..d6eba11a --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/optional @@ -0,0 +1 @@ +opt diff --git a/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/optional_multiple b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/optional_multiple new file mode 100644 index 00000000..04893522 --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/optional_multiple @@ -0,0 +1,2 @@ +optmul +optmul1 diff --git a/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/required b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/required new file mode 100644 index 00000000..da45f08d --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/required @@ -0,0 +1 @@ +req diff --git a/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/required_multiple b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/required_multiple new file mode 100644 index 00000000..28c1a19b --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__arguments_all/parameter/required_multiple @@ -0,0 +1,2 @@ +reqmul +reqmul1 diff --git a/cdist/test/emulator/fixtures/conf/type/__arguments_optional_multiple/parameter/optional_multiple b/cdist/test/emulator/fixtures/conf/type/__arguments_optional_multiple/parameter/optional_multiple new file mode 100644 index 00000000..31647628 --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__arguments_optional_multiple/parameter/optional_multiple @@ -0,0 +1 @@ +optional1 diff --git a/cdist/test/emulator/fixtures/conf/type/__arguments_required_multiple/parameter/required_multiple b/cdist/test/emulator/fixtures/conf/type/__arguments_required_multiple/parameter/required_multiple new file mode 100644 index 00000000..180d60c7 --- /dev/null +++ b/cdist/test/emulator/fixtures/conf/type/__arguments_required_multiple/parameter/required_multiple @@ -0,0 +1 @@ +required1 diff --git a/cdist/test/exec/__init__.py b/cdist/test/exec/__init__.py index e69de29b..b55cffa9 100644 --- a/cdist/test/exec/__init__.py +++ b/cdist/test/exec/__init__.py @@ -0,0 +1,7 @@ +from .local import * + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/cdist/test/exec/local.py b/cdist/test/exec/local.py index 0efdfa0a..0865b7dc 100644 --- a/cdist/test/exec/local.py +++ b/cdist/test/exec/local.py @@ -26,8 +26,12 @@ import getpass import shutil import string import random +import time +import datetime +import argparse import cdist +import cdist.configuration as cc from cdist import test from cdist.exec import local @@ -57,6 +61,7 @@ class LocalTestCase(test.CdistTestCase): self.local = local.Local( target_host=target_host, + target_host_tags=None, base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=test.cdist_exec_path @@ -120,6 +125,7 @@ class LocalTestCase(test.CdistTestCase): 'localhost', 'localhost', ), + target_host_tags=None, base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=test.cdist_exec_path, @@ -143,6 +149,7 @@ class LocalTestCase(test.CdistTestCase): 'localhost', 'localhost', ), + target_host_tags=None, base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=test.cdist_exec_path, @@ -163,15 +170,23 @@ class LocalTestCase(test.CdistTestCase): os.environ['CDIST_PATH'] = conf_dir + # bypass singleton from other tests if any + cc.Configuration.instance = None + + configuration = cc.Configuration(argparse.Namespace(), + env=os.environ) + link_test_local = local.Local( target_host=( 'localhost', 'localhost', 'localhost', ), + target_host_tags=None, base_root_path=self.host_base_path, host_dir_name=self.hostdir, exec_path=test.cdist_exec_path, + configuration=configuration.get_config(section='GLOBAL') ) link_test_local._create_conf_path_and_link_conf_dirs() @@ -183,24 +198,29 @@ class LocalTestCase(test.CdistTestCase): # other tests def test_run_success(self): + self.local.create_files_dirs() self.local.run([bin_true]) def test_run_fail(self): + self.local.create_files_dirs() self.assertRaises(cdist.Error, self.local.run, [bin_false]) def test_run_script_success(self): + self.local.create_files_dirs() handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", bin_true]) self.local.run_script(script) def test_run_script_fail(self): + self.local.create_files_dirs() handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", bin_false]) self.assertRaises(cdist.Error, self.local.run_script, script) def test_run_script_get_output(self): + self.local.create_files_dirs() handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "echo foobar"]) @@ -224,6 +244,41 @@ class LocalTestCase(test.CdistTestCase): self.assertTrue(os.path.isdir(self.local.bin_path)) self.assertTrue(os.path.isdir(self.local.conf_path)) + def test_cache_subpath(self): + start_time = time.time() + dt = datetime.datetime.fromtimestamp(start_time) + pid = str(os.getpid()) + cases = [ + ['', self.local.hostdir, ], + ['/', self.local.hostdir, ], + ['//', self.local.hostdir, ], + ['/%%h', '%h', ], + ['%%h', '%h', ], + ['%P', pid, ], + ['x%P', 'x' + pid, ], + ['%h', self.hostdir, ], + ['%h/%Y-%m-%d/%H%M%S%f%P', + dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ], + ['/%h/%Y-%m-%d/%H%M%S%f%P', + dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S%f') + pid, ], + ['%Y-%m-%d/%H%M%S%f%P/%h', + dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ], + ['///%Y-%m-%d/%H%M%S%f%P/%h', + dt.strftime('%Y-%m-%d/%H%M%S%f' + pid + os.sep + self.hostdir), ], + ['%h/%Y-%m-%d/%H%M%S-%P', + dt.strftime(self.hostdir + '/%Y-%m-%d/%H%M%S-') + pid, ], + ['%Y-%m-%d/%H%M%S-%P/%h', + dt.strftime('%Y-%m-%d/%H%M%S-') + pid + os.sep + self.hostdir, ], + ['%N', self.local.target_host[0], ], + ] + for x in cases: + x.append(self.local._cache_subpath(start_time, x[0])) + # for fmt, expected, actual in cases: + # print('\'{}\' \'{}\' \'{}\''.format(fmt, expected, actual)) + for fmt, expected, actual in cases: + self.assertEqual(expected, actual) + + if __name__ == "__main__": import unittest diff --git a/cdist/test/exec/remote.py b/cdist/test/exec/remote.py index 371d17e3..a7fe384d 100644 --- a/cdist/test/exec/remote.py +++ b/cdist/test/exec/remote.py @@ -22,8 +22,6 @@ import os import getpass import shutil -import string -import random import multiprocessing import cdist @@ -40,16 +38,30 @@ class RemoteTestCase(test.CdistTestCase): 'localhost', 'localhost', ) - self.base_path = self.temp_dir + # another temp dir for remote base path + self.base_path = self.mkdtemp() + self.remote = self.create_remote() + + def create_remote(self, *args, **kwargs): + if not args: + args = (self.target_host,) + kwargs.setdefault('base_path', self.base_path) user = getpass.getuser() - remote_exec = "ssh -o User=%s -q" % user - remote_copy = "scp -o User=%s -q" % user - self.remote = remote.Remote(self.target_host, base_path=self.base_path, - remote_exec=remote_exec, - remote_copy=remote_copy) + kwargs.setdefault('remote_exec', 'ssh -o User=%s -q' % user) + kwargs.setdefault('remote_copy', 'scp -o User=%s -q' % user) + if 'stdout_base_path' not in kwargs: + stdout_path = os.path.join(self.temp_dir, 'stdout') + os.makedirs(stdout_path, exist_ok=True) + kwargs['stdout_base_path'] = stdout_path + if 'stderr_base_path' not in kwargs: + stderr_path = os.path.join(self.temp_dir, 'stderr') + os.makedirs(stderr_path, exist_ok=True) + kwargs['stderr_base_path'] = stderr_path + return remote.Remote(*args, **kwargs) def tearDown(self): shutil.rmtree(self.temp_dir) + shutil.rmtree(self.base_path) # test api @@ -113,7 +125,8 @@ class RemoteTestCase(test.CdistTestCase): os.close(handle) target = self.mkdtemp(dir=self.temp_dir) self.remote.transfer(source, target) - self.assertTrue(os.path.isfile(target)) + self.assertTrue(os.path.isfile( + os.path.join(target, os.path.basename(source)))) def test_transfer_dir(self): source = self.mkdtemp(dir=self.temp_dir) @@ -154,8 +167,8 @@ class RemoteTestCase(test.CdistTestCase): os.chmod(remote_exec_path, 0o755) remote_exec = remote_exec_path remote_copy = "echo" - r = remote.Remote(self.target_host, base_path=self.base_path, - remote_exec=remote_exec, remote_copy=remote_copy) + r = self.create_remote(remote_exec=remote_exec, + remote_copy=remote_copy) self.assertEqual(r.run('true', return_output=True), "%s\n" % self.target_host[0]) @@ -166,8 +179,8 @@ class RemoteTestCase(test.CdistTestCase): os.chmod(remote_exec_path, 0o755) remote_exec = remote_exec_path remote_copy = "echo" - r = remote.Remote(self.target_host, base_path=self.base_path, - remote_exec=remote_exec, remote_copy=remote_copy) + r = self.create_remote(remote_exec=remote_exec, + remote_copy=remote_copy) handle, script = self.mkstemp(dir=self.temp_dir) with os.fdopen(handle, "w") as fd: fd.writelines(["#!/bin/sh\n", "true"]) @@ -188,8 +201,8 @@ class RemoteTestCase(test.CdistTestCase): os.chmod(remote_exec_path, 0o755) remote_exec = remote_exec_path remote_copy = "echo" - r = remote.Remote(self.target_host, base_path=self.base_path, - remote_exec=remote_exec, remote_copy=remote_copy) + r = self.create_remote(remote_exec=remote_exec, + remote_copy=remote_copy) output = r.run_script(script, return_output=True) self.assertEqual(output, "no_env\n") @@ -201,11 +214,12 @@ class RemoteTestCase(test.CdistTestCase): env = { '__object': 'test_object', } - r = remote.Remote(self.target_host, base_path=self.base_path, - remote_exec=remote_exec, remote_copy=remote_copy) + r = self.create_remote(remote_exec=remote_exec, + remote_copy=remote_copy) output = r.run_script(script, env=env, return_output=True) self.assertEqual(output, "test_object\n") + if __name__ == '__main__': import unittest diff --git a/cdist/test/explorer/__init__.py b/cdist/test/explorer/__init__.py index fc66020d..1c4e4bc4 100644 --- a/cdist/test/explorer/__init__.py +++ b/cdist/test/explorer/__init__.py @@ -31,6 +31,7 @@ from cdist import test from cdist.exec import local from cdist.exec import remote from cdist.core import explorer +import logging import os.path as op my_dir = op.abspath(op.dirname(__file__)) @@ -50,6 +51,7 @@ class ExplorerClassTestCase(test.CdistTestCase): self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=base_root_path, host_dir_name=hostdir, exec_path=test.cdist_exec_path, @@ -62,7 +64,9 @@ class ExplorerClassTestCase(test.CdistTestCase): target_host=self.target_host, remote_exec=self.remote_exec, remote_copy=self.remote_copy, - base_path=self.remote_base_path) + base_path=self.remote_base_path, + stdout_base_path=self.local.stdout_base_path, + stderr_base_path=self.local.stderr_base_path) self.remote.create_files_dirs() self.explorer = explorer.Explorer( @@ -209,6 +213,33 @@ class ExplorerClassTestCase(test.CdistTestCase): self.assertEqual(names, output) shutil.rmtree(out_path) + def test_explorer_environment(self): + cdist_type = core.CdistType(self.local.type_path, '__dump_env') + cdist_object = core.CdistObject(cdist_type, self.local.object_path, + self.local.object_marker_name, + 'whatever') + self.explorer.transfer_type_explorers(cdist_type) + output = self.explorer.run_type_explorer('dump', cdist_object) + + output_dict = {} + for line in output.split('\n'): + if line: + key, value = line.split(': ') + output_dict[key] = value + self.assertEqual(output_dict['__target_host'], + self.local.target_host[0]) + self.assertEqual(output_dict['__target_hostname'], + self.local.target_host[1]) + self.assertEqual(output_dict['__target_fqdn'], + self.local.target_host[2]) + self.assertEqual(output_dict['__explorer'], + self.remote.global_explorer_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) + self.assertEqual(output_dict['__cdist_log_level'], + str(logging.WARNING)) + self.assertEqual(output_dict['__cdist_log_level_name'], 'WARNING') + if __name__ == '__main__': import unittest diff --git a/cdist/test/explorer/fixtures/conf/type/__dump_env/explorer/dump b/cdist/test/explorer/fixtures/conf/type/__dump_env/explorer/dump new file mode 100755 index 00000000..3682816b --- /dev/null +++ b/cdist/test/explorer/fixtures/conf/type/__dump_env/explorer/dump @@ -0,0 +1,9 @@ +#!/bin/sh + +echo "__target_host: $__target_host" +echo "__target_hostname: $__target_hostname" +echo "__target_fqdn: $__target_fqdn" +echo "__explorer: $__explorer" +echo "__target_host_tags: $__target_host_tags" +echo "__cdist_log_level: $__cdist_log_level" +echo "__cdist_log_level_name: $__cdist_log_level_name" diff --git a/cdist/test/inventory/__init__.py b/cdist/test/inventory/__init__.py new file mode 100644 index 00000000..287c855c --- /dev/null +++ b/cdist/test/inventory/__init__.py @@ -0,0 +1,477 @@ +# -*- coding: utf-8 -*- +# +# 2016 Darko Poljak (darko.poljak at gmail.com) +# +# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +# + +import os +import shutil +import cdist +import os.path as op +import unittest +import sys +from cdist import test +from cdist import inventory +from io import StringIO + +my_dir = op.abspath(op.dirname(__file__)) +fixtures = op.join(my_dir, 'fixtures') +inventory_dir = op.join(fixtures, "inventory") + + +class InventoryTestCase(test.CdistTestCase): + + def _create_host_with_tags(self, host, tags): + os.makedirs(inventory_dir, exist_ok=True) + hostfile = op.join(inventory_dir, host) + with open(hostfile, "w") as f: + for x in tags: + f.write("{}\n".format(x)) + + def setUp(self): + self.maxDiff = None + self.db = { + "loadbalancer1": ["loadbalancer", "all", "europe", ], + "loadbalancer2": ["loadbalancer", "all", "europe", ], + "loadbalancer3": ["loadbalancer", "all", "africa", ], + "loadbalancer4": ["loadbalancer", "all", "africa", ], + "web1": ["web", "all", "static", ], + "web2": ["web", "all", "dynamic", ], + "web3": ["web", "all", "dynamic", ], + "shell1": ["shell", "all", "free", ], + "shell2": ["shell", "all", "free", ], + "shell3": ["shell", "all", "charge", ], + "shell4": ["shell", "all", "charge", ], + "monty": ["web", "python", "shell", ], + "python": ["web", "python", "shell", ], + } + for x in self.db: + self.db[x] = sorted(self.db[x]) + for host in self.db: + self._create_host_with_tags(host, self.db[host]) + self.sys_stdout = sys.stdout + out = StringIO() + sys.stdout = out + + def _get_output(self): + sys.stdout.flush() + output = sys.stdout.getvalue().strip() + return output + + def tearDown(self): + sys.stdout = self.sys_stdout + shutil.rmtree(inventory_dir) + + def test_inventory_create_db(self): + dbdir = op.join(fixtures, "foo") + inv = inventory.Inventory(db_basedir=dbdir) + self.assertTrue(os.path.isdir(dbdir)) + self.assertEqual(inv.db_basedir, dbdir) + shutil.rmtree(inv.db_basedir) + + # InventoryList + def test_inventory_list_print(self): + invList = inventory.InventoryList(db_basedir=inventory_dir) + invList.run() + output = self._get_output() + self.assertTrue(' ' in output) + + def test_inventory_list_print_host_only(self): + invList = inventory.InventoryList(db_basedir=inventory_dir, + list_only_host=True) + invList.run() + output = self._get_output() + self.assertFalse(' ' in output) + + def test_inventory_list_all(self): + invList = inventory.InventoryList(db_basedir=inventory_dir) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + self.assertEqual(db, self.db) + + def test_inventory_list_by_host_hosts(self): + hosts = ("web1", "web2", "web3",) + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = {host: sorted(self.db[host]) for host in hosts} + self.assertEqual(db, expected_db) + + def test_inventory_list_by_host_hostfile(self): + hosts = ("web1", "web2", "web3",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hosts: + f.write("{}\n".format(x)) + invList = inventory.InventoryList(db_basedir=inventory_dir, + hostfile=hostfile) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = {host: sorted(self.db[host]) for host in hosts} + self.assertEqual(db, expected_db) + os.remove(hostfile) + + def test_inventory_list_by_host_hosts_hostfile(self): + hosts = ("shell1", "shell4",) + hostsf = ("web1", "web2", "web3",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostsf: + f.write("{}\n".format(x)) + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts, hostfile=hostfile) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + import itertools + expected_db = {host: sorted(self.db[host]) for host in + itertools.chain(hostsf, hosts)} + self.assertEqual(db, expected_db) + os.remove(hostfile) + + def _gen_expected_db_for_tags(self, tags): + db = {} + for host in self.db: + for tag in tags: + if tag in self.db[host]: + db[host] = self.db[host] + break + return db + + def _gen_expected_db_for_has_all_tags(self, tags): + db = {} + for host in self.db: + if set(tags).issubset(set(self.db[host])): + db[host] = self.db[host] + return db + + def test_inventory_list_by_tag_hosts(self): + tags = ("web", "shell",) + invList = inventory.InventoryList(db_basedir=inventory_dir, + istag=True, hosts=tags) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = self._gen_expected_db_for_tags(tags) + self.assertEqual(db, expected_db) + + def test_inventory_list_by_tag_hostfile(self): + tags = ("web", "shell",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tags: + f.write("{}\n".format(x)) + invList = inventory.InventoryList(db_basedir=inventory_dir, + istag=True, hostfile=tagfile) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = self._gen_expected_db_for_tags(tags) + self.assertEqual(db, expected_db) + os.remove(tagfile) + + def test_inventory_list_by_tag_hosts_hostfile(self): + tags = ("web", "shell",) + tagsf = ("dynamic", "europe",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + invList = inventory.InventoryList(db_basedir=inventory_dir, + istag=True, hosts=tags, + hostfile=tagfile) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + import itertools + expected_db = self._gen_expected_db_for_tags(tags + tagsf) + self.assertEqual(db, expected_db) + os.remove(tagfile) + + def test_inventory_list_by_tag_has_all_tags(self): + tags = ("web", "python", "shell",) + invList = inventory.InventoryList(db_basedir=inventory_dir, + istag=True, hosts=tags, + has_all_tags=True) + entries = invList.entries() + db = {host: sorted(tags) for host, tags in entries} + expected_db = self._gen_expected_db_for_has_all_tags(tags) + self.assertEqual(db, expected_db) + + # InventoryHost + def test_inventory_host_add_hosts(self): + hosts = ("spam", "eggs", "foo",) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="add", hosts=hosts) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir) + expected_hosts = tuple(x for x in invList.host_entries() if x in hosts) + self.assertEqual(sorted(hosts), sorted(expected_hosts)) + + def test_inventory_host_add_hostfile(self): + hosts = ("spam-new", "eggs-new", "foo-new",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hosts: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="add", hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir) + expected_hosts = tuple(x for x in invList.host_entries() if x in hosts) + self.assertEqual(sorted(hosts), sorted(expected_hosts)) + os.remove(hostfile) + + def test_inventory_host_add_hosts_hostfile(self): + hosts = ("spam-spam", "eggs-spam", "foo-spam",) + hostf = ("spam-eggs-spam", "spam-foo-spam",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostf: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="add", hosts=hosts, + hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostf) + expected_hosts = tuple(invList.host_entries()) + self.assertEqual(sorted(hosts + hostf), sorted(expected_hosts)) + os.remove(hostfile) + + def test_inventory_host_del_hosts(self): + hosts = ("web1", "shell1",) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", hosts=hosts) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts) + expected_hosts = tuple(invList.host_entries()) + self.assertTupleEqual(expected_hosts, ()) + + def test_inventory_host_del_hostfile(self): + hosts = ("loadbalancer3", "loadbalancer4",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hosts: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts) + expected_hosts = tuple(invList.host_entries()) + self.assertTupleEqual(expected_hosts, ()) + os.remove(hostfile) + + def test_inventory_host_del_hosts_hostfile(self): + hosts = ("loadbalancer1", "loadbalancer2",) + hostf = ("web2", "shell2",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostf: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", hosts=hosts, + hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostf) + expected_hosts = tuple(invList.host_entries()) + self.assertTupleEqual(expected_hosts, ()) + os.remove(hostfile) + + @unittest.expectedFailure + def test_inventory_host_invalid_host(self): + try: + invalid_hostfile = op.join(inventory_dir, "invalid") + os.mkdir(invalid_hostfile) + hosts = ("invalid",) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", hosts=hosts) + invHost.run() + except e: + os.rmdir(invalid_hostfile) + raise e + + # InventoryTag + def test_inventory_tag_init(self): + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add") + self.assertTrue(invTag.allhosts) + self.assertEqual(invTag.tagfile, "-") + + def test_inventory_tag_stdin_multiple_hosts(self): + try: + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tagfile="-", + hosts=("host1", "host2",)) + except e: + self.fail() + + def test_inventory_tag_stdin_hostfile(self): + try: + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tagfile="-", + hostfile="hosts") + except e: + self.fail() + + @unittest.expectedFailure + def test_inventory_tag_stdin_both(self): + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tagfile="-", + hostfile="-") + + def test_inventory_tag_add_for_all_hosts(self): + tags = ("spam-spam-spam", "spam-spam-eggs",) + tagsf = ("spam-spam-spam-eggs", "spam-spam-eggs-spam",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tags=tags, + tagfile=tagfile) + invTag.run() + invList = inventory.InventoryList(db_basedir=inventory_dir) + failed = False + for host, taglist in invList.entries(): + for x in tagsf + tags: + if x not in taglist: + failed = True + break + if failed: + break + os.remove(tagfile) + if failed: + self.fail() + + def test_inventory_tag_add(self): + tags = ("spam-spam-spam", "spam-spam-eggs",) + tagsf = ("spam-spam-spam-eggs", "spam-spam-eggs-spam",) + hosts = ("loadbalancer1", "loadbalancer2", "shell2",) + hostsf = ("web2", "web3",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostsf: + f.write("{}\n".format(x)) + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="add", tags=tags, + tagfile=tagfile, hosts=hosts, + hostfile=hostfile) + invTag.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostsf) + failed = False + for host, taglist in invList.entries(): + if host not in hosts + hostsf: + failed = True + break + for x in tagsf + tags: + if x not in taglist: + failed = True + break + if failed: + break + os.remove(tagfile) + os.remove(hostfile) + if failed: + self.fail() + + def test_inventory_tag_del_for_all_hosts(self): + tags = ("all",) + tagsf = ("charge",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="del", tags=tags, + tagfile=tagfile) + invTag.run() + invList = inventory.InventoryList(db_basedir=inventory_dir) + failed = False + for host, taglist in invList.entries(): + for x in tagsf + tags: + if x in taglist: + failed = True + break + if failed: + break + os.remove(tagfile) + if failed: + self.fail() + + def test_inventory_tag_del(self): + tags = ("europe", "africa",) + tagsf = ("free", ) + hosts = ("loadbalancer1", "loadbalancer2", "shell2",) + hostsf = ("web2", "web3",) + tagfile = op.join(fixtures, "tags") + with open(tagfile, "w") as f: + for x in tagsf: + f.write("{}\n".format(x)) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostsf: + f.write("{}\n".format(x)) + invTag = inventory.InventoryTag(db_basedir=inventory_dir, + action="del", tags=tags, + tagfile=tagfile, hosts=hosts, + hostfile=hostfile) + invTag.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostsf) + failed = False + for host, taglist in invList.entries(): + if host not in hosts + hostsf: + failed = True + break + for x in tagsf + tags: + if x in taglist: + failed = True + break + if failed: + break + os.remove(tagfile) + os.remove(hostfile) + if failed: + self.fail() + + def test_inventory_tag_del_all_tags(self): + hosts = ("web3", "shell1",) + hostsf = ("shell2", "loadbalancer1",) + hostfile = op.join(fixtures, "hosts") + with open(hostfile, "w") as f: + for x in hostsf: + f.write("{}\n".format(x)) + invHost = inventory.InventoryHost(db_basedir=inventory_dir, + action="del", all=True, + hosts=hosts, hostfile=hostfile) + invHost.run() + invList = inventory.InventoryList(db_basedir=inventory_dir, + hosts=hosts + hostsf) + for host, htags in invList.entries(): + self.assertEqual(htags, ()) + os.remove(hostfile) + + +if __name__ == "__main__": + unittest.main() diff --git a/cdist/test/manifest/__init__.py b/cdist/test/manifest/__init__.py index 3e07c1a7..68e777a4 100644 --- a/cdist/test/manifest/__init__.py +++ b/cdist/test/manifest/__init__.py @@ -53,6 +53,7 @@ class ManifestTestCase(test.CdistTestCase): base_root_path = os.path.join(out_path, hostdir) self.local = local.Local( target_host=self.target_host, + target_host_tags=self.target_host_tags, base_root_path=base_root_path, host_dir_name=hostdir, exec_path=cdist.test.cdist_exec_path, @@ -72,7 +73,10 @@ class ManifestTestCase(test.CdistTestCase): handle, output_file = self.mkstemp(dir=self.temp_dir) os.close(handle) os.environ['__cdist_test_out'] = output_file - self.manifest.run_initial_manifest(initial_manifest) + old_loglevel = logging.root.getEffectiveLevel() + self.log.setLevel(logging.VERBOSE) + manifest = cdist.core.manifest.Manifest(self.target_host, self.local) + manifest.run_initial_manifest(initial_manifest) with open(output_file, 'r') as fd: output_string = fd.read() @@ -93,16 +97,26 @@ class ManifestTestCase(test.CdistTestCase): self.local.type_path) self.assertEqual(output_dict['__manifest'], self.local.manifest_path) self.assertEqual(output_dict['__files'], self.local.files_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) + self.assertEqual(output_dict['__cdist_log_level'], + str(logging.VERBOSE)) + self.assertEqual(output_dict['__cdist_log_level_name'], 'VERBOSE') + self.log.setLevel(old_loglevel) def test_type_manifest_environment(self): cdist_type = core.CdistType(self.local.type_path, '__dump_environment') cdist_object = core.CdistObject(cdist_type, self.local.object_path, self.local.object_marker_name, 'whatever') + cdist_object.create() handle, output_file = self.mkstemp(dir=self.temp_dir) os.close(handle) os.environ['__cdist_test_out'] = output_file - self.manifest.run_type_manifest(cdist_object) + old_loglevel = self.log.getEffectiveLevel() + self.log.setLevel(logging.VERBOSE) + manifest = cdist.core.manifest.Manifest(self.target_host, self.local) + manifest.run_type_manifest(cdist_object) with open(output_file, 'r') as fd: output_string = fd.read() @@ -126,12 +140,22 @@ class ManifestTestCase(test.CdistTestCase): self.assertEqual(output_dict['__object_id'], cdist_object.object_id) self.assertEqual(output_dict['__object_name'], cdist_object.name) self.assertEqual(output_dict['__files'], self.local.files_path) + self.assertEqual(output_dict['__target_host_tags'], + self.local.target_host_tags) + self.assertEqual(output_dict['__cdist_log_level'], + str(logging.VERBOSE)) + self.assertEqual(output_dict['__cdist_log_level_name'], 'VERBOSE') + self.log.setLevel(old_loglevel) - def test_debug_env_setup(self): + def test_loglevel_env_setup(self): current_level = self.log.getEffectiveLevel() self.log.setLevel(logging.DEBUG) manifest = cdist.core.manifest.Manifest(self.target_host, self.local) - self.assertTrue("__cdist_debug" in manifest.env) + self.assertTrue("__cdist_log_level" in manifest.env) + self.assertTrue("__cdist_log_level_name" in manifest.env) + self.assertEqual(manifest.env["__cdist_log_level"], + str(logging.DEBUG)) + self.assertEqual(manifest.env["__cdist_log_level_name"], 'DEBUG') self.log.setLevel(current_level) diff --git a/cdist/test/manifest/fixtures/conf/manifest/dump_environment b/cdist/test/manifest/fixtures/conf/manifest/dump_environment index 702145e2..8057ca42 100755 --- a/cdist/test/manifest/fixtures/conf/manifest/dump_environment +++ b/cdist/test/manifest/fixtures/conf/manifest/dump_environment @@ -9,4 +9,7 @@ __global: $__global __cdist_type_base_path: $__cdist_type_base_path __manifest: $__manifest __files: $__files +__target_host_tags: $__target_host_tags +__cdist_log_level: $__cdist_log_level +__cdist_log_level_name: $__cdist_log_level_name DONE diff --git a/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest b/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest index 757d07b5..a38050f9 100755 --- a/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest +++ b/cdist/test/manifest/fixtures/conf/type/__dump_environment/manifest @@ -13,4 +13,7 @@ __object: $__object __object_id: $__object_id __object_name: $__object_name __files: $__files +__target_host_tags: $__target_host_tags +__cdist_log_level: $__cdist_log_level +__cdist_log_level_name: $__cdist_log_level_name DONE diff --git a/cdist/util/fsproperty.py b/cdist/util/fsproperty.py index e458fd9e..5a27c9d7 100644 --- a/cdist/util/fsproperty.py +++ b/cdist/util/fsproperty.py @@ -58,7 +58,7 @@ class FileList(collections.MutableSequence): with open(self.path) as fd: for line in fd: lines.append(line.rstrip('\n')) - except EnvironmentError as e: + except EnvironmentError: # error ignored pass return lines @@ -127,7 +127,16 @@ class DirectoryDict(collections.MutableMapping): def __getitem__(self, key): try: with open(os.path.join(self.path, key), "r") as fd: - return fd.read().rstrip('\n') + value = fd.read().splitlines() + # if there is no value/empty line then return '' + # if there is only one value then return that value + # if there are multiple lines in file then return list + if not value: + return '' + elif len(value) == 1: + return value[0] + else: + return value except EnvironmentError: raise KeyError(key) diff --git a/cdist/util/ipaddr.py b/cdist/util/ipaddr.py index 71477682..9b730225 100644 --- a/cdist/util/ipaddr.py +++ b/cdist/util/ipaddr.py @@ -23,7 +23,13 @@ import socket import logging -def resolve_target_addresses(host): +def resolve_target_addresses(host, family=0): + host_name = resolve_target_host_name(host, family) + host_fqdn = resolve_target_fqdn(host) + return (host, host_name, host_fqdn) + + +def resolve_target_host_name(host, family=0): log = logging.getLogger(host) try: # getaddrinfo returns a list of 5-tuples: @@ -32,29 +38,32 @@ def resolve_target_addresses(host): # (address, port) for AF_INET, # (address, port, flow_info, scopeid) for AF_INET6 ip_addr = socket.getaddrinfo( - host, None, type=socket.SOCK_STREAM)[0][4][0] + host, None, family=family, type=socket.SOCK_STREAM)[0][4][0] # gethostbyaddr returns triple # (hostname, aliaslist, ipaddrlist) host_name = socket.gethostbyaddr(ip_addr)[0] log.debug("derived host_name for host \"{}\": {}".format( host, host_name)) except (socket.gaierror, socket.herror) as e: - log.warn("Could not derive host_name for {}" - ", $host_name will be empty. Error is: {}".format(host, e)) + log.warning("Could not derive host_name for {}" + ", $host_name will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_name = '' + return host_name + +def resolve_target_fqdn(host): + log = logging.getLogger(host) try: host_fqdn = socket.getfqdn(host) log.debug("derived host_fqdn for host \"{}\": {}".format( host, host_fqdn)) except socket.herror as e: - log.warn("Could not derive host_fqdn for {}" - ", $host_fqdn will be empty. Error is: {}".format(host, e)) + log.warning("Could not derive host_fqdn for {}" + ", $host_fqdn will be empty. Error is: {}".format(host, e)) # in case of error provide empty value host_fqdn = '' - - return (host, host_name, host_fqdn) + return host_fqdn # check whether addr is IPv6 diff --git a/cdist/util/remoteutil.py b/cdist/util/remoteutil.py index 2bd12fdf..505c4598 100644 --- a/cdist/util/remoteutil.py +++ b/cdist/util/remoteutil.py @@ -36,7 +36,7 @@ def inspect_ssh_mux_opts(): wanted_mux_opts = { "ControlPath": "{}", "ControlMaster": "auto", - "ControlPersist": "10", + "ControlPersist": "2h", } mux_opts = " ".join([" -o {}={}".format( x, wanted_mux_opts[x]) for x in wanted_mux_opts]) diff --git a/completions/bash/cdist-completion.bash b/completions/bash/cdist-completion.bash index 1c4226c2..1311384a 100644 --- a/completions/bash/cdist-completion.bash +++ b/completions/bash/cdist-completion.bash @@ -5,8 +5,8 @@ _cdist() cur="${COMP_WORDS[COMP_CWORD]}" prev="${COMP_WORDS[COMP_CWORD-1]}" prevprev="${COMP_WORDS[COMP_CWORD-2]}" - opts="-h --help -d --debug -v --verbose -V --version" - cmds="banner shell config install" + opts="-h --help -q --quiet -v --verbose -V --version" + cmds="banner config install inventory shell" case "${prevprev}" in shell) @@ -18,6 +18,41 @@ _cdist() ;; esac ;; + inventory) + case "${prev}" in + list) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --invento/y -a --all -f --file -H --host-only \ + -t --tag" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + add-host) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -f --file" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + del-host) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -a --all -f --file" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + add-tag) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -f --file -T --tag-file -t --taglist" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + del-tag) + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -a --all -f --file -T --tag-file -t --taglist" + COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) + return 0 + ;; + esac + ;; esac case "${prev}" in @@ -26,23 +61,31 @@ _cdist() return 0 ;; banner) - opts="-h --help -d --debug -v --verbose" + opts="-h --help -q --quiet -v --verbose" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; shell) - opts="-h --help -d --debug -v --verbose -s --shell" + opts="-h --help -q --quiet -v --verbose -s --shell" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; config|install) - opts="-h --help -d --debug -v --verbose -b --beta \ - -c --conf-dir -f --file -i --initial-manifest -j --jobs \ - -n --dry-run -o --out-dir -p --parallel -s --sequential \ - --remote-copy --remote-exec" + opts="-h --help -q --quiet -v --verbose -b --beta \ + -I --inventory -C --cache-path-pattern -c --conf-dir \ + -f --file -i --initial-manifest -A --all-tagged \ + -j --jobs -n --dry-run -o --out-dir -p --parallel \ + -r --remote-out-dir \ + -s --sequential --remote-copy --remote-exec -t --tag -a --all" COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) ) return 0 ;; + inventory) + cmds="list add-host del-host add-tag del-tag" + opts="-h --help -q --quiet -v --verbose" + COMPREPLY=( $(compgen -W "${opts} ${cmds}" -- ${cur}) ) + return 0 + ;; *) ;; esac diff --git a/completions/zsh/_cdist b/completions/zsh/_cdist index 001356d4..405023ed 100644 --- a/completions/zsh/_cdist +++ b/completions/zsh/_cdist @@ -11,16 +11,16 @@ _cdist() case $state in opts_cmds) - _arguments '1:Options and commands:(banner config shell install -h --help -d --debug -v --verbose -V --version)' + _arguments '1:Options and commands:(banner config install inventory shell -h --help -q --quiet -v --verbose -V --version)' ;; *) case $words[2] in -*) - opts=(-h --help -d --debug -v --verbose -V --version) + opts=(-h --help -q --quiet -v --verbose -V --version) compadd "$@" -- $opts ;; banner) - opts=(-h --help -d --debug -v --verbose) + opts=(-h --help -q --quiet -v --verbose) compadd "$@" -- $opts ;; shell) @@ -30,16 +30,45 @@ _cdist() compadd "$@" -- $shells ;; *) - opts=(-h --help -d --debug -v --verbose -s --shell) + opts=(-h --help -q --quiet -v --verbose -s --shell) compadd "$@" -- $opts ;; esac ;; config|install) - opts=(-h --help -d --debug -v --verbose -b --beta -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -s --sequential --remote-copy --remote-exec) + opts=(-h --help -q --quiet -v --verbose -a --all -b --beta -C --cache-path-pattern -c --conf-dir -f --file -i --initial-manifest -j --jobs -n --dry-run -o --out-dir -p --parallel -r --remote-out-dir -s --sequential --remote-copy --remote-exec -t --tag -I --inventory -A --all-tagged) compadd "$@" -- $opts ;; - *) + inventory) + case $words[3] in + list) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file -H --host-only -t --tag) + compadd "$@" -- $opts + ;; + add-host) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -f --file) + compadd "$@" -- $opts + ;; + del-host) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file) + compadd "$@" -- $opts + ;; + add-tag) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -f --file -T --tag-file -t --taglist) + compadd "$@" -- $opts + ;; + del-tag) + opts=(-h --help -q --quiet -v --verbose -b --beta -I --inventory -a --all -f --file -T --tag-file -t --taglist) + compadd "$@" -- $opts + ;; + *) + cmds=(list add-host del-host add-tag del-tag) + opts=(-h --help -q --quiet -v --verbose) + compadd "$@" -- $cmds $opts + ;; + esac + ;; + *) ;; esac esac diff --git a/configuration/cdist.cfg.skeleton b/configuration/cdist.cfg.skeleton new file mode 100644 index 00000000..22c1ccaf --- /dev/null +++ b/configuration/cdist.cfg.skeleton @@ -0,0 +1,73 @@ +[GLOBAL] +# archiving +# Use specified archiving. Valid values include: +# none, tar, tgz, tbz2 and txz. +# archiving = tar +# +# beta +# Enable beta functionality. It recognizes boolean values from +# yes/no, on/off, true/false and 1/0 +# beta = no +# +# cache_path_pattern +# Specify cache path pattern. +# cache_path_pattern = %h +# +# conf_dir +# List of configuration directories separated with the character conventionally +# used by the operating system to separate search path components (as in PATH), +# such as ':' for POSIX or ';' for Windows. +# If also specified at command line then values from command line are +# appended to this value. +# conf_dir = : +# +# init_manifest +# Specify default initial manifest. +# init_mainfest = +# +# inventory_dir +# Specify inventory directory. +# inventory_dir = +# +# jobs +# Specify number of jobs for parallel processing. If -1 then the default, +# number of CPU's in the system is used. If 0 then parallel processing in +# jobs is disabled. If set to positive number then specified maximum +# number of processes will be used. +# jobs = 0 +# +# local_shell +# Shell command used for local execution. +# local_shell = /bin/sh +# +# out_path +# Directory to save cdist output in. +# out_path = +# +# parallel +# Process hosts in parallel. If -1 then the default, number of CPU's in +# the system is used. If 0 then parallel processing of hosts is disabled. +# If set to positive number then specified maximum number of processes +# will be used. +# parallel = 0 +# +# remote_copy +# Command to use for remote copy (should behave like scp). +# remote_copy = +# +# remote_exec +# Command to use for remote execution (should behave like ssh). +# remote_exec = +# +# remote_out_path +# Directory to save cdist output in on the target host. +# remote_out_path = /var/lib/cdist +# +# remote_shell +# Shell command at remote host used for remote execution. +# remote_shell = /bin/sh +# +# verbosity +# Set verbosity level. Valid values are: +# ERROR, WARNING, INFO, VERBOSE, DEBUG, TRACE and OFF. +# verbosity = INFO diff --git a/docs/changelog b/docs/changelog index 4a17e583..7594a6d4 100644 --- a/docs/changelog +++ b/docs/changelog @@ -1,10 +1,315 @@ Changelog --------- +5.1.3: 2019-08-30 + * Build: Overcome bash CDPATH when building docs (Dmitry Bogatov) + * Type __grafana_dashboard: Update distribution name, package signing key URI and repository URI (Dominique Roux) + * Type __letsencrypt_cert: Add Devuan Beowulf support (Nico Schottelius) + * Type __letsencrypt_cert: Fix Devuan Ascii: support (Nico Schottelius) + * Type __docker: Add devuan support (Dominique Roux) + * Type __docker_swarm: Fix for Docker 19.03 (Ľubomír Kučera) + +5.1.2: 2019-06-21 + * Core: Add support for type parameters deprecation (Darko Poljak) + * Type __acl: Rewrite and improve (Ander Punnar) + +5.1.1: 2019-05-28 + * Type __apt_key: Use gpg key, fallback to deprecated apt-key (Ander Punnar) + * Type __acl: Fix and improve (Ander Punnar) + * Documentation: Document type stdin inside loop caveats (Darko Poljak) + +5.1.0: 2019-05-22 + * Type __consul: Add alpine support (Nico Schottelius) + * Type __consul: Add version 1.5.0 (Nico Schottelius) + * Type __consul_agent: Add alpine support (Nico Schottelius) + * New helper script: cdist-new-type (Steven Armstrong, Darko Poljak) + * Core: Add support for deprecated type marker (Darko Poljak) + +5.0.2: 2019-05-17 + * Type __package_apk: Fix @repo handling in explorer (Nico Schottelius) + * Type __postfix: Add alpine support (Nico Schottelius) + * Type __postfix_postconf: Add alpine support (Nico Schottelius) + * Type __user: Add alpine support (Nico Schottelius) + * Core: Set __cdist_dry_run env var (Ander Punnar) + +5.0.1: 2019-05-09 + * Documentation: Add 'Perils of CDIST_ORDER_DEPENDENCY' sub-section (Darko Poljak) + * Build: Clean and separate end user targets into Makefile and maintainer targets into build-helper (Darko Poljak) + * Core: Update residual references to old cdist homepage (Darko Poljak) + * Documentation: Update residual references to old cdist homepage and git source (Darko Poljak) + * Type __cdist: Fix non working 'git://' protocol source (Darko Poljak) + +5.0.0: 2019-05-05 + * Type __zypper_service: Fix spelling error in manpage (Dmitry Bogatov) + * Explorer init: Add support for OpenBSD (sideeffect42) + * Type __postgres_database: Run psql with -w (no-password) (sideeffect42) + * Type __postgres_role: Run psql with -w (no-password) (sideeffect42) + * Type __block: Quote prefix/suffix - fix when prefix/suffix contains quotes (sideeffect42) + * Build: Update due to migration to code.ungleich.ch (Darko Poljak) + * Documentation: Update due to migration to code.ungleich.ch (Darko Poljak) + * Core: Detect and report dependency cycle as soon as possible (Darko Poljak) + * Core, documentation: Release -j/--jobs option, i.e. make it non-beta (Darko Poljak) + * Documentation: Update due to new cdist website (Darko Poljak) + * Build: Update due to new cdist website (Darko Poljak) + +4.11.1: 2019-04-22 + * Core: Improve explorer error reporting (Darko Poljak) + * Type __directory: explorer stat: add support for Solaris (Ander Punnar) + * Type __file: explorer stat: add support for Solaris (Ander Punnar) + * Type __ssh_authorized_keys: Remove legacy code (Ander Punnar) + * Explorer disks: Bugfix: do not break config in case of unsupported OS + which was introduced in 4.11.0, print message to stderr and empty disk list + to stdout instead (Darko Poljak) + +4.11.0: 2019-04-20 + * Type __package: Add __package_apk support (Nico Schottelius) + * Type __directory: Add alpine support (Nico Schottelius) + * Type __file: Add alpine support (Nico Schottelius) + * Type __hostname: Add alpine support (Nico Schottelius) + * Type __locale: Add alpine support (Nico Schottelius) + * Type __start_on_boot: Add alpine support (Nico Schottelius) + * Type __timezone: Add alpine support (Nico Schottelius) + * Type __start_on_boot: gentoo: check all runlevels in explorer (Nico Schottelius) + * New type: __package_apk (Nico Schottelius) + * Type __acl: Add support for ACL mask (Dimitrios Apostolou) + * Core: Fix circular dependency for CDIST_ORDER_DEPENDENCY (Darko Poljak) + * Type __acl: Improve the type (Ander Punnar) + * Explorer interfaces: Simplify code, be more compatible (Ander Punnar) + * Explorer disks: Remove assumable default/fallback, for now explicitly support only Linux and BSDs (Ander Punnar, Darko Poljak) + +4.10.11: 2019-04-13 + * Core: Fix broken quiet mode (Darko Poljak) + * Build: Add version.py into generated raw source archive (Darko Poljak) + * Explorer disks: Fix detecting disks, fix/add support for BSDs (Ander Punnar) + * Type __file: Fix stat explorer for BSDs (Ander Punnar) + * Type __directory: Fix stat explorer for BSDs (Ander Punnar) + +4.10.10: 2019-04-11 + * New types: __ufw and __ufw_rule (Mark Polyakov) + * Type __link: Add messaging (Ander Punnar) + * Debugging: Rename debug-dump.sh to cdist-dump (Darko Poljak) + * Documentation: Add cdist-dump man page (Darko Poljak) + +4.10.9: 2019-04-09 + * Type __ssh_authorized_keys: Properly handle multiple --option params (Steven Armstrong) + * Debugging: Add debug dump helper script (Darko Poljak) + * Type __file: Bugfix: fire onchange for present and exists states if no attribute is changed (Darko Poljak) + +4.10.8: 2019-04-06 + * Type __clean_path: Fix list explorer exit code if path not directory or does not exist (Ander Punnar) + * New type: __check_messages (Ander Punnar) + +4.10.7: 2019-03-30 + * Build: Migrate from pep8 to pycodestyle (Darko Poljak) + * Type __start_on_boot: Implement state absent for OpenBSD (Daniel Néri) + * Explorers cpu_cores, disks: Add support for OpenBSD (Daniel Néri) + * Type __staged_file: Use portable -p instead of --tmpdir for mktemp (Silas Silva) + * Type __line: Add onchange parameter (Ander Punnar) + * Type __file: Add onchange parameter (Ander Punnar) + * New type: __clean_path (Ander Punnar) + +4.10.6: 2019-02-15 + * Type __prometheus_alertmanager: Add startup flag (Dominique Roux) + * Types __zypper_repo, __zypper_service: Re-add the use of echo in explorers (Daniel Heule) + +4.10.5: 2018-12-21 + * Type __group: Fix/remove '--' from echo command (Dimitrios Apostolou) + * New type: __ping (Olliver Schinagl) + * Type __postgres_role: Fix broken syntax (Nico Schottelius, Darko Poljak) + * Type __consul_agent: Add Debian 9 support (Jin-Guk Kwon) + * Documentation: Fix broken links (Rage ) + * Type __docker: Add version parameter (Jonas Weber) + * Type __sysctl: Refactor for better OS support (Takashi Yoshi) + * Types __package_*: Add messaging upon installation/removal (Takashi Yoshi) + * Type __package_pkg_openbsd: Reworked (Takashi Yoshi) + +4.10.4: 2018-11-03 + * Core: Transfer all files of a directory at once instead of calling copy once per file (myeisha) + * Core: Add timestamp (optional) to log messages (Darko Poljak) + * Explorers and types: Fix shellcheck found problems and encountered bugs (Jonas Weber, Thomas Eckert, Darko Poljak) + * Build: Add shellcheck makefile target and check when doing release (Darko Poljak) + * Type __consul: Add newest versions (Dominique Roux) + * Type __user: Remove annoying output, handle state param gracefully, add messages for removal (Thomas Eckert) + * Core: Fix checking for conflicting parameters for multiple values parameters (Darko Poljak) + * Documentation: Various fixes (Thomas Eckert) + * Various types: Improve OpenBSD support (sideeffect42) + +4.10.3: 2018-09-23 + * New global explorer: os_release (Ľubomír Kučera) + * Type __docker: Update type, install docker CE (Ľubomír Kučera) + * Type __package_apt: Write a message when a package is installed or removed; shellcheck (Jonas Weber) + * Documentation: Add 'Dive into real world cdist' walkthrough chapter (Darko Poljak) + * Core: Remove duplicate remote mkdir calls in explorer transfer (myeisha) + +4.10.2: 2018-09-06 + * Type __letsencrypt_cert: Add support for devuan ascii (Darko Poljak) + * Type __systemd_unit: Fix minor issues and add masking unit files support (Adam Dej) + * Type __grafana_dashboard: Fix devuan ascii support (Dominique Roux) + * Type __apt_source: Add nonparallel marker (Darko Poljak) + * Type __package_update_index: Fix error when using OS not using apt (Stu Zhao) + * Type __package_update_index: Support --maxage for type pacman (Stu Zhao) + * Type __letsencrypt_cert: Fix explorers: check that certbot exists before using it (Darko Poljak) + +4.10.1: 2018-06-21 + * Type __letsencrypt_cert: Fix temp file location and removal (Darko Poljak) + * Type __line: Handle missing file in __line explorer gracefully (Jonas Weber) + * Documentation: Add env vars usage idiom for writing types (Darko Poljak) + +4.10.0: 2018-06-17 + * New type: __acl (Ander Punnar) + * Core: Disable config parser interpolation (Darko Poljak) + * Type __sysctl: Use sysctl.d location if exists (Darko Poljak) + * Type __line: Rewrite and support --before and --after (Steven Armstrong) + +4.9.1: 2018-05-30 + * New type: __install_coreos (Ľubomír Kučera) + * Type __consul_agent: Add LSB init header (Nico Schottelius) + * Type __package_yum: Fix explorer when name contains package name with exact version specified (Aleksandr Dinu) + * Type __letsencrypt_cert: Use object id as domain if domain param is not specified (Darko Poljak) + +4.9.0: 2018-05-17 + * Type __docker_stack: Use --with-registry-auth option (Ľubomír Kučera) + * New type: __docker_config (Ľubomír Kučera) + * New type: __docker_secret (Ľubomír Kučera) + * Type __letsencrypt_cert: Rewritten; WARN: breaks backward compatibility (Ľubomír Kučera) + * Core: Fix NameError: name 'cdist_object' is not defined (Darko Poljak) + +4.8.4: 2018-04-20 + * Documentation, type manpages: Fix spelling (Dmitry Bogatov) + * New explorer: is-freebsd-jail (Kamila Součková) + * Types __hostname, __start_on_boot, __sysctl: Support FreeBSD (Kamila Součková) + * Type __install_config: set environment variable to distinguish between + install-config and regular config (Steven Armstrong) + * Core: Improve error reporting (Darko Poljak) + +4.8.3: 2018-03-16 + * Type __key_value: Add onchange parameter (Kamila Součková) + * Types __prometheus_server, __prometheus_alertmanager, __grafana_dashboard: + Work with packages instead of go get, remove __daemontools dependency and clean up (Kamila Součková) + * Documentation: Fix manpage generation (Darko Poljak) + * New type: __docker_swarm (Ľubomír Kučera) + * New type: __docker_stack (Ľubomír Kučera) + +4.8.2: 2018-03-10 + * Core: Fix quiet argument access for bare cdist command (Darko Poljak) + +4.8.1: 2018-03-09 + * Type __consul: Add option for directly downloading on target host (Darko Poljak) + * Core: Add -4 and -6 params to force IPv4, IPv6 addresses respectively (Darko Poljak) + * Type __package_update_index: Fix messaging (Thomas Eckert) + * Type __package_dpkg: Add state parameter and messaging (Thomas Eckert) + * Core: Fix a case when HOME is set but empty (Darko Poljak) + * Core: Fix non-existent manifest non graceful handling (Darko Poljak) + * Core: Fix main and inventory parent argparse options (Darko Poljak) + * Core: Fix lost error info with parallel jobs (option -j) (Darko Poljak) + * Core: Fix determining beta value through configuration (Darko Poljak) + * Core: Fix determining save_output_streams value through configuration (Darko Poljak) + * Core: Support in-distribution config file (Darko Poljak) + * New type: __apt_default_release (Matthijs Kooijman) + * Type __file: Add pre-exists state (Matthijs Kooijman) + * Type __grafana_dashboard: Add support for stretch + ascii (Nico Schottelius) + * Core: Fix idna (getaddrinfo) unicode tracebak for invalid host name (Darko Poljak) + +4.8.0: 2018-02-14 + * Core: Skip empty lines in parameter files (Darko Poljak) + * Explorer memory: Support OpenBSD (Philippe Gregoire) + * Type __install_config: re-export cdist log level during installation (Steven Armstrong) + * Type __sysctl: Add support for CoreOS (Ľubomír Kučera) + * Type __systemd_unit: Various improvements (Ľubomír Kučera) + * Type __line: Support regex beginning with '-' (Philippe Gregoire) + * Type __letsencrypt_cert: Add nonparallel; make admin-email required (Kamila Součková) + * Type __package_pkgng_freebsd: Redirect stdout and stderr to /dev/null instead of closing them (michal-hanu-la) + * Type __daemontools: Make it more robust and clean up the code (Kamila Součková) + * Core: Save output streams (Steven Armstrong, Darko Poljak) + * Documentation: Add local cache overview (Darko Poljak) + * Type __systemd_unit: Fix handling stdin (Jonas Weber) + * Type __package_apt: Add --purge-if-absent parameter (Jonas Weber) + * Type __package_update_index: Add --maxage parameter for apt and add message if index was updated(Thomas Eckert) + * Type __motd: Support reading from stdin (Jonas Weber) + * Type __issue: Support reading from stdin (Jonas Weber) + * Type __package_apt: Add support for --version parameter (Darko Poljak) + * Type __letsencrypt_cert: Add --renew-hook parameter(Darko Poljak) + * Core: Support disabling saving output streams (Darko Poljak) + * Type __apt_source: Remove update index dependency; call index update in gencode-remote (Darko Poljak) + +4.7.3: 2017-11-10 + * Type __ccollect_source: Add create destination parameter (Dominique Roux) + * Type __ssh_authorized_key: Add messaging (Thomas Eckert) + * New type: __letsencrypt_cert (Nico Schottelius, Kamila Součková) + * Core: Warn about invalid type in conf dir and continue instead of error (Darko Poljak) + * New type: __systemd_unit (Ľubomír Kučera) + * Type __letsencrypt_cert: Add support for debian stretch (Daniel Tschada) + * Type __line: Fix a case for absent when line contains single quotes (Darko Poljak) + * Type __config_file: Fix onchange command not being executed (Ľubomír Kučera) + +4.7.2: 2017-10-22 + * Type __hostname: Add support for CoreOS (Ľubomír Kučera) + * Type __timezone: Add support for CoreOS (Ľubomír Kučera) + * Explorer os: Fix for devuan ascii (Kamila Součková) + +4.7.1: 2017-10-01 + * Type __line: Add messaging (Thomas Eckert) + * Documentation: Fix documentation for building custom man-pages from non-standard path (Thomas Eckert) + * Core: Fix running scripts with execute bit when name without path is specified (Ander Punnar) + * Type __process: Add messaging (Thomas Eckert) + +4.7.0: 2017-09-22 + * Core: Add configuration/config file support (Darko Poljak) + * Core: Implement simple integration API (unstable) (Darko Poljak) + * Explorer machine_type: Detect kvm on proxmox (Sven Wick) + * Types __prometheus_server, __prometheus_alertmanager: Bugfixes (Kamila Součková) + * New type: __prometheus_exporter (Kamila Součková) + * Type __daemontools: Improve it on FreeBSD (Kamila Součková) + * Type __package_pkg_openbsd: Fix use of --name (Philippe Gregoire) + * Type __package_pkg_openbsd: Fix pkg_version explorer (Philippe Gregoire) + * Type __prometheus_exporter: Fixes + go version bump (Kamila Součková) + * Core, types: __cdist_loglevel -> __cdist_log_level (Darko Poljak) + * Core, types: Add __cdist_log_level_name env var with vlaue of log level name (Darko Poljak) + * Core: Make cdist honor __cdist_log_level env var (Darko Poljak) + * Core: Add -l/--log-level option (Darko Poljak) + * Type __install_stage: Fix __debug -> __cdist_log_level (Darko Poljak) + * Documentation: Document __cdist_log_level (Darko Poljak) + * Core: Log ERROR to stderr and rest to stdout (Darko Poljak, Steven Armstrong) + * Type __ssh_authorized_key: Bugfix the case where invalid key clears a file and add key validation (Darko Poljak) + +4.6.1: 2017-08-30 + * Type __user: Explore with /etc files (passwd, group, shadow) (Philippe Gregoire) + * Explorer init: Use pgrep instead of ps for Linux (Philippe Gregoire) + * Type __apt_key_uri: Redirect stderr of apt-key to /dev/null (Mark Verboom) + * Type __package_pkg_openbsd: Support the empty flavor (Philippe Gregoire) + * Type __package_pkg_openbsd: Support using /etc/installurl (Philippe Gregoire) + * Type __user_groups: Support OpenBSD (Philippe Gregoire) + * Type __hostname: Allow hostnamectl to fail silently (Steven Armstrong) + * Type __install_config: Use default default __remote_{copy,exec} in custom __remote_{copy,exec} scripts (Steven Armstrong) + * Type __ssh_authorized_key: Fix removing ssh key that is last one in the file (Darko Poljak) + +4.6.0: 2017-08-25 + * Core: Add inventory functionality (Darko Poljak) + * Core: Expose inventory host tags in __target_host_tags env var (Darko Poljak) + * Type __timezone: Check current timezone before doing anything (Ander Punnar) + * Core: Add -p HOST_MAX argument (Darko Poljak) + * Core: Add archiving support for transferring directory - new -R beta option (Darko Poljak) + * Core: Fix ssh connection multiplexing race condition (Darko Poljak) + * Core: Fix emulator race conditions with -j option (Darko Poljak) + * Documentation: Cleanup (Darko Poljak) + * Explorer os: Get ID from /etc/os-release (Philippe Gregoire) + +4.5.0: 2017-07-20 + * Types: Fix install types (Steven Armstrong) + * Core: Add -r command line option for setting remote base path (Steven Armstrong) + * Core: Allow manifest and gencode scripts to be written in any language (Darko Poljak) + * Documentation: Improvements to the english and fix typos (Mesar Hameed) + * Core: Merge -C custom cache path pattern option from beta branch (Darko Poljak) + * Core: Improve and cleanup logging (Darko Poljak, Steven Armstrong) + * Core: Remove deprecated -d option (Darko Poljak) + * Type __file: If no --source then create only if there is no file (Ander Punnar) + * Core: Ignore directory entries that begin with dot('.') (Darko Poljak) + * Core: Fix parallel object prepare and run steps and add nonparallel type marker (Darko Poljak) + 4.4.4: 2017-06-16 * Core: Support -j parallelization for object prepare and object run (Darko Poljak) * Type __install_mkfs: mkfs.vfat does not support -q (Nico Schottelius) - * Types __go_get, __daemontools*, __prometheus*: Fix missing dependencies, fix arguments(Kamila Součková) + * Types __go_get, __daemontools*, __prometheus*: Fix missing dependencies, fix arguments (Kamila Součková) 4.4.3: 2017-06-13 * Type __golang_from_vendor: Install golang from https://golang.org/dl/ (Kamila Součková) diff --git a/docs/dev/github-gitlab-migration/release b/docs/dev/github-gitlab-migration/release new file mode 100755 index 00000000..c973f0f1 --- /dev/null +++ b/docs/dev/github-gitlab-migration/release @@ -0,0 +1,50 @@ +#!/bin/sh -e + +set -x + +printf "Enter tag name: " +read tag +printf "Enter repository authentication token: " +read token + +git tag -d "${tag}" || : + +git tag "${tag}" -m "Release ${tag}" +git push origin "${tag}" + +echo 'foo' > foo +echo 'foo signature' > foo.asc + +archivename="foo" + +project="poljakowski%2Fmy-cdist-testing" +sed_cmd='s/^.*"markdown":"\([^"]*\)".*$/\1/' + +# upload archive +response_archive=$(curl -f -X POST \ + -H "PRIVATE-TOKEN: ${token}" \ + -F "file=@${archivename}" \ + "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \ + | sed "${sed_cmd}") || exit 1 + +# upload archive signature +response_archive_sig=$(curl -f -X POST \ + -H "PRIVATE-TOKEN: ${token}" \ + -F "file=@${archivename}.asc" \ + "https://code.ungleich.ch/api/v4/projects/${project}/uploads" \ + | sed "${sed_cmd}") || exit 1 + +# make release +curl -f -X POST \ + -H "PRIVATE-TOKEN: ${token}" \ + -F "description=Release ${tag}
${response_archive}
${response_archive_sig}" \ + "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}/release" \ + || exit 1 + +# get tag +curl -f -X GET \ + -H "PRIVATE-TOKEN: ${token}" \ + "https://code.ungleich.ch/api/v4/projects/${project}/repository/tags/${tag}" \ + || exit 1 + +rm -f foo foo.asc diff --git a/docs/src/_static/cdist-logo.jpeg b/docs/src/_static/cdist-logo.jpeg new file mode 100644 index 00000000..9bfa2529 Binary files /dev/null and b/docs/src/_static/cdist-logo.jpeg differ diff --git a/docs/src/cdist-logo.png b/docs/src/_static/cdist-logo.png similarity index 100% rename from docs/src/cdist-logo.png rename to docs/src/_static/cdist-logo.png diff --git a/docs/web/cdist/pgp-key-EFD2AE4EC36B6901.asc b/docs/src/_static/pgp-key-EFD2AE4EC36B6901.asc similarity index 100% rename from docs/web/cdist/pgp-key-EFD2AE4EC36B6901.asc rename to docs/src/_static/pgp-key-EFD2AE4EC36B6901.asc diff --git a/docs/src/cdist-best-practice.rst b/docs/src/cdist-best-practice.rst index 493f1506..a91f2cc0 100644 --- a/docs/src/cdist-best-practice.rst +++ b/docs/src/cdist-best-practice.rst @@ -13,15 +13,18 @@ See sshd_config(5) and ssh-keygen(1). Speeding up ssh connections --------------------------- When connecting to a new host, the initial delay with ssh connections -is pretty big. You can work around this by -"sharing of multiple sessions over a single network connection" -(quote from ssh_config(5)). The following code is suitable for -inclusion into your ~/.ssh/config:: +is pretty big. As cdist makes many connections to each host successive +connections can be sped up by "sharing of multiple sessions over a single +network connection" (quote from ssh_config(5)). This is also called "connection +multiplexing". - Host * - ControlPath ~/.ssh/master-%l-%r@%h:%p - ControlMaster auto - ControlPersist 10 +Cdist implements this since v4.0.0 by executing ssh with the appropriate +options (`-o ControlMaster=auto -o ControlPath=/tmp//s -o +ControlPersist=2h`). + +Note that the sshd_config on the server can configure the maximum number of +parallel multiplexed connections this with `MaxSessions N` (N defaults to 10 +for OpenSSH v7.4). Speeding up shell execution @@ -97,7 +100,7 @@ Including a possible common base that is reused across the different sites:: git merge common -The following **.git/config** is taken from a a real world scenario:: +The following **.git/config** is taken from a real world scenario:: # Track upstream, merge from time to time [remote "upstream"] @@ -142,7 +145,7 @@ implement this scenario with a gateway host and sudo: - Setup the ssh-pubkey for this user that has the right to configure all hosts - Create a wrapper to update the cdist configuration in ~cdist/cdist - Allow every developer to execute this script via sudo as the user cdist -- Allow run of cdist as user cdist on specific hosts on a per user/group base +- Allow run of cdist as user cdist on specific hosts on a per user/group basis. - f.i. nico ALL=(ALL) NOPASSWD: /home/cdist/bin/cdist config hostabc @@ -171,7 +174,7 @@ Templating } EOF -* in the manifest, export the relevant variables and add the following lines in your manifest: +* in the manifest, export the relevant variables and add the following lines to your manifest: .. code-block:: console @@ -213,11 +216,148 @@ Other content in cdist repository Usually the cdist repository contains all configuration items. Sometimes you may have additional resources that you would like to store in your central configuration -repositiory (like password files from KeepassX, +repository (like password files from KeepassX, Libreoffice diagrams, etc.). It is recommended to use a subfolder named "non-cdist" in the repository for such content: It allows you to -easily distinguish what is used by cdist and what not +easily distinguish what is used by cdist and what is not and also to store all important files in one repository. + + +Perils of CDIST_ORDER_DEPENDENCY +-------------------------------- +With CDIST_ORDER_DEPENDENCY all types are executed in the order in which they +are created in the manifest. The current created object automatically depends +on the previously created object. + +It essentially helps you to build up blocks of code that build upon each other +(like first creating the directory xyz than the file below the directory). + +This can be helpful, but it can also be the source of *evil*. + + +CDIST_ORDER_DEPENDENCY easily causes unobvious dependency cycles +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Let's see an example. Suppose you have special init manifest where among other +things you are assuring that remote host has packages `sudo` and `curl` +installed. + +**init1** + +.. code-block:: sh + + CDIST_ORDER_DEPENDENCY=1 + export CDIST_ORDER_DEPENDENCY + + for p in sudo curl + do + __package "${p}" + done + +Then you have some other special init manifest where among other things you are +assuring `sudo` package is installed. + +**init2** + +.. code-block:: sh + + CDIST_ORDER_DEPENDENCY=1 + export CDIST_ORDER_DEPENDENCY + + __package sudo + +Then you have third init manifest where you combine those two init manifests, +by including them: + +**init** + +.. code-block:: sh + + sh -e "$__manifest/init1" + sh -e "$__manifest/init2" + +The resulting init manifest is then equal to: + +.. code-block:: sh + + CDIST_ORDER_DEPENDENCY=1 + export CDIST_ORDER_DEPENDENCY + + for p in sudo curl + do + __package "${p}" + done + + CDIST_ORDER_DEPENDENCY=1 + export CDIST_ORDER_DEPENDENCY + + __package sudo + +In the end you get the following dependencies: + +* `__package/curl` depends on `__package/sudo` +* `__package/sudo` depends on `__package/curl` + +And here you have a circular dependency! + +In the real world manifest can be quite complex, dependencies can become +complicated and circual dependencies are not so obvious. Resolving it can +become cumbersome. + +**Practical solution?** + +Instead of managing complex init manifests you can write custom types. +Each custom type can do one thing, it has well defined dependencies that will +not leak into init manifest. In custom type you can also add special explorers +and gencode. + +Then, in init manifest you combine your complex types. It is: + +* cleaner +* easier to follow +* easier to maintain +* easier to debug. + + +CDIST_ORDER_DEPENDENCY kills parallelization +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Suppose you have defined CDIST_ORDER_DEPENDENCY and then, among other things, +you specify creation of three, by nature independent, files. + +**init** + +.. code-block:: sh + + CDIST_ORDER_DEPENDENCY=1 + export CDIST_ORDER_DEPENDENCY + + ... + __file /tmp/file1 + __file /tmp/file2 + __file /tmp/file3 + ... + +Due to defined CDIST_ORDER_DEPENDENCY cdist will execute them in specified order. +It is better to use CDIST_ORDER_DEPENDENCY in well defined blocks: + +**init** + +.. code-block:: sh + + CDIST_ORDER_DEPENDENCY=1 + export CDIST_ORDER_DEPENDENCY + ... + unset CDIST_ORDER_DEPENDENCY + + __file /tmp/file1 + __file /tmp/file2 + __file /tmp/file3 + + CDIST_ORDER_DEPENDENCY=1 + export CDIST_ORDER_DEPENDENCY + ... + unset CDIST_ORDER_DEPENDENCY diff --git a/docs/src/cdist-cache.rst b/docs/src/cdist-cache.rst new file mode 100644 index 00000000..0e5361ee --- /dev/null +++ b/docs/src/cdist-cache.rst @@ -0,0 +1,98 @@ +Local cache overview +==================== + +Description +----------- +While executing, cdist stores data to local cache. Currently this feature is +one way only. That means that cdist does not use stored data for future runs. +Anyway, those data can be used for debugging cdist, debugging types and +debugging after host configuration fails. + +Local cache is saved under $HOME/.cdist/cache directory, one directory entry +for each host. Subdirectory path is specified by +:strong:`-C/--cache-path-pattern` option, :strong:`cache_path_pattern` +configuration option or by using :strong:`CDIST_CACHE_PATH_PATTERN` +environment variable. + +For more info on cache path pattern see :strong:`CACHE PATH PATTERN FORMAT` +section in cdist man page. + + +Cache overview +-------------- +As noted above each configured host has got its subdirectory in local cache. +Entries in host's cache directory are as follows. + +bin + directory with cdist type emulators + +conf + dynamically determined cdist conf directory, union of all specified + conf directories + +explorer + directory containing global explorer named files containing explorer output + after running on target host + +messages + file containing messages + +object + directory containing subdirectory for each cdist object + +object_marker + object marker for this particular cdist run + +stderr + directory containing init manifest and remote stderr stream output + +stdout + directory containing init manifest and remote stdout stream output + +target_host + file containing target host of this cdist run, as specified when running + cdist + +typeorder + file containing types in order of execution. + + +Object cache overview +~~~~~~~~~~~~~~~~~~~~~ +Each object under :strong:`object` directory has its own structurue. + +code-local + code generated from gencode-local, present only if something is + generated + +code-remote + code generated from gencode-remote, present only if something is + generated + +explorer + directory containing type explorer named files containing explorer output + after running on target host + +files + directory with object files created during type execution + +parameter + directory containing type parameter named files containing parameter + values + +source + this type's source (init manifest) + +state + this type execution state ('done' when finished) + +stderr + directory containing type's manifest, gencode-* and code-* stderr stream + outputs + +stdin + this type stdin content + +stdout + directory containing type's manifest, gencode-* and code-* stdout stream + outputs. diff --git a/docs/src/cdist-configuration.rst b/docs/src/cdist-configuration.rst new file mode 100644 index 00000000..4c9b4d33 --- /dev/null +++ b/docs/src/cdist-configuration.rst @@ -0,0 +1,103 @@ +Configuration +============= + +Description +----------- +cdist obtains configuration data from the following sources in the following +order: + + #. command-line options + #. configuration file specified at command-line using -g command line option + #. configuration file specified in CDIST_CONFIG_FILE environment variable + #. environment variables + #. user's configuration file (first one found of ~/.cdist.cfg, $XDG_CONFIG_HOME/cdist/cdist.cfg, in specified order) + #. in-distribution configuration file (cdist/conf/cdist.cfg) + #. system-wide configuration file (/etc/cdist.cfg) + +if one exists. + +Configuration source with lower ordering number from above has a higher +precedence. Configuration option value read from source with higher +precedence will overwrite option value, if exists, read from source with +lower precedence. That means that command-line option wins them all. + +Users can decide on the local conifguration file location. It can be either +~/.cdist.cfg or $XDG_CONFIG_HOME/cdist/cdist.cfg. Note that, if both exist, +then ~/.cdist.cfg is used. + +For a per-project configuration, particular environment variables or better, +CDIST_CONFIG_FILE environment variable or -g CONFIG_FILE command line option, +can be used. + +Config file format +------------------ +cdist configuration file is in the INI file format. Currently it supports +only [GLOBAL] section. +The possible keywords and their meanings are as follows: + +:strong:`archiving` + Use specified archiving. Valid values include: + 'none', 'tar', 'tgz', 'tbz2' and 'txz'. + +:strong:`beta` + Enable beta functionality. It recognizes boolean values from + 'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0'. + +:strong:`cache_path_pattern` + Specify cache path pattern. + +:strong:`conf_dir` + List of configuration directories separated with the character conventionally + used by the operating system to separate search path components (as in PATH), + such as ':' for POSIX or ';' for Windows. + If also specified at command line then values from command line are + appended to this value. + +:strong:`init_manifest` + Specify default initial manifest. + +:strong:`inventory_dir` + Specify inventory directory. + +:strong:`jobs` + Specify number of jobs for parallel processing. If -1 then the default, + number of CPU's in the system is used. If 0 then parallel processing in + jobs is disabled. If set to positive number then specified maximum + number of processes will be used. + +:strong:`local_shell` + Shell command used for local execution. + +:strong:`out_path` + Directory to save cdist output in. + +:strong:`parallel` + Process hosts in parallel. If -1 then the default, number of CPU's in + the system is used. If 0 then parallel processing of hosts is disabled. + If set to positive number then specified maximum number of processes + will be used. + +:strong:`remote_copy` + Command to use for remote copy (should behave like scp). + +:strong:`remote_exec` + Command to use for remote execution (should behave like ssh). + +:strong:`remote_out_path` + Directory to save cdist output in on the target host. + +:strong:`remote_shell` + Shell command at remote host used for remote execution. + +:strong:`save_output_streams` + Enable/disable saving output streams (enabled by default). + It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false' + and '1'/'0'. + +:strong:`timestamp` + Timestamp log messages with the current local date and time + in the format: YYYYMMDDHHMMSS.us. + +:strong:`verbosity` + Set verbosity level. Valid values are: + 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'. diff --git a/docs/src/cdist-explorer.rst b/docs/src/cdist-explorer.rst index 4bb61d7a..a3c4f490 100644 --- a/docs/src/cdist-explorer.rst +++ b/docs/src/cdist-explorer.rst @@ -3,8 +3,8 @@ Explorer Description ----------- -Explorer are small shell scripts, which will be executed on the target -host. The aim of the explorer is to give hints to types on how to act on the +Explorers are small shell scripts, which will be executed on the target +host. The aim of each explorer is to give hints to types on how to act on the target system. An explorer outputs the result to stdout, which is usually a one liner, but may be empty or multi line especially in the case of type explorers. diff --git a/docs/src/cdist-features.rst b/docs/src/cdist-features.rst index 7018d248..be56fa22 100644 --- a/docs/src/cdist-features.rst +++ b/docs/src/cdist-features.rst @@ -8,7 +8,7 @@ Simplicity Design + Type and core cleanly separated - + Sticks completly to the KISS (keep it simple and stupid) paradigma + + Sticks completely to the KISS (keep it simple and stupid) paradigm + Meaningful error messages - do not lose time debugging error messages + Consistency in behaviour, naming and documentation + No surprise factor: Only do what is obviously clear, no magic @@ -40,9 +40,9 @@ Requirements, Simplicity UNIX Reuse of existing tools like cat, find, mv, ... -UNIX, familar environment, documentation +UNIX, familiar environment, documentation Is available as manpages and HTML -UNIX, simplicity, familar environment +UNIX, simplicity, familiar environment cdist is configured in POSIX shell diff --git a/docs/src/cdist-hacker.rst b/docs/src/cdist-hacker.rst index d7d6a056..e4252e19 100644 --- a/docs/src/cdist-hacker.rst +++ b/docs/src/cdist-hacker.rst @@ -4,7 +4,7 @@ Hacking Welcome ------- Welcome dear hacker! I invite you to a tour of pointers to -get into the usable configuration mangament system, cdist. +get into the usable configuration management system, cdist. The first thing to know is probably that cdist is brought to you by people who care about how code looks like and who think @@ -16,23 +16,23 @@ Reporting bugs -------------- If you believe you've found a bug and verified that it is in the latest version, drop a mail to the cdist mailing list, -subject prefixed with "[BUG] " or create an issue on github. +subject prefixed with "[BUG] " or create an issue on code.ungleich.ch. Coding conventions (everywhere) ------------------------------- -If something should be better done or needs to fixed, add the word FIXME +If something should be improved or needs to be fixed, add the word FIXME nearby, so grepping for FIXME gives all positions that need to be fixed. -Indention is 4 spaces (welcome to the python world). +Indentation is 4 spaces (welcome to the python world). How to submit stuff for inclusion into upstream cdist ----------------------------------------------------- -If you did some cool changes to cdist, which you value as a benefit for -everybody using cdist, you're welcome to propose inclusion into upstream. +If you did some cool changes to cdist, which you think might be of benefit to other +cdist users, you're welcome to propose inclusion into upstream. -There are though some requirements to ensure your changes don't break others +There are some requirements to ensure your changes don't break other peoples work nor kill the authors brain: - All files should contain the usual header (Author, Copying, etc.) @@ -51,7 +51,7 @@ work nor kill the authors brain: As soon as your work meets these requirements, write a mail for inclusion to the mailinglist **cdist-configuration-management at googlegroups.com** -or open a pull request at http://github.com/ungleich/cdist. +or open a merge request at https://code.ungleich.ch/ungleich-public/cdist. How to submit a new type @@ -59,10 +59,9 @@ How to submit a new type For detailed information about types, see `cdist type `_. Submitting a type works as described above, with the additional requirement -that a corresponding manpage named man.text in asciidoc format with +that a corresponding manpage named man.rst in ReSTructured text format with the manpage-name "cdist-type__NAME" is included in the type directory -AND asciidoc is able to compile it (i.e. do NOT have to many "=" in the second -line). +AND the manpage builds (`make man`). Warning: Submitting "exec" or "run" types that simply echo their parameter in **gencode** will not be accepted, because they are of no use. Every type can output @@ -77,7 +76,7 @@ The following workflow works fine for most developers .. code-block:: sh # get latest upstream master branch - git clone https://github.com/ungleich/cdist.git + git clone https://code.ungleich.ch/ungleich-public/cdist.git # update if already existing cd cdist; git fetch -v; git merge origin/master @@ -89,22 +88,22 @@ The following workflow works fine for most developers # *hack* *hack* - # clone the cdist repository on github if you haven't done so + # clone the cdist repository on code.ungleich.ch if you haven't done so # configure your repo to know about your clone (only once) - git remote add github git@github.com:YOURUSERNAME/cdist.git + git remote add ungleich git@code.ungleich.ch:YOURUSERNAME/cdist.git - # push the new branch to github - git push github documentation_cleanup + # push the new branch to ungleich gitlab + git push ungleich documentation_cleanup # (or everything) - git push --mirror github + git push --mirror ungleich - # create a pull request at github (use a browser) + # create a merge request at ungleich gitlab (use a browser) # *fixthingsbecausequalityassurancefoundissuesinourpatch* *hack* - # push code to github again + # push code to ungleich gitlab again git push ... # like above # add comment that everything should be green now (use a browser) @@ -130,7 +129,7 @@ use **git stash** to stash your changes away:: git fetch -v origin git merge origin/master -Similar when you want to develop another new feature, you go back +Similarly when you want to develop another new feature, you go back to the master branch and create another branch based on it:: .. code-block:: sh diff --git a/docs/src/cdist-install.rst b/docs/src/cdist-install.rst index 38db1a4e..a9b7d6b5 100644 --- a/docs/src/cdist-install.rst +++ b/docs/src/cdist-install.rst @@ -7,7 +7,7 @@ Requirements Source Host ~~~~~~~~~~~ -This is the machine you use to configure the target hosts. +This is the machine from which you will configure target hosts. * /bin/sh: A posix like shell (for instance bash, dash, zsh) * Python >= 3.2 @@ -36,10 +36,44 @@ To install cdist, execute the following commands: .. code-block:: sh - git clone https://github.com/ungleich/cdist.git + git clone https://code.ungleich.ch/ungleich-public/cdist.git cd cdist export PATH=$PATH:$(pwd -P)/bin +From version 4.2.0 cdist tags and releases are signed. +You can get GPG public key used for signing `here <_static/pgp-key-EFD2AE4EC36B6901.asc>`_. + +You can also get cdist from `github mirror `_. + +To install cdist with distutils from cloned repository, first you have to +create version.py: + +.. code-block:: sh + + ./bin/build-helper version + +Then you install it with: + +.. code-block:: sh + + make install + +or with: + +.. code-block:: sh + + make install-user + +to install it into user *site-packages* directory. +Or directly with distutils: + +.. code-block:: sh + + python setup.py install + +Note that `bin/build-helper` script is intended for cdist maintainers. + + Available versions in git ^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -60,14 +94,6 @@ So for instance if you want to use and stay with version 4.1, you can use git checkout -b 4.1 origin/4.1 -Git mirrors -^^^^^^^^^^^ - -If the main site is down, you can acquire cdist from one of the following sites: - - * git://github.com/telmich/cdist.git `github `_ - * git://git.code.sf.net/p/cdist/code `sourceforge `_ - Building and using documentation (man and html) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -117,7 +143,10 @@ some other custom .cdist directory, e.g. /opt/cdist then use: .. code-block:: sh - DOT_CDIST_PATH=/opt/cdist make dotman + make DOT_CDIST_PATH=/opt/cdist dotman + +Note that `dotman`-target has to be built before a `make docs`-run, otherwise +the custom man-pages are not picked up. Python package ~~~~~~~~~~~~~~ diff --git a/docs/src/cdist-integration.rst b/docs/src/cdist-integration.rst new file mode 100644 index 00000000..13880cd3 --- /dev/null +++ b/docs/src/cdist-integration.rst @@ -0,0 +1,47 @@ +cdist integration / using cdist as library +========================================== + +Description +----------- + +cdist can be integrate with other applications by importing cdist and other +cdist modules and setting all by hand. There are also helper functions which +aim to ease this integration. Just import **cdist.integration** and use its +functions: + +* :strong:`cdist.integration.configure_hosts_simple` for configuration +* :strong:`cdist.integration.install_hosts_simple` for installation. + +Functions require `host` and `manifest` parameters. +`host` can be specified as a string representing host or as iterable +of hosts. `manifest` is a path to initial manifest. For other cdist +options default values will be used. `verbose` is a desired verbosity +level which defaults to VERBOSE_INFO. `cdist_path` parameter specifies +path to cdist executable, if it is `None` then functions will try to +find it first from local lib directory and then in PATH. + +In case of cdist error :strong:`cdist.Error` exception is raised. + +:strong:`WARNING`: cdist integration helper functions are not yet stable! + +Examples +-------- + +.. code-block:: sh + + # configure host from python interactive shell + >>> import cdist.integration + >>> cdist.integration.configure_hosts_simple('185.203.114.185', + ... '~/.cdist/manifest/init') + + # configure host from python interactive shell, specifying verbosity level + >>> import cdist.integration + >>> cdist.integration.configure_hosts_simple( + ... '185.203.114.185', '~/.cdist/manifest/init', + ... verbose=cdist.argparse.VERBOSE_TRACE) + + # configure specified dns hosts from python interactive shell + >>> import cdist.integration + >>> hosts = ('dns1.ungleich.ch', 'dns2.ungleich.ch', 'dns3.ungleich.ch', ) + >>> cdist.integration.configure_hosts_simple(hosts, + ... '~/.cdist/manifest/init') diff --git a/docs/src/cdist-intro.rst b/docs/src/cdist-intro.rst deleted file mode 100644 index fad40fe5..00000000 --- a/docs/src/cdist-intro.rst +++ /dev/null @@ -1,15 +0,0 @@ -cdist - usable configuration management -======================================= - -.. image:: cdist-logo.png - :alt: cdist-logo - -cdist is a usable configuration management system. -It adheres to the KISS principle and -is being used in small up to enterprise grade environments. -cdist is an alternative to other configuration management systems like - -* `bcfg2 `_ -* `chef `_ -* `cfengine `_ -* `puppet `_. diff --git a/docs/src/cdist-inventory.rst b/docs/src/cdist-inventory.rst new file mode 100644 index 00000000..106fcdb6 --- /dev/null +++ b/docs/src/cdist-inventory.rst @@ -0,0 +1,211 @@ +Inventory +========= + +Introduction +------------ + +cdist comes with simple built-in tag based inventory. It is a simple inventory +with list of hosts and a host has a list of tags. +Inventory functionality is still in **beta** so it can be used only if beta +command line flag is specified (-b, --beta) or setting CDIST_BETA env var. + +Description +----------- + +The idea is to have simple tagging inventory. There is a list of hosts and for +each host there are tags. Inventory database is a set of files under inventory +database base directory. Filename equals hostname. Each file contains tags for +hostname with each tag on its own line. + +Using inventory you can now configure hosts by selecting them by tags. + +Tags have no values, as tags are just tags. Tag name-value would in this +context mean that host has two tags and it is selected by specifying that both +tags are present. + +This inventory is **KISS** cdist built-in inventory database. You can maintain it +using cdist inventory interface or using standard UNIX tools. + +cdist inventory interface +------------------------- + +With cdist inventory interface you can list host(s) and tag(s), add host(s), +add tag(s), delete host(s) and delete tag(s). + +Configuring hosts using inventory +--------------------------------- + +config command now has new options, **-t**, **-a** and **-A**. + +**-A** means that all hosts in tag db is selected. + +**-a** means that selected hosts must contain ALL specified tags. + +**-t** means that host specifies tag - all hosts that have specified tags are +selected. + +Examples +-------- + +.. code-block:: sh + + # List inventory content + $ cdist inventory list -b + + # List inventory for specified host localhost + $ cdist inventory list -b localhost + + # List inventory for specified tag loadbalancer + $ cdist inventory list -b -t loadbalancer + + # Add hosts to inventory + $ cdist inventory add-host -b web1 web2 web3 + + # Delete hosts from file old-hosts from inventory + $ cdist inventory del-host -b -f old-hosts + + # Add tags to specified hosts + $ cdist inventory add-tag -b -t europe,croatia,web,static web1 web2 + + # Add tag to all hosts in inventory + $ cdist inventory add-tag -b -t vm + + # Delete all tags from specified host + $ cdist inventory del-tag -b -a localhost + + # Delete tags read from stdin from hosts specified by file hosts + $ cdist inventory del-tag -b -T - -f hosts + + # Configure hosts from inventory with any of specified tags + $ cdist config -b -t web dynamic + + # Configure hosts from inventory with all specified tags + $ cdist config -b -t -a web dynamic + + # Configure all hosts from inventory db + $ cdist config -b -A + +Example of manipulating database +-------------------------------- + +.. code-block:: sh + + $ python3 scripts/cdist inventory list -b + $ python3 scripts/cdist inventory add-host -b localhost + $ python3 scripts/cdist inventory add-host -b test.mycloud.net + $ python3 scripts/cdist inventory list -b + localhost + test.mycloud.net + $ python3 scripts/cdist inventory add-host -b web1.mycloud.net web2.mycloud.net shell1.mycloud.net shell2.mycloud.net + $ python3 scripts/cdist inventory list -b + localhost + test.mycloud.net + web1.mycloud.net + web2.mycloud.net + shell1.mycloud.net + shell2.mycloud.net + $ python3 scripts/cdist inventory add-tag -b -t web web1.mycloud.net web2.mycloud.net + $ python3 scripts/cdist inventory add-tag -b -t shell shell1.mycloud.net shell2.mycloud.net + $ python3 scripts/cdist inventory add-tag -b -t cloud + $ python3 scripts/cdist inventory list -b + localhost cloud + test.mycloud.net cloud + web1.mycloud.net cloud,web + web2.mycloud.net cloud,web + shell1.mycloud.net cloud,shell + shell2.mycloud.net cloud,shell + $ python3 scripts/cdist inventory add-tag -b -t test,web,shell test.mycloud.net + $ python3 scripts/cdist inventory list -b + localhost cloud + test.mycloud.net cloud,shell,test,web + web1.mycloud.net cloud,web + web2.mycloud.net cloud,web + shell1.mycloud.net cloud,shell + shell2.mycloud.net cloud,shell + $ python3 scripts/cdist inventory del-tag -b -t shell test.mycloud.net + $ python3 scripts/cdist inventory list -b + localhost cloud + test.mycloud.net cloud,test,web + web1.mycloud.net cloud,web + web2.mycloud.net cloud,web + shell1.mycloud.net cloud,shell + shell2.mycloud.net cloud,shell + $ python3 scripts/cdist inventory add-tag -b -t all + $ python3 scripts/cdist inventory add-tag -b -t mistake + $ python3 scripts/cdist inventory list -b + localhost all,cloud,mistake + test.mycloud.net all,cloud,mistake,test,web + web1.mycloud.net all,cloud,mistake,web + web2.mycloud.net all,cloud,mistake,web + shell1.mycloud.net all,cloud,mistake,shell + shell2.mycloud.net all,cloud,mistake,shell + $ python3 scripts/cdist inventory del-tag -b -t mistake + $ python3 scripts/cdist inventory list -b + localhost all,cloud + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + shell1.mycloud.net all,cloud,shell + shell2.mycloud.net all,cloud,shell + $ python3 scripts/cdist inventory del-host -b localhost + $ python3 scripts/cdist inventory list -b + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + shell1.mycloud.net all,cloud,shell + shell2.mycloud.net all,cloud,shell + $ python3 scripts/cdist inventory list -b -t web + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + $ python3 scripts/cdist inventory list -b -t -a web test + test.mycloud.net all,cloud,test,web + $ python3 scripts/cdist inventory list -b -t -a web all + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + $ python3 scripts/cdist inventory list -b -t web all + test.mycloud.net all,cloud,test,web + web1.mycloud.net all,cloud,web + web2.mycloud.net all,cloud,web + shell1.mycloud.net all,cloud,shell + shell2.mycloud.net all,cloud,shell + $ cd cdist/inventory + $ ls -1 + shell1.mycloud.net + shell2.mycloud.net + test.mycloud.net + web1.mycloud.net + web2.mycloud.net + $ ls -l + total 20 + -rw-r--r-- 1 darko darko 16 Jun 24 12:43 shell1.mycloud.net + -rw-r--r-- 1 darko darko 16 Jun 24 12:43 shell2.mycloud.net + -rw-r--r-- 1 darko darko 19 Jun 24 12:43 test.mycloud.net + -rw-r--r-- 1 darko darko 14 Jun 24 12:43 web1.mycloud.net + -rw-r--r-- 1 darko darko 14 Jun 24 12:43 web2.mycloud.net + $ cat test.mycloud.net + test + all + web + cloud + $ cat web2.mycloud.net + all + web + cloud + +For more info about inventory commands and options see `cdist `_\ (1). + +Using external inventory +------------------------ + +cdist can be used with any external inventory where external inventory is +some storage or database from which you can get a list of hosts to configure. +cdist can then be fed with this list of hosts through stdin or file using +**-f** option. For example, if your host list is stored in sqlite3 database +hosts.db and you want to select hosts which purpose is **django** then you +can use it with cdist like: + +.. code-block:: sh + + $ sqlite3 hosts.db "select hostname from hosts where purpose = 'django';" | cdist config diff --git a/docs/src/cdist-manifest.rst b/docs/src/cdist-manifest.rst index b29cf0d8..4dd3e74b 100644 --- a/docs/src/cdist-manifest.rst +++ b/docs/src/cdist-manifest.rst @@ -51,7 +51,7 @@ The **initial manifest** is the entry point for cdist to find out, which **objects** to configure on the selected host. Cdist expects the initial manifest at **cdist/conf/manifest/init**. -Within this initial manifest you define, which objects should be +Within this initial manifest you define which objects should be created on which host. To distinguish between hosts, you can use the environment variable **__target_host** and/or **__target_hostname** and/or **__target_fqdn**. Let's have a look at a simple example:: @@ -114,7 +114,7 @@ requirements can be added white space separated. Above the "require" variable is only set for the command that is immediately following it. Dependencies should always be declared that way. -On line 4 you can see that the instantion of a type "\__link" object needs +On line 4 you can see that the instantiation of a type "\__link" object needs the object "__file/etc/cdist-configured" to be present, before it can proceed. This also means that the "\__link" command must make sure, that either @@ -149,7 +149,7 @@ All objects that are created in a type manifest are automatically required from the type that is calling them. This is called "autorequirement" in cdist jargon. -You can find an more in depth description of the flow execution of manifests +You can find a more in depth description of the flow execution of manifests in `cdist execution stages `_ and of how types work in `cdist type `_. @@ -163,6 +163,8 @@ automatically depends on the previously created object. It essentially helps you to build up blocks of code that build upon each other (like first creating the directory xyz than the file below the directory). +Read also about `perils of CDIST_ORDER_DEPENDENCY `_. + Overrides --------- diff --git a/docs/src/cdist-os.rst b/docs/src/cdist-os.rst index 4f6b4820..a8e31226 100644 --- a/docs/src/cdist-os.rst +++ b/docs/src/cdist-os.rst @@ -1,16 +1,19 @@ -Supported Operating Systems +Supported operating systems =========================== cdist was tested or is know to run on at least -* `Archlinux `_ -* `Debian `_ -* `CentOS `_ -* `Fedora `_ +* `Alpine Linux `_ +* `Archlinux `_ +* `CentOS `_ +* `Debian `_ +* `Devuan `_ +* `Fedora `_ * `FreeBSD `_ -* `Gentoo `_ -* `Mac OS X `_ +* `Gentoo `_ +* `Mac OS X `_ +* `NetBSD `_ * `OpenBSD `_ -* `Redhat `_ -* `Ubuntu `_ -* `XenServer `_ +* `Redhat `_ +* `Ubuntu `_ +* `XenServer `_ diff --git a/docs/src/cdist-parallelization.rst b/docs/src/cdist-parallelization.rst index ed3afae9..d458a128 100644 --- a/docs/src/cdist-parallelization.rst +++ b/docs/src/cdist-parallelization.rst @@ -12,8 +12,7 @@ The other way is to operate in parallel within one host where you specify the number of jobs. This is enabled with :strong:`-j/--jobs` option where you can specify the number of parallel jobs. By default, :strong:`multiprocessing.cpu_count()` is used. For this mode global explorers, -object preparation and object run are supported and this option is still in -:strong:`beta`. +object preparation and object run are supported. You can, of course, use those two options together. This means that each host will be processed by its own process. Within each process cdist will operate @@ -32,11 +31,11 @@ Examples # Configure hosts read from file hosts.file sequentially but using default # number of parallel jobs - $ cdist config -b -j -f hosts.file + $ cdist config -j -f hosts.file # Configure hosts read from file hosts.file in parallel using 16 # parallel jobs - $ cdist config -b -j 16 -p -f hosts.file + $ cdist config -j 16 -p -f hosts.file Caveats diff --git a/docs/src/cdist-quickstart.rst b/docs/src/cdist-quickstart.rst index 0020568d..99af869f 100644 --- a/docs/src/cdist-quickstart.rst +++ b/docs/src/cdist-quickstart.rst @@ -54,9 +54,7 @@ we can use cdist to configure it. You can copy and paste the following code into your shell to get started and configure localhost:: # Get cdist - # Mirrors can be found on - # http://www.nico.schottelius.org/software/cdist/install/#index2h4 - git clone git://github.com/ungleich/cdist + git clone git@code.ungleich.ch:ungleich-public/cdist.git # Create manifest (maps configuration to host(s) cd cdist diff --git a/docs/src/cdist-real-world.rst b/docs/src/cdist-real-world.rst new file mode 100644 index 00000000..8ccb0fc9 --- /dev/null +++ b/docs/src/cdist-real-world.rst @@ -0,0 +1,573 @@ +Dive into real world cdist +========================== + +Introduction +------------ + +This walkthrough shows real world cdist configuration example. + +Sample target host is named **test.ungleich.ch**. +Just replace **test.ungleich.ch** with your target hostname. + +Our goal is to configure python application hosting. For writing sample +application we will use `Bottle `_ WSGI micro web-framework. +It will use PostgreSQL database and it will list items from **items** table. +It will be served by uWSGI server. We will also use the Nginx web server +as a reverse proxy and we want HTTPS. +For HTTPS we will use Let's Encrypt certificate. + +For setting up hosting we want to use cdist so we will write a new type +for that. This type will: + +- install required packages +- create OS user, user home directory and application home directory +- create PostgreSQL database +- configure uWSGI +- configure Let's Encrypt certificate +- configure nginx. + +Our type will not create the actual python application. Its intention is only +to configure hosing for specified user and project. It is up to the user to +create his/her applications. + +So let's start. + +Creating type layout +-------------------- + +We will create a new custom type. Let's call it **__sample_bottle_hosting**. + +Go to **~/.cdist/type** directory (create it if it does not exist) and create +new type layout:: + + cd ~/.cdist/type + mkdir __sample_bottle_hosting + cd __sample_bottle_hosting + touch manifest gencode-remote + mkdir parameter + touch parameter/required + +Creating __sample_bottle_hosting type parameters +------------------------------------------------ + +Our type will be configurable through the means of parameters. Let's define +the following parameters: + +projectname + name for the project, needed for uWSGI ini file + +user + user name + +domain + target host domain, needed for Let's Encrypt certificate. + +We define parameters to make our type reusable for different projects, user and domain. + +Define required parameters:: + + printf "projectname\n" >> parameter/required + printf "user\n" >> parameter/required + printf "domain\n" >> parameter/required + +For details on type parameters see `Defining parameters `_. + +Creating __sample_bottle_hosting type manifest +---------------------------------------------- + +Next step is to define manifest (~/.cdist/type/__sample_bottle_hosting/manifest). +We also want our type to currently support only Devuan. So we will start by +checking target host OS. We will use `os `_ +global explorer:: + + os=$(cat "$__global/explorer/os") + + case "$os" in + devuan) + : + ;; + *) + echo "OS $os currently not supported" >&2 + exit 1 + ;; + esac + +If target host OS is not Devuan then we print error message to stderr +and exit. For other OS-es support we should check and change package names +we should install, because packages differ in different OS-es and in different +OS distributions like GNU/Linux distributions. There can also be a different +configuration locations (e.g. nginx config directory could be in /usr/local tree). +If we detected unsupported OS we should error out. cdist will stop configuration +process and output error message. + +Creating user and user directories +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Then we create user and his/her home directory and application home directory. +We will use existing cdist types `__user `_ and `__directory `_:: + + user="$(cat "$__object/parameter/user")" + home="/home/$user" + apphome="$home/app" + + # create user + __user "$user" --home "$home" --shell /bin/bash + # create user home dir + require="__user/$user" __directory "$home" \ + --owner "$user" --group "$user" --mode 0755 + # create app home dir + require="__user/$user __directory/$home" __directory "$apphome" \ + --state present --owner "$user" --group "$user" --mode 0755 + +First we define *user*, *home* and *apphome* variables. User is defined by type's +**user** parameter. Here we use **require** which is cdist's way to define dependencies. +User home directory should be created **after** user is created. And application +home directory is created **after** both user and user home directory are created. +For details on **require** see `Dependencies `_. + +Installing packages +~~~~~~~~~~~~~~~~~~~ + +Install required packages using existing `__package `_ type. +Before installing package we want to update apt package index using +`__apt_update_index `_:: + + # define packages that need to be installed + packages_to_install="nginx uwsgi-plugin-python3 python3-dev python3-pip postgresql postgresql-contrib libpq-dev python3-venv uwsgi python3-psycopg2" + + # update package index + __apt_update_index + # install packages + for package in $packages_to_install + do require="__apt_update_index" __package $package --state=present + done + +Here we use shell for loop. It executes **require="__apt_update_index" __package** +for each member in a list we define in **packages_to_install** variable. +This is much nicer then having as many **require="__apt_update_index" __package** +lines as there are packages we want to install. + +For python packages we use `__package_pip `_:: + + # install pip3 packages + for package in bottle bottle-pgsql; do + __package_pip --pip pip3 $package + done + +Creating PostgreSQL database +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Create PostgreSQL database using `__postgres_database `_ +and `__postgres_role `_ for creating database user:: + + #PostgreSQL db & user + postgres_server=postgresql + + # create PostgreSQL db user + require="__package/postgresql" __postgres_role $user --login --createdb + # create PostgreSQL db + require="__postgres_role/$user __package/postgresql" __postgres_database $user \ + --owner $user + +Configuring uWSGI +~~~~~~~~~~~~~~~~~ + +Configure uWSGI using `__file `_ type:: + + # configure uWSGI + projectname="$(cat "$__object/parameter/projectname")" + require="__package/uwsgi" __file /etc/uwsgi/apps-enabled/$user.ini \ + --owner root --group root --mode 0644 \ + --state present \ + --source - << EOF + [uwsgi] + socket = $apphome/uwsgi.sock + chdir = $apphome + wsgi-file = $projectname/wsgi.py + touch-reload = $projectname/wsgi.py + processes = 4 + threads = 2 + chmod-socket = 666 + daemonize=true + vacuum = true + uid = $user + gid = $user + EOF + +We require package uWSGI present in order to create **/etc/uwsgi/apps-enabled/$user.ini** file. +Installation of uWSGI also creates configuration layout: **/etc/uwsgi/apps-enabled**. +If this directory does not exist then **__file** type would error. +We also use stdin as file content source. For details see `Input from stdin `_. +For feading stdin we use here-document (**<<** operator). It allows redirection of subsequent +lines read by the shell to the input of a command until a line containing only the delimiter +and a newline, with no blank characters in between (EOF in our case). + +Configuring nginx for Let's Encrypt and HTTPS redirection +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Next configure nginx for Let's Encrypt and for HTTP -> HTTPS redirection. For this +purpose we will create new type **__sample_nginx_http_letsencrypt_and_ssl_redirect** +and use it here:: + + domain="$(cat "$__object/parameter/domain")" + webroot="/var/www/html" + __sample_nginx_http_letsencrypt_and_ssl_redirect "$domain" --webroot "$webroot" + +Configuring certificate creation +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +After HTTP nginx configuration we will create Let's Encrypt certificate using +`__letsencrypt_cert `_ type. +For Let's Encrypt cert configuration ensure that there is a DNS entry for your +domain. We assure that cert creation is applied after nginx HTTP is configured +for Let's Encrypt to work:: + + # create SSL cert + require="__package/nginx __sample_nginx_http_letsencrypt_and_ssl_redirect/$domain" \ + __letsencrypt_cert --admin-email admin@test.ungleich.ch \ + --webroot "$webroot" \ + --automatic-renewal \ + --renew-hook "service nginx reload" \ + --domain "$domain" \ + "$domain" + +Configuring nginx HTTPS server with uWSGI upstream +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Then we can configure nginx HTTPS server that will use created Let's Encrypt certificate:: + + # configure nginx + require="__package/nginx __letsencrypt_cert/$domain" \ + __file "/etc/nginx/sites-enabled/https-$domain" \ + --source - --mode 0644 << EOF + upstream _bottle { + server unix:$apphome/uwsgi.sock; + } + + server { + listen 443; + listen [::]:443; + + server_name $domain; + + access_log /var/log/nginx/access.log; + + ssl on; + ssl_certificate /etc/letsencrypt/live/$domain/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/$domain/privkey.pem; + + client_max_body_size 256m; + + location / { + try_files \$uri @uwsgi; + } + + location @uwsgi { + include uwsgi_params; + uwsgi_pass _bottle; + } + } + EOF + +Now our manifest is finished. + +Complete __sample_bottle_hosting type manifest listing +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Here is complete __sample_bottle_hosting type manifest listing, +located in ~/.cdist/type/__sample_bottle_hosting/manifest:: + + #!/bin/sh + + os=$(cat "$__global/explorer/os") + + case "$os" in + devuan) + : + ;; + *) + echo "OS $os currently not supported" >&2 + exit 1 + ;; + esac + + projectname="$(cat "$__object/parameter/projectname")" + user="$(cat "$__object/parameter/user")" + home="/home/$user" + apphome="$home/app" + domain="$(cat "$__object/parameter/domain")" + + # create user + __user "$user" --home "$home" --shell /bin/bash + # create user home dir + require="__user/$user" __directory "$home" \ + --owner "$user" --group "$user" --mode 0755 + # create app home dir + require="__user/$user __directory/$home" __directory "$apphome" \ + --state present --owner "$user" --group "$user" --mode 0755 + + # define packages that need to be installed + packages_to_install="nginx uwsgi-plugin-python3 python3-dev python3-pip postgresql postgresql-contrib libpq-dev python3-venv uwsgi python3-psycopg2" + + # update package index + __apt_update_index + # install packages + for package in $packages_to_install + do require="__apt_update_index" __package $package --state=present + done + # install pip3 packages + for package in bottle bottle-pgsql; do + __package_pip --pip pip3 $package + done + + #PostgreSQL db & user + postgres_server=postgresql + + # create PostgreSQL db user + require="__package/postgresql" __postgres_role $user --login --createdb + # create PostgreSQL db + require="__postgres_role/$user __package/postgresql" __postgres_database $user \ + --owner $user + # configure uWSGI + require="__package/uwsgi" __file /etc/uwsgi/apps-enabled/$user.ini \ + --owner root --group root --mode 0644 \ + --state present \ + --source - << EOF + [uwsgi] + socket = $apphome/uwsgi.sock + chdir = $apphome + wsgi-file = $projectname/wsgi.py + touch-reload = $projectname/wsgi.py + processes = 4 + threads = 2 + chmod-socket = 666 + daemonize=true + vacuum = true + uid = $user + gid = $user + EOF + + # setup nginx HTTP for Let's Encrypt and SSL redirect + domain="$(cat "$__object/parameter/domain")" + webroot="/var/www/html" + __sample_nginx_http_letsencrypt_and_ssl_redirect "$domain" --webroot "$webroot" + + # create SSL cert + require="__package/nginx __sample_nginx_http_letsencrypt_and_ssl_redirect/$domain" \ + __letsencrypt_cert --admin-email admin@test.ungleich.ch \ + --webroot "$webroot" \ + --automatic-renewal \ + --renew-hook "service nginx reload" \ + --domain "$domain" \ + "$domain" + + # configure nginx + require="__package/nginx __letsencrypt_cert/$domain" \ + __file "/etc/nginx/sites-enabled/https-$domain" \ + --source - --mode 0644 << EOF + upstream _bottle { + server unix:$apphome/uwsgi.sock; + } + + server { + listen 443; + listen [::]:443; + + server_name $domain; + + access_log /var/log/nginx/access.log; + + ssl on; + ssl_certificate /etc/letsencrypt/live/$domain/fullchain.pem; + ssl_certificate_key /etc/letsencrypt/live/$domain/privkey.pem; + + client_max_body_size 256m; + + location / { + try_files \$uri @uwsgi; + } + + location @uwsgi { + include uwsgi_params; + uwsgi_pass _bottle; + } + } + EOF + +Creating __sample_bottle_hosting type gencode-remote +---------------------------------------------------- + +Now define **gencode-remote** script: ~/.cdist/type/__sample_bottle_hosting/gencode-remote. +After manifest is applied it should restart uWSGI and nginx services so that our +configuration is active. Our gencode-remote looks like the following:: + + echo "service uwsgi restart" + echo "service nginx restart" + +Our **__sample_bottle_hosting** type is now finished. + +Creating __sample_nginx_http_letsencrypt_and_ssl_redirect type +-------------------------------------------------------------- + +Let's now create **__sample_nginx_http_letsencrypt_and_ssl_redirect** type:: + + cd ~/.cdist/type + mkdir __sample_nginx_http_letsencrypt_and_ssl_redirect + cd __sample_nginx_http_letsencrypt_and_ssl_redirect + mkdir parameter + echo webroot > parameter/required + touch manifest + touch gencode-remote + +Edit manifest:: + + domain="$__object_id" + webroot="$(cat "$__object/parameter/webroot")" + # make sure we have nginx package + __package nginx + # setup Let's Encrypt HTTP acme challenge, redirect HTTP to HTTPS + require="__package/nginx" __file "/etc/nginx/sites-enabled/http-$domain" \ + --source - --mode 0644 << EOF + server { + listen *:80; + listen [::]:80; + + server_name $domain; + + # Let's Encrypt + location /.well-known/acme-challenge/ { + root $webroot; + } + + # Everything else -> SSL + location / { + return 301 https://\$host\$request_uri; + } + } + + EOF + +Edit gencode-remote:: + + echo "service nginx reload" + +Creating init manifest +---------------------- + +Next create init manifest:: + + cd ~/.cdist/manifest + printf "__sample_bottle_hosting --projectname sample --user app --domain \$__target_host sample\n" > sample + +Using this init manifest our target host will be configured using our **__sample_bottle_hosting** +type with projectname *sample*, user *app* and domain equal to **__target_host**. +Here the last positional argument *sample* is type's object id. For details on +**__target_host** and **__object_id** see +`Environment variables (for reading) `_ +reference. + +Configuring host +---------------- + +Finally configure test.ungleich.ch:: + + cdist config -v -i ~/.cdist/manifest/sample test.ungleich.ch + +After cdist configuration is successfully finished our host is ready. + +Creating python bottle application +---------------------------------- + +We now need to create Bottle application. As you remember from the beginning +of this walkthrough our type does not create the actual python application, +its intention is only to configure hosing for specified user and project. +It is up to the user to create his/her applications. + +Become app user:: + + su -l app + +Preparing database +~~~~~~~~~~~~~~~~~~ + +We need to prepare database for our application. Create table and +insert some items:: + + psql -c "create table items (item varchar(255));" + + psql -c "insert into items(item) values('spam');" + psql -c "insert into items(item) values('eggs');" + psql -c "insert into items(item) values('sausage');" + +Creating application +~~~~~~~~~~~~~~~~~~~~ + +Next create sample app:: + + cd /home/app/app + mkdir sample + cd sample + +Create app.py with the following content:: + + #!/usr/bin/env python3 + + import bottle + import bottle_pgsql + + app = application = bottle.Bottle() + plugin = bottle_pgsql.Plugin('dbname=app user=app password=') + app.install(plugin) + + @app.route('/') + def show_index(db): + db.execute('select * from items') + items = db.fetchall() or [] + rv = '

Items:

    ' + for item in items: + rv += '
  • ' + str(item['item']) + '
    • ' + rv += '
' + return rv + + if __name__ == '__main__': + bottle.run(app=app, host='0.0.0.0', port=8080) + +Create wsgi.py with the following content:: + + import os + + os.chdir(os.path.dirname(__file__)) + + import app + application = app.app + +We have configured uWSGI with **touch-reload = $projectname/wsgi.py** so after +we have changed our **wsgi.py** file uWSGI reloads the application. + +Our application selects and lists items from **items** table. + +Openning application +~~~~~~~~~~~~~~~~~~~~ + +Finally try the application:: + + http://test.ungleich.ch/ + +It should redirect to HTTPS and return: + +.. container:: highlight + + .. raw:: html + +

Items:

+ +
    +
  • spam
    • +
  • eggs
    • +
  • sausage
    • +
+ +What's next? +------------ + +Continue reading next sections ;) diff --git a/docs/src/cdist-reference.rst.sh b/docs/src/cdist-reference.rst.sh index 4b94b858..f4a37816 100755 --- a/docs/src/cdist-reference.rst.sh +++ b/docs/src/cdist-reference.rst.sh @@ -63,6 +63,10 @@ cdist/conf/ The distribution configuration directory. This contains types and explorers to be used. +cdist/inventory/ + The distribution inventory directory. + This path is relative to cdist installation directory. + confdir Cdist will use all available configuration directories and create a temporary confdir containing links to the real configuration directories. @@ -194,6 +198,33 @@ Environment variables (for reading) ----------------------------------- The following environment variables are exported by cdist: +__cdist_log_level, __cdist_log_level_name + cdist log level value and cdist log level name. One of: + + +----------------+-----------------+ + | Log level name | Log level value | + +================+=================+ + | OFF | 60 | + +----------------+-----------------+ + | ERROR | 40 | + +----------------+-----------------+ + | WARNING | 30 | + +----------------+-----------------+ + | INFO | 20 | + +----------------+-----------------+ + | VERBOSE | 15 | + +----------------+-----------------+ + | DEBUG | 10 | + +----------------+-----------------+ + | TRACE | 5 | + +----------------+-----------------+ + + Available for: initial manifest, explorer, type manifest, type explorer, + type gencode. +__cdist_dry_run + Is set only when doing dry run (``-n`` flag). + Available for: initial manifest, explorer, type manifest, type explorer, + type gencode. __explorer Directory that contains all global explorers. Available for: initial manifest, explorer, type explorer, shell. @@ -239,6 +270,9 @@ __target_fqdn This variable is derived from **__target_host** (using **socket.getfqdn()**). Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. +__target_host_tags + Comma separated list of target host tags. + Available for: explorer, initial manifest, type explorer, type manifest, type gencode, shell. __type Path to the current type. Available for: type manifest, type gencode. @@ -253,6 +287,32 @@ The following environment variables influence the behaviour of cdist: require Setup dependencies between objects (see \`cdist manifest \`_). +__cdist_log_level + cdist log level value. One of: + + +----------------+-----------------+ + | Log level | Log level value | + +================+=================+ + | OFF | 60 | + +----------------+-----------------+ + | ERROR | 40 | + +----------------+-----------------+ + | WARNING | 30 | + +----------------+-----------------+ + | INFO | 20 | + +----------------+-----------------+ + | VERBOSE | 15 | + +----------------+-----------------+ + | DEBUG | 10 | + +----------------+-----------------+ + | TRACE | 5 | + +----------------+-----------------+ + + If set cdist will set this log level in + accordance with configuration rules. If cdist invokation is used + in types then nested cdist will honor this specified log level if + not specified otherwise while invoking it. + CDIST_PATH Colon delimited list of config directories. @@ -267,6 +327,7 @@ CDIST_OVERRIDE CDIST_ORDER_DEPENDENCY Create dependencies based on the execution order (see \`cdist manifest \`_). + Read also about \`perils of CDIST_ORDER_DEPENDENCY \`_. CDIST_REMOTE_EXEC Use this command for remote execution (should behave like ssh). @@ -274,6 +335,12 @@ CDIST_REMOTE_EXEC CDIST_REMOTE_COPY Use this command for remote copy (should behave like scp). +CDIST_INVENTORY_DIR + Use this directory as inventory directory. + CDIST_BETA Enable beta functionalities. + +CDIST_CACHE_PATH_PATTERN + Custom cache path pattern. eof diff --git a/docs/src/cdist-saving-output-streams.rst b/docs/src/cdist-saving-output-streams.rst new file mode 100644 index 00000000..da66f754 --- /dev/null +++ b/docs/src/cdist-saving-output-streams.rst @@ -0,0 +1,88 @@ +Saving output streams +===================== + +Description +----------- +Since version 4.8.0 cdist, by default, saves output streams to local cache. +Saving output streams is implemented because important information was lost +during a config run, hidden in all other output. +Now all created output is bound to the context where it was produced. + +Saving output streams include stdout and stderr of init manifest, remote +commands and for each object stdout and stderr of manifest, gencode-\* and code-\*. +Output stream files are created only if some output is produced. For more info +on these cache files see `Local cache overview `_. + +Also, in case of an error, cdist can now exit and show all information it has +about the error. + +For example: + +.. code-block:: sh + + $ ./bin/cdist config -v -i ~/.cdist/manifest/init-output-streams $(cat ~/ungleich/data/opennebula-debian9-test ) + INFO: 185.203.112.42: Starting configuration run + INFO: 185.203.112.42: Processing __myline/test + ERROR: 185.203.112.42: Command failed: '/bin/sh -e /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-kisrqlpw/code-local' + return code: 1 + ---- BEGIN stdout ---- + ---- END stdout ---- + + Error processing object '__myline/test' + ======================================== + name: __myline/test + path: /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-kisrqlpw + source: /home/darko/.cdist/manifest/init-output-streams + type: /tmp/tmpow6cwemh/75ee6a79e32da093da23fe4a13dd104b/data/conf/type/__myline + + ---- BEGIN manifest:stderr ---- + myline manifest stderr + + ---- END manifest:stderr ---- + + ---- BEGIN gencode-remote:stderr ---- + test gencode-remote error + + ---- END gencode-remote:stderr ---- + + ---- BEGIN code-local:stderr ---- + error + + ---- END code-local:stderr ---- + + ERROR: cdist: Failed to configure the following hosts: 185.203.112.42 + +Upon successful run execution state is saved to local cache and temporary +directory is removed. +In case of an error temporary directory is not removed and can be further +discovered. + +There is also an option :strong:`-S/--disable-saving-output-streams` for +disabling saving output streams. In this case error reporting can look +like this: + +.. code-block:: sh + + $ ./bin/cdist config -v -S -i ~/.cdist/manifest/init-output-streams $(cat ~/ungleich/data/opennebula-debian9-test ) + INFO: 185.203.112.42: Starting configuration run + test stdout output streams + test stderr output streams + myline manifest stdout + myline manifest stderr + test gencode-remote error + INFO: 185.203.112.42: Processing __myline/test + error + ERROR: 185.203.112.42: Command failed: '/bin/sh -e /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-n566pqut/code-local' + return code: 1 + ---- BEGIN stdout ---- + ---- END stdout ---- + + Error processing object '__myline/test' + ======================================== + name: __myline/test + path: /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/object/__myline/test/.cdist-n566pqut + source: /home/darko/.cdist/manifest/init-output-streams + type: /tmp/tmpzomy0wis/75ee6a79e32da093da23fe4a13dd104b/data/conf/type/__myline + + + ERROR: cdist: Failed to configure the following hosts: 185.203.112.42 diff --git a/docs/src/cdist-stages.rst b/docs/src/cdist-stages.rst index fd19ce0d..751ba9c7 100644 --- a/docs/src/cdist-stages.rst +++ b/docs/src/cdist-stages.rst @@ -3,8 +3,7 @@ Execution stages Description ----------- -Starting the execution of deployment with cdist, cdist passes -through different stages. +When cdist is started, it passes through different stages. Stage 1: target information retrieval @@ -67,5 +66,5 @@ The cache stores the information from the current run for later use. Summary ------- -If, and only if, all the stages complete without an errors, the configuration +If, and only if, all the stages complete without errors, the configuration will be applied to the target. diff --git a/docs/src/cdist-support.rst b/docs/src/cdist-support.rst index 2343500e..19afde2f 100644 --- a/docs/src/cdist-support.rst +++ b/docs/src/cdist-support.rst @@ -1,11 +1,9 @@ Support ------- -IRC -~~~ - -You can join the development ***IRC channel*** -`#cstar on irc.freenode.net `_. +Chat +~~~~ +Chat with us: `ungleich chat `_. Mailing list ~~~~~~~~~~~~ @@ -25,4 +23,4 @@ Commercial support ~~~~~~~~~~~~~~~~~~ You can request commercial support for cdist from -`my company `_. +`ungleich `_. diff --git a/docs/src/cdist-troubleshooting.rst b/docs/src/cdist-troubleshooting.rst index b016e845..e639aafd 100644 --- a/docs/src/cdist-troubleshooting.rst +++ b/docs/src/cdist-troubleshooting.rst @@ -43,3 +43,22 @@ you write to use the -e flag: % cat ~/.cdist/manifest/special #!/bin/sh -e ... + +Using debug dump helper script +------------------------------ +Since cdist stores data to local cache that can be used for debugging there +is a helper script that dumps data from local cache, +`cdist-dump `_. + +For more info see: + +.. code-block:: sh + + cdist-dump -h + +Or from cdist git cloned directory: + +.. code-block:: sh + + ./scripts/cdist-dump -h + diff --git a/docs/src/cdist-type.rst b/docs/src/cdist-type.rst index 2c5f9f6a..582c0938 100644 --- a/docs/src/cdist-type.rst +++ b/docs/src/cdist-type.rst @@ -64,21 +64,63 @@ If a type is flagged with 'install' flag then it is used only with install comma With other commands, i.e. config, these types are skipped if used. +Nonparallel types +----------------- +If a type is flagged with 'nonparallel' flag then its objects cannot be run in parallel +when using -j option. Example of such a type is __package_dpkg type where dpkg itself +prevents to be run in more than one instance. + + +Deprecated types +----------------- +If a type is flagged with 'deprecated' marker then it is considered deprecated. +When it is used cdist writes warning line. If 'deprecated' marker has content +then this content is printed as a deprecation messages, e.g.: + +.. code-block:: sh + + $ ls -l deprecated + -rw-r--r-- 1 darko darko 71 May 20 18:30 deprecated + $ cat deprecated + This type is deprecated. It will be removed in the next minor release. + $ echo '__foo foo' | ./bin/cdist config -i - 185.203.112.26 + WARNING: 185.203.112.26: Type __foo is deprecated: This type is deprecated. It will be removed in the next minor release. + +If 'deprecated' marker has no content then general message is printed, e.g.: + +.. code-block:: sh + + $ ls -l deprecated + -rw-r--r-- 1 darko darko 0 May 20 18:36 deprecated + $ echo '__bar foo' | ./bin/cdist config -i - 185.203.112.26 + WARNING: 185.203.112.26: Type __bar is deprecated. + + How to write a new type ----------------------- A type consists of -- parameter (optional) -- manifest (optional) -- singleton (optional) -- explorer (optional) -- gencode (optional) +- parameter (optional) +- manifest (optional) +- singleton (optional) +- explorer (optional) +- gencode (optional) +- nonparallel (optional) Types are stored below cdist/conf/type/. Their name should always be prefixed with two underscores (__) to prevent collisions with other executables in $PATH. To implement a new type, create the directory **cdist/conf/type/__NAME**. +Type manifest and gencode can be written in any language. They just need to be +executable and have a proper shebang. If they are not executable then cdist assumes +they are written in shell so they are executed using '/bin/sh -e' or 'CDIST_LOCAL_SHELL'. + +For executable shell code it is suggested that shebang is '#!/bin/sh -e'. + +For creating type skeleton you can use helper script +`cdist-new-type `_. + Defining parameters ------------------- @@ -144,6 +186,31 @@ Example: (e.g. in cdist/conf/type/__nginx_vhost/manifest) fi +Deprecated parameters +--------------------- +To deprecate type parameters one can declare a file for each deprecated +parameter under **parameter/deprecated** directory. + +When such parameter is used cdist writes warning line with deprecation message. +If such file has content then this content is printed as deprecation message. +If there is no content then generic parameter deprecation message is printed. + +Example: + +.. code-block:: sh + + $ ls parameter/deprecated/ + eggs spam + $ cat parameter/deprecated/eggs + eggs parameter is deprecated, please use multiple egg parameter. + $ cat parameter/deprecated/spam + $ echo '__foo foo --foo foo --eggs eggs' | ./bin/cdist config -i - 185.203.112.26 + WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated: eggs parameter is deprecated, please use multiple egg parameter. + $ echo '__foo foo --foo foo --eggs eggs --spam spam' | ./bin/cdist config -i - 185.203.112.26 + WARNING: 185.203.112.26: spam parameter of type __foo is deprecated. + WARNING: 185.203.112.26: eggs parameter of type __foo is deprecated: eggs parameter is deprecated, please use multiple egg parameter. + + Input from stdin ---------------- Every type can access what has been written on stdin when it has been called. @@ -174,6 +241,73 @@ In the __file type, stdin is used as source for the file, if - is used for sourc .... +Stdin inside a loop +~~~~~~~~~~~~~~~~~~~ +Since cdist saves type's stdin content in the object as **$__object/stdin**, +so it can be accessed in manifest and gencode-* scripts, this can lead to +unexpected behavior. For example, suppose you have some type with the following +in its manifest: + +.. code-block:: sh + + if [ -f "$__object/parameter/foo" ] + then + while read -r l + do + __file "$l" + echo "$l" >&2 + done < "$__object/parameter/foo" + fi + +and init manifest: + +.. code-block:: sh + + __foo foo --foo a --foo b --foo c + +You expect that manifest stderr content is: + +.. code-block:: sh + + a + b + c + +and that files *a*, *b* and *c* are created. But all you get in manifest stderr +is: + +.. code-block:: sh + + a + +and only *a* file is created. + +When redirecting parameter *foo* file content to while's stdin that means that all +commands in while body have this same stdin. So when *__file* type gets executed, +cdist saves its stdin which means it gets the remaining content of parameter *foo* +file, i.e.: + +.. code-block:: sh + + b + c + +The solution is to make sure that your types inside such loops get their stdin +from somewhere else, e.g. for the above problem *__file* type can get empty +stdin from */dev/null*: + +.. code-block:: sh + + if [ -f "$__object/parameter/foo" ] + then + while read -r l + do + __file "$l" < /dev/null + echo "$l" >&2 + done < "$__object/parameter/foo" + fi + + Writing the manifest -------------------- In the manifest of a type you can use other types, so your type extends @@ -234,6 +368,19 @@ install: create the (empty) file "install" in your type directory: With other commands, i.e. config, it will be skipped if used. +Nonparallel - only one instance can be run at a time +---------------------------------------------------- +If objects of a type must not or cannot be run in parallel when using -j +option, you must mark it as nonparallel: create the (empty) file "nonparallel" +in your type directory: + +.. code-block:: sh + + touch cdist/conf/type/__NAME/nonparallel + +For example, package types are nonparallel types. + + The type explorers ------------------ If a type needs to explore specific details, it can provide type specific @@ -304,6 +451,55 @@ So when you generate a script with the following content, it will work: fi +Environment variable usage idiom +-------------------------------- +In type scripts you can support environment variables with default values if +environment variable is unset or null by using **${parameter:-[word]}** +parameter expansion. + +Example using mktemp in a portable way that supports TMPDIR environment variable. + +.. code-block:: sh + + tempfile=$(mktemp "${TMPDIR:-/tmp}/cdist.XXXXXXXXXX") + + +Log level in types +------------------ +cdist log level can be accessed from __cdist_log_level variable.One of: + + +----------------+-----------------+ + | Log level | Log level value | + +================+=================+ + | OFF | 60 | + +----------------+-----------------+ + | ERROR | 40 | + +----------------+-----------------+ + | WARNING | 30 | + +----------------+-----------------+ + | INFO | 20 | + +----------------+-----------------+ + | VERBOSE | 15 | + +----------------+-----------------+ + | DEBUG | 10 | + +----------------+-----------------+ + | TRACE | 5 | + +----------------+-----------------+ + + +It is available for initial manifest, explorer, type manifest, +type explorer, type gencode. + + +Detecting dry run +----------------- + +If ``$__cdist_dry_run`` environment variable is set, then it's dry run. + +It is available for initial manifest, explorer, type manifest, +type explorer, type gencode. + + Hints for typewriters ---------------------- It must be assumed that the target is pretty dumb and thus does not have high diff --git a/docs/src/cdist-update.rst b/docs/src/cdist-upgrade.rst similarity index 99% rename from docs/src/cdist-update.rst rename to docs/src/cdist-upgrade.rst index e810d6e9..e57ed63c 100644 --- a/docs/src/cdist-update.rst +++ b/docs/src/cdist-upgrade.rst @@ -1,5 +1,5 @@ -How to update cdist -=================== +How to upgrade cdist +==================== Update the git installation --------------------------- diff --git a/docs/src/cdist-why.rst b/docs/src/cdist-why.rst index e6aefefd..1123a1de 100644 --- a/docs/src/cdist-why.rst +++ b/docs/src/cdist-why.rst @@ -40,9 +40,9 @@ call cdist types, the result is always the same. Zero dependency configuration management ---------------------------------------- -Cdist requires very litte on a target system. Even better, +Cdist requires very little on a target system. Even better, in almost all cases all dependencies are usually fulfilled. -Cdist does not require an agent or a high level programming +Cdist does not require an agent or high level programming languages on the target host: it will run on any host that has a **ssh server running** and a posix compatible shell (**/bin/sh**). Compared to other configuration management systems, @@ -52,7 +52,7 @@ Push based distribution ----------------------- Cdist uses the push based model for configuration. In this -scenario, one (or more) computers connect the target hosts +scenario, one (or more) computers connect to the target hosts and apply the configuration. That way the source host has very little requirements: Cdist can even run on a sysadmin notebook that is loosely connected to the network and has diff --git a/docs/src/conf.py b/docs/src/conf.py index a63a14ff..78f9842c 100644 --- a/docs/src/conf.py +++ b/docs/src/conf.py @@ -56,7 +56,7 @@ master_doc = 'index' # General information about the project. project = 'cdist' -# copyright = '2016, Darko Poljak' +copyright = 'ungleich GmbH 2019' # author = 'Darko Poljak' # The version info for the project you're documenting, acts as replacement for @@ -138,7 +138,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The name of an image file (relative to this directory) to place at the top # of the sidebar. -# html_logo = None +html_logo = '_static/cdist-logo.jpeg' # The name of an image file (relative to this directory) to use as a favicon of # the docs. This file should be a Windows icon file (.ico) @@ -150,6 +150,7 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". # html_static_path = ['_static'] +html_static_path = ['_static'] # Add any extra paths that contain custom files (such as robots.txt or # .htaccess) here, relative to this directory. These files are copied @@ -161,11 +162,6 @@ html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] # The empty string is equivalent to '%b %d, %Y'. # html_last_updated_fmt = None -# If true, SmartyPants will be used to convert quotes and dashes to -# typographically correct entities. -# html_use_smartypants = True -html_use_smartypants = False - # Custom sidebar templates, maps document names to template names. # html_sidebars = {} diff --git a/docs/src/docutils.conf b/docs/src/docutils.conf new file mode 100644 index 00000000..168f9e2b --- /dev/null +++ b/docs/src/docutils.conf @@ -0,0 +1,2 @@ +[parsers] +smart_quotes: false diff --git a/docs/src/index.rst b/docs/src/index.rst index b33b707d..6cbd938a 100644 --- a/docs/src/index.rst +++ b/docs/src/index.rst @@ -1,32 +1,43 @@ -Welcome to cdist documentation -============================== +cdist - usable configuration management +======================================= + +cdist is a usable configuration management system. +It adheres to the KISS principle and +is being used in small up to enterprise grade environments. -Contents: .. toctree:: - :maxdepth: 2 + :maxdepth: 3 :glob: :numbered: + :hidden: - cdist-intro cdist-why + cdist-features cdist-os cdist-install - cdist-update + cdist-upgrade cdist-support - cdist-features cdist-quickstart + cdist-real-world man1/cdist + man1/cdist-dump + man1/cdist-new-type cdist-bootstrap + cdist-configuration cdist-manifest cdist-type cdist-types cdist-explorer cdist-messaging cdist-parallelization + cdist-inventory + cdist-integration cdist-reference cdist-best-practice cdist-stages + cdist-cache + cdist-saving-output-streams cdist-remote-exec-copy cdist-hacker cdist-troubleshooting diff --git a/docs/src/man1/cdist-dump.rst b/docs/src/man1/cdist-dump.rst new file mode 100644 index 00000000..907cd192 --- /dev/null +++ b/docs/src/man1/cdist-dump.rst @@ -0,0 +1,110 @@ +cdist-dump(1) +============= + +NAME +---- +cdist-dump - Dump data from local cdist cache + + +SYNOPSIS +-------- + +:: + + cdist-dump [options] [host...] + + + +DESCRIPTION +----------- +cdist-dump is a helper script that dumps data from local cdist cache for +specified hosts. If host is not specified then all data from cache directory +is dumped. Default cache directory is '~/.cdist/cache'. + +cdist-dump can be used for debugging existing types, host configuration and +new types. + + +OPTIONS +------- +**-a** + dump all + +**-C CACHE-DIR** + use specified CACHE-DIR (default: ~/.cdist/cache) + +**-c** + dump code-* + +**-d DELIMITER** + delimiter used for filename and line number prefix (default: ':') + +**-E** + dump global explorers + +**-e** + dump type explorers + +**-F** + disable filename prefix (enabled by default) + +**-f** + enable filename prefix (default) + +**-g** + dump gencode-* + +**-h** + show this help screen and exit + +**-L** + disable line number prefix (default) + +**-l** + enable line number prefix (disabled by default) + +**-m** + dump messages + +**-o** + dump executions' stdout + +**-p** + dump parameters + +**-r** + dump executions' stderr + +**-V** + show version and exit + +**-v** + increase verbosity + + +EXAMPLES +-------- + +.. code-block:: sh + + # Dump all + % cdist-dump -a + + # Dump only code-* output + % cdist-dump -c + + +SEE ALSO +-------- +:strong:`cdist`\ (1) + + +AUTHORS +------- +Darko Poljak + + +COPYING +------- +Copyright \(C) 2019 Darko Poljak. Free use of this software is +granted under the terms of the GNU General Public License v3 or later (GPLv3+). diff --git a/docs/src/man1/cdist-new-type.rst b/docs/src/man1/cdist-new-type.rst new file mode 100644 index 00000000..f1a8b992 --- /dev/null +++ b/docs/src/man1/cdist-new-type.rst @@ -0,0 +1,74 @@ +cdist-new-type(1) +================= + +NAME +---- +cdist-new-type - Create new type skeleton + + +SYNOPSIS +-------- + +:: + + cdist-new-type TYPE-NAME AUTHOR-NAME AUTHOR-EMAIL [TYPE-BASE-PATH] + + + +DESCRIPTION +----------- +cdist-new-type is a helper script that creates new type skeleton. +It is then up to the type author to finish the type. + +It creates skeletons for the following files: + +* man.rst +* manifest +* gencode-remote. + +Upon creation it prints the path to the newly created type directory. + + +ARGUMENTS +--------- +**TYPE-NAME** + Name of the new type. + +**AUTHOR-NAME** + Type author's full name. + +**AUTHOR-NAME** + Type author's email. + +**TYPE-BASE-PATH** + Path to the base directory of the type. If not set it defaults + to '$PWD/type'. + + +EXAMPLES +-------- + +.. code-block:: sh + + # Create new type __foo in ~/.cdist directory. + $ cd ~/.cdist + $ cdist-new-type '__foo' 'Foo Bar' 'foo.bar at foobar.org' + /home/foo/.cdist/type/__foo + + +SEE ALSO +-------- +:strong:`cdist`\ (1) + + +AUTHORS +------- + +| Steven Armstrong +| Darko Poljak + + +COPYING +------- +Copyright \(C) 2019 Steven Armstrong, Darko Poljak. Free use of this software is +granted under the terms of the GNU General Public License v3 or later (GPLv3+). diff --git a/docs/src/man1/cdist.rst b/docs/src/man1/cdist.rst index a46e1e02..30832f2f 100644 --- a/docs/src/man1/cdist.rst +++ b/docs/src/man1/cdist.rst @@ -11,21 +11,55 @@ SYNOPSIS :: - cdist [-h] [-d] [-v] [-V] {banner,config,shell,install} ... + cdist [-h] [-V] {banner,config,install,inventory,shell} ... - cdist banner [-h] [-d] [-v] + cdist banner [-h] [-l LOGLEVEL] [-q] [-v] - cdist config [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE] - [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s] + cdist config [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] + [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + [-j [JOBS]] [-n] [-o OUT_PATH] [-P] + [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] + [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] + [-p [HOST_MAX]] [-s] [-t] [host [host ...]] - cdist install [-h] [-d] [-v] [-b] [-c CONF_DIR] [-f HOSTFILE] - [-i MANIFEST] [-j [JOBS]] [-n] [-o OUT_PATH] [-p] [-s] + cdist install [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] [-4] + [-6] [-C CACHE_PATH_PATTERN] [-c CONF_DIR] [-i MANIFEST] + [-j [JOBS]] [-n] [-o OUT_PATH] [-P] + [-R [{tar,tgz,tbz2,txz}]] [-r REMOTE_OUT_PATH] [--remote-copy REMOTE_COPY] [--remote-exec REMOTE_EXEC] + [-S] [-I INVENTORY_DIR] [-A] [-a] [-f HOSTFILE] + [-p [HOST_MAX]] [-s] [-t] [host [host ...]] - cdist shell [-h] [-d] [-v] [-s SHELL] + cdist inventory [-h] {add-host,add-tag,del-host,del-tag,list} ... + + cdist inventory add-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [-g CONFIG_FILE] [-I INVENTORY_DIR] + [-f HOSTFILE] + [host [host ...]] + + cdist inventory add-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [-g CONFIG_FILE] [-I INVENTORY_DIR] + [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [host [host ...]] + + cdist inventory del-host [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] + [host [host ...]] + + cdist inventory del-tag [-h] [-l LOGLEVEL] [-q] [-v] [-b] + [-g CONFIG_FILE] [-I INVENTORY_DIR] [-a] + [-f HOSTFILE] [-T TAGFILE] [-t TAGLIST] + [host [host ...]] + + cdist inventory list [-h] [-l LOGLEVEL] [-q] [-v] [-b] [-g CONFIG_FILE] + [-I INVENTORY_DIR] [-a] [-f HOSTFILE] [-H] [-t] + [host [host ...]] + + cdist shell [-h] [-l LOGLEVEL] [-q] [-v] [-s SHELL] DESCRIPTION @@ -40,24 +74,31 @@ GENERAL ------- All commands accept the following options: -.. option:: -h, --help +**-h, --help** + Show the help screen. - Show the help screen +**-l LOGLEVEL, --log-level LOGLEVEL** + Set the specified verbosity level. The levels, in + order from the lowest to the highest, are: ERROR (-1), + WARNING (0), INFO (1), VERBOSE (2), DEBUG (3) TRACE (4 + or higher). If used along with -v then -v increases + last set value and -l overwrites last set value. -.. option:: -d, --debug +**-q, --quiet** + Quiet mode: disables logging, including WARNING and ERROR. - Set log level to debug (deprecated, use -vvv instead) +**-v, --verbose** + Increase the verbosity level. Every instance of -v + increments the verbosity level by one. Its default + value is 0 which includes ERROR and WARNING levels. + The levels, in order from the lowest to the highest, + are: ERROR (-1), WARNING (0), INFO (1), VERBOSE (2), + DEBUG (3) TRACE (4 or higher). If used along with -l + then -l overwrites last set value and -v increases + last set value. -.. option:: -v, --verbose - - Increase the verbosity level. Every instance of -v increments the verbosity - level by one. Its default value is 0. There are 4 levels of verbosity. The - order of levels from the lowest to the highest are: ERROR (0), WARNING (1), - INFO (2) and DEBUG (3 or higher). - -.. option:: -V, --version - - Show version and exit +**-V, --version** + Show version and exit. BANNER @@ -69,77 +110,327 @@ cdist posters - a must have for every office. CONFIG/INSTALL -------------- Configure/install one or more hosts. +Install command is currently in beta. -.. option:: -b, --beta +**-4, --force-ipv4** + Force to use IPv4 addresses only. No influence for + custom remote commands. - Enable beta functionalities. +**-6, --force-ipv6** + Force to use IPv6 addresses only. No influence for + custom remote commands. + +**-A, --all-tagged** + Use all hosts present in tags db. Currently in beta. + +**-a, --all** + List hosts that have all specified tags, if -t/--tag + is specified. + +**-b, --beta** + Enable beta functionality. - Can also be enabled using CDIST_BETA env var. - -.. option:: -c CONF_DIR, --conf-dir CONF_DIR +**-C CACHE_PATH_PATTERN, --cache-path-pattern CACHE_PATH_PATTERN** + Specify custom cache path pattern. If it is not set then + default hostdir is used. For more info on format see + :strong:`CACHE PATH PATTERN FORMAT` below. +**-c CONF_DIR, --conf-dir CONF_DIR** Add a configuration directory. Can be specified multiple times. If configuration directories contain conflicting types, explorers or - manifests, then the last one found is used. Additionally this can also - be configured by setting the CDIST_PATH environment variable to a colon - delimited list of config directories. Directories given with the - --conf-dir argument have higher precedence over those set through the - environment variable. + manifests, then the last one found is used. -.. option:: -f HOSTFILE, --file HOSTFILE - - Read additional hosts to operate on from specified file - or from stdin if '-' (each host on separate line). +**-f HOSTFILE, --file HOSTFILE** + Read specified file for a list of additional hosts to operate on + or if '-' is given, read stdin (one host per line). If no host or host file is specified then, by default, - read hosts from stdin. For the file format see below. + read hosts from stdin. For the file format see + :strong:`HOSTFILE FORMAT` below. -.. option:: -i MANIFEST, --initial-manifest MANIFEST +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. - Path to a cdist manifest or - to read from stdin +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. -.. option:: -j [JOBS], --jobs [JOBS] +**-i MANIFEST, --initial-manifest MANIFEST** + Path to a cdist manifest or - to read from stdin. - Specify the maximum number of parallel jobs. Global - explorers, object prepare and object run are supported - (currently in beta). +**-j [JOBS], --jobs [JOBS]** + Operate in parallel in specified maximum number of + jobs. Global explorers, object prepare and object run + are supported. Without argument CPU count is used by + default. -.. option:: -n, --dry-run +**-n, --dry-run** + Do not execute code. - Do not execute code +**-o OUT_PATH, --out-dir OUT_PATH** + Directory to save cdist output in. -.. option:: -o OUT_PATH, --out-dir OUT_PATH +**-P, --timestamp** + Timestamp log messages with the current local date and time + in the format: YYYYMMDDHHMMSS.us. - Directory to save cdist output in +**-p [HOST_MAX], --parallel [HOST_MAX]** + Operate on multiple hosts in parallel for specified + maximum hosts at a time. Without argument CPU count is + used by default. -.. option:: -p, --parallel +**-R [{tar,tgz,tbz2,txz}], --use-archiving [{tar,tgz,tbz2,txz}]** + Operate by using archiving with compression where + appropriate. Supported values are: tar - tar archive, + tgz - gzip tar archive (the default), tbz2 - bzip2 tar + archive and txz - lzma tar archive. Currently in beta. - Operate on multiple hosts in parallel +**-r REMOTE_OUT_PATH, --remote-out-dir REMOTE_OUT_PATH** + Directory to save cdist output in on the target host. -.. option:: -s, --sequential +**-S, --disable-saving-output-streams** + Disable saving output streams. - Operate on multiple hosts sequentially (default) +**-s, --sequential** + Operate on multiple hosts sequentially (default). -.. option:: --remote-copy REMOTE_COPY +**--remote-copy REMOTE_COPY** + Command to use for remote copy (should behave like scp). - Command to use for remote copy (should behave like scp) - -.. option:: --remote-exec REMOTE_EXEC - - Command to use for remote execution (should behave like ssh) +**--remote-exec REMOTE_EXEC** + Command to use for remote execution (should behave like ssh). +**-t, --tag** + Host is specified by tag, not hostname/address; list + all hosts that contain any of specified tags. + Currently in beta. HOSTFILE FORMAT ~~~~~~~~~~~~~~~ -HOSTFILE contains hosts per line. -All characters after and including '#' until the end of line is a comment. -In a line, all leading and trailing whitespace characters are ignored. +The HOSTFILE contains one host per line. +A comment is started with '#' and continues to the end of the line. +Any leading and trailing whitespace on a line is ignored. Empty lines are ignored/skipped. -Hostfile line is processed like the following. First, all comments are + +The Hostfile lines are processed as follows. First, all comments are removed. Then all leading and trailing whitespace characters are stripped. If such a line results in empty line it is ignored/skipped. Otherwise, host string is used. +CACHE PATH PATTERN FORMAT +~~~~~~~~~~~~~~~~~~~~~~~~~ +Cache path pattern specifies path for a cache directory subdirectory. +In the path, '%N' will be substituted by the target host, '%h' will +be substituted by the calculated host directory, '%P' will be substituted +by the current process id. All format codes that +:strong:`python` :strong:`datetime.strftime()` function supports, except +'%h', are supported. These date/time directives format cdist config/install +start time. + +If empty pattern is specified then default calculated host directory +is used. + +Calculated host directory is a hash of a host cdist operates on. + +Resulting path is used to specify cache path subdirectory under which +current host cache data are saved. + + +INVENTORY +--------- +Manage inventory database. +Currently in beta with all sub-commands. + + +INVENTORY ADD-HOST +------------------ +Add host(s) to inventory database. + +**host** + Host(s) to add. + +**-b, --beta** + Enable beta functionality. + +**-f HOSTFILE, --file HOSTFILE** + Read additional hosts to add from specified file or + from stdin if '-' (each host on separate line). If no + host or host file is specified then, by default, read + from stdin. Hostfile format is the same as config hostfile format. + +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. + +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. + + +INVENTORY ADD-TAG +----------------- +Add tag(s) to inventory database. + +**host** + List of host(s) for which tags are added. + +**-b, --beta** + Enable beta functionality. + +**-f HOSTFILE, --file HOSTFILE** + Read additional hosts to add tags from specified file + or from stdin if '-' (each host on separate line). If + no host or host file is specified then, by default, + read from stdin. If no tags/tagfile nor hosts/hostfile + are specified then tags are read from stdin and are + added to all hosts. Hostfile format is the same as config hostfile format. + +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. + +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. + +**-T TAGFILE, --tag-file TAGFILE** + Read additional tags to add from specified file or + from stdin if '-' (each tag on separate line). If no + tag or tag file is specified then, by default, read + from stdin. If no tags/tagfile nor hosts/hostfile are + specified then tags are read from stdin and are added + to all hosts. Tagfile format is the same as config hostfile format. + +**-t TAGLIST, --taglist TAGLIST** + Tag list to be added for specified host(s), comma + separated values. + + +INVENTORY DEL-HOST +------------------ +Delete host(s) from inventory database. + +**host** + Host(s) to delete. + +**-a, --all** + Delete all hosts. + +**-b, --beta** + Enable beta functionality. + +**-f HOSTFILE, --file HOSTFILE** + Read additional hosts to delete from specified file or + from stdin if '-' (each host on separate line). If no + host or host file is specified then, by default, read + from stdin. Hostfile format is the same as config hostfile format. + +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. + +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. + + +INVENTORY DEL-TAG +----------------- +Delete tag(s) from inventory database. + +**host** + List of host(s) for which tags are deleted. + +**-a, --all** + Delete all tags for specified host(s). + +**-b, --beta** + Enable beta functionality. + +**-f HOSTFILE, --file HOSTFILE** + Read additional hosts to delete tags for from + specified file or from stdin if '-' (each host on + separate line). If no host or host file is specified + then, by default, read from stdin. If no tags/tagfile + nor hosts/hostfile are specified then tags are read + from stdin and are deleted from all hosts. Hostfile + format is the same as config hostfile format. + +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. + +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. + +**-T TAGFILE, --tag-file TAGFILE** + Read additional tags from specified file or from stdin + if '-' (each tag on separate line). If no tag or tag + file is specified then, by default, read from stdin. + If no tags/tagfile nor hosts/hostfile are specified + then tags are read from stdin and are added to all + hosts. Tagfile format is the same as config hostfile format. + +**-t TAGLIST, --taglist TAGLIST** + Tag list to be deleted for specified host(s), comma + separated values. + + +INVENTORY LIST +-------------- +List inventory database. + +**host** + Host(s) to list. + +**-a, --all** + List hosts that have all specified tags, if -t/--tag + is specified. + +**-b, --beta** + Enable beta functionality. + +**-f HOSTFILE, --file HOSTFILE** + Read additional hosts to list from specified file or + from stdin if '-' (each host on separate line). If no + host or host file is specified then, by default, list + all. Hostfile format is the same as config hostfile format. + +**-g CONFIG_FILE, --config-file CONFIG_FILE** + Use specified custom configuration file. + +**-H, --host-only** + Suppress tags listing. + +**-I INVENTORY_DIR, --inventory INVENTORY_DIR** + Use specified custom inventory directory. Inventory + directory is set up by the following rules: if cdist + configuration resolves this value then specified + directory is used, if HOME env var is set then + ~/.cdit/inventory is used, otherwise distribution + inventory directory is used. + +**-t, --tag** + Host is specified by tag, not hostname/address; list + all hosts that contain any of specified tags. + SHELL ----- @@ -148,19 +439,117 @@ to the types as commands. It can be thought as an "interactive manifest" environment. See below for example usage. Its primary use is for debugging type parameters. -.. option:: -s SHELL, --shell SHELL - +**-s SHELL, --shell SHELL** Select shell to use, defaults to current shell. Used shell should be POSIX compatible shell. + +CONFIGURATION +------------- +cdist obtains configuration data from the following sources in the following +order (from higher to lower precedence): + + #. command-line options + #. configuration file specified at command-line + #. configuration file specified in CDIST_CONFIG_FILE environment variable + #. environment variables + #. user's configuration file (first one found of ~/.cdist.cfg, $XDG_CONFIG_HOME/cdist/cdist.cfg, in specified order) + #. system-wide configuration file (/etc/cdist.cfg). + +CONFIGURATION FILE FORMAT +~~~~~~~~~~~~~~~~~~~~~~~~~ +cdist configuration file is in the INI file format. Currently it supports +only [GLOBAL] section. +The possible keywords and their meanings are as follows: + +:strong:`archiving` + Use specified archiving. Valid values include: + 'none', 'tar', 'tgz', 'tbz2' and 'txz'. + +:strong:`beta` + Enable beta functionality. It recognizes boolean values from + 'yes'/'no', 'on'/'off', 'true'/'false' and '1'/'0'. + +:strong:`cache_path_pattern` + Specify cache path pattern. + +:strong:`conf_dir` + List of configuration directories separated with the character conventionally + used by the operating system to separate search path components (as in PATH), + such as ':' for POSIX or ';' for Windows. + If also specified at command line then values from command line are + appended to this value. + +:strong:`init_manifest` + Specify default initial manifest. + +:strong:`inventory_dir` + Specify inventory directory. + +:strong:`jobs` + Specify number of jobs for parallel processing. If -1 then the default, + number of CPU's in the system is used. If 0 then parallel processing in + jobs is disabled. If set to positive number then specified maximum + number of processes will be used. + +:strong:`local_shell` + Shell command used for local execution. + +:strong:`out_path` + Directory to save cdist output in. + +:strong:`parallel` + Process hosts in parallel. If -1 then the default, number of CPU's in + the system is used. If 0 then parallel processing of hosts is disabled. + If set to positive number then specified maximum number of processes + will be used. + +:strong:`remote_copy` + Command to use for remote copy (should behave like scp). + +:strong:`remote_exec` + Command to use for remote execution (should behave like ssh). + +:strong:`remote_out_path` + Directory to save cdist output in on the target host. + +:strong:`remote_shell` + Shell command at remote host used for remote execution. + +:strong:`save_output_streams` + Enable/disable saving output streams (enabled by default). + It recognizes boolean values from 'yes'/'no', 'on'/'off', 'true'/'false' + and '1'/'0'. + +:strong:`timestamp` + Timestamp log messages with the current local date and time + in the format: YYYYMMDDHHMMSS.us. + +:strong:`verbosity` + Set verbosity level. Valid values are: + 'ERROR', 'WARNING', 'INFO', 'VERBOSE', 'DEBUG', 'TRACE' and 'OFF'. + + FILES ----- ~/.cdist Your personal cdist config directory. If exists it will be automatically used. +~/.cdist/cache + Local cache directory. +~/.cdist/inventory + The home inventory directory. If ~/.cdist exists it will be used as + default inventory directory. cdist/conf The distribution configuration directory. It contains official types and explorers. This path is relative to cdist installation directory. +cdist/inventory + The distribution inventory directory. + This path is relative to cdist installation directory. +/etc/cdist.cfg + Global cdist configuration file, if exists. +~/.cdist.cfg or $XDG_CONFIG_HOME/cdist/cdist.cfg + Local cdist configuration file, if exists. NOTES ----- @@ -174,7 +563,7 @@ EXAMPLES .. code-block:: sh # Configure ikq05.ethz.ch with debug enabled - % cdist config -d ikq05.ethz.ch + % cdist config -vvv ikq05.ethz.ch # Configure hosts in parallel and use a different configuration directory % cdist config -c ~/p/cdist-nutzung \ @@ -208,7 +597,44 @@ EXAMPLES [--group GROUP] [--owner OWNER] [--mode MODE] object_id # Install ikq05.ethz.ch with debug enabled - % cdist install -d ikq05.ethz.ch + % cdist install -vvv ikq05.ethz.ch + + # List inventory content + % cdist inventory list -b + + # List inventory for specified host localhost + % cdist inventory list -b localhost + + # List inventory for specified tag loadbalancer + % cdist inventory list -b -t loadbalancer + + # Add hosts to inventory + % cdist inventory add-host -b web1 web2 web3 + + # Delete hosts from file old-hosts from inventory + % cdist inventory del-host -b -f old-hosts + + # Add tags to specified hosts + % cdist inventory add-tag -b -t europe,croatia,web,static web1 web2 + + # Add tag to all hosts in inventory + % cdist inventory add-tag -b -t vm + + # Delete all tags from specified host + % cdist inventory del-tag -b -a localhost + + # Delete tags read from stdin from hosts specified by file hosts + % cdist inventory del-tag -b -T - -f hosts + + # Configure hosts from inventory with any of specified tags + % cdist config -b -t web dynamic + + # Configure hosts from inventory with all specified tags + % cdist config -b -t -a web dynamic + + # Configure all hosts from inventory db + $ cdist config -b -A + ENVIRONMENT ----------- @@ -239,8 +665,18 @@ CDIST_REMOTE_EXEC CDIST_REMOTE_COPY Use this command for remote copy (should behave like scp). +CDIST_INVENTORY_DIR + Use this directory as inventory directory. + CDIST_BETA - Enable beta functionalities. + Enable beta functionality. + +CDIST_CACHE_PATH_PATTERN + Custom cache path pattern. + +CDIST_CONFIG_FILE + Custom configuration file. + EXIT STATUS ----------- @@ -256,6 +692,7 @@ AUTHORS Originally written by Nico Schottelius and Steven Armstrong . + CAVEATS ------- When operating in parallel, either by operating in parallel for each host @@ -271,10 +708,10 @@ options. For more details refer to :strong:`sshd_config`\ (5). When requirements for the same object are defined in different manifests (see example below), for example, in init manifest and in some other type manifest and those requirements differ then dependency resolver cannot detect -dependencies right. This happens because cdist cannot prepare all objects first +dependencies correctly. This happens because cdist cannot prepare all objects first and run all objects afterwards. Some object can depend on the result of type explorer(s) and explorers are executed during object run. cdist will detect -such case and write warning message. Example for such a case: +such case and display a warning message. An example of such a case: .. code-block:: sh diff --git a/docs/web/cdist.mdwn b/docs/web/cdist.mdwn deleted file mode 100644 index 74457fc8..00000000 --- a/docs/web/cdist.mdwn +++ /dev/null @@ -1,21 +0,0 @@ -[[!meta title="cdist - usable configuration management"]] - -![cdist-logo](cdist-logo.png "cdist logo") - -cdist is a usable configuration management system. -It adheres to the KISS principle and -is being used in small up to enterprise grade environments. -cdist is an alternative to other configuration management systems like -[bcfg2](http://trac.mcs.anl.gov/projects/bcfg2), -[chef](http://wiki.opscode.com/display/chef/), -[cfengine](http://www.cfengine.org/) -and [puppet](http://www.puppetlabs.com/). - - * [[Why should I use cdist?|why]] - * [[Documentation|documentation]] - * [[Supported Operating Systems|os]] - * [[Installation|install]] - * [[Update|update]] - * [[Support|support]] - -[[!tag cdist unix]] diff --git a/docs/web/cdist/cdist-logo.png b/docs/web/cdist/cdist-logo.png deleted file mode 100644 index 13c27927..00000000 Binary files a/docs/web/cdist/cdist-logo.png and /dev/null differ diff --git a/docs/web/cdist/documentation.mdwn b/docs/web/cdist/documentation.mdwn deleted file mode 100644 index 6479a4bb..00000000 --- a/docs/web/cdist/documentation.mdwn +++ /dev/null @@ -1,12 +0,0 @@ -[[!meta title="Documentation"]] - -You can browse the -[latest version of the documentation](/software/cdist/man/latest) or -have a look at [all versions](/software/cdist/man). - -You can also view [speeches about cdist](/software/cdist/speeches). - -Checking out beta? Find the docs here: -[beta documentation](/software/cdist/man/beta). - -[[!tag cdist unix]] diff --git a/docs/web/cdist/features.mdwn b/docs/web/cdist/features.mdwn deleted file mode 100644 index 77c61382..00000000 --- a/docs/web/cdist/features.mdwn +++ /dev/null @@ -1,26 +0,0 @@ -But cdist ticks differently, here is the feature set that makes it unique: - -[[!table data=""" -Keywords | Description -Simplicity | There is only one type to extend cdist called ***type*** -Design | Type and core cleanly separated -Design | Sticks completly to the KISS (keep it simple and stupid) paradigma -Design | Meaningful error messages - do not lose time debugging error messages -Design | Consistency in behaviour, naming and documentation -Design | No surprise factor: Only do what is obviously clear, no magic -Design | Define target state, do not focus on methods or scripts -Design | Push architecture: Instantly apply your changes -Small core | cdist's core is very small - less code, less bugs -Fast development | Focus on straightforwardness of type creation is a main development objective -Fast development | Batteries included: A lot of requirements can be solved using standard types -Modern Programming Language | cdist is written in Python -Requirements, Scalability | No central server needed, cdist operates in push mode and can be run from any computer -Requirements, Scalability, Upgrade | cdist only needs to be updated on the master, not on the target hosts -Requirements, Security | Uses well-know [SSH](http://www.openssh.com/) as transport protocol -Requirements, Simplicity | Requires only shell and SSH server on the target -UNIX | Reuse of existing tools like cat, find, mv, ... -UNIX, familar environment, documentation | Is available as manpages and HTML -UNIX, simplicity, familar environment | cdist is configured in POSIX shell -"""]] - -[[!tag cdist unix]] diff --git a/docs/web/cdist/install.mdwn b/docs/web/cdist/install.mdwn deleted file mode 100644 index 0ced26db..00000000 --- a/docs/web/cdist/install.mdwn +++ /dev/null @@ -1,103 +0,0 @@ -[[!meta title="How to install cdist"]] -[[!toc levels=3]] - -## Requirements - -### Source Host - -This is the machine you use to configure the target hosts. - - * /bin/sh: A posix like shell (for instance bash, dash, zsh) - * Python >= 3.2 - * SSH client - * Asciidoc and xsltproc (for building the manpages) - -### Target Hosts - - * /bin/sh: A posix like shell (for instance bash, dash, zsh) - * SSH server - -## Install cdist - -You can install cdist either from git or as a python package. - -### From git - -Cloning cdist from git gives you the advantage of having -a version control in place for development of your own stuff -immediately. - -To install cdist, execute the following commands: - - git clone https://github.com/ungleich/cdist.git - cd cdist - export PATH=$PATH:$(pwd -P)/bin - -From version 4.2.0 cdist tags and github releases are signed. -You can get GPG public key used for signing [here](/software/cdist/pgp-key-EFD2AE4EC36B6901.asc). - -#### Available versions in git - - * The active development takes place in the **master** branch - * The current stable version can be found in the **2.0** branch - * The upcoming stable version can be found in the **2.1** branch - -Other branches may be available for features or bugfixes, but they -may vanish at any point. To select a specific branch use - - # Generic code - git checkout -b origin/ - -So for instance if you want to use and stay with version 2.0, you can use - - git checkout -b 2.0 origin/2.0 - -#### Git Mirrors - -If the main site is down, you can acquire cdist from one of the following sites: - - * git://github.com/telmich/cdist.git ([github](https://github.com/telmich/cdist)) - * git://git.code.sf.net/p/cdist/code ([sourceforge](https://sourceforge.net/p/cdist/code)) - -#### Building and using documentation (man and html) - -If you want to build and use the documentation, run: - - make docs - -Documentation comes in two formats, man pages and full HTML -documentation. Documentation is built into distribution's -docs/dist directory. man pages are in docs/dist/man and -HTML documentation in docs/dist/html. - -If you want to use man pages, run: - - export MANPATH=$MANPATH:$(pwd -P)/docs/dist/man - -Or you can move manpages from docs/dist/man directory to some -other directory and add it to MANPATH. - -Full HTML documentation can be accessed at docs/dist/html/index.html. - -You can also build manpages for types in your ~/.cdist directory: - - make dotman - -Built manpages are now in docs/dist/man directory. If you have -some other custom .cdist directory, e.g. /opt/cdist then use: - - DOT_CDIST_PATH=/opt/cdist make dotman - - -### Python Package - -Cdist is available as a python package at -[PyPi](http://pypi.python.org/pypi/cdist/). You can install it using - - pip install cdist - -## Use cdist - -[[Dig into the documentation|documentation]] to get started with cdist! - -[[!tag cdist unix]] diff --git a/docs/web/cdist/os.mdwn b/docs/web/cdist/os.mdwn deleted file mode 100644 index 3677f52c..00000000 --- a/docs/web/cdist/os.mdwn +++ /dev/null @@ -1,18 +0,0 @@ -[[!meta title="Supported Operating Systems"]] - -cdist was tested or is know to run on at least - - * [Archlinux](http://www.archlinux.org/) - * [Debian](http://www.debian.org/) - * [CentOS](http://www.centos.org/) - * [Scientific](https://www.scientificlinux.org/) - * [Fedora](http://fedoraproject.org/) - * [FreeBSD](http://www.freebsd.org) - * [Gentoo](http://www.gentoo.org/) - * [Mac OS X](http://www.apple.com/macosx/) - * [OpenBSD](http://www.openbsd.org) - * [Redhat](http://www.redhat.com/) - * [Ubuntu](http://www.ubuntu.com/) - * [XenServer](http://www.citrix.com/xenserver/) - -[[!tag cdist unix]] diff --git a/docs/web/cdist/support.mdwn b/docs/web/cdist/support.mdwn deleted file mode 100644 index 4f92853b..00000000 --- a/docs/web/cdist/support.mdwn +++ /dev/null @@ -1,28 +0,0 @@ -## Support - -### IRC - -You can join the development ***IRC channel*** -[#cstar on irc.freenode.net](irc://irc.freenode.org/#cstar). - -### Mailing list - -Bug reports, questions, patches, etc. should be send to the -[cdist mailing list](https://groups.google.com/forum/#!forum/cdist-configuration-management). - -### Linkedin - -If you have an account -at [Linked in](http://www.linkedin.com/), -you can join the -[cdist group](http://www.linkedin.com/groups/cdist-configuration-management-3952797). - -### Chat -Chat with us: [ungleich chat](https://chat.ungleich.ch/channel/cdist). - -### Commercial support - -You can request commercial support for cdist from -[my company](http://www.ungleich.ch/english/). - -[[!tag cdist unix]] diff --git a/docs/web/cdist/update.mdwn b/docs/web/cdist/update.mdwn deleted file mode 100644 index df4617bb..00000000 --- a/docs/web/cdist/update.mdwn +++ /dev/null @@ -1,158 +0,0 @@ -[[!meta title="How to update cdist"]] - -## Update The Git Installation - -To upgrade cdist in the current branch use - - git pull - - # Also update the manpages - ./build man - export MANPATH=$MANPATH:$(pwd -P)/doc/man - -If you stay on a version branche (i.e. 1.0, 1.1., ...), nothing should break. -The master branch on the other hand is the development branch and may not be -working, break your setup or eat the tree in your garden. - -### Safely upgrading to new versions - -To upgrade to **any** further cdist version, you can take the -following procedure to do a safe upgrade: - - # Create new branch to try out the update - git checkout -b upgrade_cdist - - # Get latest cdist version in git database - git fetch -v - - # see what will happen on merge - replace - # master with the branch you plan to merge - git diff upgrade_cdist..origin/master - - # Merge the new version - git merge origin/master - -Now you can ensure all custom types work with the new version. -Assume that you need to go back to an older version during -the migration/update, you can do so as follows: - - # commit changes - git commit -m ... - - # go back to original branch - git checkout master - -After that, you can go back and continue the upgrade: - - # git checkout upgrade_cdist - - -## Update The Python Package - -To upgrade to the lastet version do - - pip install --upgrade cdist - -## General Update Instructions - -### Updating from 3.0 to 3.1 - -The type **\_\_ssh_authorized_keys** now also manages existing keys, -not only the ones added by cdist. - -### Updating from 2.3 to 3.0 - -The **changed** attribute of objects has been removed. -Use [messaging](/software/cdist/man/3.0.0/man7/cdist-messaging.html) instead. - -### Updating from 2.2 to 2.3 - -No incompatibilities. - -### Updating from 2.1 to 2.2 - -Starting with 2.2, the syntax for requiring a singleton type changed: -Old format: - - require="__singleton_type/singleton" ... - -New format: - - require="__singleton_type" ... - -Internally the "singleton" object id was dropped to make life more easy. -You can probably fix your configuration by running the following code -snippet (currently untested, please report back if it works for you): - - find ~/.cdist/* -type f -exec sed -i 's,/singleton,,' {} \; - -### Updating from 2.0 to 2.1 - -Have a look at the update guide for [[2.0 to 2.1|2.0-to-2.1]]. - - * Type **\_\_package* and \_\_process** use --state **present** or **absent**. - The states **removed/installed** and **stopped/running** have been removed. - Support for the new states is already present in 2.0. - * Type **\_\_directory**: Parameter --parents and --recursive are now boolean - The old "yes/no" values need to be removed. - * Type **\_\_rvm_ruby**: Parameter --default is now boolean - The old "yes/no" values need to be removed. - * Type **\_\_rvm_gemset**: Parameter --default is now boolean - The old "yes/no" values need to be removed. - * Type **\_\_addifnosuchline** and **\_\_removeline** have been replaced by **\_\_line** - * The **conf** directory is now located at **cdist/conf**. - You need to migrate your types, explorers and manifests - manually to the new location. - * Replace the variable **\_\_self** by **\_\_object_name** - Support for the variable **\_\_object_name** is already present in 2.0. - * The types **\_\_autofs**, **\_\_autofs_map** and **\_\_autofs_reload** have been removed - (no maintainer, no users) - * Type **\_\_user**: Parameter --groups removed (use the new \_\_user_groups type) - * Type **\_\_ssh_authorized_key** has been replaced by more flexible type - **\_\_ssh_authorized_keys** - -### Updating from 1.7 to 2.0 - -* Ensure python (>= 3.2) is installed on the source host -* Use "cdist config host" instead of "cdist-deploy-to host" -* Use "cdist config -p host1 host2" instead of "cdist-mass-deploy" -* Use "cdist banner" for fun -* Use **\_\_object_name** instead of **\_\_self** in manifests - -### Updating from 1.6 to 1.7 - -* If you used the global explorer **hardware_type**, you need to change - your code to use **machine** instead. - -### Updating from 1.5 to 1.6 - -* If you used **\_\_package_apt --preseed**, you need to use the new - type **\_\_debconf_set_selections** instead. -* The **\_\_package** types accepted either --state deinstalled or - --state uninstaaled. Starting with 1.6, it was made consistently - to --state removed. - -### Updating from 1.3 to 1.5 - -No incompatibilities. - -### Updating from 1.2 to 1.3 - -Rename **gencode** of every type to **gencode-remote**. - -### Updating from 1.1 to 1.2 - -No incompatibilities. - -### Updating from 1.0 to 1.1 - -In 1.1 the type **\_\_file** was split into **\_\_directory**, **\_\_file** and -**\_\_link**. The parameter **--type** was removed from **\_\_file**. Thus you -need to replace **\_\_file** calls in your manifests: - - * Remove --type from all \_\_file calls - * If type was symlink, use \_\_link and --type symbolic - * If type was directory, use \_\_directory - - -[[!tag cdist unix]] diff --git a/docs/web/cdist/update/2.0-to-2.1.mdwn b/docs/web/cdist/update/2.0-to-2.1.mdwn deleted file mode 100644 index 3b5f5dc4..00000000 --- a/docs/web/cdist/update/2.0-to-2.1.mdwn +++ /dev/null @@ -1,118 +0,0 @@ -[[!meta title="Update Guide for 2.0 to 2.1"]] - -## Introduction - -When changing your installation from 2.0 to 2.1, there are -a lot of changes coming up. 2.1 is mainly a cleanup release, -which removes long time deprecated behaviour, but also makes -a lot of things more consistent and allows you to split off your types, -explorers and manifest to custom directories. - -This document will guide you to a successful update. - -## Preparation - -As for every software and system you use in production, you should first of -all make a backup of your data. To prevent any breakage, it is -recommended to create a new git branch to do the update on: - - % git checkout -b update_to_2.1 - -This also ensure that whenever you need to do a change in your -2.0 based tree, you can simply go back to that branch, apply the change -and configure your systems - independently of your update progress! - -Next fetch the latest upstream changes, I assume that -origin refers to one of the upstream mirrors (change origin if you use -another remote name for upstream cdist): - - % git fetch -v origin - -## Merge the changes - -Now try to merge upstream into the new branch. - - % git merge origin/2.1 - -Fix any conflicts that may have been occurred due to local changes -and then **git add** and *git commit** those changes. This should seldom -occur and if, it's mostly for people hacking on the cdist core. - -## Move "conf" directory - -One of the biggest changes in cdist 2.1 is that you can have multiple -**conf** directories: Indeed, the new default behaviour of cdist is to -search for conf directories - - * below the python module (cdist/conf in the source tree or in the installed location) - * at ~/.cdist/ (on conf suffix there) - -So you can now choose, where to store your types. - -### Integrate your conf/ back into the tree - -If you choose to store your types together with the upstream types, -you can just move all your stuff below **cdist/conf**: - - % git mv conf/type/* cdist/conf/type - % git mv conf/manifest/* cdist/conf/manifest - % git mv conf/explorer/* cdist/conf/explorer - % git commit -m "Re-Integrate my conf directory into cdist 2.1 tree" - -### Move your conf/ directory to ~/.cdist - -If you want to store your site specific -configuration outside of the cdist tree, you -can move your conf/ directory to your homedirectory ($HOME) under ~/.cdist: - - % mv conf ~/.cdist - % git rm -r conf - % git commit -m "Move my conf directory to ~/.cdist" - -It it still recommended to use a version control system like git in it: - - % cd ~/.cdist - % git init - % git add . - % git commit -m "Create new git repository containing my cdist configuration" - -## Test the migration - -Some of the types shipped with upstream were changed, so you may want to test -the result by running cdist on one of your staging target hosts: - - % ./bin/cdist config -v staging-host - -All incompatibilities are listed on the [[cdist update page|software/cdist/update]], -so you can browse through the list and update your configuration. - -## Final Cleanups - -When everything is tested, there are some cleanups to be done to finalise the update. - -### When continuing to keep conf/ in the tree - -You can then merge back your changes into the master tree and continue to work -as normal. - -### When using ~/.cdist - -If you decided to move your site specific code to ~/.cdist, you can now switch your -**master** branch or version branch to upstream directly. Assumnig you are in the -cdist directory, having your previous branch checked out, you can create a clean -state using the following commands: - - % upstream_branch=2.1 - % current_branch=$(git rev-parse --abbrev-ref HEAD) - % git checkout -b archive_my_own_tree - % git branch -D "$current_branch" - % git checkout -b "$current_branch" "origin/$upstream_branch" - -Afther these commands, your previous main branch is accessible at -**archive_my_own_tree** and your branch is now tracking upstream. - -## Questions? Critics? Hints? - -If you think this manual helped or misses some information, do not -hesitate to contact us on any of the usual ways (irc, mailinglist, -github issue tracker, ...). diff --git a/docs/web/cdist/why.mdwn b/docs/web/cdist/why.mdwn deleted file mode 100644 index f571555c..00000000 --- a/docs/web/cdist/why.mdwn +++ /dev/null @@ -1,69 +0,0 @@ -[[!meta title="Why should I use cdist?"]] - -[[!toc]] - -There are several motivations to use cdist, these -are probably the most popular ones. - -## Known language - -Cdist is being configured in -[shell script](https://en.wikipedia.org/wiki/Shell_script). -Shell script is used by UNIX system engineers for decades. -So when cdist is introduced, your staff does not need to learn a new -[DSL](https://en.wikipedia.org/wiki/Domain-specific_language) -or programming language. - -## Powerful language - -Not only is shell scripting widely known by system engineers, -but it is also a very powerful language. Here are some features -which make daily work easy: - - * Configuration can react dynamicly on explored values - * High level string manipulation (using sed, awk, grep) - * Conditional support (**if, case**) - * Loop support (**for, while**) - * Support for dependencies between cdist types - -## More than shell scripting - -If you compare regular shell scripting with cdist, there is one major -difference: When using cdist types, -the results are -[idempotent](https://en.wikipedia.org/wiki/Idempotence). -In practise that means it does not matter in which order you -call cdist types, the result is always the same. - -## Zero dependency configuration management - -Cdist requires very litte on a target system. Even better, -in almost all cases all dependencies are usually fulfilled. -Cdist does not require an agent or a high level programming -languages on the target host: it will run on any host that -has a **ssh server running** and a posix compatible shell -(**/bin/sh**). Compared to other configuration management systems, -it does not require to open up an additional port. - -## Push based distribution - -Cdist uses the push based model for configuration. In this -scenario, one (or more) computers connect the target hosts -and apply the configuration. That way the source host has -very little requirements: Cdist can even run on a sysadmin -notebook that is loosely connected to the network and has -limited amount of resources. - -Furthermore, from a security point of view, only one machine -needs access to the target hosts. No target hosts will ever -need to connect back to the source host, which contains the -full configuration. - -## Highly scalable - -If at some point you manage more hosts than can be handled from -a single source host, you can simply add more resources: Either -add more cores to one host or add hosts. -Cdist will utilise the given resources in parallel. - -[[!tag cdist unix]] diff --git a/scripts/cdist b/scripts/cdist index 498091b8..3110e657 100755 --- a/scripts/cdist +++ b/scripts/cdist @@ -21,31 +21,22 @@ # # -import collections import logging +import sys +import cdist +import cdist.argparse +import cdist.banner +import cdist.config +import cdist.install +import cdist.shell +import cdist.inventory def commandline(): """Parse command line""" - import cdist.argparse - import cdist.banner - import cdist.config - import cdist.install - import cdist.shell - import shutil - import os - - parser = cdist.argparse.get_parsers() - args = parser['main'].parse_args(sys.argv[1:]) - - # Loglevels are handled globally in here - retval = cdist.argparse.handle_loglevel(args) - if retval: - log.warning(retval) - - log.debug(args) - log.info("version %s" % cdist.VERSION) + parser, cfg = cdist.argparse.parse_and_configure(sys.argv[1:]) + args = cfg.get_args() # Work around python 3.3 bug: # http://bugs.python.org/issue16308 @@ -62,12 +53,10 @@ def commandline(): parser['main'].print_help() sys.exit(0) - cdist.argparse.check_beta(vars(args)) args.func(args) -if __name__ == "__main__": - import sys +if __name__ == "__main__": cdistpythonversion = '3.2' if sys.version < cdistpythonversion: print('Python >= {} is required on the source host.'.format( @@ -77,14 +66,8 @@ if __name__ == "__main__": exit_code = 0 try: - import os import re - import cdist - import cdist.log - - logging.setLoggerClass(cdist.log.Log) - logging.basicConfig(format='%(levelname)s: %(message)s') - log = logging.getLogger("cdist") + import os if re.match("__", os.path.basename(sys.argv[0])): import cdist.emulator @@ -97,6 +80,7 @@ if __name__ == "__main__": exit_code = 2 except cdist.Error as e: + log = logging.getLogger("cdist") log.error(e) exit_code = 1 diff --git a/scripts/cdist-dump b/scripts/cdist-dump new file mode 100755 index 00000000..83b09eb8 --- /dev/null +++ b/scripts/cdist-dump @@ -0,0 +1,325 @@ +#!/bin/sh + +VERSION="0.0.1" +RELEASE="" + +set -u +# set -x + +hosts= +cache_dir=~/.cdist/cache + +do_all=1 +do_global_explorer= +do_type_explorer= +do_script_stdout= +do_script_stderr= +do_gencode= +do_code= +do_messages= +do_parameter= +delimiter=':' +ln= +filename_prefix=1 +verbose=0 + +myname=${0##*/} + +print_version() +{ + printf "%s %s %s\n" "${myname}" "${VERSION}" "${RELEASE}" +} + +usage() +{ + cat << eof +${myname}: [options] [host...] +eof + + print_version + + cat << eof + +Dump data from cache directories. + +host + Dump data for specified hosts. If not specified then all data + from cache directory is dumped. + +Options + -a dump all + -C CACHE-DIR use specified CACHE-DIR (default: ~/.cdist/cache) + -c dump code-* + -d DELIMITER delimiter used for filename and line number prefix (default: ':') + -E dump global explorers + -e dump type explorers + -F disable filename prefix (enabled by default) + -f enable filename prefix (default) + -g dump gencode-* + -h show this help screen and exit + -L disable line number prefix (default) + -l enable line number prefix (disabled by default) + -m dump messages + -o dump executions' stdout + -p dump parameters + -r dump executions' stderr + -V show version and exit + -v increase verbosity +eof +} + +exit_err() +{ + printf "%s\n" "$1" + exit 1 +} + +# parse options +while [ "$#" -ge 1 ] +do + case "$1" in + -a) + do_all=1 + ;; + -C) + if [ "$#" -ge 2 ] + then + case "$2" in + -*) + exit_err "Missing cache directory" + ;; + *) + cache_dir="$2" + shift + ;; + esac + else + exit_err "Missing cache directory" + fi + ;; + -c) + do_code=1 + do_all= + ;; + -d) + if [ "$#" -ge 2 ] + then + case "$2" in + -*) + exit_err "Missing delimiter" + ;; + *) + delimiter="$2" + shift + ;; + esac + else + exit_err "Missing delimiter" + fi + ;; + -E) + do_global_explorer=1 + do_all= + ;; + -e) + do_type_explorer=1 + do_all= + ;; + -F) + filename_prefix= + ;; + -f) + filename_prefix=1 + ;; + -g) + do_gencode=1 + do_all= + ;; + -h) + usage + exit 0 + ;; + -L) + ln= + ;; + -l) + ln=1 + ;; + -m) + do_messages=1 + do_all= + ;; + -o) + do_script_stdout=1 + do_all= + ;; + -p) + do_parameter=1 + do_all= + ;; + -r) + do_script_stderr=1 + do_all= + ;; + -V) + print_version + exit 0 + ;; + -v) + verbose=$((verbose + 1)) + ;; + *) + hosts="${hosts} $1" + break + ;; + esac + shift +done + +if [ "${ln}" = "1" ] +then + ln="NR \"${delimiter}\"" +fi + +if [ "${filename_prefix}" = "1" ] +then + filename_prefix="{}${delimiter}" +fi + +if [ "${do_all}" = "1" ] +then + do_global_explorer=1 + do_type_explorer=1 + do_script_stdout=1 + do_script_stderr=1 + do_gencode=1 + do_code=1 + do_messages=1 + do_parameter=1 +fi + +set -- -size +0 +set -- "$@" \( +or= + +print_verbose() +{ + if [ "${verbose}" -ge "$1" ] + then + printf "%s\n" "$2" + fi +} + +hor_line() +{ + if [ $# -gt 0 ] + then + c="$1" + else + c='=' + fi + printf "%78s\n" "" | tr ' ' "${c}" +} + +if [ "${do_global_explorer}" ] +then + print_verbose 2 "Dumping global explorers" + set -- "$@" ${or} \( \ + -path "*/explorer/*" -a \ + ! -path "*/conf/*" -a \ + ! -path "*/object/*/explorer/*" \ + \) + or="-o" +fi + +if [ "${do_type_explorer}" ] +then + print_verbose 2 "Dumping type explorers" + set -- "$@" ${or} -path "*/object/*/explorer/*" + or="-o" +fi + +if [ "${do_script_stdout}" ] +then + print_verbose 2 "Dumping execution's stdout" + set -- "$@" ${or} -path "*/stdout/*" + or="-o" +fi + +if [ "${do_script_stderr}" ] +then + print_verbose 2 "Dumping execution's stderr" + set -- "$@" ${or} -path "*/stderr/*" + or="-o" +fi + +if [ "${do_gencode}" ] +then + print_verbose 2 "Dumping gencode-*" + set -- "$@" ${or} \( -name "gencode-*" -a ! -path "*/stdout/*" -a ! -path "*/stderr/*" \) + or="-o" +fi + +if [ "${do_code}" ] +then + print_verbose 2 "Dumping code-*" + set -- "$@" ${or} \( -name "code-*" -a ! -path "*/stdout/*" -a ! -path "*/stderr/*" \) + or="-o" +fi + +if [ "${do_messages}" ] +then + print_verbose 2 "Dumping messages" + set -- "$@" ${or} -name "messages" + or="-o" +fi + +if [ "${do_parameter}" ] +then + print_verbose 2 "Dumping parameters" + set -- "$@" ${or} -path "*/parameter/*" + or="-o" +fi + +set -- "$@" \) +set -- '.' "$@" -exec awk -v prefix="${filename_prefix}" "{print prefix ${ln} \$0}" {} \; + +# printf "+ %s\n" "$*" + +print_verbose 2 "Using cache dir: ${cache_dir}" + +OLD_PWD=$(pwd) +cd "${cache_dir}" || exit + +# If no host is specified then search all. +[ -z "${hosts}" ] && hosts="-" + +for host in ${hosts} +do + [ "${host}" = "-" ] && host= + # find host cache directory + host_dir=$(find . -name target_host -exec grep -l "${host}" {} +) + print_verbose 3 "found host directory files:" + print_verbose 3 "${host_dir}" + + OLD_IFS="${IFS}" + IFS=" + " + + for d in ${host_dir} + do + dir=$(dirname "${d}") + + print_verbose 0 "target host: $(cat "${dir}/target_host"), host directory: ${dir}" + hor_line '=' + + PREV_PWD=$(pwd) + cd "${dir}" || exit + # set -x + find "$@" + # set +x + cd "${PREV_PWD}" || exit + done + IFS="${OLD_IFS}" +done +cd "${OLD_PWD}" || exit diff --git a/scripts/cdist-new-type b/scripts/cdist-new-type new file mode 100755 index 00000000..79dcfd90 --- /dev/null +++ b/scripts/cdist-new-type @@ -0,0 +1,159 @@ +#!/bin/sh + +basename="${0##*/}" + +if [ $# -lt 3 ] +then + printf "usage: %s TYPE-NAME AUTHOR-NAME AUTHOR-EMAIL [TYPE-BASE-PATH] + TYPE-NAME Name of the type. + AUTHOR-NAME Type author's full name. + AUTHOR-EMAIL Type author's email. + TYPE-BASE-PATH Path to the base directory of the type. If not set it defaults + to '\$PWD/type'.\n" "${basename}" + exit 1 +fi + +type_name="$1" +shift +author_name="$1" +shift +author_email="$1" +shift + +if [ $# -ge 1 ] +then + type_base_path="$1" + shift +else + #type_base_path=~/.cdist/type + type_base_path="$PWD/type" +fi + +error() { + printf "%s\n" "$*" >&2 +} + +die() { + error "$@" + exit 1 +} + +cd "$type_base_path" || die "Could not change to type directory: $type_base_path. +You have to specify type base path or run me from within a cdist conf directory, +e.g. ~/.cdist." + +year=$(date +%Y) +copyright="# $year $author_name ($author_email)" + +license="# This file is part of cdist. +# +# cdist is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# cdist is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with cdist. If not, see . +# +" + +set -e + +mkdir "$type_name" +cd "$type_name" + +### man page +header="cdist-type${type_name}(7)" +header_length="${#header}" +cat >> man.rst << DONE +$header +$(while [ "${header_length}" -gt 0 ]; do printf "="; header_length=$((header_length - 1)); done; printf "\n";) + +NAME +---- +cdist-type${type_name} - TODO + + +DESCRIPTION +----------- +This space intentionally left blank. + + +REQUIRED PARAMETERS +------------------- +None. + + +OPTIONAL PARAMETERS +------------------- +None. + + +BOOLEAN PARAMETERS +------------------ +None. + + +EXAMPLES +-------- + +.. code-block:: sh + + # TODO + ${type_name} + + +SEE ALSO +-------- +:strong:\`TODO\`\\ (7) + + +AUTHORS +------- +$author_name <$author_email> + + +COPYING +------- +Copyright \(C) $year $author_name. You can redistribute it +and/or modify it under the terms of the GNU General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. +DONE + +### manifest +cat >> manifest << DONE +#!/bin/sh -e +# +${copyright} +# +${license} + +os=\$(cat "\$__global/explorer/os") + +case "\$os" in + *) + printf "Your operating system (%s) is currently not supported by this type (%s)\n" "\$os" "\${__type##*/}" >&2 + printf "Please contribute an implementation for it if you can.\n" >&2 + exit 1 + ;; +esac +DONE +chmod +x manifest + +# gencode-remote +cat >> gencode-remote << DONE +#!/bin/sh -e +# +${copyright} +# +${license} +DONE +chmod +x gencode-remote + +printf "%s/%s\n" "$type_base_path" "$type_name" diff --git a/setup.py b/setup.py index f29a8998..ae651125 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ def data_finder(data_dir): return entries + cur = os.getcwd() os.chdir("cdist") package_data = data_finder("conf") @@ -35,12 +36,12 @@ setup( name="cdist", packages=["cdist", "cdist.core", "cdist.exec", "cdist.util", ], package_data={'cdist': package_data}, - scripts=["scripts/cdist"], + scripts=["scripts/cdist", "scripts/cdist-dump", "scripts/cdist-new-type"], version=cdist.version.VERSION, description="A Usable Configuration Management System", author="Nico Schottelius", author_email="nico-cdist-pypi@schottelius.org", - url="http://www.nico.schottelius.org/software/cdist/", + url="https://www.cdi.st/", classifiers=[ "Development Status :: 6 - Mature", "Environment :: Console",