mark/cdist
1
0
Fork 0
forked from ungleich-public/cdist
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'cleanup/rearrange-makefile-and-build-helper' into 'master'

Browse source 
Re-arrange Makefile and build-helper script

See merge request ungleich-public/cdist!778
This commit is contained in:
poljakowski 2019-05-08 22:37:26 +02:00
commit 5ceb4928c7
7 changed files with 296 additions and 200 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitattributes vendored
View file

@ -4,3 +4,5 @@
docs/speeches export-ignore
docs/video export-ignore
docs/src/man7 export-ignore
bin/build-helper export-ignore
README-maintainers export-ignore

158
Makefile
View file

@ -18,25 +18,27 @@
#
#
helper=./bin/build-helper
.PHONY: help
help:
@echo "Please use \`make <target>' where <target> is one of"
@echo "man build only man user documentation"
@echo "html build only html user documentation"
@echo "docs build both man and html user documentation"
@echo "dotman build man pages for types in your ~/.cdist directory"
@echo "speeches build speeches pdf files"
@echo "install install in the system site-packages directory"
@echo "install-user install in the user site-packages directory"
@echo "docs-clean clean documentation"
@echo "clean clean"
DOCS_SRC_DIR=docs/src
SPEECHDIR=docs/speeches
TYPEDIR=cdist/conf/type
CHANGELOG_VERSION=$(shell $(helper) changelog-version)
CHANGELOG_FILE=docs/changelog
PYTHON_VERSION=cdist/version.py
SPHINXM=make -C $(DOCS_SRC_DIR) man
SPHINXH=make -C $(DOCS_SRC_DIR) html
SPHINXC=make -C $(DOCS_SRC_DIR) clean
SHELLCHECKCMD=shellcheck -s sh -f gcc -x
# Skip SC2154 for variables starting with __ since such variables are cdist
# environment variables.
SHELLCHECK_SKIP=grep -v ': __.*is referenced but not assigned.*\[SC2154\]'
################################################################################
# Manpages
#
@ -61,11 +63,16 @@ DOCSREFSH=$(DOCS_SRC_DIR)/cdist-reference.rst.sh
$(DOCSREF): $(DOCSREFSH)
$(DOCSREFSH)
version:
@[ -f "cdist/version.py" ] || { \
printf "Missing 'cdist/version.py', please generate it first.\n" && exit 1; \
}
# Manpages #3: generic part
man: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION)
man: version $(MANTYPES) $(DOCSREF)
$(SPHINXM)
html: $(MANTYPES) $(DOCSREF) $(PYTHON_VERSION)
html: version $(MANTYPES) $(DOCSREF)
$(SPHINXH)
docs: man html
@ -85,7 +92,7 @@ DOTMANTYPES=$(subst /man.rst,.rst,$(DOTMANTYPEPREFIX))
$(DOTMAN7DSTDIR)/cdist-type%.rst: $(DOTTYPEDIR)/%/man.rst
ln -sf "$^" $@
dotman: $(DOTMANTYPES)
dotman: version $(DOTMANTYPES)
$(SPHINXM)
################################################################################
@ -103,132 +110,25 @@ $(SPEECHDIR)/%.pdf: $(SPEECHDIR)/%.tex
speeches: $(SPEECHES)
################################################################################
# Release: Mailinglist
# Misc
#
ML_FILE=.lock-ml
# Only send mail once - lock until new changelog things happened
$(ML_FILE): $(CHANGELOG_FILE)
$(helper) ml-release $(CHANGELOG_VERSION)
touch $@
ml-release: $(ML_FILE)
################################################################################
# pypi
#
PYPI_FILE=.pypi-release
$(PYPI_FILE): man $(PYTHON_VERSION)
python3 setup.py sdist upload
touch $@
pypi-release: $(PYPI_FILE)
################################################################################
# archlinux
#
ARCHLINUX_FILE=.lock-archlinux
ARCHLINUXTAR=cdist-$(CHANGELOG_VERSION)-1.src.tar.gz
$(ARCHLINUXTAR): PKGBUILD
umask 022; mkaurball
PKGBUILD: PKGBUILD.in $(PYTHON_VERSION)
./PKGBUILD.in $(CHANGELOG_VERSION)
$(ARCHLINUX_FILE): $(ARCHLINUXTAR) $(PYTHON_VERSION)
burp -c system $(ARCHLINUXTAR)
touch $@
archlinux-release: $(ARCHLINUX_FILE)
################################################################################
# Release
#
$(PYTHON_VERSION) version: .git/refs/heads/master
$(helper) version
# Code that is better handled in a shell script
check-%:
$(helper) $@
release:
$(helper) $@
################################################################################
# Cleanup
#
clean:
clean: docs-clean
rm -f $(DOCS_SRC_DIR)/cdist-reference.rst
find "$(DOCS_SRC_DIR)" -mindepth 2 -type l \
| xargs rm -f
make -C $(DOCS_SRC_DIR) clean
find * -name __pycache__ | xargs rm -rf
# Archlinux
rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
rm -rf pkg/ src/
rm -f MANIFEST PKGBUILD
rm -rf dist/
# Signed release
rm -f cdist-*.tar.gz
rm -f cdist-*.tar.gz.asc
# Temp files
rm -f *.tmp
distclean: clean
rm -f cdist/version.py
# distutils
rm -rf ./build
################################################################################
# Misc
# install
#
# The pub is Nico's "push to all git remotes" way ("make pub")
pub:
git push --mirror
install:
python3 setup.py install
test:
$(helper) $@
test-remote:
$(helper) $@
pycodestyle pep8:
$(helper) $@
shellcheck-global-explorers:
@find cdist/conf/explorer -type f -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
shellcheck-type-explorers:
@find cdist/conf/type -type f -path "*/explorer/*" -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
shellcheck-manifests:
@find cdist/conf/type -type f -name manifest -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
shellcheck-local-gencodes:
@find cdist/conf/type -type f -name gencode-local -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
shellcheck-remote-gencodes:
@find cdist/conf/type -type f -name gencode-remote -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
shellcheck-scripts:
@$(SHELLCHECKCMD) scripts/cdist-dump || exit 0
shellcheck-gencodes: shellcheck-local-gencodes shellcheck-remote-gencodes
shellcheck-types: shellcheck-type-explorers shellcheck-manifests shellcheck-gencodes
shellcheck: shellcheck-global-explorers shellcheck-types shellcheck-scripts
shellcheck-type-files:
@find cdist/conf/type -type f -path "*/files/*" -exec $(SHELLCHECKCMD) {} + | $(SHELLCHECK_SKIP) || exit 0
shellcheck-with-files: shellcheck shellcheck-type-files
install-user:
python3 setup.py install --user

4
README-maintainers Normal file
View file

@ -0,0 +1,4 @@
Maintainers should use ./bin/build-helper script.
Makefile is intended for end users. It can be used for non-maintaining
targets that can be run from pure source (without git repository).

311
bin/build-helper
View file

@ -22,47 +22,101 @@
# This file contains the heavy lifting found usually in the Makefile.
#
# vars for make
helper=$0
usage() {
printf "usage: %s TARGET [RUN-AS]
Available targets:
print-runas
changelog-changes
changelog-version
check-date
check-unittest
ml-release
archlinux-release
pypi-release
release-git-tag
sign-git-release
release
test
test-remote
pycodestyle
pep8
check-pycodestyle
shellcheck-global-explorers
shellcheck-type-explorers
shellcheck-manifests
shellcheck-local-gencodes
shellcheck-remote-gencodes
shellcheck-scripts
shellcheck-gencodes
shellcheck-types
shellcheck
shellcheck-type-files
shellcheck-with-files
shellcheck-build-helper
check-shellcheck
version-branch
version
target-version
clean
distclean
Run as:
nico
darko - default, if not specified\n" "$1"
}
basedir=${0%/*}/../
# run_as is used to check how the script is called (by $0 value)
# currently supported sufixes for $0 are:
# .darko - run as darko
basename=${0##*/}
run_as=${basename#*.}
basename="${0##*/}"
if [ $# -lt 1 ]
then
usage "${basename}"
exit 1
fi
option=$1; shift
if [ $# -ge 1 ]
then
run_as="$1"
else
run_as="darko"
fi
to_a=cdist-configuration-management
to_d=googlegroups.com
case "$run_as" in
darko)
nico)
from_a=nico.schottelius
from_d=ungleich.ch
ml_name="Nico Schottelius"
ml_sig_name="Nico"
;;
darko|'')
from_a=darko.poljak
from_d=gmail.com
ml_name="Darko Poljak"
ml_sig_name="Darko"
;;
*)
from_a=nico.schottelius
from_d=ungleich.ch
ml_name="Nico Schottelius"
ml_sig_name="Nico"
printf "Unsupported RUN-AS value: '%s'.\n" "${run_as}" >&2
usage "${basename}"
exit 1
;;
esac
SHELLCHECKCMD="shellcheck -s sh -f gcc -x"
# Skip SC2154 for variables starting with __ since such variables are cdist
# environment variables.
SHELLCHECK_SKIP=': __.*is referenced but not assigned.*\[SC2154\]'
to_a="cdist-configuration-management"
to_d="googlegroups.com"
# Change to checkout directory
basedir="${0%/*}/../"
cd "$basedir"
version=$(git describe)
option=$1; shift
case "$option" in
print-make-vars)
printf "helper: ${helper}\n"
;;
print-runas)
printf "run_as: $run_as\n"
printf "run_as: '%s'\n" "$run_as"
;;
changelog-changes)
if [ "$#" -eq 1 ]; then
start=$1
@ -99,8 +153,8 @@ case "$option" in
date_changelog=$(grep '^[[:digit:]]' "$basedir/docs/changelog" | head -n1 | sed 's/.*: //')
if [ "$date_today" != "$date_changelog" ]; then
echo "Date in changelog is not today"
echo "Changelog: $date_changelog"
printf "Date in changelog is not today\n"
printf "Changelog date: %s\n" "${date_changelog}"
exit 1
fi
;;
@ -111,10 +165,15 @@ case "$option" in
ml-release)
if [ $# -ne 1 ]; then
echo "$0 ml-release version" >&2
printf "%s ml-release version\n" "$0" >&2
exit 1
fi
# Send mail only once - lock until new changelog things happened.
[ ! -f .lock-ml ] && touch .lock-ml
x=$(find 'docs' -name changelog -type f -newer .lock-ml)
[ -z "${x}" ] && exit 0
version=$1; shift
to=${to_a}@${to_d}
@ -143,20 +202,45 @@ Automatisation at its best level. With cdist.
eof
) > mailinglist.tmp
if [ "$run_as" = "build-helper" ]
if [ "$run_as" = "nico" ]
then
/usr/sbin/sendmail -f "$from" "$to" < mailinglist.tmp && rm -f mailinglist.tmp
fi
touch .lock-ml
;;
archlinux-release)
if [ $# -ne 1 ]; then
printf "%s archlinux-release version\n" "$0" >&2
exit 1
fi
version=$1; shift
ARCHLINUXTAR="cdist-${version}-1.src.tar.gz"
./PKGBUILD.in "${version}"
umask 022
mkaurball
burp -c system "${ARCHLINUXTAR}"
;;
pypi-release)
# Ensure that pypi release has the right version
"$0" version
make docs-clean
make docs
python3 setup.py sdist upload
;;
release-git-tag)
target_version=$($0 changelog-version)
if git rev-parse --verify refs/tags/$target_version 2>/dev/null; then
echo "Tag for $target_version exists, aborting"
if git rev-parse --verify "refs/tags/${target_version}" 2>/dev/null; then
printf "Tag for %s exists, aborting\n" "${target_version}"
exit 1
fi
printf "Enter tag description for ${target_version}: "
read tagmessage
printf "Enter tag description for %s: " "${target_version}"
read -r tagmessage
# setup for signed tags:
# gpg --fulL-gen-key
@ -174,7 +258,8 @@ eof
# gpg --verify <asc-file> <file>
# gpg --no-default-keyring --keyring <pubkey.gpg> --verify <asc-file> <file>
# Ensure gpg-agent is running.
export GPG_TTY=$(tty)
GPG_TTY=$(tty)
export GPG_TTY
gpg-agent
git tag -s "$target_version" -m "$tagmessage"
@ -184,14 +269,14 @@ eof
sign-git-release)
if [ $# -lt 2 ]
then
printf "usage: $0 sign-git-release TAG TOKEN [ARCHIVE]\n"
printf "usage: %s sign-git-release TAG TOKEN [ARCHIVE]\n" "$0"
printf " if ARCHIVE is not specified then it is created\n"
exit 1
fi
tag="$1"
if ! git rev-parse -q --verify "${tag}" >/dev/null 2>&1
then
printf "Tag \"${tag}\" not found.\n"
printf "Tag \"%s\" not found.\n" "${tag}"
exit 1
fi
token="$2"
@ -257,30 +342,30 @@ eof
target_version=$($0 changelog-version)
target_branch=$($0 version-branch)
echo "Beginning release process for $target_version"
printf "Beginning release process for %s\n" "${target_version}"
# First check everything is sane
"$0" check-date
"$0" check-unittest
"$0" check-pycodestyle
"$0" shellcheck
"$0" check-shellcheck
# Generate version file to be included in packaging
"$0" target-version
# Ensure the git status is clean, else abort
if ! git diff-index --name-only --exit-code HEAD ; then
echo "Unclean tree, see files above, aborting"
printf "Unclean tree, see files above, aborting.\n"
exit 1
fi
# Ensure we are on the master branch
masterbranch=yes
if [ "$(git rev-parse --abbrev-ref HEAD)" != "master" ]; then
echo "Releases are happening from the master branch, aborting"
printf "Releases are happening from the master branch, aborting.\n"
echo "Enter the magic word to release anyway"
read magicword
printf "Enter the magic word to release anyway:"
read -r magicword
if [ "$magicword" = "iknowwhatido" ]; then
masterbranch=no
@ -291,7 +376,7 @@ eof
if [ "$masterbranch" = yes ]; then
# Ensure version branch exists
if ! git rev-parse --verify refs/heads/$target_branch 2>/dev/null; then
if ! git rev-parse --verify "refs/heads/${target_branch}" 2>/dev/null; then
git branch "$target_branch"
fi
@ -306,8 +391,8 @@ eof
# Generate documentation (man and html)
# First, clean old generated docs
make helper=${helper} docs-clean
make helper=${helper} docs
make docs-clean
make docs
#############################################################
# Everything green, let's do the release
@ -322,35 +407,33 @@ eof
fi
# Publish git changes
if [ "$run_as" = "build-helper" ]
then
make helper=${helper} pub
else
# if you want to have mirror locally then uncomment this support
# if [ "$run_as" = "nico" ]
# then
# git push --mirror
# else
# if we are not Nico :) then just push, no mirror
git push
# push also new branch and set up tracking
git push -u origin "${target_branch}"
fi
# Ensure that pypi release has the right version
"$0" version
# fi
# Create and publish package for pypi
make helper=${helper} pypi-release
"$0" pypi-release
if [ "$run_as" = "build-helper" ]
if [ "$run_as" = "nico" ]
then
# Archlinux release is based on pypi
make helper=${helper} archlinux-release
"$0" archlinux-release
fi
# sign git tag
printf "Enter upstream repository authentication token: "
read token
read -r token
"$0" sign-git-release "${target_version}" "${token}"
# Announce change on ML
make helper=${helper} ml-release
"$0" ml-release "${target_version}"
cat << eof
Manual steps post release:
@ -360,7 +443,14 @@ eof
;;
test)
export PYTHONPATH="$(pwd -P)"
if [ ! -f "cdist/version.py" ]
then
printf "cdist/version.py is missing, generate it first.\n"
exit 1
fi
PYTHONPATH="$(pwd -P)"
export PYTHONPATH
if [ $# -lt 1 ]; then
python3 -m cdist.test
@ -370,7 +460,15 @@ eof
;;
test-remote)
export PYTHONPATH="$(pwd -P)"
if [ ! -f "cdist/version.py" ]
then
printf "cdist/version.py is missing, generate it first.\n"
exit 1
fi
PYTHONPATH="$(pwd -P)"
export PYTHONPATH
python3 -m cdist.test.exec.remote
;;
@ -383,9 +481,9 @@ eof
printf "\\nPlease review pycodestyle report.\\n"
while true
do
echo "Continue (yes/no)?"
printf "Continue (yes/no)?\n"
any=
read any
read -r any
case "$any" in
yes)
break
@ -394,20 +492,74 @@ eof
exit 1
;;
*)
echo "Please answer with 'yes' or 'no' explicitly."
printf "Please answer with 'yes' or 'no' explicitly.\n"
;;
esac
done
;;
shellcheck-global-explorers)
find cdist/conf/explorer -type f -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
;;
shellcheck-type-explorers)
find cdist/conf/type -type f -path "*/explorer/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
;;
shellcheck-manifests)
find cdist/conf/type -type f -name manifest -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
;;
shellcheck-local-gencodes)
find cdist/conf/type -type f -name gencode-local -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
;;
shellcheck-remote-gencodes)
find cdist/conf/type -type f -name gencode-remote -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
;;
shellcheck-scripts)
${SHELLCHECKCMD} scripts/cdist-dump || exit 0
;;
shellcheck-gencodes)
"$0" shellcheck-local-gencodes
"$0" shellcheck-remote-gencodes
;;
shellcheck-types)
"$0" shellcheck-type-explorers
"$0" shellcheck-manifests
"$0" shellcheck-gencodes
;;
shellcheck)
make helper=${helper} shellcheck
"$0" shellcheck-global-explorers
"$0" shellcheck-types
"$0" shellcheck-scripts
;;
shellcheck-type-files)
find cdist/conf/type -type f -path "*/files/*" -exec ${SHELLCHECKCMD} {} + | grep -v "${SHELLCHECK_SKIP}" || exit 0
;;
shellcheck-with-files)
"$0" shellcheck
"$0" shellcheck-type-files
;;
shellcheck-build-helper)
${SHELLCHECKCMD} ./bin/build-helper
;;
check-shellcheck)
"$0" shellcheck
printf "\\nPlease review shellcheck report.\\n"
while true
do
echo "Continue (yes/no)?"
printf "Continue (yes/no)?\n"
any=
read any
read -r any
case "$any" in
yes)
break
@ -416,7 +568,7 @@ eof
exit 1
;;
*)
echo "Please answer with 'yes' or 'no' explicitly."
printf "Please answer with 'yes' or 'no' explicitly.\n"
;;
esac
done
@ -427,16 +579,39 @@ eof
;;
version)
echo "VERSION = \"$(git describe)\"" > cdist/version.py
printf "VERSION = \"%s\"\n" "$(git describe)" > cdist/version.py
;;
target-version)
target_version=$($0 changelog-version)
echo "VERSION = \"${target_version}\"" > cdist/version.py
printf "VERSION = \"%s\"\n" "${target_version}" > cdist/version.py
;;
clean)
make clean
# Archlinux
rm -f cdist-*.pkg.tar.xz cdist-*.tar.gz
rm -rf pkg/ src/
rm -f MANIFEST PKGBUILD
rm -rf dist/
# Signed release
rm -f cdist-*.tar.gz
rm -f cdist-*.tar.gz.asc
# Temp files
rm -f ./*.tmp
;;
distclean)
"$0" clean
rm -f cdist/version.py
;;
*)
echo "Unknown helper target $@ - aborting"
printf "Unknown target: '%s'.\n" "${option}" >&2
usage "${basename}"
exit 1
;;

1
bin/build-helper.darko
View file

@ -1 +0,0 @@
build-helper

1
docs/changelog
View file

@ -3,6 +3,7 @@ Changelog
next:
* Documentation: Add 'Perils of CDIST_ORDER_DEPENDENCY' sub-section (Darko Poljak)
* Build: Clean and separate end user targets into Makefile and maintainer targets into build-helper (Darko Poljak)
5.0.0: 2019-05-05
* Type __zypper_service: Fix spelling error in manpage (Dmitry Bogatov)

19
docs/src/cdist-install.rst
View file

@ -48,14 +48,29 @@ create version.py:
.. code-block:: sh
make version
./bin/build-helper version
Then, as usual, you execute the following command:
Then you install it with:
.. code-block:: sh
make install
or with:
.. code-block:: sh
make install-user
to install it into user *site-packages* directory.
Or directly with distutils:
.. code-block:: sh
python setup.py install
Note that `bin/build-helper` script is intended for cdist maintainers.
Available versions in git
^^^^^^^^^^^^^^^^^^^^^^^^^