From 86d562857781aa80e0323b6aae67f65cc63be61f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 13 Jun 2016 13:40:32 +0200 Subject: [PATCH 01/58] Implement per source locking. --- ccollect | 87 +++++++++++++++++++++++++++++++++++++++++++++++- doc/changes/next | 1 + 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 5454f51..20498b2 100755 --- a/ccollect +++ b/ccollect @@ -54,6 +54,67 @@ CDATE="date +%Y%m%d-%H%M" DDATE="date +%Y-%m-%d-%H:%M:%S" SDATE="date +%s" +# +# LOCKING: use flock if available, otherwise mkdir +# Locking is done for each source so that only one instance per source +# can run. +# +LOCKDIR="${CSOURCES}" +# printf pattern: ccollect_.lock +LOCKFILE_PATTERN="ccollect_%s.lock" +LOCKFD=4 + +# +# locking functions using flock +# +lock_flock() +{ + # $1 = source to backup + lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" + eval "exec ${LOCKFD}> ${lockfile}" + + flock -n ${LOCKFD} && return 0 || return 1 +} + +unlock_flock() +{ + # $1 = source to backup + lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" + eval "exec ${LOCKFD}>&-" + rm -f "${lockfile}" +} + +# +# locking functions using mkdir (mkdir is atomic) +# +lock_mkdir() +{ + # $1 = source to backup + lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" + + mkdir "${lockfile}" && return 0 || return 1 +} + +unlock_mkdir() +{ + # $1 = source to backup + lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" + + rmdir "${lockfile}" +} + +# +# determine locking tool: flock or mkdir +# +if $(which flock > /dev/null 2>&1) +then + lockf="lock_flock" + unlockf="unlock_flock" +else + lockf="lock_mkdir" + unlockf="unlock_mkdir" +fi + # # unset values # @@ -63,7 +124,8 @@ USE_ALL="" # # catch signals # -trap "rm -f \"${TMP}\"" 1 2 15 +TRAPFUNC="rm -f \"${TMP}\"" +trap "${TRAPFUNC}" 1 2 15 # # Functions @@ -136,6 +198,18 @@ eof exit 0 } +# locking functions +lock() +{ + "${lockf}" "$@" || _exit_err \ + "Only one instance of ${__myname} for source \"$1\" can run at one time." +} + +unlock() +{ + "${unlockf}" "$@" +} + # # Parse options # @@ -287,6 +361,16 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _exit_err "\"${backup}\" is not a cconfig-directory. Skipping." fi + # + # Acquire lock for source. If lock cannot be acquired, lock will exit + # with error message. + # + lock "${name}" + + # redefine trap to also unlock (rm lockfile) + TRAPFUNC="${TRAPFUNC} && unlock \"${name}\"" + trap "${TRAPFUNC}" 1 2 15 + # # First execute pre_exec, which may generate destination or other parameters # @@ -561,4 +645,5 @@ if [ -x "${CPOSTEXEC}" ]; then fi rm -f "${TMP}" +unlock "${name}" _techo "Finished" diff --git a/doc/changes/next b/doc/changes/next index e865dff..05c1a37 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1,3 +1,4 @@ + * Add locking (Darko Poljak) * Fix source-is-up check (Nikita Koshikov) * Fix some minor command line parsing issues (Nico Schottelius) * Correct output, if configuration is not in cconfig format (Nico Schottelius) From 902a7d667ed41c2142083a0a64a6c0fe17c36d22 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 21 Jul 2016 12:15:37 +0200 Subject: [PATCH 02/58] Introduce -j option and deprecate -p option. --- ccollect | 115 +++++++++++++++++++++++++++++++++++++++--- doc/ccollect.text | 4 +- doc/changes/next | 1 + doc/man/ccollect.text | 12 +++-- 4 files changed, 118 insertions(+), 14 deletions(-) diff --git a/ccollect b/ccollect index 20498b2..de62aa6 100755 --- a/ccollect +++ b/ccollect @@ -20,6 +20,11 @@ # Initially written for SyGroup (www.sygroup.ch) # Date: Mon Nov 14 11:45:11 CET 2005 +# Simulate ccollect without actually performing any backup; +# conf directory need to be specifed. +# Usually used for debugging/testing locking and parallel execution. +SIMULATE="1" + # Error upon expanding unset variables: set -u @@ -41,6 +46,8 @@ CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" +CONTROL_PIPE="/tmp/${__myname}-control-pipe" + VERSION="1.0" RELEASE="2015-10-10" HALF_VERSION="ccollect ${VERSION}" @@ -59,7 +66,10 @@ SDATE="date +%s" # Locking is done for each source so that only one instance per source # can run. # -LOCKDIR="${CSOURCES}" +# Use CCOLLECT_CONF directory for lock files. +# This directory can be set arbitrary so it is writable for user +# executing ccollect. +LOCKDIR="${CCOLLECT_CONF}" # printf pattern: ccollect_.lock LOCKFILE_PATTERN="ccollect_%s.lock" LOCKFD=4 @@ -119,6 +129,7 @@ fi # unset values # PARALLEL="" +MAX_JOBS="" USE_ALL="" # @@ -185,9 +196,11 @@ ${__myname}: [args] ccollect creates (pseudo) incremental backups - -h, --help: Show this help screen - -a, --all: Backup all sources specified in ${CSOURCES} - -p, --parallel: Parallelise backup processes + -h, --help: Show this help screen + -a, --all: Backup all sources specified in ${CSOURCES} + -j [max], --jobs [max] Specifies the number of jobs to run simultaneously. + If max is not specified then parallelise all jobs. + -p, --parallel: Parallelise backup processes (deprecated from 2.0) -v, --verbose: Be very verbose (uses set -x) -V, --version: Print version information @@ -219,7 +232,24 @@ while [ "$#" -ge 1 ]; do USE_ALL=1 ;; -p|--parallel) + _techo "Warning: -p, --parallel option is deprecated," \ + "use -j, --jobs instead." PARALLEL=1 + MAX_JOBS="" + ;; + -j|--jobs) + PARALLEL=1 + if [ "$#" -ge 2 ] + then + case "$2" in + -*) + ;; + *) + MAX_JOBS=$2 + shift + ;; + esac + fi ;; -v|--verbose) set -x @@ -242,6 +272,14 @@ while [ "$#" -ge 1 ]; do shift done +# check that MAX_JOBS is natural number > 0 +# empty string means run all in parallel +echo "${MAX_JOBS}" | awk '/^$/ { exit 0 } /^[1-9][0-9]*$/ { exit 0 } { exit 1 }' +if [ "$?" -ne 0 ] +then + _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" +fi + # # Setup interval # @@ -310,6 +348,26 @@ fi # # Let's do the backup - here begins the real stuff # + +# in PARALLEL mode: +# * create control pipe +# * determine number of jobs to start at once +if [ "${PARALLEL}" ]; then + mkfifo "${CONTROL_PIPE}" + # fd 5 is tied to control pipe + eval "exec 5<>${CONTROL_PIPE}" + TRAPFUNC="${TRAPFUNC}; rm -f \"${CONTROL_PIPE}\"" + trap "${TRAPFUNC}" 0 1 2 15 + + # determine how much parallel jobs to prestart + if [ "${MAX_JOBS}" ] && [ "${MAX_JOBS}" -le "${no_sources}" ] + then + prestart="${MAX_JOBS}" + else + prestart=0 + fi +fi + source_no=0 while [ "${source_no}" -lt "${no_sources}" ]; do # @@ -322,8 +380,35 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Start ourself, if we want parallel execution # if [ "${PARALLEL}" ]; then - "$0" "${INTERVAL}" "${name}" & - continue + if [ "${SIMULATE}" ] + then + # give some time to awk's srand initialized by curr time + sleep 1 + fi + + if [ ! "${MAX_JOBS}" ] + then + # run all in parallel + "$0" "${INTERVAL}" "${name}" & + continue + elif [ "${prestart}" -gt 0 ] + then + # run prestart child if pending + { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & + prestart=$((${prestart} - 1)) + continue + else + # each time a child finishes we get a line from the pipe + # and then launch another child + while read line + do + { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & + # get out of loop so we can contnue with main loop + # for next source + break + done <&5 + continue + fi fi # @@ -366,9 +451,20 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # with error message. # lock "${name}" + if [ "${SIMULATE}" ] + then + rand_low=3 + rand_high=10 + sleep_time=$(echo '' | awk "{srand(); print int(rand()*($rand_high - $rand_low)) + $rand_low;}") + _techo "simulating backup for ${name}: ${sleep_time} secs ..." + sleep $sleep_time + unlock "${name}" + _techo "Finished backup." + break + fi # redefine trap to also unlock (rm lockfile) - TRAPFUNC="${TRAPFUNC} && unlock \"${name}\"" + TRAPFUNC="${TRAPFUNC}; unlock \"${name}\"" trap "${TRAPFUNC}" 1 2 15 # @@ -620,15 +716,19 @@ while [ "${source_no}" -lt "${no_sources}" ]; do seconds="$((${full_seconds} % 60))" _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" + + unlock "${name}" ) | add_name done # # Be a good parent and wait for our children, if they are running wild parallel +# After all children are finished then remove control pipe. # if [ "${PARALLEL}" ]; then _techo "Waiting for children to complete..." wait + rm -f "${CONTROL_PIPE}" fi # @@ -645,5 +745,4 @@ if [ -x "${CPOSTEXEC}" ]; then fi rm -f "${TMP}" -unlock "${name}" _techo "Finished" diff --git a/doc/ccollect.text b/doc/ccollect.text index 4c54191..474c491 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1189,12 +1189,12 @@ rsync -av -H --delete /mnt/archiv/ "$DDIR/archiv/" ------------------------------------------------------------------------- -Processes running when doing ccollect -p +Processes running when doing ccollect -j ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Truncated output from `ps axuwwwf`: ------------------------------------------------------------------------- - S+ 11:40 0:00 | | | \_ /bin/sh /usr/local/bin/ccollect.sh daily -p ddba034 ddba045 ddba046 ddba047 ddba049 ddna010 ddna011 + S+ 11:40 0:00 | | | \_ /bin/sh /usr/local/bin/ccollect.sh daily -j ddba034 ddba045 ddba046 ddba047 ddba049 ddna010 ddna011 S+ 11:40 0:00 | | | \_ /bin/sh /usr/local/bin/ccollect.sh daily ddba034 S+ 11:40 0:00 | | | | \_ /bin/sh /usr/local/bin/ccollect.sh daily ddba034 R+ 11:40 23:40 | | | | | \_ rsync -a --delete --numeric-ids --relative --delete-excluded --link-dest=/home/server/backup/ddba034 diff --git a/doc/changes/next b/doc/changes/next index 05c1a37..2fee139 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1,3 +1,4 @@ + * Introduce -j option for max parallel jobs, deprecate -p (Darko Poljak) * Add locking (Darko Poljak) * Fix source-is-up check (Nikita Koshikov) * Fix some minor command line parsing issues (Nico Schottelius) diff --git a/doc/man/ccollect.text b/doc/man/ccollect.text index 84c1e37..9fc9086 100644 --- a/doc/man/ccollect.text +++ b/doc/man/ccollect.text @@ -26,14 +26,18 @@ texinfo or html). OPTIONS ------- +-a, --all:: + Backup all sources specified in /etc/ccollect/sources + -h, --help:: Show the help screen --p, --parallel:: - Parallelise backup processes +-j [max], --jobs [max]:: + Specifies the number of jobs to run simultaneously. + If max is not specified then parallelise all jobs. --a, --all:: - Backup all sources specified in /etc/ccollect/sources +-p, --parallel:: + Parallelise backup processes (deprecated from 2.0) -v, --verbose:: Be very verbose (uses set -x) From 1df57c815415f1f542e6ff280be8a6621955d8a3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 21 Jul 2016 12:16:34 +0200 Subject: [PATCH 03/58] Turn simulation off. --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index de62aa6..7b5b0ca 100755 --- a/ccollect +++ b/ccollect @@ -23,7 +23,7 @@ # Simulate ccollect without actually performing any backup; # conf directory need to be specifed. # Usually used for debugging/testing locking and parallel execution. -SIMULATE="1" +SIMULATE="" # Error upon expanding unset variables: set -u From 01c36fc699f59225a9d10ab1a262d18b80039d9a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Aug 2016 15:56:19 +0200 Subject: [PATCH 04/58] Check that MAX_JOBS is positive integer or empty using grep. --- ccollect | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 7b5b0ca..2929ac3 100755 --- a/ccollect +++ b/ccollect @@ -274,11 +274,13 @@ done # check that MAX_JOBS is natural number > 0 # empty string means run all in parallel -echo "${MAX_JOBS}" | awk '/^$/ { exit 0 } /^[1-9][0-9]*$/ { exit 0 } { exit 1 }' +echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' if [ "$?" -ne 0 ] then _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" fi +printf "REMOVE ME\n" +exit 0 # # Setup interval From a18a00e773d7e982138ab6db655158d2840ce63d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Aug 2016 17:12:18 +0200 Subject: [PATCH 05/58] Drop SIMULATE stuff. --- ccollect | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/ccollect b/ccollect index 2929ac3..80b7066 100755 --- a/ccollect +++ b/ccollect @@ -20,11 +20,6 @@ # Initially written for SyGroup (www.sygroup.ch) # Date: Mon Nov 14 11:45:11 CET 2005 -# Simulate ccollect without actually performing any backup; -# conf directory need to be specifed. -# Usually used for debugging/testing locking and parallel execution. -SIMULATE="" - # Error upon expanding unset variables: set -u @@ -382,12 +377,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Start ourself, if we want parallel execution # if [ "${PARALLEL}" ]; then - if [ "${SIMULATE}" ] - then - # give some time to awk's srand initialized by curr time - sleep 1 - fi - if [ ! "${MAX_JOBS}" ] then # run all in parallel @@ -453,17 +442,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # with error message. # lock "${name}" - if [ "${SIMULATE}" ] - then - rand_low=3 - rand_high=10 - sleep_time=$(echo '' | awk "{srand(); print int(rand()*($rand_high - $rand_low)) + $rand_low;}") - _techo "simulating backup for ${name}: ${sleep_time} secs ..." - sleep $sleep_time - unlock "${name}" - _techo "Finished backup." - break - fi # redefine trap to also unlock (rm lockfile) TRAPFUNC="${TRAPFUNC}; unlock \"${name}\"" From 3049849ea698fed16ee78c11733b5fdb29cd27c4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Aug 2016 17:14:31 +0200 Subject: [PATCH 06/58] Remove testing stuff. --- ccollect | 2 -- 1 file changed, 2 deletions(-) diff --git a/ccollect b/ccollect index 80b7066..82bd8a7 100755 --- a/ccollect +++ b/ccollect @@ -274,8 +274,6 @@ if [ "$?" -ne 0 ] then _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" fi -printf "REMOVE ME\n" -exit 0 # # Setup interval From 6dca5c638d4cbc2d42fbc5a229d7cb13385d5d0f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 26 Sep 2016 17:27:33 +0200 Subject: [PATCH 07/58] Release 2.0. --- ccollect | 5 +++-- doc/ccollect.text | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ccollect b/ccollect index 82bd8a7..e1074b0 100755 --- a/ccollect +++ b/ccollect @@ -1,6 +1,7 @@ #!/bin/sh # # 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org) +# 2016 Darko Poljak (darko.poljak at gmail.com) # # This file is part of ccollect. # @@ -43,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="1.0" -RELEASE="2015-10-10" +VERSION="2.0" +RELEASE="2016-09-26" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 474c491..b1c7dba 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -0.9, for ccollect 0.9, Initial Version from 2006-01-13 +2.0, for ccollect 2.0, Initial Version from 2006-01-13 :Author Initials: NS From fbe17cae44a9b1236cc18c458e4ae16d6fcd8813 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 23 Feb 2017 20:04:53 +0100 Subject: [PATCH 08/58] Improve logging: stdout, file, syslog options. --- ccollect | 135 ++++++++++++++++++++++++++++++++++++------ doc/changes/2.0 | 16 +++++ doc/changes/next | 17 +----- doc/man/ccollect.text | 34 ++++++++++- 4 files changed, 166 insertions(+), 36 deletions(-) create mode 100644 doc/changes/2.0 diff --git a/ccollect b/ccollect index e1074b0..6369e84 100755 --- a/ccollect +++ b/ccollect @@ -127,6 +127,11 @@ fi PARALLEL="" MAX_JOBS="" USE_ALL="" +LOGFILE="" +SYSLOG="" +# e - only errors, a - all output +LOGLEVEL="a" +LOGONLYERRORS="" # # catch signals @@ -138,18 +143,11 @@ trap "${TRAPFUNC}" 1 2 15 # Functions # -# time displaying echo -_techo() +# check if we are running interactive or non-interactive +# see: http://www.tldp.org/LDP/abs/html/intandnonint.html +_is_interactive() { - echo "$(${DDATE}): $@" -} - -# exit on error -_exit_err() -{ - _techo "Error: $@" - rm -f "${TMP}" - exit 1 + [ -t 0 -o -p /dev/stdin ] } add_name() @@ -192,13 +190,16 @@ ${__myname}: [args] ccollect creates (pseudo) incremental backups - -h, --help: Show this help screen - -a, --all: Backup all sources specified in ${CSOURCES} - -j [max], --jobs [max] Specifies the number of jobs to run simultaneously. - If max is not specified then parallelise all jobs. - -p, --parallel: Parallelise backup processes (deprecated from 2.0) - -v, --verbose: Be very verbose (uses set -x) - -V, --version: Print version information + -h, --help: Show this help screen + -a, --all: Backup all sources specified in ${CSOURCES} + -e, --errors: Log only errors + -j [max], --jobs [max] Specifies the number of jobs to run simultaneously. + If max is not specified then parallelise all jobs. + -l FILE, --logfile FILE Log to specified file + -p, --parallel: Parallelise backup processes (deprecated from 2.0) + -s, --syslog: Log to syslog with tag ccollect + -v, --verbose: Be very verbose (uses set -x) + -V, --version: Print version information This is version ${VERSION} released on ${RELEASE}. @@ -219,6 +220,54 @@ unlock() "${unlockf}" "$@" } +# time displaying echo +# stdout version +_techo_stdout() +{ + echo "$(${DDATE}): $@" +} + +# syslog version +_techo_syslog() +{ + logger -t ccollect "$@" +} + +# specified file version +_techo_file() +{ + _techo_stdout "$@" >> "${LOGFILE}" +} + +# determine _techo version before parsing options +if _is_interactive +then + _techof="_techo_stdout" +else + _techof="_techo_syslog" +fi + +# _techo with determined _techo version +_techo() +{ + if [ "${LOGLEVEL}" = "a" ] + then + "${_techof}" "$@" + fi +} + +_techo_err() +{ + _techo "Error: $@" +} + +_exit_err() +{ + _techo_err "$@" + rm -f "${TMP}" + exit 1 +} + # # Parse options # @@ -247,6 +296,28 @@ while [ "$#" -ge 1 ]; do esac fi ;; + -e|--errors) + LOGONLYERRORS="1" + ;; + -l|--logfile) + if [ "$#" -ge 2 ] + then + case "$2" in + -*) + _exit_err "Missing log file" + ;; + *) + LOGFILE="$2" + shift + ;; + esac + else + _exit_err "Missing log file" + fi + ;; + -s|--syslog) + SYSLOG="1" + ;; -v|--verbose) set -x ;; @@ -268,6 +339,34 @@ while [ "$#" -ge 1 ]; do shift done +# determine _techo version and logging level after parsing options +if [ "${LOGFILE}" ] +then + _techof="_techo_file" + LOGLEVEL="a" +elif _is_interactive +then + if [ "${SYSLOG}" ] + then + _techof="_techo_syslog" + LOGLEVEL="a" + else + _techof="_techo_stdout" + LOGLEVEL="e" + fi +else + _techof="_techo_syslog" + LOGLEVEL="a" +fi + +if [ "${LOGFILE}" -o "${SYSLOG}" ] +then + if [ "${LOGONLYERRORS}" ] + then + LOGLEVEL="e" + fi +fi + # check that MAX_JOBS is natural number > 0 # empty string means run all in parallel echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' diff --git a/doc/changes/2.0 b/doc/changes/2.0 new file mode 100644 index 0000000..2fee139 --- /dev/null +++ b/doc/changes/2.0 @@ -0,0 +1,16 @@ + * Introduce -j option for max parallel jobs, deprecate -p (Darko Poljak) + * Add locking (Darko Poljak) + * Fix source-is-up check (Nikita Koshikov) + * Fix some minor command line parsing issues (Nico Schottelius) + * Correct output, if configuration is not in cconfig format (Nico Schottelius) + * Minor code cleanups and optimisations (Nico Schottelius) + * ccollect_analyse_logs.sh traps more errors and warnings (Patrick Drolet) + * Remove -v for mkdir and rm, as they are not POSIX (Patrick Drolet) + * Export destination_* to source specific post_exec (Nico Schottelius) + * Update documentation regarding exported variables (Nico Schottelius) + * Simplify time calculation (Nico Schottelius) + * Documentate pre_exec error handling (Nico Schottelius) + * Added start script (Thorsten Elle) + * Documentate autofs hint (Nico Schottelius) + * Speedup source-is-up check and remove --archive (Nico Schottelius) + * Removed support for remote backup (see doc) (Nico Schottelius) diff --git a/doc/changes/next b/doc/changes/next index 2fee139..ec42d4b 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1,16 +1 @@ - * Introduce -j option for max parallel jobs, deprecate -p (Darko Poljak) - * Add locking (Darko Poljak) - * Fix source-is-up check (Nikita Koshikov) - * Fix some minor command line parsing issues (Nico Schottelius) - * Correct output, if configuration is not in cconfig format (Nico Schottelius) - * Minor code cleanups and optimisations (Nico Schottelius) - * ccollect_analyse_logs.sh traps more errors and warnings (Patrick Drolet) - * Remove -v for mkdir and rm, as they are not POSIX (Patrick Drolet) - * Export destination_* to source specific post_exec (Nico Schottelius) - * Update documentation regarding exported variables (Nico Schottelius) - * Simplify time calculation (Nico Schottelius) - * Documentate pre_exec error handling (Nico Schottelius) - * Added start script (Thorsten Elle) - * Documentate autofs hint (Nico Schottelius) - * Speedup source-is-up check and remove --archive (Nico Schottelius) - * Removed support for remote backup (see doc) (Nico Schottelius) +* Add options for stdout, file and syslog logging (Darko Poljak) diff --git a/doc/man/ccollect.text b/doc/man/ccollect.text index 9fc9086..67f8f47 100644 --- a/doc/man/ccollect.text +++ b/doc/man/ccollect.text @@ -29,6 +29,9 @@ OPTIONS -a, --all:: Backup all sources specified in /etc/ccollect/sources +-e, --errors:: + Log only errors + -h, --help:: Show the help screen @@ -36,14 +39,41 @@ OPTIONS Specifies the number of jobs to run simultaneously. If max is not specified then parallelise all jobs. +-l FILE, --logfile FILE:: + Log to specified file + -p, --parallel:: Parallelise backup processes (deprecated from 2.0) +-s, --syslog:: + Log to syslog with tag ccollect + +-V, --version:: + Show version and exit + -v, --verbose:: Be very verbose (uses set -x) --V, --version:: - Show version and exit + +LOGGING MECHANISM +----------------- +ccollect logging depends on running in non-interactive/interactive mode +and on specified optins. The mechanism behaves as the following: + +non-interactive mode:: + + * standard output goes to syslog + * optional: specify logging into file + * log all output by default + * optional: log only errors + +interactive mode:: + + * standard output goes to stdout + * log only errors + * optional: log into syslog or file + - log all output by default + - optional: log only errors SEE ALSO From 89a82ba55ea72d988eae424c6c711143902b44cc Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 22 Mar 2017 10:24:18 +0100 Subject: [PATCH 09/58] Release 2.1 --- ccollect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 6369e84..47526ad 100755 --- a/ccollect +++ b/ccollect @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.0" -RELEASE="2016-09-26" +VERSION="2.1" +RELEASE="2017-03-22" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" From 07c925de5d174b59fd64a9a15f26a0db2caeedf2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 22 Mar 2017 10:46:57 +0100 Subject: [PATCH 10/58] Release 2.1 --- doc/ccollect.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index b1c7dba..74fcab6 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.0, for ccollect 2.0, Initial Version from 2006-01-13 +2.1, for ccollect 2.1, Initial Version from 2006-01-13 :Author Initials: NS From 04bf9aff3987c8f2543ff887cbb9f4bc6737e871 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 3 Sep 2017 20:00:54 +0200 Subject: [PATCH 11/58] Bugfix: empty rsync_options line causes destroying source. (#8) Bugfix: empty rsync_options line causes destroying source. --- ccollect | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 47526ad..edae187 100755 --- a/ccollect +++ b/ccollect @@ -655,7 +655,17 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # if [ -f "${c_rsync_options}" ]; then while read line; do - set -- "$@" "${line}" + # Trim line. + ln=$(echo "${line}" | awk '{$1=$1;print;}') + # Only if ln is non zero length string. + # + # If ln is empty then rsync '' DEST evaluates + # to transfer current directory to DEST which would + # with specific options destroy DEST content. + if [ -n "${ln}" ] + then + set -- "$@" "${ln}" + fi done < "${c_rsync_options}" fi From b0f131771387cf8eb46d44eb108c891527f5ee68 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 3 Sep 2017 20:02:42 +0200 Subject: [PATCH 12/58] changelog++ --- doc/changes/2.1 | 1 + doc/changes/next | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 doc/changes/2.1 diff --git a/doc/changes/2.1 b/doc/changes/2.1 new file mode 100644 index 0000000..ec42d4b --- /dev/null +++ b/doc/changes/2.1 @@ -0,0 +1 @@ +* Add options for stdout, file and syslog logging (Darko Poljak) diff --git a/doc/changes/next b/doc/changes/next index ec42d4b..0ee09d6 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1 @@ -* Add options for stdout, file and syslog logging (Darko Poljak) +* Bugfix: empty rsync_options line causes destroying source (Darko Poljak) From e504d1f42b4e4ce73e47e7cdb4d1ae24842e6f85 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 3 Sep 2017 22:29:13 +0200 Subject: [PATCH 13/58] Release 2.2 --- ccollect | 4 ++-- doc/changes/2.2 | 1 + doc/changes/next | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 doc/changes/2.2 diff --git a/ccollect b/ccollect index edae187..35af419 100755 --- a/ccollect +++ b/ccollect @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.1" -RELEASE="2017-03-22" +VERSION="2.2" +RELEASE="2017-09-03" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/changes/2.2 b/doc/changes/2.2 new file mode 100644 index 0000000..0ee09d6 --- /dev/null +++ b/doc/changes/2.2 @@ -0,0 +1 @@ +* Bugfix: empty rsync_options line causes destroying source (Darko Poljak) diff --git a/doc/changes/next b/doc/changes/next index 0ee09d6..a767caf 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1 @@ -* Bugfix: empty rsync_options line causes destroying source (Darko Poljak) +* From 890b166a43c0824d66dfdefd0a38e4643401ec13 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 3 Sep 2017 23:11:13 +0200 Subject: [PATCH 14/58] Release 2.2 --- doc/ccollect.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index 74fcab6..b97409f 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.1, for ccollect 2.1, Initial Version from 2006-01-13 +2.2, for ccollect 2.2, Initial Version from 2006-01-13 :Author Initials: NS From bd0fe050038e3137d2ede6db5126a7fb92cdc64a Mon Sep 17 00:00:00 2001 From: Jun Futagawa Date: Tue, 19 Dec 2017 17:42:49 +0900 Subject: [PATCH 15/58] Update rpm spec file to version 2.2 (#9) --- contrib/ccollect.spec | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/contrib/ccollect.spec b/contrib/ccollect.spec index 0e35bb0..91bb83b 100644 --- a/contrib/ccollect.spec +++ b/contrib/ccollect.spec @@ -1,6 +1,6 @@ Summary: (pseudo) incremental backup with different exclude lists using hardlinks and rsync Name: ccollect -Version: 0.8.1 +Version: 2.2 Release: 0 URL: http://www.nico.schottelius.org/software/ccollect Source0: http://www.nico.schottelius.org/software/ccollect/%{name}-%{version}.tar.bz2 @@ -23,10 +23,10 @@ Only the inodes used by the hardlinks and the changed files need additional spac %install rm -rf $RPM_BUILD_ROOT -#Installing main ccollect.sh and /etc directory +#Installing main ccollect and /etc directory %__install -d 755 %buildroot%_bindir %__install -d 755 %buildroot%_sysconfdir/%name -%__install -m 755 ccollect.sh %buildroot%_bindir/ +%__install -m 755 ccollect %buildroot%_bindir/ #bin files from tools directory for t in $(ls tools/ccollect_*) ; do @@ -45,19 +45,13 @@ done #Addition documentation and some config tools %__install -d 755 %buildroot%_datadir/%name/tools +%__install -m 755 tools/called_from_remote_pre_exec %buildroot%_datadir/%name/tools %__cp -pr tools/config-pre-* %buildroot%_datadir/%name/tools -%__install -m 755 tools/gnu-du-backup-size-compare.sh %buildroot%_datadir/%name/tools -%__install -m 755 tools/report_success.sh %buildroot%_datadir/%name/tools +%__install -m 755 tools/report_success %buildroot%_datadir/%name/tools %clean rm -rf $RPM_BUILD_ROOT -%post -%__ln_s %_bindir/ccollect.sh %_bindir/ccollect - -%preun -unlink %_bindir/ccollect - %files %defattr(-,root,root) %_bindir/ccollect* From fc0b86005c6231eac06204351e9601393aff2e7a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 30 Jan 2018 09:48:43 +0100 Subject: [PATCH 16/58] Fix parallel mode deadlock when MAX_JOBS > number of sources. --- ccollect | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 35af419..6b48606 100755 --- a/ccollect +++ b/ccollect @@ -455,9 +455,14 @@ if [ "${PARALLEL}" ]; then trap "${TRAPFUNC}" 0 1 2 15 # determine how much parallel jobs to prestart - if [ "${MAX_JOBS}" ] && [ "${MAX_JOBS}" -le "${no_sources}" ] + if [ "${MAX_JOBS}" ] then - prestart="${MAX_JOBS}" + if [ "${MAX_JOBS}" -le "${no_sources}" ] + then + prestart="${MAX_JOBS}" + else + prestart="${no_sources}" + fi else prestart=0 fi From eeccc0b26019f3d9d5f4a93e97522f5190b1827f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 2 Feb 2018 07:57:41 +0100 Subject: [PATCH 17/58] Release 2.3 --- ccollect | 4 ++-- contrib/ccollect.spec | 2 +- doc/ccollect.text | 2 +- doc/changes/2.3 | 1 + doc/changes/next | 1 - 5 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 doc/changes/2.3 diff --git a/ccollect b/ccollect index 6b48606..70a998e 100755 --- a/ccollect +++ b/ccollect @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.2" -RELEASE="2017-09-03" +VERSION="2.3" +RELEASE="2018-02-02" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/contrib/ccollect.spec b/contrib/ccollect.spec index 91bb83b..8916b7e 100644 --- a/contrib/ccollect.spec +++ b/contrib/ccollect.spec @@ -1,6 +1,6 @@ Summary: (pseudo) incremental backup with different exclude lists using hardlinks and rsync Name: ccollect -Version: 2.2 +Version: 2.3 Release: 0 URL: http://www.nico.schottelius.org/software/ccollect Source0: http://www.nico.schottelius.org/software/ccollect/%{name}-%{version}.tar.bz2 diff --git a/doc/ccollect.text b/doc/ccollect.text index b97409f..b267cdb 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.2, for ccollect 2.2, Initial Version from 2006-01-13 +2.3, for ccollect 2.3, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/2.3 b/doc/changes/2.3 new file mode 100644 index 0000000..960110c --- /dev/null +++ b/doc/changes/2.3 @@ -0,0 +1 @@ +* Bugfix: Fix parallel mode deadlock when max jobs > number of sources (Darko Poljak) diff --git a/doc/changes/next b/doc/changes/next index a767caf..e69de29 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +0,0 @@ -* From 51f468182f5e68147c7c0bbbfb8611750b8cc90c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Apr 2018 17:27:56 +0200 Subject: [PATCH 18/58] Document Windows(Cygwin) as supported OS. --- doc/ccollect.text | 1 + doc/changes/next | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/ccollect.text b/doc/ccollect.text index b267cdb..5585a97 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -26,6 +26,7 @@ Supported and tested operating systems and architectures - Mac OS X 10.5 - NetBSD on alpha/amd64/i386/sparc/sparc64 - OpenBSD on amd64 +- Windows by installing Cygwin, OpenSSH and rsync It *should* run on any Unix that supports `rsync` and has a POSIX-compatible bourne shell. If your platform is not listed above and you have it successfully diff --git a/doc/changes/next b/doc/changes/next index e69de29..784eac6 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -0,0 +1 @@ +* Add Windows(Cygwin) as supported OS From 420dc3fe7fcf5bfb879e70675cb4d412b969f337 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 20 Sep 2018 16:08:00 +0200 Subject: [PATCH 19/58] Add source name tag in log line --- ccollect | 53 ++++++++++++++++++++++++------------------------ doc/changes/next | 1 + 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/ccollect b/ccollect index 70a998e..04454a1 100755 --- a/ccollect +++ b/ccollect @@ -514,6 +514,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do c_dest="${backup}/destination" c_pre_exec="${backup}/pre_exec" c_post_exec="${backup}/post_exec" + tag="[${name}]" # # Stderr to stdout, so we can produce nice logs @@ -524,13 +525,13 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Record start of backup: internal and for the user # begin_s="$(${SDATE})" - _techo "Beginning to backup" + _techo "${tag}" "Beginning to backup" # # Standard configuration checks # if [ ! -e "${backup}" ]; then - _exit_err "Source does not exist." + _exit_err "Source \"${backup}\" does not exist." fi # @@ -554,11 +555,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # First execute pre_exec, which may generate destination or other parameters # if [ -x "${c_pre_exec}" ]; then - _techo "Executing ${c_pre_exec} ..." + _techo "${tag}" "Executing ${c_pre_exec} ..." "${c_pre_exec}"; ret="$?" - _techo "Finished ${c_pre_exec} (return code ${ret})." + _techo "${tag}" "Finished ${c_pre_exec} (return code ${ret})." - [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." + [ "${ret}" -eq 0 ] || _exit_err "${tag}" "${c_pre_exec} failed. Skipping." fi # @@ -583,7 +584,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" if [ -z "${c_interval}" ]; then - _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." + _exit_err "${tag}" "No definition for interval \"${INTERVAL}\" found. Skipping." fi fi @@ -612,11 +613,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Destination is a path # if [ ! -f "${c_dest}" ]; then - _exit_err "Destination ${c_dest} is not a file. Skipping." + _exit_err "${tag}" "Destination ${c_dest} is not a file. Skipping." else ddir="$(cat "${c_dest}")"; ret="$?" if [ "${ret}" -ne 0 ]; then - _exit_err "Destination ${c_dest} is not readable. Skipping." + _exit_err "${tag}" "Destination ${c_dest} is not readable. Skipping." fi fi @@ -681,7 +682,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi - _exit_err "Source ${source} is not readable. Skipping." + _exit_err "${tag}" "Source ${source} is not readable. Skipping." fi # @@ -692,7 +693,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: destination exists? # - cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping." + cd "${ddir}" || _exit_err "${tag}" "Cannot change to ${ddir}. Skipping." # # Check incomplete backups (needs echo to remove newlines) @@ -700,7 +701,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? if [ "$ret" -eq 0 ]; then - _techo "Incomplete backups: $(echo $(cat "${TMP}"))" + _techo "${tag}" "Incomplete backups: $(echo $(cat "${TMP}"))" if [ -f "${c_delete_incomplete}" ]; then delete_from_file "${TMP}" "${CMARKER}" fi @@ -711,14 +712,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # count="$(ls -1 | grep -c "^${INTERVAL}\\.")" - _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" + _techo "${tag}" "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then remove="$((${count} - ${c_interval} + 1))" - _techo "Removing ${remove} backup(s)..." + _techo "${tag}" "Removing ${remove} backup(s)..." ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + _exit_err "${tag}" "Listing old backups failed" delete_from_file "${TMP}" fi @@ -727,14 +728,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check for backup directory to clone from: Always clone from the latest one! # last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ - _exit_err "Failed to list contents of ${ddir}." + _exit_err "${tag}" "Failed to list contents of ${ddir}." # # Clone from old backup, if existing # if [ "${last_dir}" ]; then set -- "$@" "--link-dest=${ddir}/${last_dir}" - _techo "Hard linking from ${last_dir}" + _techo "${tag}" "Hard linking from ${last_dir}" fi # @@ -747,14 +748,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Mark backup running and go back to original directory # touch "${destination_dir}${CMARKER}" - cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}." + cd "${__abs_mydir}" || _exit_err "${tag}" "Cannot go back to ${__abs_mydir}." # # the rsync part # - _techo "Transferring files..." + _techo "${tag}" "Transferring files..." rsync "$@" "${source}" "${destination_dir}"; ret=$? - _techo "Finished backup (rsync return code: $ret)." + _techo "${tag}" "Finished backup (rsync return code: $ret)." # # Set modification time (mtime) to current time, if sorting by mtime is enabled @@ -778,24 +779,24 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # if [ -z "$fail" ]; then rm "${destination_dir}${CMARKER}" || \ - _exit_err "Removing ${destination_dir}${CMARKER} failed." + _exit_err "${tag}" "Removing ${destination_dir}${CMARKER} failed." if [ "${ret}" -ne 0 ]; then - _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." + _techo "${tag}" "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." fi else - _techo "Warning: rsync failed with return code $ret." + _techo "${tag}" "Warning: rsync failed with return code $ret." fi # # post_exec # if [ -x "${c_post_exec}" ]; then - _techo "Executing ${c_post_exec} ..." + _techo "${tag}" "Executing ${c_post_exec} ..." "${c_post_exec}"; ret=$? - _techo "Finished ${c_post_exec}." + _techo "${tag}" "Finished ${c_post_exec}." if [ "${ret}" -ne 0 ]; then - _exit_err "${c_post_exec} failed." + _exit_err "${tag}" "${c_post_exec} failed." fi fi @@ -808,7 +809,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do minutes="$(((${full_seconds} % 3600) / 60))" seconds="$((${full_seconds} % 60))" - _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" + _techo "${tag}" "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" ) | add_name diff --git a/doc/changes/next b/doc/changes/next index 784eac6..07ffe9d 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1,2 @@ * Add Windows(Cygwin) as supported OS +* Add source name tag in log line From b47a828af0e5e6b3b230aa7338054799767dde2a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 20 Sep 2018 16:08:00 +0200 Subject: [PATCH 20/58] Add source name tag in log line --- ccollect | 66 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/ccollect b/ccollect index 04454a1..b565355 100755 --- a/ccollect +++ b/ccollect @@ -150,11 +150,6 @@ _is_interactive() [ -t 0 -o -p /dev/stdin ] } -add_name() -{ - awk "{ print \"[${name}] \" \$0 }" -} - # # ssh-"feature": we cannot do '... read ...; ssh ...; < file', # because ssh reads stdin! -n does not work -> does not ask for password @@ -252,7 +247,13 @@ _techo() { if [ "${LOGLEVEL}" = "a" ] then - "${_techof}" "$@" + tag="${name:-}" + if [ "${tag}" ]; then + tag="[${tag}]" + else + tag="" + fi + "${_techof}" ${tag} "$@" fi } @@ -514,7 +515,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do c_dest="${backup}/destination" c_pre_exec="${backup}/pre_exec" c_post_exec="${backup}/post_exec" - tag="[${name}]" # # Stderr to stdout, so we can produce nice logs @@ -525,7 +525,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Record start of backup: internal and for the user # begin_s="$(${SDATE})" - _techo "${tag}" "Beginning to backup" + _techo "Beginning to backup" # # Standard configuration checks @@ -555,11 +555,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # First execute pre_exec, which may generate destination or other parameters # if [ -x "${c_pre_exec}" ]; then - _techo "${tag}" "Executing ${c_pre_exec} ..." + _techo "Executing ${c_pre_exec} ..." "${c_pre_exec}"; ret="$?" - _techo "${tag}" "Finished ${c_pre_exec} (return code ${ret})." + _techo "Finished ${c_pre_exec} (return code ${ret})." - [ "${ret}" -eq 0 ] || _exit_err "${tag}" "${c_pre_exec} failed. Skipping." + [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." fi # @@ -584,7 +584,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" if [ -z "${c_interval}" ]; then - _exit_err "${tag}" "No definition for interval \"${INTERVAL}\" found. Skipping." + _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." fi fi @@ -613,11 +613,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Destination is a path # if [ ! -f "${c_dest}" ]; then - _exit_err "${tag}" "Destination ${c_dest} is not a file. Skipping." + _exit_err "Destination ${c_dest} is not a file. Skipping." else ddir="$(cat "${c_dest}")"; ret="$?" if [ "${ret}" -ne 0 ]; then - _exit_err "${tag}" "Destination ${c_dest} is not readable. Skipping." + _exit_err "Destination ${c_dest} is not readable. Skipping." fi fi @@ -682,7 +682,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi - _exit_err "${tag}" "Source ${source} is not readable. Skipping." + _exit_err "Source ${source} is not readable. Skipping." fi # @@ -693,7 +693,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: destination exists? # - cd "${ddir}" || _exit_err "${tag}" "Cannot change to ${ddir}. Skipping." + cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping." # # Check incomplete backups (needs echo to remove newlines) @@ -701,7 +701,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? if [ "$ret" -eq 0 ]; then - _techo "${tag}" "Incomplete backups: $(echo $(cat "${TMP}"))" + _techo "Incomplete backups: $(echo $(cat "${TMP}"))" if [ -f "${c_delete_incomplete}" ]; then delete_from_file "${TMP}" "${CMARKER}" fi @@ -712,14 +712,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # count="$(ls -1 | grep -c "^${INTERVAL}\\.")" - _techo "${tag}" "Existing backups: ${count} Total keeping backups: ${c_interval}" + _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then remove="$((${count} - ${c_interval} + 1))" - _techo "${tag}" "Removing ${remove} backup(s)..." + _techo "Removing ${remove} backup(s)..." ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "${tag}" "Listing old backups failed" + _exit_err "Listing old backups failed" delete_from_file "${TMP}" fi @@ -728,14 +728,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check for backup directory to clone from: Always clone from the latest one! # last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ - _exit_err "${tag}" "Failed to list contents of ${ddir}." + _exit_err "Failed to list contents of ${ddir}." # # Clone from old backup, if existing # if [ "${last_dir}" ]; then set -- "$@" "--link-dest=${ddir}/${last_dir}" - _techo "${tag}" "Hard linking from ${last_dir}" + _techo "Hard linking from ${last_dir}" fi # @@ -748,14 +748,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Mark backup running and go back to original directory # touch "${destination_dir}${CMARKER}" - cd "${__abs_mydir}" || _exit_err "${tag}" "Cannot go back to ${__abs_mydir}." + cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}." # # the rsync part # - _techo "${tag}" "Transferring files..." + _techo "Transferring files..." rsync "$@" "${source}" "${destination_dir}"; ret=$? - _techo "${tag}" "Finished backup (rsync return code: $ret)." + _techo "Finished backup (rsync return code: $ret)." # # Set modification time (mtime) to current time, if sorting by mtime is enabled @@ -779,24 +779,24 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # if [ -z "$fail" ]; then rm "${destination_dir}${CMARKER}" || \ - _exit_err "${tag}" "Removing ${destination_dir}${CMARKER} failed." + _exit_err "Removing ${destination_dir}${CMARKER} failed." if [ "${ret}" -ne 0 ]; then - _techo "${tag}" "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." + _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." fi else - _techo "${tag}" "Warning: rsync failed with return code $ret." + _techo "Warning: rsync failed with return code $ret." fi # # post_exec # if [ -x "${c_post_exec}" ]; then - _techo "${tag}" "Executing ${c_post_exec} ..." + _techo "Executing ${c_post_exec} ..." "${c_post_exec}"; ret=$? - _techo "${tag}" "Finished ${c_post_exec}." + _techo "Finished ${c_post_exec}." if [ "${ret}" -ne 0 ]; then - _exit_err "${tag}" "${c_post_exec} failed." + _exit_err "${c_post_exec} failed." fi fi @@ -809,10 +809,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do minutes="$(((${full_seconds} % 3600) / 60))" seconds="$((${full_seconds} % 60))" - _techo "${tag}" "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" + _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" -) | add_name +) done # From a63e16efc5a4db733b5505067468f702e48b3e1e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 23 Sep 2018 11:42:09 +0200 Subject: [PATCH 21/58] Use better expansion and parameter setting --- ccollect | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ccollect b/ccollect index b565355..c3f6328 100755 --- a/ccollect +++ b/ccollect @@ -247,13 +247,8 @@ _techo() { if [ "${LOGLEVEL}" = "a" ] then - tag="${name:-}" - if [ "${tag}" ]; then - tag="[${tag}]" - else - tag="" - fi - "${_techof}" ${tag} "$@" + set -- ${name:+"[${name}]"} $@ + "${_techof}" "$@" fi } From 71eabe2f23f9a9c5fcd3f4d9a94a3d82d2f5cb2a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 25 Sep 2018 21:14:55 +0200 Subject: [PATCH 22/58] Release 2.4 --- ccollect | 6 +++--- doc/ccollect.text | 2 +- doc/changes/{next => 2.4} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename doc/changes/{next => 2.4} (100%) diff --git a/ccollect b/ccollect index c3f6328..3c39d9c 100755 --- a/ccollect +++ b/ccollect @@ -1,7 +1,7 @@ #!/bin/sh # # 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org) -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2018 Darko Poljak (darko.poljak at gmail.com) # # This file is part of ccollect. # @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.3" -RELEASE="2018-02-02" +VERSION="2.4" +RELEASE="2018-09-25" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 5585a97..9851d08 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.3, for ccollect 2.3, Initial Version from 2006-01-13 +2.4, for ccollect 2.4, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.4 similarity index 100% rename from doc/changes/next rename to doc/changes/2.4 From 835e21c56c6580debb2d661e3b5145601280294c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 26 Apr 2019 14:03:01 +0200 Subject: [PATCH 23/58] exit in case of subshell error --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 3c39d9c..335b51d 100755 --- a/ccollect +++ b/ccollect @@ -807,7 +807,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" -) +) || exit done # From 2725a1ced49d8af47abde2618bc53a08f08499f6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 26 Apr 2019 14:47:21 +0200 Subject: [PATCH 24/58] github -> code.ungleich.ch --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 908bd6c..dc98502 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,6 @@ t2: # pub: git push - git push github publish-doc: documentation @echo "Transferring files to ${host}" From 10dcf076a9f9ccd7e8cca6a514835c1ee0c955ea Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 1 May 2019 14:38:35 +0200 Subject: [PATCH 25/58] Release 2.5 --- ccollect | 6 +++--- doc/ccollect.text | 2 +- doc/changes/2.5 | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 doc/changes/2.5 diff --git a/ccollect b/ccollect index 335b51d..bb44ed6 100755 --- a/ccollect +++ b/ccollect @@ -1,7 +1,7 @@ #!/bin/sh # # 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org) -# 2016-2018 Darko Poljak (darko.poljak at gmail.com) +# 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of ccollect. # @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.4" -RELEASE="2018-09-25" +VERSION="2.5" +RELEASE="2019-05-01" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 9851d08..f252b25 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.4, for ccollect 2.4, Initial Version from 2006-01-13 +2.5, for ccollect 2.5, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/2.5 b/doc/changes/2.5 new file mode 100644 index 0000000..a6b319c --- /dev/null +++ b/doc/changes/2.5 @@ -0,0 +1 @@ +* Bugfix: exit in case of subshell error From 1628ce58c768211052b9d8d02cd0c4d3c7ea3a17 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 5 Oct 2019 10:48:19 +0200 Subject: [PATCH 26/58] Replace rm with faster rsync --delete with empty src dir --- ccollect | 18 +++++++++++++++--- doc/changes/next | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 doc/changes/next diff --git a/ccollect b/ccollect index bb44ed6..05b7014 100755 --- a/ccollect +++ b/ccollect @@ -160,16 +160,28 @@ delete_from_file() file="$1"; shift suffix="" # It will be set, if deleting incomplete backups. [ $# -eq 1 ] && suffix="$1" && shift + # dirs for deletion will be moved to this trash dir inside destination dir + # - for fast mv operation + trash="$(mktemp -d "trash.XXXXXX")" while read to_remove; do + mv "${to_remove}" "${trash}" || + _exit_err "Moving ${to_remove} to ${trash} failed." set -- "$@" "${to_remove}" if [ "${suffix}" ]; then to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" + mv "${to_remove_no_suffix}" "${trash}" || + _exit_err "Moving ${to_remove_no_suffix} to ${trash} failed." set -- "$@" "${to_remove_no_suffix}" fi done < "${file}" - _techo "Removing $@ ..." - [ "${VVERBOSE}" ] && echo rm "$@" - rm -rf "$@" || _exit_err "Removing $@ failed." + _techo "Removing $@ in ${trash}..." + empty_dir="empty-dir" + mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." + [ "${VVERBOSE}" ] && echo "rsync -a --delete ${empty_dir} ${trash}" + # rsync needs ending slash for directory content + rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." + rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" + rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" } display_version() diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..df06edc --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Performance: replace rm -rf with faster rsync --delete empty src dir From ca6d06c2c32c0296dfc8dcb107d469f863fd67e6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 10 Oct 2019 10:54:56 +0200 Subject: [PATCH 27/58] Add more verbose logging --- ccollect | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 05b7014..8e02524 100755 --- a/ccollect +++ b/ccollect @@ -177,11 +177,12 @@ delete_from_file() _techo "Removing $@ in ${trash}..." empty_dir="empty-dir" mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." - [ "${VVERBOSE}" ] && echo "rsync -a --delete ${empty_dir} ${trash}" + [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}" # rsync needs ending slash for directory content rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" + [ "${VVERBOSE}" ] && echo "Finished: rsync -a --delete ${empty_dir} ${trash}" } display_version() From 30abef474dea75705ee005a0453ea536e73d2fb8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 16 Oct 2019 14:03:13 +0200 Subject: [PATCH 28/58] Delete in background and finally wait for children --- ccollect | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 8e02524..7ede4dc 100755 --- a/ccollect +++ b/ccollect @@ -711,7 +711,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ "$ret" -eq 0 ]; then _techo "Incomplete backups: $(echo $(cat "${TMP}"))" if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" "${CMARKER}" + delete_from_file "${TMP}" "${CMARKER}" & fi fi @@ -729,7 +729,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ _exit_err "Listing old backups failed" - delete_from_file "${TMP}" + delete_from_file "${TMP}" & fi # @@ -820,6 +820,9 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" + + # wait for children (doing delete_from_file) if any still running + wait ) || exit done From bfb3c6338c3e7a5bb1f4dcf8c3e1aa9b1117fad0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 16 Oct 2019 15:50:54 +0200 Subject: [PATCH 29/58] _techo instead of very verbose --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 7ede4dc..f424f11 100755 --- a/ccollect +++ b/ccollect @@ -182,7 +182,7 @@ delete_from_file() rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" - [ "${VVERBOSE}" ] && echo "Finished: rsync -a --delete ${empty_dir} ${trash}" + _techo "Removing $@ in ${trash} finished." } display_version() From 702cdf931e6b76be8aaad5c8cbfaaf4f9e2dc168 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 06:52:51 +0200 Subject: [PATCH 30/58] Use hidden directory for deletion --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index f424f11..41f23c5 100755 --- a/ccollect +++ b/ccollect @@ -162,7 +162,7 @@ delete_from_file() [ $# -eq 1 ] && suffix="$1" && shift # dirs for deletion will be moved to this trash dir inside destination dir # - for fast mv operation - trash="$(mktemp -d "trash.XXXXXX")" + trash="$(mktemp -d ".trash.XXXXXX")" while read to_remove; do mv "${to_remove}" "${trash}" || _exit_err "Moving ${to_remove} to ${trash} failed." From 51dcf4a02f3b57bc94c1bc15b955ef097bfbf3a6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 06:52:51 +0200 Subject: [PATCH 31/58] Use hidden empty directory for deletion --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 41f23c5..c744e27 100755 --- a/ccollect +++ b/ccollect @@ -175,7 +175,7 @@ delete_from_file() fi done < "${file}" _techo "Removing $@ in ${trash}..." - empty_dir="empty-dir" + empty_dir=".empty-dir" mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}" # rsync needs ending slash for directory content From 1e18e71b9de892b255516cdd9fbfdfa4304f7c57 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 07:42:31 +0200 Subject: [PATCH 32/58] Use oldest backup as destination dir without deletion --- ccollect | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/ccollect b/ccollect index c744e27..cf7abe9 100755 --- a/ccollect +++ b/ccollect @@ -715,6 +715,12 @@ while [ "${source_no}" -lt "${no_sources}" ]; do fi fi + # + # Include current time in name, not the time when we began to remove above + # + export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" + export destination_dir="${ddir}/${destination_name}" + # # Check: maximum number of backups is reached? # @@ -723,13 +729,26 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then - remove="$((${count} - ${c_interval} + 1))" - _techo "Removing ${remove} backup(s)..." + # Use oldest directory as new backup destination directory. + # It need not to be deleted, rsync will sync its content. + oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ + _exit_err "Listing oldest backup failed") + _techo "Using ${oldest_bak} for destination dir" + mv "${oldest_bak}" "${destination_dir}" || + _exit_err "Moving oldest backup ${oldest_bak} to ${destination_dir} failed." + # Touch dest dir so it is not sorted wrong in listings below. + touch "${destination_dir}" - ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + # We have something to remove only if count > interval. + if [ "${count}" -gt "${c_interval}" ]; then + remove="$((${count} - ${c_interval}))" + _techo "Removing ${remove} backup(s)..." - delete_from_file "${TMP}" & + ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + + delete_from_file "${TMP}" & + fi fi # @@ -746,12 +765,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Hard linking from ${last_dir}" fi - # - # Include current time in name, not the time when we began to remove above - # - export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" - export destination_dir="${ddir}/${destination_name}" - # # Mark backup running and go back to original directory # From 2788de47b8a7f5a23192b48f421ed37a059f8bb0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 07:52:53 +0200 Subject: [PATCH 33/58] Improve log line --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index cf7abe9..af7a2c1 100755 --- a/ccollect +++ b/ccollect @@ -733,7 +733,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # It need not to be deleted, rsync will sync its content. oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ _exit_err "Listing oldest backup failed") - _techo "Using ${oldest_bak} for destination dir" + _techo "Using ${oldest_bak} for destination dir ${destination_dir}" mv "${oldest_bak}" "${destination_dir}" || _exit_err "Moving oldest backup ${oldest_bak} to ${destination_dir} failed." # Touch dest dir so it is not sorted wrong in listings below. From c39205d30898110b40040c6fad2d6a026013b977 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 08:03:12 +0200 Subject: [PATCH 34/58] Exclude destintion dir from listing for last dir --- ccollect | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index af7a2c1..35d2071 100755 --- a/ccollect +++ b/ccollect @@ -753,8 +753,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check for backup directory to clone from: Always clone from the latest one! + # Exclude destination_dir from listing, it can be touched reused and renamed + # oldest existing destination directory. # - last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ + last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${destination_dir}" | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." # From 7701bdb0a87413bc756ea408656e32d65d7dfb20 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 08:06:13 +0200 Subject: [PATCH 35/58] Use destination dir basename --- ccollect | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 35d2071..f5aa145 100755 --- a/ccollect +++ b/ccollect @@ -756,7 +756,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Exclude destination_dir from listing, it can be touched reused and renamed # oldest existing destination directory. # - last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${destination_dir}" | head -n 1)" || \ + dest_dir_name=$(basename "${destination_dir}") + last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${dest_dir_name}" | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." # From e44dede92f9114db1ffec24a46f215fb270ed08a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 09:05:55 +0200 Subject: [PATCH 36/58] ++changelog --- doc/changes/next | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/next b/doc/changes/next index df06edc..a68cbbc 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1 @@ -* Performance: replace rm -rf with faster rsync --delete empty src dir +* Improve performance, improve process of deletion of old backups From de720ecfe9a71bd2fad3e5ef429629007e8ea6ed Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 11:53:46 +0200 Subject: [PATCH 37/58] If renaming oldest bak dir fails then fallback to removing it --- ccollect | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ccollect b/ccollect index f5aa145..b8011ba 100755 --- a/ccollect +++ b/ccollect @@ -734,14 +734,17 @@ while [ "${source_no}" -lt "${no_sources}" ]; do oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ _exit_err "Listing oldest backup failed") _techo "Using ${oldest_bak} for destination dir ${destination_dir}" - mv "${oldest_bak}" "${destination_dir}" || - _exit_err "Moving oldest backup ${oldest_bak} to ${destination_dir} failed." - # Touch dest dir so it is not sorted wrong in listings below. - touch "${destination_dir}" + if mv "${oldest_bak}" "${destination_dir}"; then + # Touch dest dir so it is not sorted wrong in listings below. + touch "${destination_dir}" - # We have something to remove only if count > interval. - if [ "${count}" -gt "${c_interval}" ]; then + # We have something to remove only if count > interval. remove="$((${count} - ${c_interval}))" + else + _techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it." + remove="$((${count} - ${c_interval} + 1))" + fi + if [ "${remove}" -gt 0 ]; then _techo "Removing ${remove} backup(s)..." ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ From 74e3b26790e4b77f2761d62bd6811aab9bfb9bbd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 12 Nov 2019 17:55:15 +0100 Subject: [PATCH 38/58] Release 2.6 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/{next => 2.6} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename doc/changes/{next => 2.6} (100%) diff --git a/ccollect b/ccollect index b8011ba..4fa6da3 100755 --- a/ccollect +++ b/ccollect @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.5" -RELEASE="2019-05-01" +VERSION="2.6" +RELEASE="2019-11-12" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index f252b25..37e4eaf 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.5, for ccollect 2.5, Initial Version from 2006-01-13 +2.6, for ccollect 2.6, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.6 similarity index 100% rename from doc/changes/next rename to doc/changes/2.6 From 2cefdaa1a5f6adac7240534e558ba9196872cbfa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 18 Oct 2019 16:24:22 +0200 Subject: [PATCH 39/58] Fix shellcheck reported issues --- ccollect | 115 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 40 deletions(-) diff --git a/ccollect b/ccollect index 4fa6da3..6dd3fc0 100755 --- a/ccollect +++ b/ccollect @@ -27,9 +27,9 @@ set -u # # Standard variables (stolen from cconf) # -__pwd="$(pwd -P)" -__mydir="${0%/*}"; __abs_mydir="$(cd "$__mydir" && pwd -P)" -__myname=${0##*/}; __abs_myname="$__abs_mydir/$__myname" +__mydir="${0%/*}" +__abs_mydir="$(cd "$__mydir" && pwd -P)" +__myname=${0##*/} # # where to find our configuration and temporary file @@ -41,7 +41,8 @@ CPREEXEC="${CDEFAULTS}/pre_exec" CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" -export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" +TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" +export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" VERSION="2.6" @@ -76,6 +77,7 @@ LOCKFD=4 lock_flock() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" eval "exec ${LOCKFD}> ${lockfile}" @@ -85,6 +87,7 @@ lock_flock() unlock_flock() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" eval "exec ${LOCKFD}>&-" rm -f "${lockfile}" @@ -96,6 +99,7 @@ unlock_flock() lock_mkdir() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" mkdir "${lockfile}" && return 0 || return 1 @@ -104,6 +108,7 @@ lock_mkdir() unlock_mkdir() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" rmdir "${lockfile}" @@ -112,7 +117,7 @@ unlock_mkdir() # # determine locking tool: flock or mkdir # -if $(which flock > /dev/null 2>&1) +if command -v flock > /dev/null 2>&1 then lockf="lock_flock" unlockf="unlock_flock" @@ -137,6 +142,7 @@ LOGONLYERRORS="" # catch signals # TRAPFUNC="rm -f \"${TMP}\"" +# shellcheck disable=SC2064 trap "${TRAPFUNC}" 1 2 15 # @@ -147,7 +153,7 @@ trap "${TRAPFUNC}" 1 2 15 # see: http://www.tldp.org/LDP/abs/html/intandnonint.html _is_interactive() { - [ -t 0 -o -p /dev/stdin ] + [ -t 0 ] || [ -p /dev/stdin ] } # @@ -163,26 +169,26 @@ delete_from_file() # dirs for deletion will be moved to this trash dir inside destination dir # - for fast mv operation trash="$(mktemp -d ".trash.XXXXXX")" - while read to_remove; do + while read -r to_remove; do mv "${to_remove}" "${trash}" || _exit_err "Moving ${to_remove} to ${trash} failed." set -- "$@" "${to_remove}" if [ "${suffix}" ]; then - to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" + to_remove_no_suffix="$(echo "${to_remove}" | sed "s/$suffix\$//")" mv "${to_remove_no_suffix}" "${trash}" || _exit_err "Moving ${to_remove_no_suffix} to ${trash} failed." set -- "$@" "${to_remove_no_suffix}" fi done < "${file}" - _techo "Removing $@ in ${trash}..." + _techo "Removing $* in ${trash}..." empty_dir=".empty-dir" mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}" # rsync needs ending slash for directory content - rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." + rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $* failed." rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" - _techo "Removing $@ in ${trash} finished." + _techo "Removing $* in ${trash} finished." } display_version() @@ -232,7 +238,7 @@ unlock() # stdout version _techo_stdout() { - echo "$(${DDATE}): $@" + echo "$(${DDATE}): $*" } # syslog version @@ -260,14 +266,16 @@ _techo() { if [ "${LOGLEVEL}" = "a" ] then - set -- ${name:+"[${name}]"} $@ + # name is exported before calling this function + # shellcheck disable=SC2154 + set -- ${name:+"[${name}]"} "$@" "${_techof}" "$@" fi } _techo_err() { - _techo "Error: $@" + _techo "Error: $*" } _exit_err() @@ -368,7 +376,7 @@ else LOGLEVEL="a" fi -if [ "${LOGFILE}" -o "${SYSLOG}" ] +if [ "${LOGFILE}" ] || [ "${SYSLOG}" ] then if [ "${LOGONLYERRORS}" ] then @@ -378,8 +386,7 @@ fi # check that MAX_JOBS is natural number > 0 # empty string means run all in parallel -echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' -if [ "$?" -ne 0 ] +if ! echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' then _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" fi @@ -412,19 +419,22 @@ if [ "${USE_ALL}" = 1 ]; then ( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \ _exit_err "Listing of sources failed. Aborting." - while read tmp; do - eval export source_${no_sources}=\"${tmp}\" - no_sources=$((${no_sources}+1)) + while read -r tmp; do + eval export "source_${no_sources}=\"${tmp}\"" + no_sources=$((no_sources + 1)) done < "${TMP}" else # # Get sources from command line # while [ "$#" -ge 1 ]; do - eval arg=\"\$1\"; shift + eval "arg=\"\$1\"" + shift - eval export source_${no_sources}=\"${arg}\" - no_sources="$((${no_sources}+1))" + # arg is assigned in the eval above + # shellcheck disable=SC2154 + eval export "source_${no_sources}=\"${arg}\"" + no_sources="$((no_sources + 1))" done fi @@ -461,6 +471,7 @@ if [ "${PARALLEL}" ]; then # fd 5 is tied to control pipe eval "exec 5<>${CONTROL_PIPE}" TRAPFUNC="${TRAPFUNC}; rm -f \"${CONTROL_PIPE}\"" + # shellcheck disable=SC2064 trap "${TRAPFUNC}" 0 1 2 15 # determine how much parallel jobs to prestart @@ -483,7 +494,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Get current source # eval export name=\"\$source_${source_no}\" - source_no=$((${source_no}+1)) + source_no=$((source_no + 1)) # # Start ourself, if we want parallel execution @@ -498,12 +509,12 @@ while [ "${source_no}" -lt "${no_sources}" ]; do then # run prestart child if pending { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & - prestart=$((${prestart} - 1)) + prestart=$((prestart - 1)) continue else # each time a child finishes we get a line from the pipe # and then launch another child - while read line + while read -r line do { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & # get out of loop so we can contnue with main loop @@ -557,6 +568,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # redefine trap to also unlock (rm lockfile) TRAPFUNC="${TRAPFUNC}; unlock \"${name}\"" + # shellcheck disable=SC2064 trap "${TRAPFUNC}" 1 2 15 # @@ -576,10 +588,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do for opt in verbose very_verbose summary exclude rsync_options \ delete_incomplete rsync_failure_codes \ mtime quiet_if_down ; do - if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}" ]; then - eval c_$opt=\"${backup}/$opt\" + if [ -f "${backup}/${opt}" ] || [ -f "${backup}/no_${opt}" ]; then + eval "c_$opt=\"${backup}/$opt\"" else - eval c_$opt=\"${CDEFAULTS}/$opt\" + eval "c_$opt=\"${CDEFAULTS}/$opt\"" fi done @@ -599,6 +611,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Sort by ctime (default) or mtime (configuration option) # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_mtime}" ] ; then TSORT="t" else @@ -642,6 +656,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Exclude list # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_exclude}" ]; then set -- "$@" "--exclude-from=${c_exclude}" fi @@ -649,6 +665,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Output a summary # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_summary}" ]; then set -- "$@" "--stats" fi @@ -657,6 +675,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Verbosity for rsync, rm, and mkdir # VVERBOSE="" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_very_verbose}" ]; then set -- "$@" "-vv" VVERBOSE="-v" @@ -667,8 +687,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Extra options for rsync provided by the user # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_rsync_options}" ]; then - while read line; do + while read -r line; do # Trim line. ln=$(echo "${line}" | awk '{$1=$1;print;}') # Only if ln is non zero length string. @@ -687,6 +709,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check: source is up and accepting connections (before deleting old backups!) # if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then + # variable is assigned using eval + # shellcheck disable=SC2154 if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi @@ -706,10 +730,13 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check incomplete backups (needs echo to remove newlines) # + # shellcheck disable=SC2010 ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? if [ "$ret" -eq 0 ]; then - _techo "Incomplete backups: $(echo $(cat "${TMP}"))" + _techo "Incomplete backups: $(cat "${TMP}")" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_delete_incomplete}" ]; then delete_from_file "${TMP}" "${CMARKER}" & fi @@ -718,12 +745,15 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Include current time in name, not the time when we began to remove above # - export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" - export destination_dir="${ddir}/${destination_name}" + destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" + export destination_name + destination_dir="${ddir}/${destination_name}" + export destination_dir # # Check: maximum number of backups is reached? # + # shellcheck disable=SC2010 count="$(ls -1 | grep -c "^${INTERVAL}\\.")" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" @@ -731,6 +761,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ "${count}" -ge "${c_interval}" ]; then # Use oldest directory as new backup destination directory. # It need not to be deleted, rsync will sync its content. + # shellcheck disable=SC2010 oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ _exit_err "Listing oldest backup failed") _techo "Using ${oldest_bak} for destination dir ${destination_dir}" @@ -739,14 +770,15 @@ while [ "${source_no}" -lt "${no_sources}" ]; do touch "${destination_dir}" # We have something to remove only if count > interval. - remove="$((${count} - ${c_interval}))" + remove="$((count - c_interval))" else _techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it." - remove="$((${count} - ${c_interval} + 1))" + remove="$((count - c_interval + 1))" fi if [ "${remove}" -gt 0 ]; then _techo "Removing ${remove} backup(s)..." + # shellcheck disable=SC2010 ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ _exit_err "Listing old backups failed" @@ -760,6 +792,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # oldest existing destination directory. # dest_dir_name=$(basename "${destination_dir}") + # shellcheck disable=SC2010 last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${dest_dir_name}" | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." @@ -793,8 +826,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check if rsync exit code indicates failure. # fail="" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "$c_rsync_failure_codes" ]; then - while read code ; do + while read -r code ; do if [ "$ret" = "$code" ]; then fail=1 fi @@ -831,10 +866,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Time calculation # end_s="$(${SDATE})" - full_seconds="$((${end_s} - ${begin_s}))" - hours="$((${full_seconds} / 3600))" - minutes="$(((${full_seconds} % 3600) / 60))" - seconds="$((${full_seconds} % 60))" + full_seconds="$((end_s - begin_s))" + hours="$((full_seconds / 3600))" + minutes="$(((full_seconds % 3600) / 60))" + seconds="$((full_seconds % 60))" _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" From a5e565b5d69106c11c8090825038b8af51c6fdb5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 14 Nov 2019 19:16:11 +0100 Subject: [PATCH 40/58] Add shellcheck makefile target --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index dc98502..9b07e09 100644 --- a/Makefile +++ b/Makefile @@ -199,6 +199,9 @@ dist: distclean documentation /tmp/ccollect: mkdir -p /tmp/ccollect +shellcheck: ./ccollect + shellcheck -s sh -f gcc -x ./ccollect + test: $(CCOLLECT_SOURCE) /tmp/ccollect cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect daily "$$s"; done touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker From f818f011e3cdd5aa556ee140913f607381555cf5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 14 Nov 2019 19:20:16 +0100 Subject: [PATCH 41/58] Release 2.7 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/2.7 | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 doc/changes/2.7 diff --git a/ccollect b/ccollect index 6dd3fc0..9077e0f 100755 --- a/ccollect +++ b/ccollect @@ -45,8 +45,8 @@ TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.6" -RELEASE="2019-11-12" +VERSION="2.7" +RELEASE="2019-11-14" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 37e4eaf..2478265 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.6, for ccollect 2.6, Initial Version from 2006-01-13 +2.7, for ccollect 2.7, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/2.7 b/doc/changes/2.7 new file mode 100644 index 0000000..bafbb01 --- /dev/null +++ b/doc/changes/2.7 @@ -0,0 +1 @@ +* Fix shellcheck reported issues From 401dd4fa8e4a50e7f8a706c35bf5c505b874a205 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 15 Nov 2019 08:29:21 +0100 Subject: [PATCH 42/58] Fix path with spaces in eval --- ccollect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 9077e0f..d89aec1 100755 --- a/ccollect +++ b/ccollect @@ -79,7 +79,7 @@ lock_flock() # $1 = source to backup # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" - eval "exec ${LOCKFD}> ${lockfile}" + eval "exec ${LOCKFD}> '${lockfile}'" flock -n ${LOCKFD} && return 0 || return 1 } @@ -469,7 +469,7 @@ fi if [ "${PARALLEL}" ]; then mkfifo "${CONTROL_PIPE}" # fd 5 is tied to control pipe - eval "exec 5<>${CONTROL_PIPE}" + eval "exec 5<>'${CONTROL_PIPE}'" TRAPFUNC="${TRAPFUNC}; rm -f \"${CONTROL_PIPE}\"" # shellcheck disable=SC2064 trap "${TRAPFUNC}" 0 1 2 15 From 8f5d9b2c978916d3093ab7c2ce7bb0a94e580018 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 15 Nov 2019 08:28:55 +0100 Subject: [PATCH 43/58] Add unit testing --- .gitignore | 1 - Makefile | 44 ++++++++++++++++++- test/conf/ccollect_local-with | 0 test/conf/ccollect_source | 0 test/conf/defaults/intervals/daily | 1 + test/conf/defaults/intervals/monthly | 1 + test/conf/defaults/intervals/normal | 1 + test/conf/defaults/intervals/weekly | 1 + test/conf/defaults/post_exec | 5 +++ test/conf/defaults/pre_exec | 5 +++ test/conf/defaults/sources/exclude | 1 + test/conf/defaults/sources/rsync_options | 0 test/conf/defaults/sources/verbose | 0 test/conf/defaults/verbose | 0 .../delete_incomplete/delete_incomplete | 0 .../sources/delete_incomplete/destination | 1 + test/conf/sources/delete_incomplete/exclude | 1 + test/conf/sources/delete_incomplete/source | 1 + .../sources/local-with&ersand/destination | 1 + .../conf/sources/local-with&ersand/exclude | 1 + test/conf/sources/local-with&ersand/source | 1 + .../local-with-interval/delete_incomplete | 0 .../sources/local-with-interval/destination | 1 + test/conf/sources/local-with-interval/exclude | 1 + .../local-with-interval/intervals/daily | 1 + test/conf/sources/local-with-interval/source | 1 + test/conf/sources/local-with-interval/verbose | 0 test/conf/sources/local/destination | 1 + test/conf/sources/local/exclude | 1 + test/conf/sources/local/no_verbose | 0 test/conf/sources/local/source | 1 + .../delete_incomplete | 0 .../destination | 1 + .../source with spaces and interval/exclude | 1 + .../source with spaces and interval/source | 1 + .../source with spaces and interval/verbose | 0 test/conf/sources/very_verbose/destination | 1 + test/conf/sources/very_verbose/exclude | 1 + test/conf/sources/very_verbose/source | 1 + test/conf/sources/very_verbose/summary | 0 test/conf/sources/very_verbose/verbose | 0 test/conf/sources/very_verbose/very_verbose | 0 test/conf/sources/with_exec/destination | 1 + test/conf/sources/with_exec/post_exec | 5 +++ test/conf/sources/with_exec/pre_exec | 5 +++ test/conf/sources/with_exec/source | 1 + test/exec.sh | 18 -------- test/local.sh | 1 - test/remote.sh | 1 - test/return-value.sh | 23 ---------- test/test-ccollect-tools.sh | 29 ------------ test/test-ccollect1.sh | 44 ------------------- 52 files changed, 89 insertions(+), 118 deletions(-) create mode 100644 test/conf/ccollect_local-with create mode 100644 test/conf/ccollect_source create mode 100644 test/conf/defaults/intervals/daily create mode 100644 test/conf/defaults/intervals/monthly create mode 100644 test/conf/defaults/intervals/normal create mode 100644 test/conf/defaults/intervals/weekly create mode 100755 test/conf/defaults/post_exec create mode 100755 test/conf/defaults/pre_exec create mode 100644 test/conf/defaults/sources/exclude create mode 100644 test/conf/defaults/sources/rsync_options create mode 100644 test/conf/defaults/sources/verbose create mode 100644 test/conf/defaults/verbose create mode 100644 test/conf/sources/delete_incomplete/delete_incomplete create mode 100644 test/conf/sources/delete_incomplete/destination create mode 100644 test/conf/sources/delete_incomplete/exclude create mode 100644 test/conf/sources/delete_incomplete/source create mode 100644 test/conf/sources/local-with&ersand/destination create mode 100644 test/conf/sources/local-with&ersand/exclude create mode 100644 test/conf/sources/local-with&ersand/source create mode 100644 test/conf/sources/local-with-interval/delete_incomplete create mode 100644 test/conf/sources/local-with-interval/destination create mode 100644 test/conf/sources/local-with-interval/exclude create mode 100644 test/conf/sources/local-with-interval/intervals/daily create mode 100644 test/conf/sources/local-with-interval/source create mode 100644 test/conf/sources/local-with-interval/verbose create mode 100644 test/conf/sources/local/destination create mode 100644 test/conf/sources/local/exclude create mode 100644 test/conf/sources/local/no_verbose create mode 100644 test/conf/sources/local/source create mode 100644 test/conf/sources/source with spaces and interval/delete_incomplete create mode 100644 test/conf/sources/source with spaces and interval/destination create mode 100644 test/conf/sources/source with spaces and interval/exclude create mode 100644 test/conf/sources/source with spaces and interval/source create mode 100644 test/conf/sources/source with spaces and interval/verbose create mode 100644 test/conf/sources/very_verbose/destination create mode 100644 test/conf/sources/very_verbose/exclude create mode 100644 test/conf/sources/very_verbose/source create mode 100644 test/conf/sources/very_verbose/summary create mode 100644 test/conf/sources/very_verbose/verbose create mode 100644 test/conf/sources/very_verbose/very_verbose create mode 100644 test/conf/sources/with_exec/destination create mode 100755 test/conf/sources/with_exec/post_exec create mode 100755 test/conf/sources/with_exec/pre_exec create mode 100644 test/conf/sources/with_exec/source delete mode 100755 test/exec.sh delete mode 100755 test/local.sh delete mode 100755 test/remote.sh delete mode 100755 test/return-value.sh delete mode 100644 test/test-ccollect-tools.sh delete mode 100755 test/test-ccollect1.sh diff --git a/.gitignore b/.gitignore index 4e9c74d..e5c18f6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ doc/man/*.html doc/man/*.htm doc/man/*.texi doc/man/*.man -test/* .*.swp doc/man/*.[0-9] doc/*.xml diff --git a/Makefile b/Makefile index 9b07e09..9b9f3e5 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,8 @@ DOCBDOCS = ${DOCS:.text=.docbook} DOC_ALL = ${HTMLDOCS} ${DBHTMLDOCS} ${TEXIDOCS} ${MANPDOCS} ${PDFDOCS} +TEST_LOG_FILE = /tmp/ccollect/ccollect.log + # # End user targets # @@ -202,9 +204,49 @@ dist: distclean documentation shellcheck: ./ccollect shellcheck -s sh -f gcc -x ./ccollect -test: $(CCOLLECT_SOURCE) /tmp/ccollect +test-nico: $(CCOLLECT_SOURCE) /tmp/ccollect cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect daily "$$s"; done touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect -a daily touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect -a -p daily + +test-dir-source: + mkdir -p /tmp/ccollect/source + cp -R -f ./* /tmp/ccollect/source + +test-dir-destination: + mkdir -p /tmp/ccollect/backup + +test-dir-destination-chint: + mkdir -p /tmp/ccollect/backup-chint + +test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint + for s in ./test/conf/sources/*; do \ + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename $$s)"; \ + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0"; \ + done + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -p daily + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" + @printf "\nFixed intervals test ended successfully\n" + +test-interval-changing: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination-chint + rm -rf /tmp/ccollect/backup-chint/* + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "0" + printf "3" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "3" + printf "5" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5 6 7; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "5" + printf "4" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5 6; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "4" + printf "3" > ./test/conf/sources/local-with-interval/intervals/daily + @printf "\nInterval changing test ended successfully\n" + +test: test-fixed-intervals test-interval-changing + test -f "${TEST_LOG_FILE}" + @printf "\nTests ended successfully\n" diff --git a/test/conf/ccollect_local-with b/test/conf/ccollect_local-with new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/ccollect_source b/test/conf/ccollect_source new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/intervals/daily b/test/conf/defaults/intervals/daily new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/test/conf/defaults/intervals/daily @@ -0,0 +1 @@ +5 diff --git a/test/conf/defaults/intervals/monthly b/test/conf/defaults/intervals/monthly new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/test/conf/defaults/intervals/monthly @@ -0,0 +1 @@ +4 diff --git a/test/conf/defaults/intervals/normal b/test/conf/defaults/intervals/normal new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/test/conf/defaults/intervals/normal @@ -0,0 +1 @@ +4 diff --git a/test/conf/defaults/intervals/weekly b/test/conf/defaults/intervals/weekly new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/test/conf/defaults/intervals/weekly @@ -0,0 +1 @@ +2 diff --git a/test/conf/defaults/post_exec b/test/conf/defaults/post_exec new file mode 100755 index 0000000..8aee58f --- /dev/null +++ b/test/conf/defaults/post_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +General post_exec executed. +###################################################################### diff --git a/test/conf/defaults/pre_exec b/test/conf/defaults/pre_exec new file mode 100755 index 0000000..ecd2857 --- /dev/null +++ b/test/conf/defaults/pre_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +General pre__exec executed. +###################################################################### diff --git a/test/conf/defaults/sources/exclude b/test/conf/defaults/sources/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/defaults/sources/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/defaults/sources/rsync_options b/test/conf/defaults/sources/rsync_options new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/sources/verbose b/test/conf/defaults/sources/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/verbose b/test/conf/defaults/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/delete_incomplete/delete_incomplete b/test/conf/sources/delete_incomplete/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/delete_incomplete/destination b/test/conf/sources/delete_incomplete/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/delete_incomplete/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/delete_incomplete/exclude b/test/conf/sources/delete_incomplete/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/delete_incomplete/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/delete_incomplete/source b/test/conf/sources/delete_incomplete/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/delete_incomplete/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with&ersand/destination b/test/conf/sources/local-with&ersand/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/local-with&ersand/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/local-with&ersand/exclude b/test/conf/sources/local-with&ersand/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local-with&ersand/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local-with&ersand/source b/test/conf/sources/local-with&ersand/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local-with&ersand/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with-interval/delete_incomplete b/test/conf/sources/local-with-interval/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local-with-interval/destination b/test/conf/sources/local-with-interval/destination new file mode 100644 index 0000000..4de7e06 --- /dev/null +++ b/test/conf/sources/local-with-interval/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup-chint diff --git a/test/conf/sources/local-with-interval/exclude b/test/conf/sources/local-with-interval/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local-with-interval/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local-with-interval/intervals/daily b/test/conf/sources/local-with-interval/intervals/daily new file mode 100644 index 0000000..e440e5c --- /dev/null +++ b/test/conf/sources/local-with-interval/intervals/daily @@ -0,0 +1 @@ +3 \ No newline at end of file diff --git a/test/conf/sources/local-with-interval/source b/test/conf/sources/local-with-interval/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local-with-interval/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with-interval/verbose b/test/conf/sources/local-with-interval/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local/destination b/test/conf/sources/local/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/local/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/local/exclude b/test/conf/sources/local/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local/no_verbose b/test/conf/sources/local/no_verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local/source b/test/conf/sources/local/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/source with spaces and interval/delete_incomplete b/test/conf/sources/source with spaces and interval/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/source with spaces and interval/destination b/test/conf/sources/source with spaces and interval/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/source with spaces and interval/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/source with spaces and interval/exclude b/test/conf/sources/source with spaces and interval/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/source with spaces and interval/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/source with spaces and interval/source b/test/conf/sources/source with spaces and interval/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/source with spaces and interval/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/source with spaces and interval/verbose b/test/conf/sources/source with spaces and interval/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/destination b/test/conf/sources/very_verbose/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/very_verbose/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/very_verbose/exclude b/test/conf/sources/very_verbose/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/very_verbose/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/very_verbose/source b/test/conf/sources/very_verbose/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/very_verbose/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/very_verbose/summary b/test/conf/sources/very_verbose/summary new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/verbose b/test/conf/sources/very_verbose/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/very_verbose b/test/conf/sources/very_verbose/very_verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/with_exec/destination b/test/conf/sources/with_exec/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/with_exec/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/with_exec/post_exec b/test/conf/sources/with_exec/post_exec new file mode 100755 index 0000000..abc0a40 --- /dev/null +++ b/test/conf/sources/with_exec/post_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +Source post_exec executed. +###################################################################### diff --git a/test/conf/sources/with_exec/pre_exec b/test/conf/sources/with_exec/pre_exec new file mode 100755 index 0000000..ba7b2af --- /dev/null +++ b/test/conf/sources/with_exec/pre_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +Source pre_exec executed. +###################################################################### diff --git a/test/conf/sources/with_exec/source b/test/conf/sources/with_exec/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/with_exec/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/exec.sh b/test/exec.sh deleted file mode 100755 index bdf601d..0000000 --- a/test/exec.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -host="home.schottelius.org" -host="" -set -x -pcmd() -{ - echo "$#", "$@" - if [ "$host" ]; then - ssh "$host" "$@" - else - $@ - fi -} - -#pcmd ls / -#pcmd cd /; ls "/is not there" -pcmd cd / && ls diff --git a/test/local.sh b/test/local.sh deleted file mode 100755 index c2430fd..0000000 --- a/test/local.sh +++ /dev/null @@ -1 +0,0 @@ -CCOLLECT_CONF=./conf ./ccollect.sh daily -v local1 diff --git a/test/remote.sh b/test/remote.sh deleted file mode 100755 index 2af364e..0000000 --- a/test/remote.sh +++ /dev/null @@ -1 +0,0 @@ -CCOLLECT_CONF=./conf ./ccollect.sh daily -v remote1 diff --git a/test/return-value.sh b/test/return-value.sh deleted file mode 100755 index 554def0..0000000 --- a/test/return-value.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -ls /surely-not-existent$$ 2>/dev/null - -if [ "$?" -ne 0 ]; then - echo "$?" -fi - -ls /surely-not-existent$$ 2>/dev/null - -ret=$? - -if [ "$ret" -ne 0 ]; then - echo "$ret" -fi - -# if is true, ls is fales -if [ "foo" = "foo" ]; then - ls /surely-not-existent$$ 2>/dev/null -fi - -# but that's still the return of ls and not of fi -echo $? diff --git a/test/test-ccollect-tools.sh b/test/test-ccollect-tools.sh deleted file mode 100644 index 5980d04..0000000 --- a/test/test-ccollect-tools.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# 2009 Nico Schottelius (nico-ccollect at schottelius.org) -# -# This file is part of ccollect. -# -# ccollect is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ccollect is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with ccollect. If not, see . -# -# -# Test the ccollect tools suite -# - -set -x - -tmp="$(mktemp /tmp/ccollect-tools.XXXXXXXXXXX)" - - -rm -rf "${tmp}" diff --git a/test/test-ccollect1.sh b/test/test-ccollect1.sh deleted file mode 100755 index c5acf54..0000000 --- a/test/test-ccollect1.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# -# Nico Schottelius -# Date: 27-Jan-2007 -# Last Modified: - -# Description: -# - -ccollect=../ccollect.sh -testdir="$(dirname $0)/test-backups" -confdir="$(dirname $0)/test-config" -source="$(hostname)" -source_source="/tmp" -interval="taeglich" - - -# backup destination -mkdir -p "$testdir" -source_dest="$(cd "$testdir"; pwd -P)" - -# configuration -mkdir -p "${confdir}/sources/${source}" -ln -s "$source_dest" "${confdir}/sources/${source}/destination" -echo "$source_source" > "${confdir}/sources/${source}/source" -touch "${confdir}/sources/${source}/summary" -touch "${confdir}/sources/${source}/verbose" - -mkdir -p "${confdir}/defaults/intervals/" -echo 3 > "${confdir}/defaults/intervals/$interval" - -# create backups - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a -touch "${source_source}/$(date +%s)-$$.1982" - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a -touch "${source_source}/$(date +%s)-$$.42" - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a - -du -sh "$testdir" -du -shl "$testdir" - -echo "Delete $testdir and $confdir after test" From 5ce3fddf6272cb443f37cde21512bce5eedf9d21 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 19 Nov 2019 18:25:10 +0100 Subject: [PATCH 44/58] Define gitlab CI --- .gitlab-ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..b499ddc --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,20 @@ +stages: + - test + +unit_tests: + stage: test + before_script: + - 'apk update' + - 'apk add make rsync' + script: + - make test + +shellcheck: + stage: test + before_script: + - 'apk update' + - 'apk add make' + - 'wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz' + - 'tar xf shellcheck-stable.linux.x86_64.tar.xz && mv shellcheck-stable/shellcheck /usr/bin/' + script: + - make shellcheck From 9ed5912461dcd9b56836ad643d1cfc8b9ff95983 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 08:42:42 +0100 Subject: [PATCH 45/58] Improve unit tests --- Makefile | 14 +++++++------- test/conf/defaults/post_exec | 6 ++---- test/conf/defaults/pre_exec | 6 ++---- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 9b9f3e5..ced0d28 100644 --- a/Makefile +++ b/Makefile @@ -224,26 +224,26 @@ test-dir-destination-chint: test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint for s in ./test/conf/sources/*; do \ CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename $$s)"; \ - test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0"; \ + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; }; \ done CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily - test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; } CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -p daily - test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; } @printf "\nFixed intervals test ended successfully\n" test-interval-changing: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination-chint rm -rf /tmp/ccollect/backup-chint/* - test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "0" + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "0" || { cat ${TEST_LOG_FILE}; exit 1; } printf "3" > ./test/conf/sources/local-with-interval/intervals/daily for x in 1 2 3 4 5; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done - test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "3" + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "3" || { cat ${TEST_LOG_FILE}; exit 1; } printf "5" > ./test/conf/sources/local-with-interval/intervals/daily for x in 1 2 3 4 5 6 7; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done - test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "5" + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "5" || { cat ${TEST_LOG_FILE}; exit 1; } printf "4" > ./test/conf/sources/local-with-interval/intervals/daily for x in 1 2 3 4 5 6; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done - test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "4" + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "4" || { cat ${TEST_LOG_FILE}; exit 1; } printf "3" > ./test/conf/sources/local-with-interval/intervals/daily @printf "\nInterval changing test ended successfully\n" diff --git a/test/conf/defaults/post_exec b/test/conf/defaults/post_exec index 8aee58f..0dac0ed 100755 --- a/test/conf/defaults/post_exec +++ b/test/conf/defaults/post_exec @@ -1,5 +1,3 @@ -#!/bin/cat +#!/bin/sh -###################################################################### -General post_exec executed. -###################################################################### +echo 'General post_exec executed.' diff --git a/test/conf/defaults/pre_exec b/test/conf/defaults/pre_exec index ecd2857..451fdad 100755 --- a/test/conf/defaults/pre_exec +++ b/test/conf/defaults/pre_exec @@ -1,5 +1,3 @@ -#!/bin/cat +#!/bin/sh -###################################################################### -General pre__exec executed. -###################################################################### +echo 'General pre__exec executed.' From 42bd1afb09ce17d84520ae2649c8af1dd91ced8b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 13:35:22 +0100 Subject: [PATCH 46/58] Fix quoting in tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ced0d28..52b7a56 100644 --- a/Makefile +++ b/Makefile @@ -223,7 +223,7 @@ test-dir-destination-chint: test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint for s in ./test/conf/sources/*; do \ - CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename $$s)"; \ + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename '$$s')"; \ test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; }; \ done CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily From 6c24e8a7d35fd7132d2f2b6bbe3977159cf3054b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 13:35:22 +0100 Subject: [PATCH 47/58] Fix quoting in tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 52b7a56..1e783fc 100644 --- a/Makefile +++ b/Makefile @@ -223,7 +223,7 @@ test-dir-destination-chint: test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint for s in ./test/conf/sources/*; do \ - CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename '$$s')"; \ + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename "$$s")"; \ test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; }; \ done CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily From 61ab45fc651b9fafb6da4de208fe0a7d5769c447 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 13:50:30 +0100 Subject: [PATCH 48/58] Fix excluding destination dir from removal Touch can lead to wrong ls order, and destination dir gets selected for removal. Use grep -v to exclude, instead of touch. --- ccollect | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ccollect b/ccollect index d89aec1..61e4ebd 100755 --- a/ccollect +++ b/ccollect @@ -767,20 +767,27 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Using ${oldest_bak} for destination dir ${destination_dir}" if mv "${oldest_bak}" "${destination_dir}"; then # Touch dest dir so it is not sorted wrong in listings below. - touch "${destination_dir}" + ls_rm_exclude=$(basename "${destination_dir}") # We have something to remove only if count > interval. remove="$((count - c_interval))" else _techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it." remove="$((count - c_interval + 1))" + ls_rm_exclude="" fi if [ "${remove}" -gt 0 ]; then _techo "Removing ${remove} backup(s)..." - # shellcheck disable=SC2010 - ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + if [ -z "${ls_rm_exclude}" ]; then + # shellcheck disable=SC2010 + ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + else + # shellcheck disable=SC2010 + ls -${TSORT}1r | grep -v "${ls_rm_exclude}" | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + fi delete_from_file "${TMP}" & fi From 589fed6107e4cd0c719e523b81f1ae7ff5b55677 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 14:41:41 +0100 Subject: [PATCH 49/58] ++changelog --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/next diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..563b438 --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Fix excluding destination dir from removal From 987277f1cf2d02f2c2e527c423f90becd0e0b4f9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 20:52:48 +0100 Subject: [PATCH 50/58] Update Makefile Simplify and generalize. --- Makefile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 1e783fc..789e099 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ ASCIIDOC=asciidoc DOCBOOKTOTEXI=docbook2x-texi DOCBOOKTOMAN=docbook2x-man XSLTPROC=xsltproc -XSL=/usr/share/xml/docbook/stylesheet/nwalsh/html/docbook.xsl +XSL=/usr/local/share/xsl/docbook/html/docbook.xsl A2X=a2x prefix=/usr/packages/ccollect-git @@ -41,11 +41,7 @@ manlink=/usr/local/man/man1 path_dir=/usr/local/bin path_destination=${path_dir}/${CCOLLECT_DEST} - -# where to publish -host=localhost -dir=/home/users/nico/privat/rechner/netz/seiten/www.nico.schottelius.org/src/software/ccollect -docdir=${dir}/documentation +docs_archive_name=docs.tar # # Asciidoc will be used to generate other formats later @@ -91,6 +87,8 @@ all: @echo "info: only generate Texinfo" @echo "man: only generate manpage{s}" @echo "install: install ccollect to ${prefix}" + @echo "shellcheck: shellcheck ccollect script" + @echo "test: run unit tests" html: ${HTMLDOCS} htm: ${DBHTMLDOCS} @@ -179,9 +177,9 @@ pub: git push publish-doc: documentation - @echo "Transferring files to ${host}" @chmod a+r ${DOCS} ${DOC_ALL} - @tar c ${DOCS} ${DOC_ALL} | ssh ${host} "cd ${dir}; tar xv" + @tar cf ${docs_archive_name} ${DOCS} ${DOC_ALL} + @echo "Documentation files are in ${docs_archive_name}" # # Distribution From 5341de86fb331512dd442bbbcc8d68d20e6b06a5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 26 Nov 2019 06:10:17 +0100 Subject: [PATCH 51/58] Release 2.8 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/{next => 2.8} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename doc/changes/{next => 2.8} (100%) diff --git a/ccollect b/ccollect index 61e4ebd..39799cd 100755 --- a/ccollect +++ b/ccollect @@ -45,8 +45,8 @@ TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.7" -RELEASE="2019-11-14" +VERSION="2.8" +RELEASE="2019-11-26" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 2478265..84c326e 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.7, for ccollect 2.7, Initial Version from 2006-01-13 +2.8, for ccollect 2.8, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.8 similarity index 100% rename from doc/changes/next rename to doc/changes/2.8 From 109b70ea769755fbb576e3ee83af399fb5735e9e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 09:26:47 +0100 Subject: [PATCH 52/58] gitlab runner should have necessary tools --- .gitlab-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b499ddc..5391f4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,18 +3,10 @@ stages: unit_tests: stage: test - before_script: - - 'apk update' - - 'apk add make rsync' script: - make test shellcheck: stage: test - before_script: - - 'apk update' - - 'apk add make' - - 'wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz' - - 'tar xf shellcheck-stable.linux.x86_64.tar.xz && mv shellcheck-stable/shellcheck /usr/bin/' script: - make shellcheck From a261ef841eb76595d169eef334b8f004a2d3737c Mon Sep 17 00:00:00 2001 From: Steffen Zieger Date: Sun, 24 May 2020 16:40:04 +0200 Subject: [PATCH 53/58] make rsync return code available in post_exec --- ccollect | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ccollect b/ccollect index 39799cd..679e3ca 100755 --- a/ccollect +++ b/ccollect @@ -824,6 +824,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do rsync "$@" "${source}" "${destination_dir}"; ret=$? _techo "Finished backup (rsync return code: $ret)." + # + # export rsync return code, might be useful in post_exec + # + export rsync_return_code=$ret + # # Set modification time (mtime) to current time, if sorting by mtime is enabled # From 28dec3694ae11b3924ba76c7de830183857d08bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 24 May 2020 17:29:29 +0200 Subject: [PATCH 54/58] ++changelog --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/next diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..45097c7 --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Make rsync return code available in post_exec (Steffen Zieger) From 7a7dec7751862ca1eec30f2041c80e40746766b3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 May 2020 12:05:35 +0200 Subject: [PATCH 55/58] Release 2.9 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/{next => 2.9} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename doc/changes/{next => 2.9} (100%) diff --git a/ccollect b/ccollect index 679e3ca..01c5d05 100755 --- a/ccollect +++ b/ccollect @@ -45,8 +45,8 @@ TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.8" -RELEASE="2019-11-26" +VERSION="2.9" +RELEASE="2020-05-25" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 84c326e..bc8326a 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.8, for ccollect 2.8, Initial Version from 2006-01-13 +2.9, for ccollect 2.9, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.9 similarity index 100% rename from doc/changes/next rename to doc/changes/2.9 From 616b1d9e3e9cf996e1f90bfc8ede58143ed0e7e7 Mon Sep 17 00:00:00 2001 From: Steffen Zieger Date: Mon, 25 May 2020 16:16:32 +0200 Subject: [PATCH 56/58] Add 'current' symlink to backup destinations --- ccollect | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ccollect b/ccollect index 01c5d05..b4af2f1 100755 --- a/ccollect +++ b/ccollect @@ -861,6 +861,16 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Warning: rsync failed with return code $ret." fi + # + # Create symlink to newest backup + # + # shellcheck disable=SC2010 + latest_dir="$(ls -${TSORT}p1 "${ddir}" | grep '/$' | head -n 1)" || \ + _exit_err "Failed to list content of ${ddir}." + + ln -snf "${ddir}${latest_dir}" "${ddir}current" || \ + _exit_err "Failed to create 'current' symlink." + # # post_exec # From 309d8dc773182a053619a93bc099f88f087b4c0e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 May 2020 17:52:23 +0200 Subject: [PATCH 57/58] ++changelog --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/next diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..1d1e85d --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Add 'current' symlink to backup destinations (Steffen Zieger) From 08cb857664a6dbcbacaffc020c747db554ebbf15 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 26 Aug 2020 08:36:43 +0200 Subject: [PATCH 58/58] Release 2.10 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/{next => 2.10} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename doc/changes/{next => 2.10} (100%) diff --git a/ccollect b/ccollect index b4af2f1..2deaa2d 100755 --- a/ccollect +++ b/ccollect @@ -45,8 +45,8 @@ TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.9" -RELEASE="2020-05-25" +VERSION="2.10" +RELEASE="2020-08-26" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index bc8326a..f075b9a 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.9, for ccollect 2.9, Initial Version from 2006-01-13 +2.10, for ccollect 2.10, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.10 similarity index 100% rename from doc/changes/next rename to doc/changes/2.10