diff --git a/.gitignore b/.gitignore index 4e9c74d..e5c18f6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ doc/man/*.html doc/man/*.htm doc/man/*.texi doc/man/*.man -test/* .*.swp doc/man/*.[0-9] doc/*.xml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..5391f4d --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,12 @@ +stages: + - test + +unit_tests: + stage: test + script: + - make test + +shellcheck: + stage: test + script: + - make shellcheck diff --git a/Makefile b/Makefile index 908bd6c..789e099 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ ASCIIDOC=asciidoc DOCBOOKTOTEXI=docbook2x-texi DOCBOOKTOMAN=docbook2x-man XSLTPROC=xsltproc -XSL=/usr/share/xml/docbook/stylesheet/nwalsh/html/docbook.xsl +XSL=/usr/local/share/xsl/docbook/html/docbook.xsl A2X=a2x prefix=/usr/packages/ccollect-git @@ -41,11 +41,7 @@ manlink=/usr/local/man/man1 path_dir=/usr/local/bin path_destination=${path_dir}/${CCOLLECT_DEST} - -# where to publish -host=localhost -dir=/home/users/nico/privat/rechner/netz/seiten/www.nico.schottelius.org/src/software/ccollect -docdir=${dir}/documentation +docs_archive_name=docs.tar # # Asciidoc will be used to generate other formats later @@ -79,6 +75,8 @@ DOCBDOCS = ${DOCS:.text=.docbook} DOC_ALL = ${HTMLDOCS} ${DBHTMLDOCS} ${TEXIDOCS} ${MANPDOCS} ${PDFDOCS} +TEST_LOG_FILE = /tmp/ccollect/ccollect.log + # # End user targets # @@ -89,6 +87,8 @@ all: @echo "info: only generate Texinfo" @echo "man: only generate manpage{s}" @echo "install: install ccollect to ${prefix}" + @echo "shellcheck: shellcheck ccollect script" + @echo "test: run unit tests" html: ${HTMLDOCS} htm: ${DBHTMLDOCS} @@ -175,12 +175,11 @@ t2: # pub: git push - git push github publish-doc: documentation - @echo "Transferring files to ${host}" @chmod a+r ${DOCS} ${DOC_ALL} - @tar c ${DOCS} ${DOC_ALL} | ssh ${host} "cd ${dir}; tar xv" + @tar cf ${docs_archive_name} ${DOCS} ${DOC_ALL} + @echo "Documentation files are in ${docs_archive_name}" # # Distribution @@ -200,9 +199,52 @@ dist: distclean documentation /tmp/ccollect: mkdir -p /tmp/ccollect -test: $(CCOLLECT_SOURCE) /tmp/ccollect +shellcheck: ./ccollect + shellcheck -s sh -f gcc -x ./ccollect + +test-nico: $(CCOLLECT_SOURCE) /tmp/ccollect cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect daily "$$s"; done touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect -a daily touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect -a -p daily + +test-dir-source: + mkdir -p /tmp/ccollect/source + cp -R -f ./* /tmp/ccollect/source + +test-dir-destination: + mkdir -p /tmp/ccollect/backup + +test-dir-destination-chint: + mkdir -p /tmp/ccollect/backup-chint + +test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint + for s in ./test/conf/sources/*; do \ + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename "$$s")"; \ + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; }; \ + done + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; } + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -p daily + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; } + @printf "\nFixed intervals test ended successfully\n" + +test-interval-changing: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination-chint + rm -rf /tmp/ccollect/backup-chint/* + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "0" || { cat ${TEST_LOG_FILE}; exit 1; } + printf "3" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "3" || { cat ${TEST_LOG_FILE}; exit 1; } + printf "5" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5 6 7; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "5" || { cat ${TEST_LOG_FILE}; exit 1; } + printf "4" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5 6; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "4" || { cat ${TEST_LOG_FILE}; exit 1; } + printf "3" > ./test/conf/sources/local-with-interval/intervals/daily + @printf "\nInterval changing test ended successfully\n" + +test: test-fixed-intervals test-interval-changing + test -f "${TEST_LOG_FILE}" + @printf "\nTests ended successfully\n" diff --git a/ccollect b/ccollect index e1074b0..2deaa2d 100755 --- a/ccollect +++ b/ccollect @@ -1,7 +1,7 @@ #!/bin/sh # # 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org) -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of ccollect. # @@ -27,9 +27,9 @@ set -u # # Standard variables (stolen from cconf) # -__pwd="$(pwd -P)" -__mydir="${0%/*}"; __abs_mydir="$(cd "$__mydir" && pwd -P)" -__myname=${0##*/}; __abs_myname="$__abs_mydir/$__myname" +__mydir="${0%/*}" +__abs_mydir="$(cd "$__mydir" && pwd -P)" +__myname=${0##*/} # # where to find our configuration and temporary file @@ -41,11 +41,12 @@ CPREEXEC="${CDEFAULTS}/pre_exec" CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" -export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" +TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" +export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.0" -RELEASE="2016-09-26" +VERSION="2.10" +RELEASE="2020-08-26" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" @@ -76,8 +77,9 @@ LOCKFD=4 lock_flock() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" - eval "exec ${LOCKFD}> ${lockfile}" + eval "exec ${LOCKFD}> '${lockfile}'" flock -n ${LOCKFD} && return 0 || return 1 } @@ -85,6 +87,7 @@ lock_flock() unlock_flock() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" eval "exec ${LOCKFD}>&-" rm -f "${lockfile}" @@ -96,6 +99,7 @@ unlock_flock() lock_mkdir() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" mkdir "${lockfile}" && return 0 || return 1 @@ -104,6 +108,7 @@ lock_mkdir() unlock_mkdir() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" rmdir "${lockfile}" @@ -112,7 +117,7 @@ unlock_mkdir() # # determine locking tool: flock or mkdir # -if $(which flock > /dev/null 2>&1) +if command -v flock > /dev/null 2>&1 then lockf="lock_flock" unlockf="unlock_flock" @@ -127,34 +132,28 @@ fi PARALLEL="" MAX_JOBS="" USE_ALL="" +LOGFILE="" +SYSLOG="" +# e - only errors, a - all output +LOGLEVEL="a" +LOGONLYERRORS="" # # catch signals # TRAPFUNC="rm -f \"${TMP}\"" +# shellcheck disable=SC2064 trap "${TRAPFUNC}" 1 2 15 # # Functions # -# time displaying echo -_techo() +# check if we are running interactive or non-interactive +# see: http://www.tldp.org/LDP/abs/html/intandnonint.html +_is_interactive() { - echo "$(${DDATE}): $@" -} - -# exit on error -_exit_err() -{ - _techo "Error: $@" - rm -f "${TMP}" - exit 1 -} - -add_name() -{ - awk "{ print \"[${name}] \" \$0 }" + [ -t 0 ] || [ -p /dev/stdin ] } # @@ -167,16 +166,29 @@ delete_from_file() file="$1"; shift suffix="" # It will be set, if deleting incomplete backups. [ $# -eq 1 ] && suffix="$1" && shift - while read to_remove; do + # dirs for deletion will be moved to this trash dir inside destination dir + # - for fast mv operation + trash="$(mktemp -d ".trash.XXXXXX")" + while read -r to_remove; do + mv "${to_remove}" "${trash}" || + _exit_err "Moving ${to_remove} to ${trash} failed." set -- "$@" "${to_remove}" if [ "${suffix}" ]; then - to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" + to_remove_no_suffix="$(echo "${to_remove}" | sed "s/$suffix\$//")" + mv "${to_remove_no_suffix}" "${trash}" || + _exit_err "Moving ${to_remove_no_suffix} to ${trash} failed." set -- "$@" "${to_remove_no_suffix}" fi done < "${file}" - _techo "Removing $@ ..." - [ "${VVERBOSE}" ] && echo rm "$@" - rm -rf "$@" || _exit_err "Removing $@ failed." + _techo "Removing $* in ${trash}..." + empty_dir=".empty-dir" + mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." + [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}" + # rsync needs ending slash for directory content + rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $* failed." + rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" + rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" + _techo "Removing $* in ${trash} finished." } display_version() @@ -192,13 +204,16 @@ ${__myname}: [args] ccollect creates (pseudo) incremental backups - -h, --help: Show this help screen - -a, --all: Backup all sources specified in ${CSOURCES} - -j [max], --jobs [max] Specifies the number of jobs to run simultaneously. - If max is not specified then parallelise all jobs. - -p, --parallel: Parallelise backup processes (deprecated from 2.0) - -v, --verbose: Be very verbose (uses set -x) - -V, --version: Print version information + -h, --help: Show this help screen + -a, --all: Backup all sources specified in ${CSOURCES} + -e, --errors: Log only errors + -j [max], --jobs [max] Specifies the number of jobs to run simultaneously. + If max is not specified then parallelise all jobs. + -l FILE, --logfile FILE Log to specified file + -p, --parallel: Parallelise backup processes (deprecated from 2.0) + -s, --syslog: Log to syslog with tag ccollect + -v, --verbose: Be very verbose (uses set -x) + -V, --version: Print version information This is version ${VERSION} released on ${RELEASE}. @@ -219,6 +234,57 @@ unlock() "${unlockf}" "$@" } +# time displaying echo +# stdout version +_techo_stdout() +{ + echo "$(${DDATE}): $*" +} + +# syslog version +_techo_syslog() +{ + logger -t ccollect "$@" +} + +# specified file version +_techo_file() +{ + _techo_stdout "$@" >> "${LOGFILE}" +} + +# determine _techo version before parsing options +if _is_interactive +then + _techof="_techo_stdout" +else + _techof="_techo_syslog" +fi + +# _techo with determined _techo version +_techo() +{ + if [ "${LOGLEVEL}" = "a" ] + then + # name is exported before calling this function + # shellcheck disable=SC2154 + set -- ${name:+"[${name}]"} "$@" + "${_techof}" "$@" + fi +} + +_techo_err() +{ + _techo "Error: $*" +} + +_exit_err() +{ + _techo_err "$@" + rm -f "${TMP}" + exit 1 +} + # # Parse options # @@ -247,6 +313,28 @@ while [ "$#" -ge 1 ]; do esac fi ;; + -e|--errors) + LOGONLYERRORS="1" + ;; + -l|--logfile) + if [ "$#" -ge 2 ] + then + case "$2" in + -*) + _exit_err "Missing log file" + ;; + *) + LOGFILE="$2" + shift + ;; + esac + else + _exit_err "Missing log file" + fi + ;; + -s|--syslog) + SYSLOG="1" + ;; -v|--verbose) set -x ;; @@ -268,10 +356,37 @@ while [ "$#" -ge 1 ]; do shift done +# determine _techo version and logging level after parsing options +if [ "${LOGFILE}" ] +then + _techof="_techo_file" + LOGLEVEL="a" +elif _is_interactive +then + if [ "${SYSLOG}" ] + then + _techof="_techo_syslog" + LOGLEVEL="a" + else + _techof="_techo_stdout" + LOGLEVEL="e" + fi +else + _techof="_techo_syslog" + LOGLEVEL="a" +fi + +if [ "${LOGFILE}" ] || [ "${SYSLOG}" ] +then + if [ "${LOGONLYERRORS}" ] + then + LOGLEVEL="e" + fi +fi + # check that MAX_JOBS is natural number > 0 # empty string means run all in parallel -echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' -if [ "$?" -ne 0 ] +if ! echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' then _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" fi @@ -304,19 +419,22 @@ if [ "${USE_ALL}" = 1 ]; then ( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \ _exit_err "Listing of sources failed. Aborting." - while read tmp; do - eval export source_${no_sources}=\"${tmp}\" - no_sources=$((${no_sources}+1)) + while read -r tmp; do + eval export "source_${no_sources}=\"${tmp}\"" + no_sources=$((no_sources + 1)) done < "${TMP}" else # # Get sources from command line # while [ "$#" -ge 1 ]; do - eval arg=\"\$1\"; shift + eval "arg=\"\$1\"" + shift - eval export source_${no_sources}=\"${arg}\" - no_sources="$((${no_sources}+1))" + # arg is assigned in the eval above + # shellcheck disable=SC2154 + eval export "source_${no_sources}=\"${arg}\"" + no_sources="$((no_sources + 1))" done fi @@ -351,14 +469,20 @@ fi if [ "${PARALLEL}" ]; then mkfifo "${CONTROL_PIPE}" # fd 5 is tied to control pipe - eval "exec 5<>${CONTROL_PIPE}" + eval "exec 5<>'${CONTROL_PIPE}'" TRAPFUNC="${TRAPFUNC}; rm -f \"${CONTROL_PIPE}\"" + # shellcheck disable=SC2064 trap "${TRAPFUNC}" 0 1 2 15 # determine how much parallel jobs to prestart - if [ "${MAX_JOBS}" ] && [ "${MAX_JOBS}" -le "${no_sources}" ] + if [ "${MAX_JOBS}" ] then - prestart="${MAX_JOBS}" + if [ "${MAX_JOBS}" -le "${no_sources}" ] + then + prestart="${MAX_JOBS}" + else + prestart="${no_sources}" + fi else prestart=0 fi @@ -370,7 +494,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Get current source # eval export name=\"\$source_${source_no}\" - source_no=$((${source_no}+1)) + source_no=$((source_no + 1)) # # Start ourself, if we want parallel execution @@ -385,12 +509,12 @@ while [ "${source_no}" -lt "${no_sources}" ]; do then # run prestart child if pending { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & - prestart=$((${prestart} - 1)) + prestart=$((prestart - 1)) continue else # each time a child finishes we get a line from the pipe # and then launch another child - while read line + while read -r line do { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & # get out of loop so we can contnue with main loop @@ -426,7 +550,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Standard configuration checks # if [ ! -e "${backup}" ]; then - _exit_err "Source does not exist." + _exit_err "Source \"${backup}\" does not exist." fi # @@ -444,6 +568,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # redefine trap to also unlock (rm lockfile) TRAPFUNC="${TRAPFUNC}; unlock \"${name}\"" + # shellcheck disable=SC2064 trap "${TRAPFUNC}" 1 2 15 # @@ -463,10 +588,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do for opt in verbose very_verbose summary exclude rsync_options \ delete_incomplete rsync_failure_codes \ mtime quiet_if_down ; do - if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}" ]; then - eval c_$opt=\"${backup}/$opt\" + if [ -f "${backup}/${opt}" ] || [ -f "${backup}/no_${opt}" ]; then + eval "c_$opt=\"${backup}/$opt\"" else - eval c_$opt=\"${CDEFAULTS}/$opt\" + eval "c_$opt=\"${CDEFAULTS}/$opt\"" fi done @@ -486,6 +611,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Sort by ctime (default) or mtime (configuration option) # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_mtime}" ] ; then TSORT="t" else @@ -529,6 +656,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Exclude list # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_exclude}" ]; then set -- "$@" "--exclude-from=${c_exclude}" fi @@ -536,6 +665,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Output a summary # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_summary}" ]; then set -- "$@" "--stats" fi @@ -544,6 +675,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Verbosity for rsync, rm, and mkdir # VVERBOSE="" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_very_verbose}" ]; then set -- "$@" "-vv" VVERBOSE="-v" @@ -554,9 +687,21 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Extra options for rsync provided by the user # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_rsync_options}" ]; then - while read line; do - set -- "$@" "${line}" + while read -r line; do + # Trim line. + ln=$(echo "${line}" | awk '{$1=$1;print;}') + # Only if ln is non zero length string. + # + # If ln is empty then rsync '' DEST evaluates + # to transfer current directory to DEST which would + # with specific options destroy DEST content. + if [ -n "${ln}" ] + then + set -- "$@" "${ln}" + fi done < "${c_rsync_options}" fi @@ -564,6 +709,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check: source is up and accepting connections (before deleting old backups!) # if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then + # variable is assigned using eval + # shellcheck disable=SC2154 if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi @@ -583,36 +730,77 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check incomplete backups (needs echo to remove newlines) # + # shellcheck disable=SC2010 ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? if [ "$ret" -eq 0 ]; then - _techo "Incomplete backups: $(echo $(cat "${TMP}"))" + _techo "Incomplete backups: $(cat "${TMP}")" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" "${CMARKER}" + delete_from_file "${TMP}" "${CMARKER}" & fi fi + # + # Include current time in name, not the time when we began to remove above + # + destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" + export destination_name + destination_dir="${ddir}/${destination_name}" + export destination_dir + # # Check: maximum number of backups is reached? # + # shellcheck disable=SC2010 count="$(ls -1 | grep -c "^${INTERVAL}\\.")" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then - remove="$((${count} - ${c_interval} + 1))" - _techo "Removing ${remove} backup(s)..." + # Use oldest directory as new backup destination directory. + # It need not to be deleted, rsync will sync its content. + # shellcheck disable=SC2010 + oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ + _exit_err "Listing oldest backup failed") + _techo "Using ${oldest_bak} for destination dir ${destination_dir}" + if mv "${oldest_bak}" "${destination_dir}"; then + # Touch dest dir so it is not sorted wrong in listings below. + ls_rm_exclude=$(basename "${destination_dir}") - ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + # We have something to remove only if count > interval. + remove="$((count - c_interval))" + else + _techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it." + remove="$((count - c_interval + 1))" + ls_rm_exclude="" + fi + if [ "${remove}" -gt 0 ]; then + _techo "Removing ${remove} backup(s)..." - delete_from_file "${TMP}" + if [ -z "${ls_rm_exclude}" ]; then + # shellcheck disable=SC2010 + ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + else + # shellcheck disable=SC2010 + ls -${TSORT}1r | grep -v "${ls_rm_exclude}" | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + fi + + delete_from_file "${TMP}" & + fi fi # # Check for backup directory to clone from: Always clone from the latest one! + # Exclude destination_dir from listing, it can be touched reused and renamed + # oldest existing destination directory. # - last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ + dest_dir_name=$(basename "${destination_dir}") + # shellcheck disable=SC2010 + last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${dest_dir_name}" | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." # @@ -623,12 +811,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Hard linking from ${last_dir}" fi - # - # Include current time in name, not the time when we began to remove above - # - export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" - export destination_dir="${ddir}/${destination_name}" - # # Mark backup running and go back to original directory # @@ -642,6 +824,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do rsync "$@" "${source}" "${destination_dir}"; ret=$? _techo "Finished backup (rsync return code: $ret)." + # + # export rsync return code, might be useful in post_exec + # + export rsync_return_code=$ret + # # Set modification time (mtime) to current time, if sorting by mtime is enabled # @@ -651,8 +838,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check if rsync exit code indicates failure. # fail="" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "$c_rsync_failure_codes" ]; then - while read code ; do + while read -r code ; do if [ "$ret" = "$code" ]; then fail=1 fi @@ -672,6 +861,16 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Warning: rsync failed with return code $ret." fi + # + # Create symlink to newest backup + # + # shellcheck disable=SC2010 + latest_dir="$(ls -${TSORT}p1 "${ddir}" | grep '/$' | head -n 1)" || \ + _exit_err "Failed to list content of ${ddir}." + + ln -snf "${ddir}${latest_dir}" "${ddir}current" || \ + _exit_err "Failed to create 'current' symlink." + # # post_exec # @@ -689,15 +888,18 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Time calculation # end_s="$(${SDATE})" - full_seconds="$((${end_s} - ${begin_s}))" - hours="$((${full_seconds} / 3600))" - minutes="$(((${full_seconds} % 3600) / 60))" - seconds="$((${full_seconds} % 60))" + full_seconds="$((end_s - begin_s))" + hours="$((full_seconds / 3600))" + minutes="$(((full_seconds % 3600) / 60))" + seconds="$((full_seconds % 60))" _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" -) | add_name + + # wait for children (doing delete_from_file) if any still running + wait +) || exit done # diff --git a/contrib/ccollect.spec b/contrib/ccollect.spec index 0e35bb0..8916b7e 100644 --- a/contrib/ccollect.spec +++ b/contrib/ccollect.spec @@ -1,6 +1,6 @@ Summary: (pseudo) incremental backup with different exclude lists using hardlinks and rsync Name: ccollect -Version: 0.8.1 +Version: 2.3 Release: 0 URL: http://www.nico.schottelius.org/software/ccollect Source0: http://www.nico.schottelius.org/software/ccollect/%{name}-%{version}.tar.bz2 @@ -23,10 +23,10 @@ Only the inodes used by the hardlinks and the changed files need additional spac %install rm -rf $RPM_BUILD_ROOT -#Installing main ccollect.sh and /etc directory +#Installing main ccollect and /etc directory %__install -d 755 %buildroot%_bindir %__install -d 755 %buildroot%_sysconfdir/%name -%__install -m 755 ccollect.sh %buildroot%_bindir/ +%__install -m 755 ccollect %buildroot%_bindir/ #bin files from tools directory for t in $(ls tools/ccollect_*) ; do @@ -45,19 +45,13 @@ done #Addition documentation and some config tools %__install -d 755 %buildroot%_datadir/%name/tools +%__install -m 755 tools/called_from_remote_pre_exec %buildroot%_datadir/%name/tools %__cp -pr tools/config-pre-* %buildroot%_datadir/%name/tools -%__install -m 755 tools/gnu-du-backup-size-compare.sh %buildroot%_datadir/%name/tools -%__install -m 755 tools/report_success.sh %buildroot%_datadir/%name/tools +%__install -m 755 tools/report_success %buildroot%_datadir/%name/tools %clean rm -rf $RPM_BUILD_ROOT -%post -%__ln_s %_bindir/ccollect.sh %_bindir/ccollect - -%preun -unlink %_bindir/ccollect - %files %defattr(-,root,root) %_bindir/ccollect* diff --git a/doc/ccollect.text b/doc/ccollect.text index b1c7dba..f075b9a 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.0, for ccollect 2.0, Initial Version from 2006-01-13 +2.10, for ccollect 2.10, Initial Version from 2006-01-13 :Author Initials: NS @@ -26,6 +26,7 @@ Supported and tested operating systems and architectures - Mac OS X 10.5 - NetBSD on alpha/amd64/i386/sparc/sparc64 - OpenBSD on amd64 +- Windows by installing Cygwin, OpenSSH and rsync It *should* run on any Unix that supports `rsync` and has a POSIX-compatible bourne shell. If your platform is not listed above and you have it successfully diff --git a/doc/changes/next b/doc/changes/2.0 similarity index 100% rename from doc/changes/next rename to doc/changes/2.0 diff --git a/doc/changes/2.1 b/doc/changes/2.1 new file mode 100644 index 0000000..ec42d4b --- /dev/null +++ b/doc/changes/2.1 @@ -0,0 +1 @@ +* Add options for stdout, file and syslog logging (Darko Poljak) diff --git a/doc/changes/2.10 b/doc/changes/2.10 new file mode 100644 index 0000000..1d1e85d --- /dev/null +++ b/doc/changes/2.10 @@ -0,0 +1 @@ +* Add 'current' symlink to backup destinations (Steffen Zieger) diff --git a/doc/changes/2.2 b/doc/changes/2.2 new file mode 100644 index 0000000..0ee09d6 --- /dev/null +++ b/doc/changes/2.2 @@ -0,0 +1 @@ +* Bugfix: empty rsync_options line causes destroying source (Darko Poljak) diff --git a/doc/changes/2.3 b/doc/changes/2.3 new file mode 100644 index 0000000..960110c --- /dev/null +++ b/doc/changes/2.3 @@ -0,0 +1 @@ +* Bugfix: Fix parallel mode deadlock when max jobs > number of sources (Darko Poljak) diff --git a/doc/changes/2.4 b/doc/changes/2.4 new file mode 100644 index 0000000..07ffe9d --- /dev/null +++ b/doc/changes/2.4 @@ -0,0 +1,2 @@ +* Add Windows(Cygwin) as supported OS +* Add source name tag in log line diff --git a/doc/changes/2.5 b/doc/changes/2.5 new file mode 100644 index 0000000..a6b319c --- /dev/null +++ b/doc/changes/2.5 @@ -0,0 +1 @@ +* Bugfix: exit in case of subshell error diff --git a/doc/changes/2.6 b/doc/changes/2.6 new file mode 100644 index 0000000..a68cbbc --- /dev/null +++ b/doc/changes/2.6 @@ -0,0 +1 @@ +* Improve performance, improve process of deletion of old backups diff --git a/doc/changes/2.7 b/doc/changes/2.7 new file mode 100644 index 0000000..bafbb01 --- /dev/null +++ b/doc/changes/2.7 @@ -0,0 +1 @@ +* Fix shellcheck reported issues diff --git a/doc/changes/2.8 b/doc/changes/2.8 new file mode 100644 index 0000000..563b438 --- /dev/null +++ b/doc/changes/2.8 @@ -0,0 +1 @@ +* Fix excluding destination dir from removal diff --git a/doc/changes/2.9 b/doc/changes/2.9 new file mode 100644 index 0000000..45097c7 --- /dev/null +++ b/doc/changes/2.9 @@ -0,0 +1 @@ +* Make rsync return code available in post_exec (Steffen Zieger) diff --git a/doc/man/ccollect.text b/doc/man/ccollect.text index 9fc9086..67f8f47 100644 --- a/doc/man/ccollect.text +++ b/doc/man/ccollect.text @@ -29,6 +29,9 @@ OPTIONS -a, --all:: Backup all sources specified in /etc/ccollect/sources +-e, --errors:: + Log only errors + -h, --help:: Show the help screen @@ -36,14 +39,41 @@ OPTIONS Specifies the number of jobs to run simultaneously. If max is not specified then parallelise all jobs. +-l FILE, --logfile FILE:: + Log to specified file + -p, --parallel:: Parallelise backup processes (deprecated from 2.0) +-s, --syslog:: + Log to syslog with tag ccollect + +-V, --version:: + Show version and exit + -v, --verbose:: Be very verbose (uses set -x) --V, --version:: - Show version and exit + +LOGGING MECHANISM +----------------- +ccollect logging depends on running in non-interactive/interactive mode +and on specified optins. The mechanism behaves as the following: + +non-interactive mode:: + + * standard output goes to syslog + * optional: specify logging into file + * log all output by default + * optional: log only errors + +interactive mode:: + + * standard output goes to stdout + * log only errors + * optional: log into syslog or file + - log all output by default + - optional: log only errors SEE ALSO diff --git a/test/conf/ccollect_local-with b/test/conf/ccollect_local-with new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/ccollect_source b/test/conf/ccollect_source new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/intervals/daily b/test/conf/defaults/intervals/daily new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/test/conf/defaults/intervals/daily @@ -0,0 +1 @@ +5 diff --git a/test/conf/defaults/intervals/monthly b/test/conf/defaults/intervals/monthly new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/test/conf/defaults/intervals/monthly @@ -0,0 +1 @@ +4 diff --git a/test/conf/defaults/intervals/normal b/test/conf/defaults/intervals/normal new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/test/conf/defaults/intervals/normal @@ -0,0 +1 @@ +4 diff --git a/test/conf/defaults/intervals/weekly b/test/conf/defaults/intervals/weekly new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/test/conf/defaults/intervals/weekly @@ -0,0 +1 @@ +2 diff --git a/test/conf/defaults/post_exec b/test/conf/defaults/post_exec new file mode 100755 index 0000000..0dac0ed --- /dev/null +++ b/test/conf/defaults/post_exec @@ -0,0 +1,3 @@ +#!/bin/sh + +echo 'General post_exec executed.' diff --git a/test/conf/defaults/pre_exec b/test/conf/defaults/pre_exec new file mode 100755 index 0000000..451fdad --- /dev/null +++ b/test/conf/defaults/pre_exec @@ -0,0 +1,3 @@ +#!/bin/sh + +echo 'General pre__exec executed.' diff --git a/test/conf/defaults/sources/exclude b/test/conf/defaults/sources/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/defaults/sources/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/defaults/sources/rsync_options b/test/conf/defaults/sources/rsync_options new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/sources/verbose b/test/conf/defaults/sources/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/verbose b/test/conf/defaults/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/delete_incomplete/delete_incomplete b/test/conf/sources/delete_incomplete/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/delete_incomplete/destination b/test/conf/sources/delete_incomplete/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/delete_incomplete/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/delete_incomplete/exclude b/test/conf/sources/delete_incomplete/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/delete_incomplete/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/delete_incomplete/source b/test/conf/sources/delete_incomplete/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/delete_incomplete/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with&ersand/destination b/test/conf/sources/local-with&ersand/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/local-with&ersand/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/local-with&ersand/exclude b/test/conf/sources/local-with&ersand/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local-with&ersand/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local-with&ersand/source b/test/conf/sources/local-with&ersand/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local-with&ersand/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with-interval/delete_incomplete b/test/conf/sources/local-with-interval/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local-with-interval/destination b/test/conf/sources/local-with-interval/destination new file mode 100644 index 0000000..4de7e06 --- /dev/null +++ b/test/conf/sources/local-with-interval/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup-chint diff --git a/test/conf/sources/local-with-interval/exclude b/test/conf/sources/local-with-interval/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local-with-interval/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local-with-interval/intervals/daily b/test/conf/sources/local-with-interval/intervals/daily new file mode 100644 index 0000000..e440e5c --- /dev/null +++ b/test/conf/sources/local-with-interval/intervals/daily @@ -0,0 +1 @@ +3 \ No newline at end of file diff --git a/test/conf/sources/local-with-interval/source b/test/conf/sources/local-with-interval/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local-with-interval/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with-interval/verbose b/test/conf/sources/local-with-interval/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local/destination b/test/conf/sources/local/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/local/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/local/exclude b/test/conf/sources/local/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local/no_verbose b/test/conf/sources/local/no_verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local/source b/test/conf/sources/local/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/source with spaces and interval/delete_incomplete b/test/conf/sources/source with spaces and interval/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/source with spaces and interval/destination b/test/conf/sources/source with spaces and interval/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/source with spaces and interval/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/source with spaces and interval/exclude b/test/conf/sources/source with spaces and interval/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/source with spaces and interval/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/source with spaces and interval/source b/test/conf/sources/source with spaces and interval/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/source with spaces and interval/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/source with spaces and interval/verbose b/test/conf/sources/source with spaces and interval/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/destination b/test/conf/sources/very_verbose/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/very_verbose/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/very_verbose/exclude b/test/conf/sources/very_verbose/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/very_verbose/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/very_verbose/source b/test/conf/sources/very_verbose/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/very_verbose/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/very_verbose/summary b/test/conf/sources/very_verbose/summary new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/verbose b/test/conf/sources/very_verbose/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/very_verbose b/test/conf/sources/very_verbose/very_verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/with_exec/destination b/test/conf/sources/with_exec/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/with_exec/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/with_exec/post_exec b/test/conf/sources/with_exec/post_exec new file mode 100755 index 0000000..abc0a40 --- /dev/null +++ b/test/conf/sources/with_exec/post_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +Source post_exec executed. +###################################################################### diff --git a/test/conf/sources/with_exec/pre_exec b/test/conf/sources/with_exec/pre_exec new file mode 100755 index 0000000..ba7b2af --- /dev/null +++ b/test/conf/sources/with_exec/pre_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +Source pre_exec executed. +###################################################################### diff --git a/test/conf/sources/with_exec/source b/test/conf/sources/with_exec/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/with_exec/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/exec.sh b/test/exec.sh deleted file mode 100755 index bdf601d..0000000 --- a/test/exec.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -host="home.schottelius.org" -host="" -set -x -pcmd() -{ - echo "$#", "$@" - if [ "$host" ]; then - ssh "$host" "$@" - else - $@ - fi -} - -#pcmd ls / -#pcmd cd /; ls "/is not there" -pcmd cd / && ls diff --git a/test/local.sh b/test/local.sh deleted file mode 100755 index c2430fd..0000000 --- a/test/local.sh +++ /dev/null @@ -1 +0,0 @@ -CCOLLECT_CONF=./conf ./ccollect.sh daily -v local1 diff --git a/test/remote.sh b/test/remote.sh deleted file mode 100755 index 2af364e..0000000 --- a/test/remote.sh +++ /dev/null @@ -1 +0,0 @@ -CCOLLECT_CONF=./conf ./ccollect.sh daily -v remote1 diff --git a/test/return-value.sh b/test/return-value.sh deleted file mode 100755 index 554def0..0000000 --- a/test/return-value.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -ls /surely-not-existent$$ 2>/dev/null - -if [ "$?" -ne 0 ]; then - echo "$?" -fi - -ls /surely-not-existent$$ 2>/dev/null - -ret=$? - -if [ "$ret" -ne 0 ]; then - echo "$ret" -fi - -# if is true, ls is fales -if [ "foo" = "foo" ]; then - ls /surely-not-existent$$ 2>/dev/null -fi - -# but that's still the return of ls and not of fi -echo $? diff --git a/test/test-ccollect-tools.sh b/test/test-ccollect-tools.sh deleted file mode 100644 index 5980d04..0000000 --- a/test/test-ccollect-tools.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# 2009 Nico Schottelius (nico-ccollect at schottelius.org) -# -# This file is part of ccollect. -# -# ccollect is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ccollect is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with ccollect. If not, see . -# -# -# Test the ccollect tools suite -# - -set -x - -tmp="$(mktemp /tmp/ccollect-tools.XXXXXXXXXXX)" - - -rm -rf "${tmp}" diff --git a/test/test-ccollect1.sh b/test/test-ccollect1.sh deleted file mode 100755 index c5acf54..0000000 --- a/test/test-ccollect1.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# -# Nico Schottelius -# Date: 27-Jan-2007 -# Last Modified: - -# Description: -# - -ccollect=../ccollect.sh -testdir="$(dirname $0)/test-backups" -confdir="$(dirname $0)/test-config" -source="$(hostname)" -source_source="/tmp" -interval="taeglich" - - -# backup destination -mkdir -p "$testdir" -source_dest="$(cd "$testdir"; pwd -P)" - -# configuration -mkdir -p "${confdir}/sources/${source}" -ln -s "$source_dest" "${confdir}/sources/${source}/destination" -echo "$source_source" > "${confdir}/sources/${source}/source" -touch "${confdir}/sources/${source}/summary" -touch "${confdir}/sources/${source}/verbose" - -mkdir -p "${confdir}/defaults/intervals/" -echo 3 > "${confdir}/defaults/intervals/$interval" - -# create backups - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a -touch "${source_source}/$(date +%s)-$$.1982" - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a -touch "${source_source}/$(date +%s)-$$.42" - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a - -du -sh "$testdir" -du -shl "$testdir" - -echo "Delete $testdir and $confdir after test"