From f5493342260c6ead1e9b847317df917e0fe71967 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 21 Aug 2009 13:03:38 +0200 Subject: [PATCH 001/177] Add rsync arguments for source checking Fix proposed by Nikita Koshikov: -------------------------------------------------------------------------------- [...] [guard] + set -- --archive --delete --numeric-ids --relative --delete-excluded --sparse --stats -v [guard] + '[' -f /etc/ccollect/sources/guard/rsync_options ']' [guard] + read line [guard] + set -- --archive --delete --numeric-ids --relative --delete-excluded --sparse --stats -v --password-file=/etc/ccollect/passwd [guard] + read line [guard] + rsync backup@192.168.1.8::etc Password: In this place it's sit and waiting for password. If I comment out related ccollect code section everything start working as expected. I had to modify this part to be look like: if ! rsync "${source}" "$@" >/dev/null 2>"${TMP}" ; then if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi _exit_err "Source ${source} is not readable. Skipping." fi Then checking host activity is passed and backup process running successful. -------------------------------------------------------------------------------- Changed it to "$@" "$source" instead, to be consistent and use options before source. Signed-off-by: Nico Schottelius --- ccollect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect.sh b/ccollect.sh index 13ceb33..11998af 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -415,7 +415,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # # Check: source is up and accepting connections (before deleting old backups!) # - if ! rsync "${source}" >/dev/null 2>"${TMP}" ; then + if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi From 1f84f87888196b6b0aaf77d2aa298c9bb44ff7c4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 21 Aug 2009 13:05:49 +0200 Subject: [PATCH 002/177] changes for the next version Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/next diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..b20d40c --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Fix source-is-up check (Nikita Koshikov) From 59f880ea86530c3e506de5c293cd87531bfc62cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 21 Aug 2009 13:06:01 +0200 Subject: [PATCH 003/177] [DOC] fix typo Signed-off-by: Nico Schottelius --- doc/ccollect.text | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index 720bcc8..100118b 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -620,6 +620,7 @@ If you create the file `delete_incomplete` in a source specification directory, was interrupted) and remove them. Without this file `ccollect` will only warn the user. + Detailed description of "rsync_failure_codes" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you have the file `rsync_failure_codes` in your source configuration @@ -629,6 +630,7 @@ be left in the destination directory indicating failure of this backup. If you have enabled delete_incomplete, then this backup will be deleted during the next ccollect run on the same interval. + Detailed description of "mtime" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, ccollect.sh chooses the most recent backup directory for cloning or @@ -642,6 +644,7 @@ option because the ctimes are not preserved during such operations. If you have any backups in your repository made with ccollect version 0.7.1 or earlier, do not use this option. + Detailed description of "quiet_if_down" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, ccollect.sh emits a series of error messages if a source is not @@ -825,7 +828,7 @@ If you want to see what changed between two backups, you can use [12:00] u0255:ddba034.netstream.ch# rsync -n -a --delete --stats --progress daily.20080324-0313.17841/ daily.20080325-0313.31148/ -------------------------------------------------------------------------------- This results in a listing of changes. Because we pass -n to rsync no transfer -is made (i.e. report only mode)" +is made (i.e. report only mode). This hint was reported by Daniel Aubry. From 93b56025fa560fb1c86bd48e884f2523603fafd6 Mon Sep 17 00:00:00 2001 From: Nikita Koshikov Date: Thu, 10 Sep 2009 12:48:28 +0200 Subject: [PATCH 004/177] Add ccollect rpm spec Thanks a lot, Nikita! --- contrib/ccollect.spec | 85 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 contrib/ccollect.spec diff --git a/contrib/ccollect.spec b/contrib/ccollect.spec new file mode 100644 index 0000000..477a878 --- /dev/null +++ b/contrib/ccollect.spec @@ -0,0 +1,85 @@ +Summary: (pseudo) incremental backup with different exclude lists using hardlinks and rsync +Name: ccollect +Version: 0.8 +Release: 0 +URL: http://www.nico.schottelius.org/software/ccollect +Source0: http://www.nico.schottelius.org/software/ccollect/%{name}-%{version}.tar.bz2 + +License: LGPL-3 +Group: Applications/System +Vendor: Nico Schottelius +BuildRoot: %{_tmppath}/%{name}-%(id -un) +BuildArch: noarch +Requires: rsync + +%description +Ccollect backups data from local and remote hosts to your local harddisk. +Although ccollect creates full backups, it requires very less space on the backup medium, because ccollect uses hardlinks to create an initial copy of the last backup. +Only the inodes used by the hardlinks and the changed files need additional space. + +%prep +%setup -q + +%install +rm -rf $RPM_BUILD_ROOT + +#Installing main ccollect.sh and /etc directory +%__install -d 755 %buildroot%_bindir +%__install -d 755 %buildroot%_sysconfdir/%name +%__install -m 755 ccollect.sh %buildroot%_bindir/ + +#bin files from tools directory +for t in $(ls tools/ccollect_*) ; do + %__install -m 755 ${t} %buildroot%_bindir/ +done + +#Configuration examples and docs +%__install -d 755 %buildroot%_datadir/doc/%name-%version/examples + +%__install -m 644 README %buildroot%_datadir/doc/%name-%version +%__install -m 644 COPYING %buildroot%_datadir/doc/%name-%version +%__install -m 644 CREDITS %buildroot%_datadir/doc/%name-%version +%__install -m 644 conf/README %buildroot%_datadir/doc/%name-%version/examples +%__cp -pr conf/defaults %buildroot%_datadir/doc/%name-%version/examples/ +%__cp -pr conf/sources %buildroot%_datadir/doc/%name-%version/examples/ + +#Addition documentation and some config tools +%__install -d 755 %buildroot%_datadir/%name/tools +%__cp -pr tools/config-pre-* %buildroot%_datadir/%name/tools +%__install -m 755 tools/gnu-du-backup-size-compare.sh %buildroot%_datadir/%name/tools +%__install -m 755 tools/report_success.sh %buildroot%_datadir/%name/tools + +%clean +rm -rf $RPM_BUILD_ROOT + +%post +%__ln_s %_bindir/ccollect.sh %_bindir/ccollect + +%preun +unlink %_bindir/ccollect + +%files +%defattr(-,root,root) +%_bindir/ccollect* +%_datadir/doc/%name-%version +%_datadir/%name/tools +%docdir %_datadir/doc/%name-%version +%dir %_sysconfdir/%name + +%changelog +* Thu Aug 20 2009 Nico Schottelius 0.8 +- Introduce consistenst time sorting (John Lawless) +- Check for source connectivity before trying backup (John Lawless) +- Defensive programming patch (John Lawless) +- Some code cleanups (argument parsing, usage) (Nico Schottelius) +- Only consider directories as sources when using -a (Nico Schottelius) +- Fix general parsing problem with -a (Nico Schottelius) +- Fix potential bug when using remote_host, delete_incomplete and ssh (Nico Schottelius) +- Improve removal performance: minimised number of 'rm' calls (Nico Schottelius) +- Support sorting by mtime (John Lawless) +- Improve option handling (John Lawless) +- Add support for quiet operation for dead devices (quiet_if_down) (John Lawless) +- Add smart option parsing, including support for default values (John Lawless) +- Updated and cleaned up documentation (Nico Schottelius) +- Fixed bug "removal of current directory" in ccollect_delete_source.sh (Found by G????nter St????hr, fixed by Nico Schottelius) + From 5d41dea79dad7e0a69fa78270f38d6aceed5d86e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 12 Oct 2009 08:04:22 +0200 Subject: [PATCH 005/177] update manpage of ccollect: update argument order Signed-off-by: Nico Schottelius --- doc/man/ccollect.text | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/doc/man/ccollect.text b/doc/man/ccollect.text index 84e95bb..84c1e37 100644 --- a/doc/man/ccollect.text +++ b/doc/man/ccollect.text @@ -10,7 +10,7 @@ ccollect - (pseudo) incremental backup with different exclude lists using hardli SYNOPSIS -------- -'ccollect.sh' [args] +'ccollect.sh' [args] DESCRIPTION @@ -36,7 +36,10 @@ OPTIONS Backup all sources specified in /etc/ccollect/sources -v, --verbose:: - Be very verbose (uses set -x). + Be very verbose (uses set -x) + +-V, --version:: + Show version and exit SEE ALSO From 9eba4e8b8e28d4b43d305ee399bd88bb6bdc6308 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 15:21:19 +0100 Subject: [PATCH 006/177] update destination directory Signed-off-by: Nico Schottelius --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 82bab0e..cfef1b4 100644 --- a/Makefile +++ b/Makefile @@ -44,7 +44,7 @@ path_destination=${path_dir}/${CCOLLECT_DEST} # where to publish host=localhost -dir=/home/users/nico/privat/computer/net/netzseiten/www.nico.schottelius.org/src/software/ccollect +dir=/home/users/nico/privat/rechner/netz/seiten/www.nico.schottelius.org/src/software/ccollect docdir=${dir}/doc # From 25d8a2e2fb8a193ecfcb7f60ee8fb702d5201c64 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 15:24:18 +0100 Subject: [PATCH 007/177] remove double --help section Signed-off-by: Nico Schottelius --- ccollect.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 11998af..db350a6 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -152,9 +152,6 @@ while [ "$#" -ge 1 ]; do -p|--parallel) PARALLEL=1 ;; - -h|--help) - usage - ;; -V|--version) display_version ;; From 23b2fcee08745f7cac748d532cad51d1616a4ff9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 15:26:17 +0100 Subject: [PATCH 008/177] move -* check at the right location Signed-off-by: Nico Schottelius --- ccollect.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index db350a6..ca4e890 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -155,14 +155,14 @@ while [ "$#" -ge 1 ]; do -V|--version) display_version ;; - -h|--help|-*) - usage - ;; --) # ignore the -- itself shift break ;; + -h|--help|-*) + usage + ;; *) break ;; From 9d94beec68886517a0104bdebbed248484ee7f3a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 15:27:11 +0100 Subject: [PATCH 009/177] more changes for the next version Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index b20d40c..7a344a1 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1,2 @@ * Fix source-is-up check (Nikita Koshikov) +* Fix some minor command line parsing issues (Nico Schottelius) From 36f413173aa3e1cc4453f4799c331258fd430bd3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 15:30:55 +0100 Subject: [PATCH 010/177] minor argument reordering - beautify Signed-off-by: Nico Schottelius --- ccollect.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index ca4e890..3d13bfc 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -125,8 +125,8 @@ ${__myname}: [args] ccollect creates (pseudo) incremental backups -h, --help: Show this help screen - -p, --parallel: Parallelise backup processes -a, --all: Backup all sources specified in ${CSOURCES} + -p, --parallel: Parallelise backup processes -v, --verbose: Be very verbose (uses set -x) -V, --version: Print version information @@ -146,12 +146,12 @@ while [ "$#" -ge 1 ]; do -a|--all) USE_ALL=1 ;; - -v|--verbose) - set -x - ;; -p|--parallel) PARALLEL=1 ;; + -v|--verbose) + set -x + ;; -V|--version) display_version ;; From 422b22049436ce322fbde6cef9f01fc1a3214717 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 15:40:37 +0100 Subject: [PATCH 011/177] more smaller cleanups Signed-off-by: Nico Schottelius --- ccollect.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 3d13bfc..f4fc319 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -195,9 +195,8 @@ if [ "${USE_ALL}" = 1 ]; then # # Get sources from source configuration # - ( cd "${CSOURCES}" && ls -1 > "${TMP}" ); ret=$? - - [ "${ret}" -eq 0 ] || _exit_err "Listing of sources failed. Aborting." + ( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \ + _exit_err "Listing of sources failed. Aborting." while read tmp; do eval export source_${no_sources}=\"${tmp}\" @@ -235,8 +234,9 @@ if [ -x "${CPREEXEC}" ]; then [ "${ret}" -eq 0 ] || _exit_err "${CPREEXEC} failed. Aborting" fi +################################################################################ # -# Let's do the backup +# Let's do the backup - here begins the real stuff # i=0 while [ "${i}" -lt "${no_sources}" ]; do @@ -249,7 +249,7 @@ while [ "${i}" -lt "${no_sources}" ]; do export name # - # start ourself, if we want parallel execution + # Start ourself, if we want parallel execution # if [ "${PARALLEL}" ]; then "$0" "${INTERVAL}" "${name}" & From 229e25148218f4204f60475b84701df147d0f12e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 15:40:57 +0100 Subject: [PATCH 012/177] report about whole path, not the backup name Signed-off-by: Nico Schottelius --- ccollect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect.sh b/ccollect.sh index f4fc319..25aba17 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -283,7 +283,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # Configuration _must_ be a directory (cconfig style) # if [ ! -d "${backup}" ]; then - _exit_err "\"${name}\" is not a cconfig-directory. Skipping." + _exit_err "\"${backup}\" is not a cconfig-directory. Skipping." fi # From e8a977720f8ca18f5cdec17a96e257af907522ae Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 15:44:16 +0100 Subject: [PATCH 013/177] one more bugfix for the next release Signed-off-by: Nico Schottelius --- doc/changes/next | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/changes/next b/doc/changes/next index 7a344a1..5605242 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1,2 +1,3 @@ -* Fix source-is-up check (Nikita Koshikov) -* Fix some minor command line parsing issues (Nico Schottelius) + * Fix source-is-up check (Nikita Koshikov) + * Fix some minor command line parsing issues (Nico Schottelius) + * Correct output, if configuration is not in cconfig format (Nico Schottelius) From d7c4834dce8ab37b46bf369149a1198335b9690a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 19:34:13 +0100 Subject: [PATCH 014/177] move c_marker to the global section, it's not specfic to a source Signed-off-by: Nico Schottelius --- ccollect.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 25aba17..b1c85ee 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -38,6 +38,7 @@ CSOURCES="${CCOLLECT_CONF}/sources" CDEFAULTS="${CCOLLECT_CONF}/defaults" CPREEXEC="${CDEFAULTS}/pre_exec" CPOSTEXEC="${CDEFAULTS}/post_exec" +CMARKER="ccollect-marker" export TMP=$(mktemp "/tmp/${__myname}.XXXXXX") VERSION="0.8" @@ -293,7 +294,6 @@ while [ "${i}" -lt "${no_sources}" ]; do c_dest="${backup}/destination" c_pre_exec="${backup}/pre_exec" c_post_exec="${backup}/post_exec" - c_marker="ccollect-marker" for opt in verbose very_verbose summary exclude rsync_options \ delete_incomplete remote_host rsync_failure_codes \ mtime quiet_if_down ; do @@ -429,7 +429,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # incomplete="$(echo \ $(pcmd ls -1 "${ddir}/" | \ - awk "/\.${c_marker}\$/ { print \$0; gsub(\"\.${c_marker}\$\",\"\",\$0); print \$0 }" | \ + awk "/\.${CMARKER}\$/ { print \$0; gsub(\"\.${CMARKER}\$\",\"\",\$0); print \$0 }" | \ tee "${TMP}"))" if [ "${incomplete}" ]; then @@ -502,7 +502,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # # added marking in 0.6 (and remove it, if successful later) # - pcmd touch "${destination_dir}.${c_marker}" + pcmd touch "${destination_dir}.${CMARKER}" # # the rsync part @@ -532,8 +532,8 @@ while [ "${i}" -lt "${no_sources}" ]; do # Remove marking here unless rsync failed. # if [ -z "$fail" ]; then - pcmd rm "${destination_dir}.${c_marker}" || \ - _exit_err "Removing ${destination_dir}.${c_marker} failed." + pcmd rm "${destination_dir}.${CMARKER}" || \ + _exit_err "Removing ${destination_dir}.${CMARKER} failed." if [ "${ret}" -ne 0 ]; then _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." fi From 1c8a0808a6868629cfc9c7f4282153f2cd22488b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 19:37:36 +0100 Subject: [PATCH 015/177] and more stuff for the next release Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index 5605242..d321d1e 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1,3 +1,4 @@ * Fix source-is-up check (Nikita Koshikov) * Fix some minor command line parsing issues (Nico Schottelius) * Correct output, if configuration is not in cconfig format (Nico Schottelius) + * Minor code cleanups and optimisations (Nico Schottelius) From 3d571e915a716ddc563b5f6c97a3335dc65e9394 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 19:50:04 +0100 Subject: [PATCH 016/177] add braces Signed-off-by: Nico Schottelius --- ccollect.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index b1c85ee..0c478ab 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -307,7 +307,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # # Sort by ctime (default) or mtime (configuration option) # - if [ -f "$c_mtime" ] ; then + if [ -f "${c_mtime}" ] ; then TSORT="t" else TSORT="tc" @@ -525,7 +525,7 @@ while [ "${i}" -lt "${no_sources}" ]; do if [ "$ret" = "$code" ]; then fail=1 fi - done <"$c_rsync_failure_codes" + done <"${c_rsync_failure_codes}" fi # From 262ceabca380c7c48b65624da27233daf5d74f97 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 19:53:26 +0100 Subject: [PATCH 017/177] shorten some lines Signed-off-by: Nico Schottelius --- ccollect.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 0c478ab..5ef06a3 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -321,9 +321,7 @@ while [ "${i}" -lt "${no_sources}" ]; do "${c_pre_exec}"; ret="$?" _techo "Finished ${c_pre_exec} (return code ${ret})." - if [ "${ret}" -ne 0 ]; then - _exit_err "${c_pre_exec} failed. Skipping." - fi + [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." fi # @@ -356,7 +354,7 @@ while [ "${i}" -lt "${no_sources}" ]; do if [ -f "${c_remote_host}" ]; then remote_host="$(cat "${c_remote_host}")"; ret="$?" if [ "${ret}" -ne 0 ]; then - _exit_err "Remote host file ${c_remote_host} exists, but is not readable. Skipping." + _exit_err "Remote host file ${c_remote_host} is unreadable. Skipping." fi destination="${remote_host}:${ddir}" else From 08331387b7db9ace036bfa7c17645e8da358909a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 20:05:50 +0100 Subject: [PATCH 018/177] +braces Signed-off-by: Nico Schottelius --- ccollect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect.sh b/ccollect.sh index 5ef06a3..f940fef 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -403,7 +403,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # if [ -f "${c_rsync_options}" ]; then while read line; do - set -- "$@" "$line" + set -- "$@" "${line}" done < "${c_rsync_options}" fi From ce6157efee8f3e69a9996f847a3aea4c0e5f6311 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 20:13:50 +0100 Subject: [PATCH 019/177] begin to cleanup ugly awk line Using ENVIRON[] instead of ugly quoting Signed-off-by: Nico Schottelius --- ccollect.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ccollect.sh b/ccollect.sh index f940fef..7e16dd6 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -427,7 +427,11 @@ while [ "${i}" -lt "${no_sources}" ]; do # incomplete="$(echo \ $(pcmd ls -1 "${ddir}/" | \ - awk "/\.${CMARKER}\$/ { print \$0; gsub(\"\.${CMARKER}\$\",\"\",\$0); print \$0 }" | \ + awk '/\.ENVIRON["CMARKER"]$/ { + print $0; + gsub("\." ENVIRON["CMARKER"]$","",$0); + print $0 + }' | \ tee "${TMP}"))" if [ "${incomplete}" ]; then From 84c732bfc0db1372e249c70a9080afb652846c22 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 25 Oct 2009 20:15:14 +0100 Subject: [PATCH 020/177] now put the dot back into CMARKER Removed it some hundred commits before Signed-off-by: Nico Schottelius --- ccollect.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 7e16dd6..f2396dc 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -38,7 +38,7 @@ CSOURCES="${CCOLLECT_CONF}/sources" CDEFAULTS="${CCOLLECT_CONF}/defaults" CPREEXEC="${CDEFAULTS}/pre_exec" CPOSTEXEC="${CDEFAULTS}/post_exec" -CMARKER="ccollect-marker" +CMARKER=".ccollect-marker" export TMP=$(mktemp "/tmp/${__myname}.XXXXXX") VERSION="0.8" @@ -427,9 +427,9 @@ while [ "${i}" -lt "${no_sources}" ]; do # incomplete="$(echo \ $(pcmd ls -1 "${ddir}/" | \ - awk '/\.ENVIRON["CMARKER"]$/ { + awk '/ENVIRON["CMARKER"]$/ { print $0; - gsub("\." ENVIRON["CMARKER"]$","",$0); + gsub(ENVIRON["CMARKER"]$,"",$0); print $0 }' | \ tee "${TMP}"))" @@ -504,7 +504,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # # added marking in 0.6 (and remove it, if successful later) # - pcmd touch "${destination_dir}.${CMARKER}" + pcmd touch "${destination_dir}${CMARKER}" # # the rsync part @@ -534,8 +534,8 @@ while [ "${i}" -lt "${no_sources}" ]; do # Remove marking here unless rsync failed. # if [ -z "$fail" ]; then - pcmd rm "${destination_dir}.${CMARKER}" || \ - _exit_err "Removing ${destination_dir}.${CMARKER} failed." + pcmd rm "${destination_dir}${CMARKER}" || \ + _exit_err "Removing ${destination_dir}${CMARKER} failed." if [ "${ret}" -ne 0 ]; then _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." fi From 4af94d9e71f3e76b2190b8a9d6563d15e1c1d09d Mon Sep 17 00:00:00 2001 From: Patrick Drolet Date: Fri, 30 Oct 2009 10:18:58 +0100 Subject: [PATCH 021/177] traps more errors and warnings. Signed-off-by: Nico Schottelius --- tools/ccollect_analyse_logs.sh | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) mode change 100755 => 100644 tools/ccollect_analyse_logs.sh diff --git a/tools/ccollect_analyse_logs.sh b/tools/ccollect_analyse_logs.sh old mode 100755 new mode 100644 index 7ff9916..7d48d54 --- a/tools/ccollect_analyse_logs.sh +++ b/tools/ccollect_analyse_logs.sh @@ -72,6 +72,11 @@ if [ "$search_err" ]; then set -- "$@" "-e" 'ssh: connect to host .*: Connection timed out' set -- "$@" "-e" 'rsync error: unexplained error (code 255)' set -- "$@" "-e" 'rsync: connection unexpectedly closed' + set -- "$@" "-e" 'rsync: send_files failed to open' + set -- "$@" "-e" 'rsync: readlink_stat .* failed: File name too long' + set -- "$@" "-e" 'IO error encountered .* skipping file deletion' + set -- "$@" "-e" 'rsync: read error: Connection reset by peer' + set -- "$@" "-e" 'rsync error: error in rsync protocol data stream' fi # known error strings: @@ -103,6 +108,7 @@ fi if [ "$search_warn" ]; then # warn on non-zero exit code set -- "$@" "-e" 'Finished backup (rsync return code: [^0]' + set -- "$@" "-e" 'rsync: file has vanished' set -- "$@" "-e" 'WARNING: .* failed verification -- update discarded (will try again).' fi # known warnings: @@ -113,9 +119,12 @@ fi # Interesting strings in the logs: informational # ---------------------------------------------- if [ "$search_info" ]; then + set -- "$@" "-e" 'Number of files: [[:digit:]]*' + set -- "$@" "-e" 'Number of files transferred: [[:digit:]]*' + set -- "$@" "-e" 'Total transferred file size: [[:digit:]]* bytes' set -- "$@" "-e" 'total size is [[:digit:]]* speedup is' set -- "$@" "-e" 'Backup lasted: [[:digit:]]*:[[:digit:]]\{1,2\}:[[:digit:]]* (h:m:s)$' - set -- "$@" "-e" 'send [[:digit:]]* bytes received [0-9]* bytes [0-9]* bytes/sec$' + set -- "$@" "-e" 'sent [[:digit:]]* bytes received [0-9]* bytes' fi # info includes: @@ -123,4 +132,6 @@ fi # [u0160.nshq.ch.netstream.com] 2007-08-20-18:26:06: Backup lasted: 0:43:34 (h:m:s) #[ddba012.netstream.ch] sent 3303866 bytes received 1624630525 bytes 122700.92 bytes/sec +#echo Parameters: "$@" + grep "$@" From 67aead1db2ab776427f9c6fd4eeebe8e99fc07c8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 30 Oct 2009 10:19:44 +0100 Subject: [PATCH 022/177] more changes for the next release Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index d321d1e..34c9530 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -2,3 +2,4 @@ * Fix some minor command line parsing issues (Nico Schottelius) * Correct output, if configuration is not in cconfig format (Nico Schottelius) * Minor code cleanups and optimisations (Nico Schottelius) + * ccollect_analyse_logs.sh traps more errors and warnings (Patrick Drolet) From 3d92f30574429b9e24b6973d09e16d6b12cb200f Mon Sep 17 00:00:00 2001 From: Patrick Drolet Date: Fri, 30 Oct 2009 10:30:30 +0100 Subject: [PATCH 023/177] Add ccollect_mgr.sh, rdu and S60ccollect_example (initscript for ccollect) Signed-off-by: Nico Schottelius --- contrib/patrick/README | 47 +++ contrib/patrick/S60ccollect_example | 21 ++ contrib/patrick/ccollect_mgr.sh | 471 ++++++++++++++++++++++++++++ contrib/patrick/rdu | 65 ++++ 4 files changed, 604 insertions(+) create mode 100644 contrib/patrick/README create mode 100644 contrib/patrick/S60ccollect_example create mode 100644 contrib/patrick/ccollect_mgr.sh create mode 100644 contrib/patrick/rdu diff --git a/contrib/patrick/README b/contrib/patrick/README new file mode 100644 index 0000000..63bba1a --- /dev/null +++ b/contrib/patrick/README @@ -0,0 +1,47 @@ +[Almost complete Copy of an e-mail from Patrick Drolet] + +Hello again, + + + +I have created a script to better manage the backups since my +upload/download ratio and my bandwidth is limited by my ISP, and my hard +disk space is also somewhat limited. The script is called +"ccollect_mgr.sh". + + + +Provides the following features + +1) Determine the interval (daily/weekly/monthly) + +a. Define when you want weekly and monthly backups. It takes care of +the rest + +2) Perform the backups using ccollect + +3) Copy the ccollect log output to the first backup of the set + +a. Keeping the detailed log of each backup is always handy! + +4) Build a periodic report and include the real amount of disk used + +a. Computes the real amount of disk used (eg: no double counting of +hard links) + +b. Shows the actual amount of data transferred + +5) Send an email if there has been errors or warnings + +6) Send a periodic email to show transfer size, real backup size, etc + +a. Weekly reports are nice.! + +[...] + +- rdu (real du), which computes the real amount of disk used (no +double/triple counting hard links), same code as in ccollect_mgr.sh. + +- S60ccollect_example, an example script to put in etc/init.d to +add ccollect_mgr to the crontab + diff --git a/contrib/patrick/S60ccollect_example b/contrib/patrick/S60ccollect_example new file mode 100644 index 0000000..08e839e --- /dev/null +++ b/contrib/patrick/S60ccollect_example @@ -0,0 +1,21 @@ +#!/bin/sh + +# Standard Linux: put in /etc/init.d +# Busybox: put in /opt/etc/init.d + +# Add ccollect_mgr job to crontab +# Syntax reminder from crontab: +# minute 0-59 +# hour 0-23 +# day of month 1-31 +# month 1-12 (or names, see below) +# day of week 0-7 (0 or 7 is Sun, or use names) + +crontab -l | grep -v ccollect_mgr > /tmp/crontab.tmp + +# Backup every day at 1 am. +echo "00 01 * * * /usr/local/sbin/ccollect_mgr.sh -from nas@myemail.net -to me@myemail.net -server relay_or_smtp_server NAS > /usr/local/var/log/ccollect.cron &" >> /tmp/crontab.tmp + +crontab /tmp/crontab.tmp +rm /tmp/crontab.tmp + diff --git a/contrib/patrick/ccollect_mgr.sh b/contrib/patrick/ccollect_mgr.sh new file mode 100644 index 0000000..5678779 --- /dev/null +++ b/contrib/patrick/ccollect_mgr.sh @@ -0,0 +1,471 @@ +#!/bin/sh +# +# ---------------------------------------------------------------------------- +# Last update: 2009-10-29 +# By : pdrolet (ccollect_mgr@drolet.name) +# ---------------------------------------------------------------------------- +# Job manager to the ccollect utilities +# (ccollect written by Nico Schottelius) +# +# Provides the following features +# 1) Determine the interval (daily/weekly/monthly) +# 2) Perform the backups using ccollect +# 4) Copy the ccollect log to the first backup of the set +# 5) Build a periodic report and include the real amount of disk used +# 6) Send an email if there has been errors or warnings +# 7) Send a periodic email to show transfer size, real backup size, etc +# ---------------------------------------------------------------------------- +# +# This script was written primarily to gain better visibility of backups in +# an environment where data transfer is limited and so is bandwidth +# (eg: going through an ISP). The primary target of this script were a +# DNS323 and a QNAP T209 (eg: Busybox devices and not standard Linux devices) +# but it should run on any POSIX compliant device. +# +# Note: This is one of my first script in over a decade... don't use this as a +# reference (but take a look at ccollect.sh... very well written!) +# ---------------------------------------------------------------------------- +# +# ------------------------------------------- +# TO MAKE THIS SCRIPT RUN ON A BUSYBOX DEVICE +# ------------------------------------------- +# - You may need to install Optware and the following packages: +# - findutils (to get a find utility which supports printf) +# - procps (to get a ps utility that is standard) +# - mini-sendmail (this is what I used to send emails... you could easily +# modify this to use sendmail, mutt, putmail, etc...). +# - On DNS323 only: Your Busybox is very limited. For details, see +# http://wiki.dns323.info/howto:ffp#shells. You need to redirect /bin/sh +# to the Busybox provided with ffp (Fun Plug). To do this, type: +# ln -fs /ffp/bin/sh /bin/sh +# +# -------------------------------------------------- +# TO MAKE THIS SCRIPT RUN ON A STANDARD LINUX DEVICE +# -------------------------------------------------- +# - You will need install mini_sendmail or rewrite the send_email routine. +# +# ---------------------------------------------------------------------------- + +# Send warning if the worst case data transfer will be larger than (in MB)... +warning_transfer_size=1024 + +# Define paths and default file names +ADD_TO_PATH=/opt/bin:/opt/sbin:/usr/local/bin:/usr/local/sbin +CCOLLECT=ccollect.sh +CCOLLECT_CONF=/usr/local/etc/ccollect + +per_report=${CCOLLECT_CONF}/periodic_report.log +tmp_report=/tmp/ccollect.$$ +tmp_email=/tmp/email.$$ + +# Sub routines... + +find_interval() +{ + # ---------------------------------------------------- + # Find interval for ccollect backup. + # optional parameters: + # - Day of the week to do weekly backups + # - Do monthly instead of weekly on the Nth week + # ---------------------------------------------------- + + weekly_backup=$1 + monthly_backup=$2 + + weekday=`date "+%w"` + if [ ${weekday} -eq ${weekly_backup} ]; then + dom=`date "+%d"` + weeknum=$(( ( ${dom} / 7 ) + 1 )) + if [ ${weeknum} -eq ${monthly_backup} ]; then + interval=monthly + else + interval=weekly + fi + else + interval=daily + fi +} + +move_log() +{ + for backup in $@ ; do + ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" + if [ "${ret}" -ne 0 ]; then + echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping." + backup_dir="" + else + backup_dir=`cat ${TEMP_LOG} | grep "\[${backup}\] .*: Creating ${ddir}" | awk '{ print $4 }'` + fi + if [ "${backup_dir}" != "" ]; then + new_log=${backup_dir}/ccollect.log + mv ${TEMP_LOG} ${new_log} + echo New Log Location: ${new_log} + return 0 + fi + done + echo "WARNING: none of the backup set have been created" + new_log=${TEMP_LOG} +} + +compute_rdu() +{ + # WARNING: Don't pass a directory with a space as parameter (I'm too new at scripting!) + + kdivider=1 + find_options="" + + while [ "$#" -ge 1 ]; do + case "$1" in + -m) + kdivider=1024 + ;; + -g) + kdivider=1048576 + ;; + *) + find_options="${find_options} $1" + ;; + esac + shift + done + + # ------------------------------------------------------------------------------------------------------ + # Compute the real disk usage (eg: hard links do files outside the backup set don't count) + # ------------------------------------------------------------------------------------------------------ + # 1) Find selected files and list link count, inodes, file type and size + # 2) Sort (sorts on inodes since link count is constant per inode) + # 3) Merge duplicates using uniq + # (result is occurence count, link count, inode, file type and size) + # 4) Use awk to sum up the file size of each inodes when the occurence count + # and link count are the same. Use %k for size since awk's printf is 32 bits + # 5) Present the result with additional dividers based on command line parameters + # + + rdu=$(( ( `/opt/bin/find ${find_options} -printf '%n %i %y %k \n' \ + | sort -n \ + | uniq -c \ + | awk '{ if (( $1 == $2 ) || ($4 == "d")) { sum += $5; } } END { printf "%u\n",(sum); }'` \ + + ${kdivider} - 1 ) / ${kdivider} )) + echo RDU for ${find_options} is ${rdu} +} + +compute_total_rdu() +{ + real_usage=0 + + # ------------------------------------------ + # Get the real disk usage for the backup set + # ------------------------------------------ + for backup in $@ ; do + ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" + echo ${backup} - Adding ${ddir} to backup list + backup_dir_list="${backup_dir_list} ${ddir}" + if [ "${ret}" -ne 0 ]; then + echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping." + else + backup_dir=`find ${ddir}/${interval}.* -maxdepth 0 -type d -print | sort -r | head -n 1` + compute_rdu -m ${backup_dir} + real_usage=$(( ${real_usage} + ${rdu} )) + fi + done + echo Backup list - ${backup_dir_list} +} + +send_email() +{ + # Send a simple email using mini-sendmail. + + msg_body_file=$1 + shift + + # ------------------------------ + # Quit if we can't send an email + # ------------------------------ + if [ "${to}" == "" ] || [ "${mail_server}" == "" ]; then + echo "Missing mail server or destination email. No email sent with subject: $@" + exit 1 + fi + + echo from: ${from} > ${tmp_email} + echo subject: $@ >> ${tmp_email} + echo to: ${to} >> ${tmp_email} + echo cc: >> ${tmp_email} + echo bcc: >> ${tmp_email} + echo "" >> ${tmp_email} + echo "" >> ${tmp_email} + cat ${msg_body_file} >> ${tmp_email} + echo "" >> ${tmp_email} + + echo Sending email to ${to} to report the following error: + echo ----------------------------------------------------- + cat ${tmp_email} + cat ${tmp_email} | mini_sendmail -f${from} -s${mail_server} ${to} + rm ${tmp_email} +} + +check_running_backups() +{ + # Check if a backup is already ongoing. If so, skip and send email + # Don't use the ccollect marker as this is no indication if it is still running + + for backup in ${ccollect_backups} ; do + PID=$$ + /opt/bin/ps -e -o pid,ppid,args 2> /dev/null | grep -v -e grep -e "${PID}.*ccollect.*${backup}" | grep "ccollect.*${backup}" > /tmp/ccollect_mgr.$$ 2> /dev/null + running_proc=`cat /tmp/ccollect_mgr.$$ | wc -l` + if [ ${running_proc} -gt 0 ]; then + running_backups="${running_backups}${backup} " + echo "Process:" + cat /tmp/ccollect.$$ + else + backups_to_do="${backups_to_do}${backup} " + fi + rm /tmp/ccollect_mgr.$$ + done + ccollect_backups=${backups_to_do} + + if [ "${running_backups}" != "" ]; then + echo "skipping ccollect backups already running: ${running_backups}" | tee ${tmp_report} + send_email ${tmp_report} "WARNING - skipping ccollect backups already running: ${running_backups}" + rm ${tmp_report} + fi +} + +precheck_transfer_size() +{ + # Check the estimated (worst case) transfer size and send email if larger than certain size + # + # Be nice and add error checking one day... + + for backup in ${ccollect_backups} ; do + ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" + last_dir="$(ls -tcp1 "${ddir}" | grep '/$' | head -n 1)" + sdir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/source)"; ret="$?" + if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/exclude ]; then + exclude="--exclude-from=${CCOLLECT_CONF}/sources/${backup}/exclude"; + else + exclude="" + fi + if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/rsync_options ]; then + while read line; do + rsync_options="${rsync_options} ${line}" + done < ${CCOLLECT_CONF}/sources/${backup}/rsync_options + fi + rsync -n -a --delete --stats ${rsync_options} ${exclude} ${sdir} ${ddir}/${last_dir} > ${tmp_report} + tx_rx=`cat ${tmp_report} | grep "Total transferred file size" | \ + awk '{ { tx += $5 } } END { printf "%u",(((tx)+1024*1024-1)/1024/1024); }'` + total_xfer=$(( ${total_xfer} + ${tx_rx} )) + done + echo Transfer estimation for ${ccollect_backups}: ${total_xfer} MB + + if [ ${total_xfer} -gt ${warning_transfer_size} ]; then + # -------------------------------------------------- + # Send a warning if transfer is expected to be large + # -------------------------------------------------- + # Useful to detect potential issues when there is transfer quota (ex: with ISP) + + echo Data transfer larger than ${warning_transfer_size} MB is expected for ${ccollect_backups} > ${tmp_report} + + send_email ${tmp_report} "WARNING ccollect for ${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB" + rm ${tmp_report} + fi +} + +send_report() +{ + log=$1 + + # Analyze log for periodic report and for error status report + cat ${log} | ccollect_analyse_logs.sh iwe > ${tmp_report} + + # ------------------------- + # Build the periodic report + # ------------------------- + + # Compute the total number of MB sent and received for all the backup sets + tx_rx=`cat ${tmp_report} | \ + grep 'sent [[:digit:]]* bytes received [0-9]* bytes' | \ + awk '{ { tx += $3 } { rx += $6} } END \ + { printf "%u",(((tx+rx)+1024*1024-1)/1024/1024); }'` + current_date=`date +'20%y/%m/%d %Hh%M -- '` + + # --------------------------------------------------------- + # Get the disk usage for all backups of each backup sets... + # ** be patient ** + # --------------------------------------------------------- + compute_rdu -g ${backup_dir_list} + + echo ${current_date} Tx+Rx: ${tx_rx} MB -- \ + Disk Usage: ${real_usage} MB -- \ + Backup set \(${interval}\): ${ccollect_backups} -- \ + Historical backups usage: ${rdu} GB >> ${per_report} + + # ---------------------------------------- + # Send a status email if there is an error + # ---------------------------------------- + ccollect_we=`cat ${log} | ccollect_analyse_logs.sh we | wc -l` + if [ ${ccollect_we} -ge 1 ]; then + send_email ${tmp_report} "ERROR ccollect for ${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" + fi + + # -------------------- + # Send periodic report + # -------------------- + if [ ${report_interval} == ${interval} ] || [ ${interval} == "monthly" ]; then + + # Make reporting atomic to handle concurrent ccollect_mgr instances + mv ${per_report} ${per_report}.$$ + cat ${per_report}.$$ >> ${per_report}.history + + # Calculate total amount of bytes sent and received + tx_rx=`cat ${per_report}.$$ | \ + awk '{ { transfer += $5 } } END \ + { printf "%u",(transfer); }'` + + # Send email + send_email ${per_report}.$$ "${report_interval} ccollect status for ${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" + rm ${per_report}.$$ + fi + + rm ${tmp_report} +} + +# ------------------------------------------------ +# Add to PATH in case we're launching from crontab +# ------------------------------------------------ + +PATH=${ADD_TO_PATH}:${PATH} + +# -------------- +# Default Values +# -------------- + +# Set on which interval status emails are sent (daily, weekly, monthly) +report_interval=weekly + +# Set day of the week for weekly backups. Default is Monday +# 0=Sun, 1=Mon, 2=Tue, 3=Wed, 4=Thu, 5=Fri, 6=Sat +weekly_backup=1 + +# Set the monthly backup interval. Default is 4th Monday of every month +monthly_backup=4 + +show_help=0 + +# --------------------------------- +# Parse command line +# --------------------------------- + +while [ "$#" -ge 1 ]; do + case "$1" in + -help) + show_help=1 + ;; + -from) + from=$2 + shift + ;; + -to) + to=$2 + shift + ;; + -server|mail_server) + mail_server=$2 + shift + ;; + -weekly) + weekly_backup=$2 + shift + ;; + -monthly) + monthly_backup=$2 + shift + ;; + -warning_size) + warning_transfer_size=$2 + shift + ;; + -report) + report_interval=$2 + shift + ;; + -*) + ccollect_options="${ccollect_options} $1" + ;; + daily|weekly|monthly) + ;; + *) + ccollect_backups="${ccollect_backups}$1 " + ;; + esac + shift +done + +if [ "${ccollect_backups}" == "" ] || [ ${show_help} -eq 1 ]; then + echo + echo "$0: Syntax" + echo " -help This help" + echo " -from From email address (ex.: -from nas@home.com)" + echo " -to Send email to this address (ex.: -to me@home.com)" + echo " -server SMTP server used for sending emails" + echo " -weekly Define wich day of the week is the weekly backup" + echo " Default is ${weekly_backup}. Sunday = 0, Saturday = 6" + echo " -monthly Define on which week # is the monthly backup" + echo " Default is ${monthly_backup}. Value = 1 to 5" + echo " -report Frequency of report email (daily, weekly or monthly)" + echo " Default is ${report_interval}" + echo " -warning_size Send a warning email if the transfer size exceed this" + echo " Default is ${warning_transfer_size}" + echo "" + echo " other parameters are transfered to ccollect" + echo + exit 0 +fi + +# ------------------------------------------------------------------ +# Check if ccollect_mgr is already running for the given backup sets +# ------------------------------------------------------------------ + +check_running_backups + +if [ "${ccollect_backups}" == "" ]; then + echo "No backup sets are reachable" + exit 1 +fi + +# ---------------------------------------------------------- +# Set the interval type +# +# Here, weeklys are Mondays, and Monthlys are the 4th Monday +# ---------------------------------------------------------- + +find_interval ${weekly_backup} ${monthly_backup} +echo Interval: ${interval} + +# -------------- +# Do the backups +# -------------- +TEMP_LOG=${CCOLLECT_CONF}/log.$$ +echo Backup sets: ${ccollect_backups} + +# Check the transfer size (to issue email warning) +precheck_transfer_size + +${CCOLLECT} ${ccollect_options} ${interval} ${ccollect_backups} | tee ${TEMP_LOG} + +# --------------------------------------- +# Move log to the last backup of the set +# --------------------------------------- + +move_log ${ccollect_backups} + +# ----------------------------------------- +# Compute the physical amount of disk usage +# ----------------------------------------- + +compute_total_rdu ${ccollect_backups} + +# ----------------- +# Send status email +# ----------------- + +send_report ${new_log} diff --git a/contrib/patrick/rdu b/contrib/patrick/rdu new file mode 100644 index 0000000..aa5ffe4 --- /dev/null +++ b/contrib/patrick/rdu @@ -0,0 +1,65 @@ +#!/bin/sh +# +# ------------------------------------------------------------- +# Get the real disk usage for a group of selected files +# +# This script counts the size of the files and directories +# listed, but exclude files that have hard links referenced outside +# the list. +# +# The undelying objective of this script is to report the +# real amount of disk used for backup solutions that are heavily +# using hard links to save disk space on identical files (I use +# ccollect, but this likely works with rsnapshot) +# ------------------------------------------------------------- +# 20091002 - initial release - pdrolet (rdu@drolet.name) + +# -------------------- +# Parse options +# -------------------- +# Known problem: +# - Command line cannot get a directory with a space in it +# +kdivider=1 +find_options="" +while [ "$#" -ge 1 ]; do + case "$1" in + -m) + kdivider=1024 + ;; + -g) + kdivider=1048576 + ;; + -h|--help) + echo + echo $0: \ \[options below and any \"find\" options\] + echo \ \ -m: result in mega bytes \(rounded up\) + echo \ \ -g: result in giga bytes \(rounded up\) + echo \ \ -h: this help + echo + exit 0 + ;; + *) + find_options="${find_options} $1" + ;; + esac + shift +done + +# ------------------------------------------------------------------------------------------------------ +# Compute the size +# ------------------------------------------------------------------------------------------------------ +# 1) Find selected files and list link count, inodes, file type and size +# 2) Sort (sorts on inodes since link count is constant per inode) +# 3) Merge duplicates using uniq +# (result is occurence count, link count, inode, file type and size) +# 4) Use awk to sum up the file size of each inodes when the occurence count +# and link count are the same. Use %k for size since awk's printf is 32 bits +# 5) Present the result with additional dividers based on command line parameters +# +echo $((( `find ${find_options} -printf '%n %i %y %k \n' \ + | sort -n \ + | uniq -c \ + | awk '{ if (( $1 == $2 ) || ($4 == "d")) { sum += $5; } } END { printf "%u\n",(sum); }'` \ + + ${kdivider} -1 ) / ${kdivider} )) + From 4865f3c8c605b80732f2ff8fe4f2a5df7a3220d8 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 30 Oct 2009 19:25:23 +0100 Subject: [PATCH 024/177] extend delete_from_file() to allow removal of suffix files Signed-off-by: Nico Schottelius --- ccollect.sh | 45 +++++++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index f2396dc..08cf756 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -100,16 +100,22 @@ pcmd() fi } +# +# ssh-"feature": we cannot do '... read ...; ssh ...; < file', +# because ssh reads stdin! -n does not work -> does not ask for password +# Allow deletion for files with suffix and without suffix +# delete_from_file() { - # - # ssh-"feature": we cannot do '... read ...; ssh ...; < file', - # because ssh reads stdin! -n does not work -> does not ask for password - # file="$1"; shift - while read to_remove; do set -- "$@" "${ddir}/${to_remove}"; done < "${file}" + suffix="$1" # only set for delete_incomplete +# [ "$#" = 1 ] && suffix="$1" && shift + while read to_remove; do + set -- "$@" "${ddir}/${to_remove}" + [ "$suffix" ] && set -- "$@" "$(echo ${to_remove} | sed "s/$suffix\$//")" + done < "${file}" _techo "Removing $@ ..." - pcmd rm ${VVERBOSE} -rf "$@" || _exit_err "Removing $@ failed." + pcmd echo rm ${VVERBOSE} -rf "$@" || _exit_err "Removing $@ failed." } display_version() @@ -425,14 +431,29 @@ while [ "${i}" -lt "${no_sources}" ]; do # # Check: incomplete backups? (needs echo to remove newlines) # + # *.marker: not possible, creates an error, if no *.marker exists + # -> catch return value + + pcmd ls -1 "${ddir}/"*"${CMARKER}" > "${TMP}" 2>/dev/null; ret=$? + + if [ "$ret" -eq 0 ]; then + _techo "Incomplete backups: ${incomplete}" + if [ -f "${c_delete_incomplete}" ]; then + delete_from_file "${TMP}" + fi + fi + incomplete="$(echo \ $(pcmd ls -1 "${ddir}/" | \ - awk '/ENVIRON["CMARKER"]$/ { - print $0; - gsub(ENVIRON["CMARKER"]$,"",$0); - print $0 - }' | \ - tee "${TMP}"))" + awk ' + BEGIN { regexp=ENVIRON["CMARKER"] "$" } + $0 ~ regexp { + print $0; + gsub(ENVIRON["CMARKER"]$,"",$0); + print $0 + }' | tee "${TMP}"))" + +FIXME: stopped here! -> gsub broken if [ "${incomplete}" ]; then _techo "Incomplete backups: ${incomplete}" From 8a70e30d971a3564e722bd3b4dcbb65fc4836e5d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 30 Oct 2009 19:26:32 +0100 Subject: [PATCH 025/177] cleanup delete_from_file(): clean suffix stuff Signed-off-by: Nico Schottelius --- ccollect.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 08cf756..b90c2c6 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -108,10 +108,9 @@ pcmd() delete_from_file() { file="$1"; shift - suffix="$1" # only set for delete_incomplete -# [ "$#" = 1 ] && suffix="$1" && shift + suffix="$1" # only set, if deleting incomplete backups while read to_remove; do - set -- "$@" "${ddir}/${to_remove}" + set -- "$@" "${to_remove}" [ "$suffix" ] && set -- "$@" "$(echo ${to_remove} | sed "s/$suffix\$//")" done < "${file}" _techo "Removing $@ ..." From 086af1497c49faba99f6e64580c8147ee8815404 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 30 Oct 2009 19:42:13 +0100 Subject: [PATCH 026/177] need to unset $@ completly in delete_from_file() Signed-off-by: Nico Schottelius --- ccollect.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index b90c2c6..d060156 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -103,12 +103,12 @@ pcmd() # # ssh-"feature": we cannot do '... read ...; ssh ...; < file', # because ssh reads stdin! -n does not work -> does not ask for password -# Allow deletion for files with suffix and without suffix +# Alsa allow deletion for files without the given suffix # delete_from_file() { file="$1"; shift - suffix="$1" # only set, if deleting incomplete backups + [ $# -eq 1 ] && suffix="$1" && shift # set if deleting incomplete backups while read to_remove; do set -- "$@" "${to_remove}" [ "$suffix" ] && set -- "$@" "$(echo ${to_remove} | sed "s/$suffix\$//")" From ef4291c72293db89900155de1a11fd001dc5001f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 30 Oct 2009 19:42:32 +0100 Subject: [PATCH 027/177] use delete_from_file with suffix Signed-off-by: Nico Schottelius --- ccollect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect.sh b/ccollect.sh index d060156..efd0440 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -438,7 +438,7 @@ while [ "${i}" -lt "${no_sources}" ]; do if [ "$ret" -eq 0 ]; then _techo "Incomplete backups: ${incomplete}" if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" + delete_from_file "${TMP}" "${CMARKER}" fi fi From 9d65e1b1bf8a13e74965a55e6245279b87c3df06 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 30 Oct 2009 19:43:21 +0100 Subject: [PATCH 028/177] remove now obsolete code :-) Signed-off-by: Nico Schottelius --- ccollect.sh | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index efd0440..13cc7b2 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -442,25 +442,6 @@ while [ "${i}" -lt "${no_sources}" ]; do fi fi - incomplete="$(echo \ - $(pcmd ls -1 "${ddir}/" | \ - awk ' - BEGIN { regexp=ENVIRON["CMARKER"] "$" } - $0 ~ regexp { - print $0; - gsub(ENVIRON["CMARKER"]$,"",$0); - print $0 - }' | tee "${TMP}"))" - -FIXME: stopped here! -> gsub broken - - if [ "${incomplete}" ]; then - _techo "Incomplete backups: ${incomplete}" - if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" - fi - fi - # # Interval definition: First try source specific, fallback to default # From 39e8eb4c948bb11cec0b117c47e3e92c3d286480 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 30 Oct 2009 19:45:03 +0100 Subject: [PATCH 029/177] fix incomplete output message Signed-off-by: Nico Schottelius --- ccollect.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 13cc7b2..2fe7a33 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -435,8 +435,8 @@ while [ "${i}" -lt "${no_sources}" ]; do pcmd ls -1 "${ddir}/"*"${CMARKER}" > "${TMP}" 2>/dev/null; ret=$? - if [ "$ret" -eq 0 ]; then - _techo "Incomplete backups: ${incomplete}" + if [ "${ret}" -eq 0 ]; then + _techo "Incomplete backups: $(echo $(cat "${TMP}"))" if [ -f "${c_delete_incomplete}" ]; then delete_from_file "${TMP}" "${CMARKER}" fi From 2a6ab4c125398f917c26f161d361f226a4da6bba Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 30 Oct 2009 19:49:17 +0100 Subject: [PATCH 030/177] adjust ls for existing backups to new delete_from_file behaviour Signed-off-by: Nico Schottelius --- ccollect.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 2fe7a33..5697003 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -457,9 +457,8 @@ while [ "${i}" -lt "${no_sources}" ]; do # # Check: maximum number of backups is reached? - # If so remove. Use grep and ls -p so we only look at directories # - count="$(pcmd ls -p1 "${ddir}" | grep "^${INTERVAL}\..*/\$" | wc -l \ + count="$(pcmd ls -p1 "${ddir}/${INTERVAL}.*/" | wc -l \ | sed 's/^ *//g')" || _exit_err "Counting backups failed" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" From 1b591a040c27f6886c074ed4de722692ac9e05a0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 07:31:23 +0100 Subject: [PATCH 031/177] -p is not needed, when matching directories with */ Signed-off-by: Nico Schottelius --- ccollect.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccollect.sh b/ccollect.sh index 5697003..6395004 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -458,7 +458,8 @@ while [ "${i}" -lt "${no_sources}" ]; do # # Check: maximum number of backups is reached? # - count="$(pcmd ls -p1 "${ddir}/${INTERVAL}.*/" | wc -l \ + + count="$(pcmd ls -1 "${ddir}/${INTERVAL}.*/" | wc -l \ | sed 's/^ *//g')" || _exit_err "Counting backups failed" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" From 145c6de2fb75156f8805ddaac90f647f679b25f2 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 07:40:55 +0100 Subject: [PATCH 032/177] double check quotes, remove grep Signed-off-by: Nico Schottelius --- ccollect.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 6395004..bac7f2f 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -459,7 +459,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # Check: maximum number of backups is reached? # - count="$(pcmd ls -1 "${ddir}/${INTERVAL}.*/" | wc -l \ + count="$(pcmd ls -1 "${ddir}/${INTERVAL}."*"/" | wc -l \ | sed 's/^ *//g')" || _exit_err "Counting backups failed" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" @@ -469,9 +469,9 @@ while [ "${i}" -lt "${no_sources}" ]; do remove="$((${count} - ${substract}))" _techo "Removing ${remove} backup(s)..." - pcmd ls -${TSORT}p1r "${ddir}" | grep "^${INTERVAL}\..*/\$" | \ - head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + pcmd ls -${TSORT}1r "${ddir}/${INTERVAL}."*"/" | + head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" delete_from_file "${TMP}" fi From 435f2140dadf0ff68ffe918b962b22b07172bf2f Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 08:14:59 +0100 Subject: [PATCH 033/177] cleanly set suffix Signed-off-by: Nico Schottelius --- ccollect.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index bac7f2f..a0c6489 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -108,13 +108,17 @@ pcmd() delete_from_file() { file="$1"; shift - [ $# -eq 1 ] && suffix="$1" && shift # set if deleting incomplete backups + suffix="" # It will be set, if deleting incomplete backups. + [ $# -eq 1 ] && suffix="$1" && shift while read to_remove; do set -- "$@" "${to_remove}" - [ "$suffix" ] && set -- "$@" "$(echo ${to_remove} | sed "s/$suffix\$//")" + if [ "$suffix" ]; then + to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" + set -- "$@" "${to_remove_no_suffix}" + fi done < "${file}" _techo "Removing $@ ..." - pcmd echo rm ${VVERBOSE} -rf "$@" || _exit_err "Removing $@ failed." + pcmd rm ${VVERBOSE} -rf "$@" || _exit_err "Removing $@ failed." } display_version() From 9e801582d53efd7d4fba00780f34620824c6834c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 09:03:58 +0100 Subject: [PATCH 034/177] use -d to prevent displaying the content of a directory Signed-off-by: Nico Schottelius --- ccollect.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index a0c6489..0247c00 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -437,7 +437,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # *.marker: not possible, creates an error, if no *.marker exists # -> catch return value - pcmd ls -1 "${ddir}/"*"${CMARKER}" > "${TMP}" 2>/dev/null; ret=$? + pcmd ls -d1 "${ddir}/"*"${CMARKER}" > "${TMP}" 2>/dev/null; ret=$? if [ "${ret}" -eq 0 ]; then _techo "Incomplete backups: $(echo $(cat "${TMP}"))" @@ -463,7 +463,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # Check: maximum number of backups is reached? # - count="$(pcmd ls -1 "${ddir}/${INTERVAL}."*"/" | wc -l \ + count="$(pcmd ls -d1 "${ddir}/${INTERVAL}."*"/" | wc -l \ | sed 's/^ *//g')" || _exit_err "Counting backups failed" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" @@ -473,7 +473,7 @@ while [ "${i}" -lt "${no_sources}" ]; do remove="$((${count} - ${substract}))" _techo "Removing ${remove} backup(s)..." - pcmd ls -${TSORT}1r "${ddir}/${INTERVAL}."*"/" | + pcmd ls -${TSORT}d1r "${ddir}/${INTERVAL}."*"/" | head -n "${remove}" > "${TMP}" || \ _exit_err "Listing old backups failed" From 59c894137310fb14c6dcf29c4bde3c89ab6a801a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 09:15:53 +0100 Subject: [PATCH 035/177] cleanup destination setting :-) Signed-off-by: Nico Schottelius --- ccollect.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 0247c00..9e779b9 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -495,9 +495,9 @@ while [ "${i}" -lt "${no_sources}" ]; do fi # set time when we really begin to backup, not when we began to remove above - destination_date="$(${CDATE})" - destination_dir="${ddir}/${INTERVAL}.${destination_date}.$$" - destination_full="${destination}/${INTERVAL}.${destination_date}.$$" + destination_name="${INTERVAL}.$(${CDATE}).$$" + destination_dir="${ddir}/${destination_name}" + destination_full="${destination}/${destination_name}" # give some info _techo "Beginning to backup, this may take some time..." From e4dea56e4957ebd2617ab4d864bad8094014d920 Mon Sep 17 00:00:00 2001 From: Patrick Drolet Date: Sat, 31 Oct 2009 09:52:33 +0100 Subject: [PATCH 036/177] Begin to replace false verbose handling Signed-off-by: Nico Schottelius --- ccollect.sh | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 9e779b9..e797c0c 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -118,7 +118,11 @@ delete_from_file() fi done < "${file}" _techo "Removing $@ ..." - pcmd rm ${VVERBOSE} -rf "$@" || _exit_err "Removing $@ failed." +# pcmd rm ${VVERBOSE} -rf "$@" || _exit_err "Removing $@ failed." + if [ ${VVERBOSE} ]; then + echo rm "$@" + fi + pcmd rm -rf "$@" || _exit_err "Removing $@ failed." } display_version() @@ -503,7 +507,12 @@ while [ "${i}" -lt "${no_sources}" ]; do _techo "Beginning to backup, this may take some time..." _techo "Creating ${destination_dir} ..." - pcmd mkdir ${VVERBOSE} "${destination_dir}" || \ +# pcmd mkdir ${VVERBOSE} "${destination_dir}" || \ +# _exit_err "Creating ${destination_dir} failed. Skipping." + if [ ${VVERBOSE} ]; then + echo mkdir "${destination_dir}" + fi + pcmd mkdir "${destination_dir}" || \ _exit_err "Creating ${destination_dir} failed. Skipping." # @@ -515,7 +524,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # the rsync part # _techo "Transferring files..." - rsync "$@" "${source}" "${destination_full}"; ret=$? + rsync $@ "${source}" "${destination_full}"; ret=$? _techo "Finished backup (rsync return code: $ret)." # @@ -553,7 +562,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # if [ -x "${c_post_exec}" ]; then _techo "Executing ${c_post_exec} ..." - "${c_post_exec}"; ret=$? + "${c_post_exec}" "${destination_full}"; ret=$? _techo "Finished ${c_post_exec}." if [ "${ret}" -ne 0 ]; then @@ -571,7 +580,6 @@ while [ "${i}" -lt "${no_sources}" ]; do seconds="$((${seconds} - (${minutes} * 60)))" _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" - ) | add_name done From e6d89d57fcfd4e334a5a374a9487633e891ba18d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 10:01:11 +0100 Subject: [PATCH 037/177] Fix Patricks patch: remove comments, remove if..then, add quotes for $@, do not add destination_full for post_exec Signed-off-by: Nico Schottelius --- ccollect.sh | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index e797c0c..6959fa0 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -118,10 +118,7 @@ delete_from_file() fi done < "${file}" _techo "Removing $@ ..." -# pcmd rm ${VVERBOSE} -rf "$@" || _exit_err "Removing $@ failed." - if [ ${VVERBOSE} ]; then - echo rm "$@" - fi + [ "${VVERBOSE}" ] && echo rm "$@" pcmd rm -rf "$@" || _exit_err "Removing $@ failed." } @@ -507,11 +504,7 @@ while [ "${i}" -lt "${no_sources}" ]; do _techo "Beginning to backup, this may take some time..." _techo "Creating ${destination_dir} ..." -# pcmd mkdir ${VVERBOSE} "${destination_dir}" || \ -# _exit_err "Creating ${destination_dir} failed. Skipping." - if [ ${VVERBOSE} ]; then - echo mkdir "${destination_dir}" - fi + [ "${VVERBOSE}" ] && echo "mkdir ${destination_dir}" pcmd mkdir "${destination_dir}" || \ _exit_err "Creating ${destination_dir} failed. Skipping." @@ -524,7 +517,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # the rsync part # _techo "Transferring files..." - rsync $@ "${source}" "${destination_full}"; ret=$? + rsync "$@" "${source}" "${destination_full}"; ret=$? _techo "Finished backup (rsync return code: $ret)." # @@ -562,7 +555,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # if [ -x "${c_post_exec}" ]; then _techo "Executing ${c_post_exec} ..." - "${c_post_exec}" "${destination_full}"; ret=$? + "${c_post_exec}"; ret=$? _techo "Finished ${c_post_exec}." if [ "${ret}" -ne 0 ]; then From cf62a0fada3a5daa37c90e2b5c4d916e2a1fb78a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 10:01:58 +0100 Subject: [PATCH 038/177] even more changes for the next version Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index 34c9530..a3dc769 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -3,3 +3,4 @@ * Correct output, if configuration is not in cconfig format (Nico Schottelius) * Minor code cleanups and optimisations (Nico Schottelius) * ccollect_analyse_logs.sh traps more errors and warnings (Patrick Drolet) + * Remove -v for mkdir and rm, as they are not POSIX (Patrick Drolet) From 375f9ebafe265cff7a898509911984bdbc37e13b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 10:10:09 +0100 Subject: [PATCH 039/177] add some more notes on post-configuring ccollect Signed-off-by: Nico Schottelius --- doc/ccollect.text | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index 100118b..3bfef96 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -264,8 +264,10 @@ After having installed and used ccollect, report success using Configuring ----------- For configuration aid have a look at the above mentioned tools, which can assist -you quite well. When you are successfully using `ccollect`, report success using -`tools/report_success.sh`. +you quite well. When you are successfully using `ccollect`, I would be happy if +you add a link to your website, stating "I backup with ccollect", which points +to the ccollect homepage. So more people now about ccollect, use it and +improve it. You can also report success using `tools/report_success.sh`. Runtime options From c314f284a2b129eed8e324f0284548e9d19fb1b4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 10:18:23 +0100 Subject: [PATCH 040/177] export destination_* for usage in post_exec in documentate it Signed-off-by: Nico Schottelius --- ccollect.sh | 6 +++--- doc/ccollect.text | 6 ++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 6959fa0..651cd8c 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -496,9 +496,9 @@ while [ "${i}" -lt "${no_sources}" ]; do fi # set time when we really begin to backup, not when we began to remove above - destination_name="${INTERVAL}.$(${CDATE}).$$" - destination_dir="${ddir}/${destination_name}" - destination_full="${destination}/${destination_name}" + export destination_name="${INTERVAL}.$(${CDATE}).$$" + export destination_dir="${ddir}/${destination_name}" + export destination_full="${destination}/${destination_name}" # give some info _techo "Beginning to backup, this may take some time..." diff --git a/doc/ccollect.text b/doc/ccollect.text index 3bfef96..38a93fb 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -599,6 +599,12 @@ respectively after doing the backup for *this specific* source. If you want to have pre-/post-exec before and after *all* backups, see above for general configuration. +The `post_exec` script can access the following exported variables from +ccollect: + +- destination_name: contains the base directory name (`daily.20091031-1013.24496`) +- destination_dir: full path (`/tmp/ccollect/daily.20091031-1013.24496`) +- destination_full: like 'destination_dir', but prepended with the remote_host, if set (`host:/tmp/ccollect/daily.20091031-1013.24496` or `/tmp/ccollect/daily.20091031-1013.24496`) Example: -------------------------------------------------------------------------------- From 7d1669827a5727dae2ee01433cad9169af7b2cd3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 31 Oct 2009 10:19:03 +0100 Subject: [PATCH 041/177] and... even more changes.. this will be a great one Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index a3dc769..45972f9 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -4,3 +4,4 @@ * Minor code cleanups and optimisations (Nico Schottelius) * ccollect_analyse_logs.sh traps more errors and warnings (Patrick Drolet) * Remove -v for mkdir and rm, as they are not POSIX (Patrick Drolet) + * Export destination_* to source specific post_exec (Nico Schottelius) From e1ccba4f576855010b4c2a35f370abe7265f35e0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 13:36:50 +0100 Subject: [PATCH 042/177] update documentation with all exported variables Signed-off-by: Nico Schottelius --- doc/ccollect.text | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index 38a93fb..e2090a9 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -335,6 +335,12 @@ If you add '$CCOLLECT_CONF/defaults/`pre_exec`' or will start `pre_exec` before the whole backup process and `post_exec` after backup of all sources is done. +The `pre_exec` and `post_exec` script can access the following exported variables: + +- 'INTERVAL': the interval selected (`daily`) +- 'no_sources': number of sources to backup (`2`) +- 'source_$no': name of the source, '$no' starts at 0 + The following example describes how to report free disk space in human readable format before and after the whole backup process: ------------------------------------------------------------------------- @@ -602,9 +608,10 @@ backups, see above for general configuration. The `post_exec` script can access the following exported variables from ccollect: -- destination_name: contains the base directory name (`daily.20091031-1013.24496`) -- destination_dir: full path (`/tmp/ccollect/daily.20091031-1013.24496`) -- destination_full: like 'destination_dir', but prepended with the remote_host, if set (`host:/tmp/ccollect/daily.20091031-1013.24496` or `/tmp/ccollect/daily.20091031-1013.24496`) +- 'name': name of the source that is being backed up +- 'destination_name': contains the base directory name (`daily.20091031-1013.24496`) +- 'destination_dir': full path (`/tmp/ccollect/daily.20091031-1013.24496`) +- 'destination_full': like 'destination_dir', but prepended with the remote_host, if set (`host:/tmp/ccollect/daily.20091031-1013.24496` or `/tmp/ccollect/daily.20091031-1013.24496`) Example: -------------------------------------------------------------------------------- @@ -710,12 +717,16 @@ Using source names or interval in pre_/post_exec scripts ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ The pre-/post_exec scripts can access some internal variables from `ccollect`: -- INTERVAL: The interval specified on the command line -- no_sources: number of sources -- source_$NUM: the name of the source -- name: the name of the currently being backuped source (not available for +- 'INTERVAL': The interval specified on the command line +- 'no_sources': number of sources +- 'source_$NUM': the name of the source +- 'name': the name of the currently being backuped source (not available for generic pre_exec script) +Only available for `post_exec`: + +- 'remote_host': name of host we backup to (empty if unused) + Using rsync protocol without ssh ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ From 6a8ff3f1d2f0828a59918819e6e60991d9a4cdc5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 13:37:02 +0100 Subject: [PATCH 043/177] better quote TMP Signed-off-by: Nico Schottelius --- ccollect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect.sh b/ccollect.sh index 651cd8c..9cbc96c 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -40,7 +40,7 @@ CPREEXEC="${CDEFAULTS}/pre_exec" CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" -export TMP=$(mktemp "/tmp/${__myname}.XXXXXX") +export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" VERSION="0.8" RELEASE="2009-08-20" HALF_VERSION="ccollect ${VERSION}" From 8b01949f4bd271edce29d48e25f41b47a677db4a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 13:44:43 +0100 Subject: [PATCH 044/177] begin to claeunp the restorign document Signed-off-by: Nico Schottelius --- doc/ccollect-restoring.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ccollect-restoring.text b/doc/ccollect-restoring.text index 7bb29ea..8a3e11b 100644 --- a/doc/ccollect-restoring.text +++ b/doc/ccollect-restoring.text @@ -30,7 +30,7 @@ Boot the system to be rescued from a media that contains low level tools for your OS (like partitioning, formatting) and the necessary tools (ssh, tar or rsync). Use -- create the necessary partition table (or however it is called +- create the necessary volumes (like partitions, slices, ...) Get a live-cd, that ships with - rsync / tar From 7e155f4219c964be51dbc512843bf190d70d96f9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:00:54 +0100 Subject: [PATCH 045/177] prepare the next release Signed-off-by: Nico Schottelius --- ccollect.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 9cbc96c..163c96e 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -41,8 +41,8 @@ CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" -VERSION="0.8" -RELEASE="2009-08-20" +VERSION="0.8.1" +RELEASE="2009-11-XX" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" From ec61905fc4f72ce7b02d10299b7bbe47f3a7a715 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:02:50 +0100 Subject: [PATCH 046/177] always call $@, which is now setup correctly Signed-off-by: Nico Schottelius --- ccollect.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 163c96e..7747fcc 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -89,15 +89,13 @@ add_name() } # -# Execute on remote host, if backing up to a remote host +# Prepend "ssh ${remote_host}", if backing up to a remote host # pcmd() { - if [ "${remote_host}" ]; then - ssh "${remote_host}" "$@" - else - "$@" - fi + [ "${remote_host}" ] && set -- "ssh" "${remote_host}" "$@" + + "$@" } # From afe732a69f27d85ad5a330bbbe222972f162af1b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:04:27 +0100 Subject: [PATCH 047/177] more changes for the next version Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index 45972f9..2a9a906 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -5,3 +5,4 @@ * ccollect_analyse_logs.sh traps more errors and warnings (Patrick Drolet) * Remove -v for mkdir and rm, as they are not POSIX (Patrick Drolet) * Export destination_* to source specific post_exec (Nico Schottelius) + * Update documentation regarding exported variables (Nico Schottelius) From 49ef5871bceaee013b0a45af02f4ab536ce4a3e9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:10:30 +0100 Subject: [PATCH 048/177] remove old braindumps Signed-off-by: Nico Schottelius --- doc/braindumps/LOCAL_vs._REMOTE | 35 --------------------------------- doc/braindumps/README | 1 - 2 files changed, 36 deletions(-) delete mode 100644 doc/braindumps/LOCAL_vs._REMOTE delete mode 100644 doc/braindumps/README diff --git a/doc/braindumps/LOCAL_vs._REMOTE b/doc/braindumps/LOCAL_vs._REMOTE deleted file mode 100644 index f2a40b7..0000000 --- a/doc/braindumps/LOCAL_vs._REMOTE +++ /dev/null @@ -1,35 +0,0 @@ - to Local to Remote - backup destination is exiting - pre/postexec runs locally - --link-dest? - /delete_incomplete - can chech ddir - - can check destination dir - -> dooooooo it before! - - - remote_host! - => rddir_ls: - incomplete: ls -1 "${INTERVAL}"*".${c_marker}" - - host support? - ssh-host-support? - - => ssh_host => save to host - execute commands there! - - rm! - - --link-dest? - - --link-dest=DIR - => remote dirs, rsync remote - => works!!!! - - local_destination - remote_destination - => remote_* - - both - configuration is local (what to where) - diff --git a/doc/braindumps/README b/doc/braindumps/README deleted file mode 100644 index 973addc..0000000 --- a/doc/braindumps/README +++ /dev/null @@ -1 +0,0 @@ -Do not read the files in this directory From e18c9fa94d13999c23cf1be2e18e2a98397fe74d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:18:28 +0100 Subject: [PATCH 049/177] remove old todos, create new todo list Signed-off-by: Nico Schottelius --- doc/todo/0.5.2 | 4 --- doc/todo/0.5.3 | 34 ------------------ doc/todo/0.5.3.tonnerre | 11 ------ doc/todo/0.6 | 63 --------------------------------- doc/todo/0.6.1 | 23 ------------ doc/todo/0.6.2 | 1 - doc/todo/0.7.1 | 35 ------------------ doc/todo/0.7.2 | 63 --------------------------------- doc/todo/0.7.3 | 2 -- doc/todo/0.7.4 | 1 - doc/todo/0.8.0 | 6 ---- doc/{todo/extern => todos.text} | 4 +++ 12 files changed, 4 insertions(+), 243 deletions(-) delete mode 100644 doc/todo/0.5.2 delete mode 100644 doc/todo/0.5.3 delete mode 100644 doc/todo/0.5.3.tonnerre delete mode 100644 doc/todo/0.6 delete mode 100644 doc/todo/0.6.1 delete mode 100644 doc/todo/0.6.2 delete mode 100644 doc/todo/0.7.1 delete mode 100644 doc/todo/0.7.2 delete mode 100644 doc/todo/0.7.3 delete mode 100644 doc/todo/0.7.4 delete mode 100644 doc/todo/0.8.0 rename doc/{todo/extern => todos.text} (92%) diff --git a/doc/todo/0.5.2 b/doc/todo/0.5.2 deleted file mode 100644 index 0c1c225..0000000 --- a/doc/todo/0.5.2 +++ /dev/null @@ -1,4 +0,0 @@ -x Fix $? problem -x Check last dir searching -x Check count generation -x Check general functionality (remove testing) diff --git a/doc/todo/0.5.3 b/doc/todo/0.5.3 deleted file mode 100644 index 0019581..0000000 --- a/doc/todo/0.5.3 +++ /dev/null @@ -1,34 +0,0 @@ -Done: -==> screenshot -u0219 zrha166.netstream.ch # ~chdscni9/ccollect.sh weekly zrha166.netstream.ch -2007-08-16-21:49:44: ccollect 0.6: Beginning backup using interval weekly -[zrha166.netstream.ch] 2007-08-16-21:49:44: Beginning to backup -[zrha166.netstream.ch] 2007-08-16-21:49:45: Existing backups: 0 Total keeping backups: 8 -[zrha166.netstream.ch] 2007-08-16-21:49:45: Did not find existing backups for interval weekly. -[zrha166.netstream.ch] 2007-08-16-21:49:45: Using backup from daily. -[zrha166.netstream.ch] 2007-08-16-21:49:45: Beginning to backup, this may take some time... -[zrha166.netstream.ch] 2007-08-16-21:49:45: Creating /etc/ccollect/sources/zrha166.netstream.ch/destination/weekly.20070816-2149.22188 ... -[zrha166.netstream.ch] 2007-08-16-21:49:45: Transferring files... - -- beep-after-delete-hack? - -> tonnerre / #cLinux - -- replace nico-linux-ccollect with nico-ccollect - * ccollect is not Linux specific - -- remove exit-calls - * will leave behind unused temporary file! - * use _exit_err - -X join todos - -- fix possible quoting problems - * rsync_extra (redefine format) - * exclude - * other - * create _rsync, filtering args, creating new $@ - -- check and export variables for use in scripts! - -Contact Julian later: - * integrate updated german documentation diff --git a/doc/todo/0.5.3.tonnerre b/doc/todo/0.5.3.tonnerre deleted file mode 100644 index 4e7b7db..0000000 --- a/doc/todo/0.5.3.tonnerre +++ /dev/null @@ -1,11 +0,0 @@ -NetBSD/i386,amd64,sparc,sparc64 - -13:13 < Tonnerre> telmich, die kleine wä, 2 Variablen aus $(CCOLLECT) zu - machen -13:13 < Tonnerre> telmich, eine fü Sourcepfad und eine fü -Destinationfpad -13:13 < Tonnerre> pfa -13:13 < Tonnerre> d -13:14 < Tonnerre> telmich, die gröre wä die $() durch ${} zu ersetzen, so -dass der Kram auch mit !GNU-Make geht - diff --git a/doc/todo/0.6 b/doc/todo/0.6 deleted file mode 100644 index ee8e97f..0000000 --- a/doc/todo/0.6 +++ /dev/null @@ -1,63 +0,0 @@ -not did, did not remember why I wanted to do that: - -- do sed-time check: - -1.2. replace sed? -compare timing: -_echo () { echo "$name $msg" } -and create -_techo () { echo "$timestamp $name $msg" } -perhaps create -_eecho () { _techo "ERROR $msg" } -? - - - -done: -add note for - 09:24 < telmich> Obri: ~/.ssh/config! - -- implement use of different intervals as source for cloning - * use 'woechentlich' if available and no backup exists for 'daily' - * add 'prefer_latest' to prefer different interval, that's newer than - ours - * or add 'prefer_same_interval' instead? -- implement detection of partial backups -3. detect unfinished backups -sven / markierung - - wie seht der Marker aus? - -> .ccollect-YEAR-MM-DD.HHmm.pid (like the directory) - --> assume incomplete, when we did not finish. - --> means it's complete,when rsync was successful ----> partial implemented in 0.5.2 (commented out) -- do not check by name, but by time - * is ls -t posix? - * also check: -p -1 -- ccollect Zeitausgabe verbessern - - Wofuer wie lange gebraucht - * rm - * klonen (gibt es so ja nicht) - Wenn Summary angegeben ist am Ende ausgeben -- do we want rsync -H by default? - * no: takes much more memory -ssh / port change: - change rsync_extra format -- Variables: - source_$n - no_sources - name -- changed naming of sources: - YYYYMMDD-HHMM.PID (better readable) - => source / backup converter! => not needed! -config: - set destination-base - /etc/ccollect/defaults/source_config - -Documentation: -- hint: backuping backup -- update documentation: - - german doc? - - exit pre/post exec -> error codes (after implementation!) (in 0.4) - - write about fast changing fs -- delete_incomplete - diff --git a/doc/todo/0.6.1 b/doc/todo/0.6.1 deleted file mode 100644 index 99a17eb..0000000 --- a/doc/todo/0.6.1 +++ /dev/null @@ -1,23 +0,0 @@ -- Add filter support - * filter -- add source/ignore_failed_pre_exec -- add source/ignore_failed_post_exec -- do not delete_incomplete, when there's only one backup left! -- .ccollect-marker is deleted by rsync at the beginning! - - fix marking -- add logwrapper -- add loganalyser - speedup is - error codes - vanished files (see netstream)!!! - -Done: -- Improve finding backup from another interval: - o strip of interval name - o run sort -n - o use the last entry -- add --version, -V - -not needed: -- think about 'prefer_same_interval' / 'prefer_latest' - diff --git a/doc/todo/0.6.2 b/doc/todo/0.6.2 deleted file mode 100644 index 05798ff..0000000 --- a/doc/todo/0.6.2 +++ /dev/null @@ -1 +0,0 @@ -- fix delete_incomplete marker diff --git a/doc/todo/0.7.1 b/doc/todo/0.7.1 deleted file mode 100644 index 49df154..0000000 --- a/doc/todo/0.7.1 +++ /dev/null @@ -1,35 +0,0 @@ - - --------------------------------------------------------------------------------- -done: -- tools in makefile -14:15 also alle tools/ccollect_* -14:15 mach mal n besseres Makefile :) -14:15 hatte die extra deswegen umbenannt -14:15 nehme ich mal als hinweis für 0.7.1 - - -- add global delete_incomplete (/etc/ccollect/defaults/delete_incomplete) - -09:31 < Obri> telmich: hab nen kleinen tipp für ccollect -09:32 < Obri> telmich: ich habe hier hosts die nicht immer online sind, das ist dumm weil so das backup - kaputtgeht... -09:32 < Obri> telmich: ich habe jetzt das ein preexec script gebastelt: -09:32 < Obri> ping -c1 -q `cat /etc/ccollect/sources/$name/source | cut -d"@" -f2 | cut -d":" -f1` -09:33 < Obri> telmich: so bricht das backup ab wenn der host nicht erreichbar ist -09:33 < Obri> ohne dass ein altes backup entsorgt wird - - -- remove basename - -> include config from cconf! -> standard! - -reject: --------------------------------------------------------------------------------- -- fix german documentation! - => I'm the coder, somebody else can fix it. -- add wrapper, to do logging and analyse - * output filename in the wrapper, save into variable - => mktemp - * call analyser - => output stdout? - => no use for that currently diff --git a/doc/todo/0.7.2 b/doc/todo/0.7.2 deleted file mode 100644 index 79a50aa..0000000 --- a/doc/todo/0.7.2 +++ /dev/null @@ -1,63 +0,0 @@ --------------------------------------------------------------------------------- -Stats version --------------------------------------------------------------------------------- - -Add tools/ccollect_stats.sh, clearify license --------------------------------------------------------------------------------- -Preamble: - Netstream (www.netstream.ch) may consider using ccollect for backing up many - unix servers and needs some clean reporting. The following things - need to be done, so ccollect will be useful for netstream: - -Logger: - - Needs to write small mails (sysadmins don't have time to read mails) - - Needs to be able to only write a mail on error - * needs configuration option to also mail on warnings - - Should be able to send one mail per backup source - * or one for the whole backup job - -Messages: (to be used for filtering) - Errors: - Read from remote host .*: Connection timed out - - Warnings: - rsync: mknod .* failed: Invalid argument (22) - file has vanished: ".*" - --------------------------------------------------------------------------------- - -Analyzer: - - grosse Dateien - - grosse Veraenderungen --------------------------------------------------------------------------------- -exclude-lists-doku: -freebsd: - /usr/ports/* - /proc/* - /dev/* - /tmp/* - /var/tmp/* -linux: - /sys/* - /proc/* - /dev/* - /tmp/* - --------------------------------------------------------------------------------- -done: - -rsync_extra global! -- \n delimeted --------------------------------------------------------------------------------- - -S, --sparse - Try to handle sparse files efficiently so they take up less space on the des‐ - tination. Conflicts with --inplace because it’s not possible to overwrite - data in a sparse fashion. - --------------------------------------------------------------------------------- - Always report return code! - -[12:00] u0255:ddba034.netstream.ch# rsync -n -a --delete --stats --progress daily.20080324-0313.17841/ daily.20080325-0313.31148/ - -$tool - diff --git a/doc/todo/0.7.3 b/doc/todo/0.7.3 deleted file mode 100644 index 73e5ffc..0000000 --- a/doc/todo/0.7.3 +++ /dev/null @@ -1,2 +0,0 @@ -- add -a (archive) to ccollect_delete_source.text -- add listing of intervals to ccollect_list_intervals diff --git a/doc/todo/0.7.4 b/doc/todo/0.7.4 deleted file mode 100644 index 70515e7..0000000 --- a/doc/todo/0.7.4 +++ /dev/null @@ -1 +0,0 @@ -add support for wipe diff --git a/doc/todo/0.8.0 b/doc/todo/0.8.0 deleted file mode 100644 index 2b5130d..0000000 --- a/doc/todo/0.8.0 +++ /dev/null @@ -1,6 +0,0 @@ -- restructure code to easily allow global versus source options: - f_name=.... - check_option $f_name - => source - => defaults -- support all senseful options as default and source specific diff --git a/doc/todo/extern b/doc/todos.text similarity index 92% rename from doc/todo/extern rename to doc/todos.text index cb42a07..5be3799 100644 --- a/doc/todo/extern +++ b/doc/todos.text @@ -24,3 +24,7 @@ to have. This means I won't code them, so somebody can code them. - add option ("run_only_once") to prevent ccollect to run twice (per source/general) + -> or create wrapper, that takes care of it + +- add support for wipe + From 5d0a3c73d2408fc85eb26f82a66ff15ff5cc9d5e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:20:24 +0100 Subject: [PATCH 050/177] begin to cleanup the todo list Signed-off-by: Nico Schottelius --- doc/todos.text | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/todos.text b/doc/todos.text index 5be3799..0182040 100644 --- a/doc/todos.text +++ b/doc/todos.text @@ -1,6 +1,12 @@ +ccollect todos +============== +Nico Schottelius +0.1, for the current ccollect version +:Author Initials: NS + + These todos are things that would be senseful todo, but are just nice to have. This means I won't code them, so somebody can code them. --------------------------------------------------------------------------------- - Add ccollect-restore.sh (new project, perhaps coordinated with jchome's ccollect-config) @@ -28,3 +34,5 @@ to have. This means I won't code them, so somebody can code them. - add support for wipe +- cleanup doc/: remove old stuff, make clear what can be installeld, + cleanup ccollect-restoring.text From 8ae649c761ae6c22ca0478f53ede9ddeeb2423d4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:22:08 +0100 Subject: [PATCH 051/177] update release checklist Signed-off-by: Nico Schottelius --- doc/release-checklist | 3 +-- doc/todos.text | 3 +++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/release-checklist b/doc/release-checklist index 7cba30c..71b40ee 100644 --- a/doc/release-checklist +++ b/doc/release-checklist @@ -1,8 +1,7 @@ * Change version and date in ccollect.sh * Change version in documentation/ccollect.text -* Regenerate documentation * Create tarball -* Transfer to home.schottelius.org +* Transfer to website * Extract files * Update website * Announce on freshmeat diff --git a/doc/todos.text b/doc/todos.text index 0182040..05bb56e 100644 --- a/doc/todos.text +++ b/doc/todos.text @@ -36,3 +36,6 @@ to have. This means I won't code them, so somebody can code them. - cleanup doc/: remove old stuff, make clear what can be installeld, cleanup ccollect-restoring.text + +- clean logwrapper.text and probably create logwrapper, that does + what is being stated in logwrapper.text From e508ef052fce1ad13d14627721d7301caf04a5a1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:25:06 +0100 Subject: [PATCH 052/177] reorder operating systems in documentation Signed-off-by: Nico Schottelius --- doc/ccollect.text | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index e2090a9..8bdd81a 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -21,9 +21,9 @@ Supported and tested operating systems and architectures ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ `ccollect` was successfully tested on the following platforms: -- GNU/Linux on amd64/hppa/i386/ppc/ARM - FreeBSD on amd64/i386 -- Mac OS X 10.5 +- GNU/Linux on amd64/arm/hppa/i386/ppc +- Mac OS X 10.5 - NetBSD on alpha/amd64/i386/sparc/sparc64 - OpenBSD on amd64 From d79c2b0a28fa19ff92a5f5cc16128f37b6aaaf57 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:36:49 +0100 Subject: [PATCH 053/177] begin to create bigger automated tests Signed-off-by: Nico Schottelius --- Makefile | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index cfef1b4..494dbcb 100644 --- a/Makefile +++ b/Makefile @@ -190,8 +190,10 @@ distclean: clean dist: distclean documentation #test: ccollect.sh documentation -test: ccollect.sh +/tmp/ccollect: mkdir -p /tmp/ccollect + +test: ccollect.sh /tmp/ccollect CCOLLECT_CONF=./conf ./ccollect.sh daily from-remote CCOLLECT_CONF=./conf ./ccollect.sh daily local CCOLLECT_CONF=./conf ./ccollect.sh daily "local-with&ersand" @@ -203,5 +205,7 @@ test: ccollect.sh touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect.sh daily delete_incomplete CCOLLECT_CONF=./conf ./ccollect.sh daily no-source-must-fail -# for s in $$(ls ./conf/sources); do CCOLLECT_CONF=./conf echo ./ccollect.sh daily $$s; done -# CCOLLECT_CONF=./conf ./ccollect.sh -a daily + +test2: ccollect.sh /tmp/ccollect + cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect.sh daily "$$s"; done + CCOLLECT_CONF=./conf ./ccollect.sh -a daily From a48fe6d41b98b4bee2b89c4a02c18533ace6c1ea Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 14:37:32 +0100 Subject: [PATCH 054/177] +braces Signed-off-by: Nico Schottelius --- Makefile | 1 + ccollect.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 494dbcb..e509290 100644 --- a/Makefile +++ b/Makefile @@ -208,4 +208,5 @@ test: ccollect.sh /tmp/ccollect test2: ccollect.sh /tmp/ccollect cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect.sh daily "$$s"; done + touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect.sh -a daily diff --git a/ccollect.sh b/ccollect.sh index 7747fcc..41411da 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -110,7 +110,7 @@ delete_from_file() [ $# -eq 1 ] && suffix="$1" && shift while read to_remove; do set -- "$@" "${to_remove}" - if [ "$suffix" ]; then + if [ "${suffix}" ]; then to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" set -- "$@" "${to_remove_no_suffix}" fi From 0f7891de8d04c38434e7e310803acb5e5c7c3c4e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 15:03:37 +0100 Subject: [PATCH 055/177] rename $i to $current_source Signed-off-by: Nico Schottelius --- ccollect.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 41411da..9fb73c3 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -247,13 +247,13 @@ fi # # Let's do the backup - here begins the real stuff # -i=0 -while [ "${i}" -lt "${no_sources}" ]; do +current_source=0 +while [ "${current_source}" -lt "${no_sources}" ]; do # # Get current source # - eval name=\"\$source_${i}\" - i=$((${i}+1)) + eval name=\"\$source_${current_source}\" + current_source=$((${current_source}+1)) export name From c2226f913411efb3280c359d683be64f5cc4e736 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 15:07:50 +0100 Subject: [PATCH 056/177] BUGFIX: Use different destination directory based on the source number If you have muliple sources, that go into the same destination directory or specify a source mulitple times on the command line and the backup is done within a minute, the destination directory will still be the name: 497 export destination_name="${INTERVAL}.$(${CDATE}).$$" None of the values will change for the next backup. Thus the new format since ccollect 0.8.1 is 497 export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" which contains the number of the current source. Signed-off-by: Nico Schottelius --- ccollect.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 9fb73c3..491710d 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -247,13 +247,13 @@ fi # # Let's do the backup - here begins the real stuff # -current_source=0 -while [ "${current_source}" -lt "${no_sources}" ]; do +source_no=0 +while [ "${source_no}" -lt "${no_sources}" ]; do # # Get current source # - eval name=\"\$source_${current_source}\" - current_source=$((${current_source}+1)) + eval name=\"\$source_${source_no}\" + source_no=$((${source_no}+1)) export name @@ -494,7 +494,7 @@ while [ "${current_source}" -lt "${no_sources}" ]; do fi # set time when we really begin to backup, not when we began to remove above - export destination_name="${INTERVAL}.$(${CDATE}).$$" + export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" export destination_dir="${ddir}/${destination_name}" export destination_full="${destination}/${destination_name}" From 564ef0bd873044e2c7530bd2cd0e923b8a3ac1d5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 15:11:26 +0100 Subject: [PATCH 057/177] add testing with parallel execution Signed-off-by: Nico Schottelius --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index e509290..4590d29 100644 --- a/Makefile +++ b/Makefile @@ -210,3 +210,4 @@ test2: ccollect.sh /tmp/ccollect cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect.sh daily "$$s"; done touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect.sh -a daily + CCOLLECT_CONF=./conf ./ccollect.sh -a -p daily From 7de72e5e8d5a60d082ae12177959c79aefc95f57 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 15:12:16 +0100 Subject: [PATCH 058/177] insert more faults into test setup and remove previous test Signed-off-by: Nico Schottelius --- Makefile | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 4590d29..1ca8502 100644 --- a/Makefile +++ b/Makefile @@ -194,20 +194,8 @@ dist: distclean documentation mkdir -p /tmp/ccollect test: ccollect.sh /tmp/ccollect - CCOLLECT_CONF=./conf ./ccollect.sh daily from-remote - CCOLLECT_CONF=./conf ./ccollect.sh daily local - CCOLLECT_CONF=./conf ./ccollect.sh daily "local-with&ersand" - CCOLLECT_CONF=./conf ./ccollect.sh daily source-without-destination - CCOLLECT_CONF=./conf ./ccollect.sh daily "source with spaces and interval" - CCOLLECT_CONF=./conf ./ccollect.sh daily to-remote - CCOLLECT_CONF=./conf ./ccollect.sh daily with_exec - CCOLLECT_CONF=./conf ./ccollect.sh daily very_verbose - touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker - CCOLLECT_CONF=./conf ./ccollect.sh daily delete_incomplete - CCOLLECT_CONF=./conf ./ccollect.sh daily no-source-must-fail - -test2: ccollect.sh /tmp/ccollect cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect.sh daily "$$s"; done touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect.sh -a daily + touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect.sh -a -p daily From e9c02b8e2d0cc9ed2d41f2ea9e2f3536c219cb73 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 15:16:51 +0100 Subject: [PATCH 059/177] plural Signed-off-by: Nico Schottelius --- README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README b/README index 558594d..1b5341d 100644 --- a/README +++ b/README @@ -7,7 +7,7 @@ ccollect backups (local or remote) data to local or remote destinations. You can retrieve the latest version of ccollect at [0]. ccollect was inspired by rsnapshot [1], which has some problems: - - configuration parameters has to be TAB seperated + - configuration parameters have to be TAB seperated - you can not specify per source exclude lists - no per source pre/post execution support - no parallel execution From 49cb1f92eeaec36e0d3068ca8845161d5f0fb4c1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 15:17:01 +0100 Subject: [PATCH 060/177] join line Signed-off-by: Nico Schottelius --- ccollect.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 491710d..23bab24 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -462,8 +462,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check: maximum number of backups is reached? # - count="$(pcmd ls -d1 "${ddir}/${INTERVAL}."*"/" | wc -l \ - | sed 's/^ *//g')" || _exit_err "Counting backups failed" + count="$(pcmd ls -d1 "${ddir}/${INTERVAL}."*"/" | wc -l | sed 's/^ *//g')" \ + || _exit_err "Counting backups failed" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" From ccf86defaf0ed71cc131d8b1d872902b958745c6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 15:26:57 +0100 Subject: [PATCH 061/177] move pre_exec part to the top Signed-off-by: Nico Schottelius --- ccollect.sh | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 23bab24..4e04bde 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -270,6 +270,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # ( backup="${CSOURCES}/${name}" + c_source="${backup}/source" + c_dest="${backup}/destination" + c_pre_exec="${backup}/pre_exec" + c_post_exec="${backup}/post_exec" + # # Stderr to stdout, so we can produce nice logs # @@ -296,12 +301,19 @@ while [ "${source_no}" -lt "${no_sources}" ]; do fi # - # Read / create configuration + # First execute pre_exec, which may generate destination or other parameters + # + if [ -x "${c_pre_exec}" ]; then + _techo "Executing ${c_pre_exec} ..." + "${c_pre_exec}"; ret="$?" + _techo "Finished ${c_pre_exec} (return code ${ret})." + + [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." + fi + + # + # Read source configuration # - c_source="${backup}/source" - c_dest="${backup}/destination" - c_pre_exec="${backup}/pre_exec" - c_post_exec="${backup}/post_exec" for opt in verbose very_verbose summary exclude rsync_options \ delete_incomplete remote_host rsync_failure_codes \ mtime quiet_if_down ; do @@ -321,17 +333,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do TSORT="tc" fi - # - # First execute pre_exec, which may generate destination or other parameters - # - if [ -x "${c_pre_exec}" ]; then - _techo "Executing ${c_pre_exec} ..." - "${c_pre_exec}"; ret="$?" - _techo "Finished ${c_pre_exec} (return code ${ret})." - - [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." - fi - # # Source configuration checks # From f98853379ee8a55e112f6111454539353749a688 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 15:28:33 +0100 Subject: [PATCH 062/177] also move the interval check further to the beginning Signed-off-by: Nico Schottelius --- ccollect.sh | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 4e04bde..8ef2183 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -324,6 +324,19 @@ while [ "${source_no}" -lt "${no_sources}" ]; do fi done + # + # Interval definition: First try source specific, fallback to default + # + c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)" + + if [ -z "${c_interval}" ]; then + c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" + + if [ -z "${c_interval}" ]; then + _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." + fi + fi + # # Sort by ctime (default) or mtime (configuration option) # @@ -446,19 +459,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do fi fi - # - # Interval definition: First try source specific, fallback to default - # - c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)" - - if [ -z "${c_interval}" ]; then - c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" - - if [ -z "${c_interval}" ]; then - _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." - fi - fi - # # Check: maximum number of backups is reached? # From 545158b56fc6e461b8dc0b9513b032e3c0eb7bc7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 16:14:29 +0100 Subject: [PATCH 063/177] cleanup for messages and remove unecessary message message=Beginning to backup, this may take some time... Signed-off-by: Nico Schottelius --- ccollect.sh | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 8ef2183..4b8214f 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -462,7 +462,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: maximum number of backups is reached? # - count="$(pcmd ls -d1 "${ddir}/${INTERVAL}."*"/" | wc -l | sed 's/^ *//g')" \ || _exit_err "Counting backups failed" @@ -487,25 +486,24 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _exit_err "Failed to list contents of ${ddir}." # - # clone from old backup, if existing + # Clone from old backup, if existing # if [ "${last_dir}" ]; then set -- "$@" "--link-dest=${ddir}/${last_dir}" _techo "Hard linking from ${last_dir}" fi - # set time when we really begin to backup, not when we began to remove above + # + # Include current time in name, not the time when we began to remove above + # export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" export destination_dir="${ddir}/${destination_name}" export destination_full="${destination}/${destination_name}" - # give some info - _techo "Beginning to backup, this may take some time..." - - _techo "Creating ${destination_dir} ..." + _techo "Creating directory ${destination_dir} ..." [ "${VVERBOSE}" ] && echo "mkdir ${destination_dir}" pcmd mkdir "${destination_dir}" || \ - _exit_err "Creating ${destination_dir} failed. Skipping." + _exit_err "Creating directory ${destination_dir} failed. Skipping." # # added marking in 0.6 (and remove it, if successful later) From f630bef3b5a549ae5092e24f7f00576ac95af8cd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 16:15:00 +0100 Subject: [PATCH 064/177] Finally simplify the time calculations! According to http://www.opengroup.org/onlinepubs/009695399/utilities/xcu_chap01.html a shell must be able to do modulus (=remainder). Zsh, bash and dash confirm this and thus I can save another line and get rid of a very ugly one. Signed-off-by: Nico Schottelius --- ccollect.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 4b8214f..ba91e49 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -560,14 +560,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do fi fi - # Calculation + # + # Time calculation + # end_s="$(${SDATE})" - full_seconds="$((${end_s} - ${begin_s}))" hours="$((${full_seconds} / 3600))" - seconds="$((${full_seconds} - (${hours} * 3600)))" - minutes="$((${seconds} / 60))" - seconds="$((${seconds} - (${minutes} * 60)))" + minutes="$(((${full_seconds} % 3600) / 60))" + seconds="$((${full_seconds} % 60))" _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" ) | add_name From 0b8e6409cf45b8a8af3485ecfced804c29e875f4 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 16:18:36 +0100 Subject: [PATCH 065/177] more changes for the upcoming release Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index 2a9a906..42f3007 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -6,3 +6,4 @@ * Remove -v for mkdir and rm, as they are not POSIX (Patrick Drolet) * Export destination_* to source specific post_exec (Nico Schottelius) * Update documentation regarding exported variables (Nico Schottelius) + * Simplify time calculation (Nico Schottelius) From 9c474129917b72aade20a9791156319bc61f56d0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 16:21:21 +0100 Subject: [PATCH 066/177] integrate pdf support for documents soon Signed-off-by: Nico Schottelius --- doc/todos.text | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todos.text b/doc/todos.text index 05bb56e..4fa2a18 100644 --- a/doc/todos.text +++ b/doc/todos.text @@ -39,3 +39,5 @@ to have. This means I won't code them, so somebody can code them. - clean logwrapper.text and probably create logwrapper, that does what is being stated in logwrapper.text + +- include pdf support again (Makefile:72) From 23c395bcbddefe4fb7c4ea6c524df977a9dec75c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 1 Nov 2009 16:27:23 +0100 Subject: [PATCH 067/177] remove some old stuff from the makefile Signed-off-by: Nico Schottelius --- Makefile | 2 -- 1 file changed, 2 deletions(-) diff --git a/Makefile b/Makefile index 1ca8502..3c092f3 100644 --- a/Makefile +++ b/Makefile @@ -128,7 +128,6 @@ TOOLSMAN = $(subst .sh,.text,$(TOOLSMAN1)) TOOLSFP = $(subst ccollect,tools/ccollect,$(TOOLS)) -#t2: $(TOOLSMAN) t2: echo $(TOOLS) - $(TOOLSMAN) - $(TOOLSFP) @@ -189,7 +188,6 @@ distclean: clean # dist: distclean documentation -#test: ccollect.sh documentation /tmp/ccollect: mkdir -p /tmp/ccollect From 44442a09c943036e7d976a3d72b9706c085d8fdd Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 Nov 2009 07:45:51 +0100 Subject: [PATCH 068/177] todo for the version after the next version Signed-off-by: Nico Schottelius --- doc/todos.text | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/todos.text b/doc/todos.text index 4fa2a18..cf687cb 100644 --- a/doc/todos.text +++ b/doc/todos.text @@ -41,3 +41,5 @@ to have. This means I won't code them, so somebody can code them. what is being stated in logwrapper.text - include pdf support again (Makefile:72) + +- integrate website into ccollect -> updates are done here From ca9106054b462d2446deff9bb068e3e95fc316ce Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 Nov 2009 07:46:39 +0100 Subject: [PATCH 069/177] update version in the documentation Signed-off-by: Nico Schottelius --- doc/ccollect.text | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index 8bdd81a..a751abf 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -0.8, for ccollect 0.8, Initial Version from 2006-01-13 +0.8.1, for ccollect 0.8.1, Initial Version from 2006-01-13 :Author Initials: NS @@ -159,7 +159,7 @@ For those who do not want to read the whole long document: -------------------------------------------------------------------------------- # get latest ccollect tarball from http://www.nico.schottelius.org/software/ccollect/ # replace value for CCV with the current version -export CCV=0.8 +export CCV=0.8.1 # # replace 'wget' with 'fetch' on bsd From 86992b9787e7f99750cf1a8671cea87fc98db013 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 2 Nov 2009 07:49:57 +0100 Subject: [PATCH 070/177] begin to update rpm spec file Signed-off-by: Nico Schottelius --- contrib/ccollect.spec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/ccollect.spec b/contrib/ccollect.spec index 477a878..0e35bb0 100644 --- a/contrib/ccollect.spec +++ b/contrib/ccollect.spec @@ -1,11 +1,11 @@ Summary: (pseudo) incremental backup with different exclude lists using hardlinks and rsync Name: ccollect -Version: 0.8 +Version: 0.8.1 Release: 0 URL: http://www.nico.schottelius.org/software/ccollect Source0: http://www.nico.schottelius.org/software/ccollect/%{name}-%{version}.tar.bz2 -License: LGPL-3 +License: GPL-3 Group: Applications/System Vendor: Nico Schottelius BuildRoot: %{_tmppath}/%{name}-%(id -un) From e47fb78603172b8636956c95025705346ef9bb9c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 23 Dec 2009 18:43:40 +0100 Subject: [PATCH 071/177] import new ccollect_mgr version from patrick Signed-off-by: Nico Schottelius --- contrib/patrick/ccollect_mgr.sh | 467 ++++++++++++++++++-------------- 1 file changed, 269 insertions(+), 198 deletions(-) diff --git a/contrib/patrick/ccollect_mgr.sh b/contrib/patrick/ccollect_mgr.sh index 5678779..fe69286 100644 --- a/contrib/patrick/ccollect_mgr.sh +++ b/contrib/patrick/ccollect_mgr.sh @@ -1,7 +1,7 @@ #!/bin/sh # # ---------------------------------------------------------------------------- -# Last update: 2009-10-29 +# Last update: 2009-12-11 # By : pdrolet (ccollect_mgr@drolet.name) # ---------------------------------------------------------------------------- # Job manager to the ccollect utilities @@ -9,7 +9,8 @@ # # Provides the following features # 1) Determine the interval (daily/weekly/monthly) -# 2) Perform the backups using ccollect +# 2) Check the estimated file transfer size +# 3) Perform the backups using ccollect # 4) Copy the ccollect log to the first backup of the set # 5) Build a periodic report and include the real amount of disk used # 6) Send an email if there has been errors or warnings @@ -48,69 +49,74 @@ # Send warning if the worst case data transfer will be larger than (in MB)... warning_transfer_size=1024 +abort_transfer_size=5120 # Define paths and default file names -ADD_TO_PATH=/opt/bin:/opt/sbin:/usr/local/bin:/usr/local/sbin -CCOLLECT=ccollect.sh -CCOLLECT_CONF=/usr/local/etc/ccollect +ADD_TO_PATH="/opt/bin:/opt/sbin:/usr/local/bin:/usr/local/sbin" +CCOLLECT="ccollect.sh" +CCOLLECT_CONF="/usr/local/etc/ccollect" -per_report=${CCOLLECT_CONF}/periodic_report.log -tmp_report=/tmp/ccollect.$$ -tmp_email=/tmp/email.$$ +PS="/opt/bin/ps" +FIND="/opt/bin/find" + +TEMP_LOG="${CCOLLECT_CONF}"/log.$$ +per_report="${CCOLLECT_CONF}/periodic_report.log" +tmp_report="/tmp/ccollect.$$" +tmp_mgr="/tmp/ccollect_mgr.$$" +tmp_email="/tmp/email.$$" + +backups_not_found="" # Sub routines... -find_interval() +send_email() { - # ---------------------------------------------------- - # Find interval for ccollect backup. - # optional parameters: - # - Day of the week to do weekly backups - # - Do monthly instead of weekly on the Nth week - # ---------------------------------------------------- - - weekly_backup=$1 - monthly_backup=$2 + # Send a simple email using mini-sendmail. - weekday=`date "+%w"` - if [ ${weekday} -eq ${weekly_backup} ]; then - dom=`date "+%d"` - weeknum=$(( ( ${dom} / 7 ) + 1 )) - if [ ${weeknum} -eq ${monthly_backup} ]; then - interval=monthly - else - interval=weekly - fi - else - interval=daily + msg_body_file="$1" + shift + + # ------------------------------ + # Quit if we can't send an email + # ------------------------------ + if [ "${to}" == "" ] || [ "${mail_server}" == "" ]; then + echo "Missing mail server or destination email. No email sent with subject: $@" + exit 1 fi + + echo from: "${from}" > "${tmp_email}" + echo subject: "$@" >> "${tmp_email}" + echo to: "${to}" >> "${tmp_email}" + echo cc: >> "${tmp_email}" + echo bcc: >> "${tmp_email}" + echo "" >> "${tmp_email}" + echo "" >> "${tmp_email}" + cat "${msg_body_file}" >> "${tmp_email}" + echo "" >> "${tmp_email}" + + echo "" + echo Sending email to ${to} to report the following: + echo ----------------------------------------------- + cat "${tmp_email}" + cat "${tmp_email}" | mini_sendmail -f"${from}" -s"${mail_server}" "${to}" + rm "${tmp_email}" } -move_log() +remove_source() { - for backup in $@ ; do - ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" - if [ "${ret}" -ne 0 ]; then - echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping." - backup_dir="" - else - backup_dir=`cat ${TEMP_LOG} | grep "\[${backup}\] .*: Creating ${ddir}" | awk '{ print $4 }'` - fi - if [ "${backup_dir}" != "" ]; then - new_log=${backup_dir}/ccollect.log - mv ${TEMP_LOG} ${new_log} - echo New Log Location: ${new_log} - return 0 - fi - done - echo "WARNING: none of the backup set have been created" - new_log=${TEMP_LOG} + remove_no=$1 + eval echo Removing backup \"\$source_$1\" + + no_sources="$(( ${no_sources} - 1 ))" + while [ "${remove_no}" -lt "${no_sources}" ]; do + eval source_${remove_no}=\"\$source_$(( ${remove_no} + 1))\" + eval ddir_${remove_no}=\"\$ddir_$(( ${remove_no} + 1))\" + remove_no=$(( ${remove_no} + 1 )) + done } compute_rdu() { - # WARNING: Don't pass a directory with a space as parameter (I'm too new at scripting!) - kdivider=1 find_options="" @@ -123,12 +129,17 @@ compute_rdu() kdivider=1048576 ;; *) - find_options="${find_options} $1" + break ;; esac shift done + if [ "$#" == 0 ]; then + rdu=0 + return 1 + fi + # ------------------------------------------------------------------------------------------------------ # Compute the real disk usage (eg: hard links do files outside the backup set don't count) # ------------------------------------------------------------------------------------------------------ @@ -141,103 +152,88 @@ compute_rdu() # 5) Present the result with additional dividers based on command line parameters # - rdu=$(( ( `/opt/bin/find ${find_options} -printf '%n %i %y %k \n' \ + rdu=$(( ( `"${FIND}" "$@" -printf '%n %i %y %k \n' \ | sort -n \ | uniq -c \ | awk '{ if (( $1 == $2 ) || ($4 == "d")) { sum += $5; } } END { printf "%u\n",(sum); }'` \ + ${kdivider} - 1 ) / ${kdivider} )) - echo RDU for ${find_options} is ${rdu} -} - -compute_total_rdu() -{ - real_usage=0 - - # ------------------------------------------ - # Get the real disk usage for the backup set - # ------------------------------------------ - for backup in $@ ; do - ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" - echo ${backup} - Adding ${ddir} to backup list - backup_dir_list="${backup_dir_list} ${ddir}" - if [ "${ret}" -ne 0 ]; then - echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping." - else - backup_dir=`find ${ddir}/${interval}.* -maxdepth 0 -type d -print | sort -r | head -n 1` - compute_rdu -m ${backup_dir} - real_usage=$(( ${real_usage} + ${rdu} )) - fi - done - echo Backup list - ${backup_dir_list} -} - -send_email() -{ - # Send a simple email using mini-sendmail. - - msg_body_file=$1 - shift - - # ------------------------------ - # Quit if we can't send an email - # ------------------------------ - if [ "${to}" == "" ] || [ "${mail_server}" == "" ]; then - echo "Missing mail server or destination email. No email sent with subject: $@" - exit 1 - fi - - echo from: ${from} > ${tmp_email} - echo subject: $@ >> ${tmp_email} - echo to: ${to} >> ${tmp_email} - echo cc: >> ${tmp_email} - echo bcc: >> ${tmp_email} - echo "" >> ${tmp_email} - echo "" >> ${tmp_email} - cat ${msg_body_file} >> ${tmp_email} - echo "" >> ${tmp_email} - - echo Sending email to ${to} to report the following error: - echo ----------------------------------------------------- - cat ${tmp_email} - cat ${tmp_email} | mini_sendmail -f${from} -s${mail_server} ${to} - rm ${tmp_email} } check_running_backups() { # Check if a backup is already ongoing. If so, skip and send email # Don't use the ccollect marker as this is no indication if it is still running - - for backup in ${ccollect_backups} ; do + + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup=\"\$source_${source_no}\" + PID=$$ - /opt/bin/ps -e -o pid,ppid,args 2> /dev/null | grep -v -e grep -e "${PID}.*ccollect.*${backup}" | grep "ccollect.*${backup}" > /tmp/ccollect_mgr.$$ 2> /dev/null - running_proc=`cat /tmp/ccollect_mgr.$$ | wc -l` + "${PS}" -e -o pid,ppid,args 2> /dev/null \ + | grep -v -e grep -e "${PID}.*ccollect.*${backup}" \ + | grep "ccollect.*${backup}" > "${tmp_mgr}" 2> /dev/null + running_proc=`cat "${tmp_mgr}" | wc -l` + if [ ${running_proc} -gt 0 ]; then + # Remove backup from list running_backups="${running_backups}${backup} " - echo "Process:" - cat /tmp/ccollect.$$ + + echo "Process already running:" + cat "${tmp_mgr}" + + remove_source ${source_no} else - backups_to_do="${backups_to_do}${backup} " + source_no=$(( ${source_no} + 1 )) fi - rm /tmp/ccollect_mgr.$$ + rm "${tmp_mgr}" done - ccollect_backups=${backups_to_do} if [ "${running_backups}" != "" ]; then - echo "skipping ccollect backups already running: ${running_backups}" | tee ${tmp_report} - send_email ${tmp_report} "WARNING - skipping ccollect backups already running: ${running_backups}" - rm ${tmp_report} + echo "skipping ccollect backups already running: ${running_backups}" | tee "${tmp_report}" + send_email "${tmp_report}" "WARNING - skipping ccollect backups already running: ${running_backups}" + rm "${tmp_report}" fi } +find_interval() +{ + # ---------------------------------------------------- + # Find interval for ccollect backup. + # optional parameters: + # - Day of the week to do weekly backups + # - Do monthly instead of weekly on the Nth week + # ---------------------------------------------------- + + weekly_backup="$1" + monthly_backup="$2" + + weekday=`date "+%w"` + if [ ${weekday} -eq ${weekly_backup} ]; then + dom=`date "+%e"` + weeknum=$(( ( ${dom} / 7 ) + 1 )) + if [ "${weeknum}" -eq "${monthly_backup}" ]; then + interval=monthly + else + interval=weekly + fi + else + interval=daily + fi +} + precheck_transfer_size() { # Check the estimated (worst case) transfer size and send email if larger than certain size + # Abort backup if total transfer is larger than maximum limit (ex: an error somewhere + # requires to do full backup and not incremental, which could blow the quota with ISP) # # Be nice and add error checking one day... - for backup in ${ccollect_backups} ; do - ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup=\"\$source_${source_no}\" + eval ddir=\"\$ddir_${source_no}\" + last_dir="$(ls -tcp1 "${ddir}" | grep '/$' | head -n 1)" sdir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/source)"; ret="$?" if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/exclude ]; then @@ -245,66 +241,152 @@ precheck_transfer_size() else exclude="" fi + rsync_options="" if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/rsync_options ]; then while read line; do rsync_options="${rsync_options} ${line}" done < ${CCOLLECT_CONF}/sources/${backup}/rsync_options fi - rsync -n -a --delete --stats ${rsync_options} ${exclude} ${sdir} ${ddir}/${last_dir} > ${tmp_report} - tx_rx=`cat ${tmp_report} | grep "Total transferred file size" | \ + + rsync -n -a --delete --stats ${rsync_options} "${exclude}" "${sdir}" "${ddir}/${last_dir}" > "${tmp_report}" + + tx_rx=`cat "${tmp_report}" | grep "Total transferred file size" | \ awk '{ { tx += $5 } } END { printf "%u",(((tx)+1024*1024-1)/1024/1024); }'` total_xfer=$(( ${total_xfer} + ${tx_rx} )) + + source_no=$(( ${source_no} + 1 )) done - echo Transfer estimation for ${ccollect_backups}: ${total_xfer} MB - if [ ${total_xfer} -gt ${warning_transfer_size} ]; then + echo "Transfer estimation for${ccollect_backups}: ${total_xfer} MB" + + if [ ${total_xfer} -gt ${abort_transfer_size} ]; then + # -------------------------------------------------- + # Send an error if transfer is larger than max limit + # -------------------------------------------------- + # Useful to detect potential issues when there is transfer quota (ex: with ISP) + + echo "Data transfer larger than ${abort_transfer_size} MB is expected for${ccollect_backups}" >> "${tmp_report}" + echo "** BACKUP ABORTED **" >> "${tmp_report}" + + send_email "${tmp_report}" "ERROR: aborted ccollect for${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB" + rm "${tmp_report}" + exit 1 + elif [ ${total_xfer} -gt ${warning_transfer_size} ]; then # -------------------------------------------------- # Send a warning if transfer is expected to be large # -------------------------------------------------- # Useful to detect potential issues when there is transfer quota (ex: with ISP) - echo Data transfer larger than ${warning_transfer_size} MB is expected for ${ccollect_backups} > ${tmp_report} + echo "Data transfer larger than ${warning_transfer_size} MB is expected for${ccollect_backups}" > "${tmp_report}" - send_email ${tmp_report} "WARNING ccollect for ${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB" - rm ${tmp_report} + send_email "${tmp_report}" "WARNING ccollect for${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB" + rm "${tmp_report}" + fi +} + +build_backup_dir_list() +{ + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup=\"\$source_${source_no}\" + eval ddir=\"\$ddir_${source_no}\" + + backup_dir="`cat "${TEMP_LOG}" \ + | grep "\[${backup}\] .*: Creating.* ${ddir}" \ + | head -n 1 \ + | sed 's/[^\/]*\//\//; s/ \.\.\.//'`" + + if [ ! -d "${backup_dir}" ]; then + backups_not_found="${backups_not_found}\"${backup}\" " + echo -n "Backup directory for \"${backup}\" not found. " + remove_source "${source_no}" + else + eval export backup_dir_list_${source_no}="${backup_dir}" +# eval echo Backup Dir List: \"\$backup_dir_list_${source_no}\" + source_no=$(( ${source_no} + 1 )) + fi + done +} + +move_log() +{ + if [ "${no_sources}" -gt 0 ]; then + eval log_file=\"\$backup_dir_list_1\"/ccollect.log + mv "${TEMP_LOG}" "${log_file}" + echo New Log Location: "${log_file}" + else + echo "WARNING: none of the backup set have been created" + log_file="${TEMP_LOG}" fi } send_report() { - log=$1 - # Analyze log for periodic report and for error status report - cat ${log} | ccollect_analyse_logs.sh iwe > ${tmp_report} + cat "${log_file}" | ccollect_analyse_logs.sh iwe > "${tmp_report}" # ------------------------- # Build the periodic report # ------------------------- - # Compute the total number of MB sent and received for all the backup sets - tx_rx=`cat ${tmp_report} | \ + tx_rx=`cat "${tmp_report}" | \ grep 'sent [[:digit:]]* bytes received [0-9]* bytes' | \ awk '{ { tx += $3 } { rx += $6} } END \ - { printf "%u",(((tx+rx)+1024*1024-1)/1024/1024); }'` + { printf "%u",(((tx+rx)+(1024*1024)-1)/1024/1024); }'` current_date=`date +'20%y/%m/%d %Hh%M -- '` - + + # ------------------------------------------ + # Get the real disk usage for the backup set + # ------------------------------------------ + total_rdu=0 + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup_dir=\"\$backup_dir_list_${source_no}\" + compute_rdu -m "${backup_dir}" + total_rdu=$(( ${total_rdu} + ${rdu} )) + source_no=$(( ${source_no} + 1 )) + done + # --------------------------------------------------------- # Get the disk usage for all backups of each backup sets... - # ** be patient ** + # ** BE PATIENT!!! ** # --------------------------------------------------------- - compute_rdu -g ${backup_dir_list} + historical_rdu=0 + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup_dir=\"\$ddir_${source_no}\" + compute_rdu -m "${backup_dir}" + historical_rdu=$(( ${historical_rdu} + ${rdu} )) + source_no=$(( ${source_no} + 1 )) + done + + historical_rdu=$(( (${historical_rdu}+1023) / 1024 )) + + if [ "${no_sources}" -gt 0 ]; then + ccollect_backups="" + else + ccollect_backups="(none performed) " + fi + + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup=\"\$source_${source_no}\" + ccollect_backups="${ccollect_backups}\"${backup}\" " + source_no=$(( ${source_no} + 1 )) + done echo ${current_date} Tx+Rx: ${tx_rx} MB -- \ - Disk Usage: ${real_usage} MB -- \ - Backup set \(${interval}\): ${ccollect_backups} -- \ - Historical backups usage: ${rdu} GB >> ${per_report} - + Disk Usage: ${total_rdu} MB -- \ + Backup set \(${interval}\):${ccollect_backups} -- \ + Historical backups usage: ${historical_rdu} GB >> "${per_report}" + echo "Total Data Transfer: ${tx_rx} MB -- Total Disk Usage: ${total_rdu} MB -- Total Historical backups usage: ${historical_rdu} GB" + # ---------------------------------------- # Send a status email if there is an error # ---------------------------------------- - ccollect_we=`cat ${log} | ccollect_analyse_logs.sh we | wc -l` + ccollect_we=`cat "${log_file}" | ccollect_analyse_logs.sh we | wc -l` if [ ${ccollect_we} -ge 1 ]; then - send_email ${tmp_report} "ERROR ccollect for ${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" + send_email "${tmp_report}" "ERROR ccollect for${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" fi # -------------------- @@ -313,27 +395,27 @@ send_report() if [ ${report_interval} == ${interval} ] || [ ${interval} == "monthly" ]; then # Make reporting atomic to handle concurrent ccollect_mgr instances - mv ${per_report} ${per_report}.$$ - cat ${per_report}.$$ >> ${per_report}.history + mv "${per_report}" "${per_report}".$$ + cat "${per_report}".$$ >> "${per_report}".history # Calculate total amount of bytes sent and received - tx_rx=`cat ${per_report}.$$ | \ + tx_rx=`cat "${per_report}".$$ | \ awk '{ { transfer += $5 } } END \ { printf "%u",(transfer); }'` # Send email - send_email ${per_report}.$$ "${report_interval} ccollect status for ${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" - rm ${per_report}.$$ + send_email "${per_report}.$$" "${report_interval} ccollect status for${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" + rm "${per_report}.$$" fi - rm ${tmp_report} + rm "${tmp_report}" } # ------------------------------------------------ # Add to PATH in case we're launching from crontab # ------------------------------------------------ -PATH=${ADD_TO_PATH}:${PATH} +PATH="${ADD_TO_PATH}:${PATH}" # -------------- # Default Values @@ -349,59 +431,74 @@ weekly_backup=1 # Set the monthly backup interval. Default is 4th Monday of every month monthly_backup=4 -show_help=0 - # --------------------------------- # Parse command line # --------------------------------- +show_help=0 +export no_sources=0 + while [ "$#" -ge 1 ]; do case "$1" in -help) show_help=1 ;; -from) - from=$2 + from="$2" shift ;; -to) - to=$2 + to="$2" shift ;; -server|mail_server) - mail_server=$2 + mail_server="$2" shift ;; -weekly) - weekly_backup=$2 + weekly_backup="$2" shift ;; -monthly) - monthly_backup=$2 + monthly_backup="$2" shift ;; -warning_size) - warning_transfer_size=$2 + warning_transfer_size="$2" + shift + ;; + -abort_size) + abort_transfer_size="$2" shift ;; -report) - report_interval=$2 + report_interval="$2" shift ;; -*) - ccollect_options="${ccollect_options} $1" + ccollect_options="${ccollect_options}$1 " ;; daily|weekly|monthly) ;; *) - ccollect_backups="${ccollect_backups}$1 " + eval backup=\"\$1\" + ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" + if [ "${ret}" -ne 0 ]; then + echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping." + else + ccollect_backups="${ccollect_backups} \"$1\"" + eval export source_${no_sources}=\"\$1\" + eval export ddir_${no_sources}="${ddir}" +# eval echo Adding source \"\$source_${no_sources}\" -- \"\$ddir_${no_sources}\" + no_sources="$(( ${no_sources} + 1 ))" + fi ;; esac shift done -if [ "${ccollect_backups}" == "" ] || [ ${show_help} -eq 1 ]; then - echo +if [ "${no_sources}" -lt 1 ] || [ ${show_help} -eq 1 ]; then + echo "" echo "$0: Syntax" echo " -help This help" echo " -from From email address (ex.: -from nas@home.com)" @@ -414,58 +511,32 @@ if [ "${ccollect_backups}" == "" ] || [ ${show_help} -eq 1 ]; then echo " -report Frequency of report email (daily, weekly or monthly)" echo " Default is ${report_interval}" echo " -warning_size Send a warning email if the transfer size exceed this" - echo " Default is ${warning_transfer_size}" + echo " Default is ${warning_transfer_size} MB" + echo " -abort_size Abort and send an error email if the transfer size exceed this" + echo " Default is ${abort_transfer_size} MB" echo "" echo " other parameters are transfered to ccollect" - echo + echo "" exit 0 fi -# ------------------------------------------------------------------ -# Check if ccollect_mgr is already running for the given backup sets -# ------------------------------------------------------------------ - +#echo Backup sets:"${ccollect_backups}" check_running_backups -if [ "${ccollect_backups}" == "" ]; then +if [ "${no_sources}" -lt 1 ]; then echo "No backup sets are reachable" exit 1 fi -# ---------------------------------------------------------- -# Set the interval type -# -# Here, weeklys are Mondays, and Monthlys are the 4th Monday -# ---------------------------------------------------------- - find_interval ${weekly_backup} ${monthly_backup} echo Interval: ${interval} -# -------------- -# Do the backups -# -------------- -TEMP_LOG=${CCOLLECT_CONF}/log.$$ -echo Backup sets: ${ccollect_backups} - -# Check the transfer size (to issue email warning) precheck_transfer_size -${CCOLLECT} ${ccollect_options} ${interval} ${ccollect_backups} | tee ${TEMP_LOG} +"${CCOLLECT}" ${ccollect_options} ${interval} ${ccollect_backups} | tee "${TEMP_LOG}" -# --------------------------------------- -# Move log to the last backup of the set -# --------------------------------------- +build_backup_dir_list +move_log -move_log ${ccollect_backups} +send_report -# ----------------------------------------- -# Compute the physical amount of disk usage -# ----------------------------------------- - -compute_total_rdu ${ccollect_backups} - -# ----------------- -# Send status email -# ----------------- - -send_report ${new_log} From 72034cb0420fce291b9cc95fc1baba1050b8f9ef Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 26 Mar 2010 09:36:22 +0100 Subject: [PATCH 072/177] add hints on pre_exec error handling Signed-off-by: Nico Schottelius --- contrib/{patrick => ccollect_mgr}/README | 0 contrib/{patrick => ccollect_mgr}/S60ccollect_example | 0 contrib/{patrick => ccollect_mgr}/ccollect_mgr.sh | 0 contrib/{patrick => ccollect_mgr}/rdu | 0 doc/ccollect.text | 8 +++++++- 5 files changed, 7 insertions(+), 1 deletion(-) rename contrib/{patrick => ccollect_mgr}/README (100%) rename contrib/{patrick => ccollect_mgr}/S60ccollect_example (100%) rename contrib/{patrick => ccollect_mgr}/ccollect_mgr.sh (100%) rename contrib/{patrick => ccollect_mgr}/rdu (100%) diff --git a/contrib/patrick/README b/contrib/ccollect_mgr/README similarity index 100% rename from contrib/patrick/README rename to contrib/ccollect_mgr/README diff --git a/contrib/patrick/S60ccollect_example b/contrib/ccollect_mgr/S60ccollect_example similarity index 100% rename from contrib/patrick/S60ccollect_example rename to contrib/ccollect_mgr/S60ccollect_example diff --git a/contrib/patrick/ccollect_mgr.sh b/contrib/ccollect_mgr/ccollect_mgr.sh similarity index 100% rename from contrib/patrick/ccollect_mgr.sh rename to contrib/ccollect_mgr/ccollect_mgr.sh diff --git a/contrib/patrick/rdu b/contrib/ccollect_mgr/rdu similarity index 100% rename from contrib/patrick/rdu rename to contrib/ccollect_mgr/rdu diff --git a/doc/ccollect.text b/doc/ccollect.text index a751abf..56e575b 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -335,11 +335,14 @@ If you add '$CCOLLECT_CONF/defaults/`pre_exec`' or will start `pre_exec` before the whole backup process and `post_exec` after backup of all sources is done. +If `pre_exec` exits with a non-zero return code, the whole backup +process will be aborted. + The `pre_exec` and `post_exec` script can access the following exported variables: - 'INTERVAL': the interval selected (`daily`) - 'no_sources': number of sources to backup (`2`) -- 'source_$no': name of the source, '$no' starts at 0 +- 'source_$no': name of the source, '$no' starts at 0 (`$source_0`) The following example describes how to report free disk space in human readable format before and after the whole backup process: @@ -605,6 +608,9 @@ respectively after doing the backup for *this specific* source. If you want to have pre-/post-exec before and after *all* backups, see above for general configuration. +If `pre_exec` exits with a non-zero return code, the backup +process of `this source` will be aborted (i.e. backup skipped). + The `post_exec` script can access the following exported variables from ccollect: From 63686c3598ab7fd84b2167a59491e7f29fc8ceb7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 26 Mar 2010 09:37:25 +0100 Subject: [PATCH 073/177] more changes for the next release Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index 42f3007..119514f 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -7,3 +7,4 @@ * Export destination_* to source specific post_exec (Nico Schottelius) * Update documentation regarding exported variables (Nico Schottelius) * Simplify time calculation (Nico Schottelius) + * Documentate pre_exec error handling (Nico Schottelius) From fdb68e1adeeedc22ed74cd0209a59251c784b9f6 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 8 Apr 2010 22:00:45 +0200 Subject: [PATCH 074/177] add long path for documentation Signed-off-by: Nico Schottelius --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3c092f3..bd5869d 100644 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ path_destination=${path_dir}/${CCOLLECT_DEST} # where to publish host=localhost dir=/home/users/nico/privat/rechner/netz/seiten/www.nico.schottelius.org/src/software/ccollect -docdir=${dir}/doc +docdir=${dir}/documentation # # Asciidoc will be used to generate other formats later From a8c34581ea90203d90f45e8a8c9bb261ed446af9 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Apr 2010 21:05:40 +0200 Subject: [PATCH 075/177] cleanup --- ccollect.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index ba91e49..172e256 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -42,7 +42,7 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" VERSION="0.8.1" -RELEASE="2009-11-XX" +RELEASE="2010-03-26" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" @@ -101,7 +101,7 @@ pcmd() # # ssh-"feature": we cannot do '... read ...; ssh ...; < file', # because ssh reads stdin! -n does not work -> does not ask for password -# Alsa allow deletion for files without the given suffix +# Also allow deletion for files without the given suffix # delete_from_file() { @@ -252,11 +252,9 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Get current source # - eval name=\"\$source_${source_no}\" + eval export name=\"\$source_${source_no}\" source_no=$((${source_no}+1)) - export name - # # Start ourself, if we want parallel execution # From 43bba003b2ef15547d6abd25f1a23166d3b9304a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 9 Apr 2010 21:06:24 +0200 Subject: [PATCH 076/177] more cleanups Signed-off-by: Nico Schottelius --- doc/ccollect.text | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index 56e575b..b265459 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -646,7 +646,7 @@ Detailed description of "rsync_failure_codes" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ If you have the file `rsync_failure_codes` in your source configuration directory, it should contain a newline-separated list of numbers representing -rsync exit codes. If rsync exits with any code in this list, a marker will +rsync exit codes. If rsync exits with any code in this list, a marker will be left in the destination directory indicating failure of this backup. If you have enabled delete_incomplete, then this backup will be deleted during the next ccollect run on the same interval. @@ -656,7 +656,7 @@ Detailed description of "mtime" ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ By default, ccollect.sh chooses the most recent backup directory for cloning or the oldest for deletion based on the directory's last change time (ctime). -With this option, the sorting is done based on modification time (mtime). With +With this option, the sorting is done based on modification time (mtime). With this version of ccollect, the ctime and mtime of your backups will normally be the same and this option has no effect. However, if you, for example, move your backups to another hard disk using cp -a or rsync -a, you should use this From b13ed10eafaf68bbd30c6343662d57fbb095c79d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 27 May 2010 17:11:32 +0200 Subject: [PATCH 077/177] add start script from thorsten Signed-off-by: Nico Schottelius --- .../thorsten_start_ccollect/start_ccollect | 72 +++++++++++++++++++ doc/changes/next | 1 + 2 files changed, 73 insertions(+) create mode 100644 contrib/thorsten_start_ccollect/start_ccollect diff --git a/contrib/thorsten_start_ccollect/start_ccollect b/contrib/thorsten_start_ccollect/start_ccollect new file mode 100644 index 0000000..5a774ed --- /dev/null +++ b/contrib/thorsten_start_ccollect/start_ccollect @@ -0,0 +1,72 @@ +#!/bin/bash + +# Backup-Ordner +BACKUP_DIR="/mnt" + +# ccollect_logwrapper-Skript +CCOLLECT_LOGWRAPPER="./ccollect_logwrapper.sh" + +# letzte Sicherung für Gruppe daily, weekly und monthly in Backup-Ordner ermitteln +DATE_DAILY=` ls $BACKUP_DIR | grep daily | sort -r | sed -e'2,$d' | cut -f 2 -d.` +DATE_WEEKLY=` ls $BACKUP_DIR | grep weekly | sort -r | sed -e'2,$d' | cut -f 2 -d.` +DATE_MONTHLY=`ls $BACKUP_DIR | grep monthly | sort -r | sed -e'2,$d' | cut -f 2 -d.` +DATE_YEARLY=` ls $BACKUP_DIR | grep yearly | sort -r | sed -e'2,$d' | cut -f 2 -d.` + +# Falls Leerstring diesen mit "altem Datum" füllen +if [ -z "$DATE_DAILY" ] ; then DATE_DAILY="20000101-0101" ; fi +if [ -z "$DATE_WEEKLY" ] ; then DATE_WEEKLY="20000101-0101" ; fi +if [ -z "$DATE_MONTHLY" ] ; then DATE_MONTHLY="20000101-0101" ; fi +if [ -z "$DATE_YEARLY" ] ; then DATE_YEARLY="20000101-0101" ; fi + +echo current: $DATE_CUR +echo last daily: $DATE_DAILY +echo last weekly: $DATE_WEEKLY +echo last monthly: $DATE_MONTHLY +echo last yearly: $DATE_YEARLY + +# Datum date-konform wandeln +# Achtung: mit bash - nicht mit sh möglich! +# Alternativ mit expr... konvertieren + +DATE_DAILY=${DATE_DAILY:0:4}-${DATE_DAILY:4:2}-${DATE_DAILY:6:2}" "${DATE_DAILY:9:2}:${DATE_DAILY:11:2}:00 +DATE_WEEKLY=${DATE_WEEKLY:0:4}-${DATE_WEEKLY:4:2}-${DATE_WEEKLY:6:2}" "${DATE_WEEKLY:9:2}:${DATE_WEEKLY:11:2}:00 +DATE_MONTHLY=${DATE_MONTHLY:0:4}-${DATE_MONTHLY:4:2}-${DATE_MONTHLY:6:2}" "${DATE_MONTHLY:9:2}:${DATE_MONTHLY:11:2}:00 +DATE_YEARLY=${DATE_YEARLY:0:4}-${DATE_YEARLY:4:2}-${DATE_YEARLY:6:2}" "${DATE_YEARLY:9:2}:${DATE_YEARLY:11:2}:00 +DATE_CUR=`date "+%Y-%m-%d %T"` + +# Bei Bedarf Backups durchführen + +if [ `date --date "$DATE_YEARLY" +%Y` -ne `date --date "$DATE_CUR" +%Y` ] +then + + # Jahresbackup erzeugen + echo monthly backup started + source $CCOLLECT_LOGWRAPPER -a yearly + +elif [ `date --date "$DATE_MONTHLY" +%Y%m` -ne `date --date "$DATE_CUR" +%Y%m` ] +then + + # Monatsbackup erzeugen + echo monthly backup started + source $CCOLLECT_LOGWRAPPER -a monthly + +elif [ `date --date "$DATE_WEEKLY" +%Y%W` -ne `date --date "$DATE_CUR" +%Y%W` ] +then + + # Wochenbackup erzeugen + echo weekly backup started + source $CCOLLECT_LOGWRAPPER -a weekly + +elif [ `date --date "$DATE_DAILY" +%Y%j` -ne `date --date "$DATE_CUR" +%Y%j` ] +then + + # Tagesbackup erzeugen + echo daily backup started + source $CCOLLECT_LOGWRAPPER -a daily + +else + + # nichts zu tun + echo nothing to do + +fi diff --git a/doc/changes/next b/doc/changes/next index 119514f..8226b02 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -8,3 +8,4 @@ * Update documentation regarding exported variables (Nico Schottelius) * Simplify time calculation (Nico Schottelius) * Documentate pre_exec error handling (Nico Schottelius) + * Added start script (Thorsten Elle) From ac7c703ff07753690b4406a9754ea4ffd53a2afc Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 27 May 2010 17:20:11 +0200 Subject: [PATCH 078/177] add autofs hint Signed-off-by: Nico Schottelius --- doc/ccollect.text | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/ccollect.text b/doc/ccollect.text index b265459..a8b83f2 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -971,6 +971,31 @@ you can enter your password (have a look at screen(1), especially "C-a M" and "C-a _", for more information). +Backup fails, if autofs is running, but sources not reachable +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +If you are trying to backup a system containing paths that are managed +by autofs, you may run into this error: + +------------------------------------------------------------------------------- +2009-12-01-23:14:15: ccollect 0.8.1: Beginning backup using interval monatlich +[ikn] 2009-12-01-23:14:15: Beginning to backup +[ikn] 2009-12-01-23:14:15: Executing /home/users/nico/ethz/ccollect/sources/ikn/pre_exec ... +Enter LUKS passphrase: +[ikn] Command successful. +[ikn] key slot 0 unlocked. +[ikn] 2009-12-01-23:14:23: Finished /home/users/nico/ethz/ccollect/sources/ikn/pre_exec (return code 0). [ikn] directory has vanished: "/home/users/nico/privat/firmen/ethz/autofs/projects" +[ikn] directory has vanished: "/home/users/nico/privat/firmen/ethz/autofs/scratch" +[ikn] directory has vanished: "/home/users/nico/privat/firmen/ethz/autofs/sgscratch" +[ikn] directory has vanished: "/home/users/nico/privat/firmen/ethz/autofs/supp" +[ikn] directory has vanished: "/home/users/nico/privat/firmen/ethz/autofs/sysadmin" +[ikn] rsync warning: some files vanished before they could be transferred (code 24) at main.c(1057) [sender=3.0.6] +[ikn] 2009-12-01-23:44:23: Source / is not readable. Skipping. +------------------------------------------------------------------------------- + +Thus, if you are unsure whether autofs paths can be mounted during backup, +stop autofs in pre_exec and reenable it in post_exec. + + Examples -------- From 64824cb3b16b33377d112186a9f8837446b7e98c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 27 May 2010 17:22:29 +0200 Subject: [PATCH 079/177] more changes Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index 8226b02..a8f9d3c 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -9,3 +9,4 @@ * Simplify time calculation (Nico Schottelius) * Documentate pre_exec error handling (Nico Schottelius) * Added start script (Thorsten Elle) + * Documentate autofs hint (Nico Schottelius) From 9819c718b1a971f4ac55f700fd5851f072aa823b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Thu, 27 May 2010 17:37:20 +0200 Subject: [PATCH 080/177] begin rename from .sh to "" Signed-off-by: Nico Schottelius --- ccollect.sh => ccollect | 0 tools/README | 30 +++++++++---------- ...lect_add_source.sh => ccollect_add_source} | 0 ..._analyse_logs.sh => ccollect_analyse_logs} | 0 ...hive_config.sh => ccollect_archive_config} | 0 ..._check_config.sh => ccollect_check_config} | 0 ...elete_source.sh => ccollect_delete_source} | 0 ...t_intervals.sh => ccollect_list_intervals} | 0 ...lect_logwrapper.sh => ccollect_logwrapper} | 4 +-- tools/{ccollect_stats.sh => ccollect_stats} | 0 ...re-0.4-to-0.4.sh => config-pre-0.4-to-0.4} | 0 ...o-0.4.sub.sh => config-pre-0.4-to-0.4.sub} | 0 ...re-0.6-to-0.6.sh => config-pre-0.6-to-0.6} | 2 +- ...o-0.6.sub.sh => config-pre-0.6-to-0.6.sub} | 0 ...re-0.7-to-0.7.sh => config-pre-0.7-to-0.7} | 2 +- ...o-0.7.sub.sh => config-pre-0.7-to-0.7.sub} | 0 ...-compare.sh => gnu-du-backup-size-compare} | 0 tools/{report_success.sh => report_success} | 0 18 files changed, 19 insertions(+), 19 deletions(-) rename ccollect.sh => ccollect (100%) rename tools/{ccollect_add_source.sh => ccollect_add_source} (100%) rename tools/{ccollect_analyse_logs.sh => ccollect_analyse_logs} (100%) rename tools/{ccollect_archive_config.sh => ccollect_archive_config} (100%) rename tools/{ccollect_check_config.sh => ccollect_check_config} (100%) rename tools/{ccollect_delete_source.sh => ccollect_delete_source} (100%) rename tools/{ccollect_list_intervals.sh => ccollect_list_intervals} (100%) rename tools/{ccollect_logwrapper.sh => ccollect_logwrapper} (94%) rename tools/{ccollect_stats.sh => ccollect_stats} (100%) rename tools/{config-pre-0.4-to-0.4.sh => config-pre-0.4-to-0.4} (100%) rename tools/{config-pre-0.4-to-0.4.sub.sh => config-pre-0.4-to-0.4.sub} (100%) rename tools/{config-pre-0.6-to-0.6.sh => config-pre-0.6-to-0.6} (96%) rename tools/{config-pre-0.6-to-0.6.sub.sh => config-pre-0.6-to-0.6.sub} (100%) rename tools/{config-pre-0.7-to-0.7.sh => config-pre-0.7-to-0.7} (96%) rename tools/{config-pre-0.7-to-0.7.sub.sh => config-pre-0.7-to-0.7.sub} (100%) rename tools/{gnu-du-backup-size-compare.sh => gnu-du-backup-size-compare} (100%) rename tools/{report_success.sh => report_success} (100%) diff --git a/ccollect.sh b/ccollect similarity index 100% rename from ccollect.sh rename to ccollect diff --git a/tools/README b/tools/README index 7312018..cc25edb 100644 --- a/tools/README +++ b/tools/README @@ -1,19 +1,19 @@ Files and their tasks / destinations: -ccollect_add_source.sh: bin -ccollect_analyse_logs.sh: bin -ccollect_create_source2.sh: ??? -ccollect_create_source.sh: ??? -ccollect_delete_source.sh: bin -ccollect_list_intervals.sh: bin -ccollect_logwrapper.sh: bin -ccollect-stats.sh: ??? +ccollect_add_source: bin +ccollect_analyse_logs: bin +ccollect_create_source2: ??? +ccollect_create_source: ??? +ccollect_delete_source: bin +ccollect_list_intervals: bin +ccollect_logwrapper: bin +ccollect-stats: ??? config-pre-0.4-to-0.4.BUGS: only to be used for converting -config-pre-0.4-to-0.4.sh: only to be used for converting -config-pre-0.4-to-0.4.sub.sh: only to be used for converting -config-pre-0.6-to-0.6.sh: only to be used for converting -config-pre-0.6-to-0.6.sub.sh: only to be used for converting -config-pre-0.7-to-0.7.sh: only to be used for converting -config-pre-0.7-to-0.7.sub.sh: only to be used for converting -gnu-du-backup-size-compare.sh +config-pre-0.4-to-0.4: only to be used for converting +config-pre-0.4-to-0.4.sub: only to be used for converting +config-pre-0.6-to-0.6: only to be used for converting +config-pre-0.6-to-0.6.sub: only to be used for converting +config-pre-0.7-to-0.7: only to be used for converting +config-pre-0.7-to-0.7.sub: only to be used for converting +gnu-du-backup-size-compare README diff --git a/tools/ccollect_add_source.sh b/tools/ccollect_add_source similarity index 100% rename from tools/ccollect_add_source.sh rename to tools/ccollect_add_source diff --git a/tools/ccollect_analyse_logs.sh b/tools/ccollect_analyse_logs similarity index 100% rename from tools/ccollect_analyse_logs.sh rename to tools/ccollect_analyse_logs diff --git a/tools/ccollect_archive_config.sh b/tools/ccollect_archive_config similarity index 100% rename from tools/ccollect_archive_config.sh rename to tools/ccollect_archive_config diff --git a/tools/ccollect_check_config.sh b/tools/ccollect_check_config similarity index 100% rename from tools/ccollect_check_config.sh rename to tools/ccollect_check_config diff --git a/tools/ccollect_delete_source.sh b/tools/ccollect_delete_source similarity index 100% rename from tools/ccollect_delete_source.sh rename to tools/ccollect_delete_source diff --git a/tools/ccollect_list_intervals.sh b/tools/ccollect_list_intervals similarity index 100% rename from tools/ccollect_list_intervals.sh rename to tools/ccollect_list_intervals diff --git a/tools/ccollect_logwrapper.sh b/tools/ccollect_logwrapper similarity index 94% rename from tools/ccollect_logwrapper.sh rename to tools/ccollect_logwrapper index ac7f496..95e6fbb 100755 --- a/tools/ccollect_logwrapper.sh +++ b/tools/ccollect_logwrapper @@ -58,7 +58,7 @@ _echo "Starting with arguments: $@" touch "${ccollect_logfile}" || _exit_err "Failed to create ${ccollect_logfile}" # First line in the logfile is always the commandline -echo ccollect.sh "$@" > "${ccollect_logfile}" 2>&1 -ccollect.sh "$@" >> "${ccollect_logfile}" 2>&1 +echo ccollect "$@" > "${ccollect_logfile}" 2>&1 +ccollect "$@" >> "${ccollect_logfile}" 2>&1 _echo "Finished." diff --git a/tools/ccollect_stats.sh b/tools/ccollect_stats similarity index 100% rename from tools/ccollect_stats.sh rename to tools/ccollect_stats diff --git a/tools/config-pre-0.4-to-0.4.sh b/tools/config-pre-0.4-to-0.4 similarity index 100% rename from tools/config-pre-0.4-to-0.4.sh rename to tools/config-pre-0.4-to-0.4 diff --git a/tools/config-pre-0.4-to-0.4.sub.sh b/tools/config-pre-0.4-to-0.4.sub similarity index 100% rename from tools/config-pre-0.4-to-0.4.sub.sh rename to tools/config-pre-0.4-to-0.4.sub diff --git a/tools/config-pre-0.6-to-0.6.sh b/tools/config-pre-0.6-to-0.6 similarity index 96% rename from tools/config-pre-0.6-to-0.6.sh rename to tools/config-pre-0.6-to-0.6 index 58a538a..1a4bc2f 100755 --- a/tools/config-pre-0.6-to-0.6.sh +++ b/tools/config-pre-0.6-to-0.6 @@ -30,7 +30,7 @@ if [ $# -ne 1 ]; then fi dir="$1" -script=$(echo $0 | sed 's/\.sh$/.sub.sh/') +script="${0}.sub" find "${dir}/sources/" -type f -name rsync_options -exec "${script}" {} \; diff --git a/tools/config-pre-0.6-to-0.6.sub.sh b/tools/config-pre-0.6-to-0.6.sub similarity index 100% rename from tools/config-pre-0.6-to-0.6.sub.sh rename to tools/config-pre-0.6-to-0.6.sub diff --git a/tools/config-pre-0.7-to-0.7.sh b/tools/config-pre-0.7-to-0.7 similarity index 96% rename from tools/config-pre-0.7-to-0.7.sh rename to tools/config-pre-0.7-to-0.7 index 6d540ab..3098728 100755 --- a/tools/config-pre-0.7-to-0.7.sh +++ b/tools/config-pre-0.7-to-0.7 @@ -29,7 +29,7 @@ if [ $# -ne 1 ]; then fi dir="$1" -script=$(echo $0 | sed 's/\.sh$/.sub.sh/') +script="${0}.sub" find "${dir}/sources/" -name destination -exec "${script}" {} \; diff --git a/tools/config-pre-0.7-to-0.7.sub.sh b/tools/config-pre-0.7-to-0.7.sub similarity index 100% rename from tools/config-pre-0.7-to-0.7.sub.sh rename to tools/config-pre-0.7-to-0.7.sub diff --git a/tools/gnu-du-backup-size-compare.sh b/tools/gnu-du-backup-size-compare similarity index 100% rename from tools/gnu-du-backup-size-compare.sh rename to tools/gnu-du-backup-size-compare diff --git a/tools/report_success.sh b/tools/report_success similarity index 100% rename from tools/report_success.sh rename to tools/report_success From 073b1138b76a867eb7cd0337853425c4c5578031 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 12 Jan 2011 09:14:58 +0100 Subject: [PATCH 081/177] move gnu-du-backup-size-compare to old/ Signed-off-by: Nico Schottelius --- tools/{ => old}/gnu-du-backup-size-compare | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tools/{ => old}/gnu-du-backup-size-compare (100%) diff --git a/tools/gnu-du-backup-size-compare b/tools/old/gnu-du-backup-size-compare similarity index 100% rename from tools/gnu-du-backup-size-compare rename to tools/old/gnu-du-backup-size-compare From cbf34deade296075f83453fa27e8ac70395bc43c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 12 Jan 2011 09:15:46 +0100 Subject: [PATCH 082/177] also remove gnu-du-backup-size-compare from docs Signed-off-by: Nico Schottelius --- tools/README | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/README b/tools/README index cc25edb..bbc9706 100644 --- a/tools/README +++ b/tools/README @@ -15,5 +15,4 @@ config-pre-0.6-to-0.6: only to be used for converting config-pre-0.6-to-0.6.sub: only to be used for converting config-pre-0.7-to-0.7: only to be used for converting config-pre-0.7-to-0.7.sub: only to be used for converting -gnu-du-backup-size-compare README From 540d860e284633ee52fd25f1498883803d209b79 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 01:39:00 +0100 Subject: [PATCH 083/177] Do not use --archive on source check Signed-off-by: Nico Schottelius --- ccollect | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 172e256..a7ce55b 100755 --- a/ccollect +++ b/ccollect @@ -388,9 +388,9 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # - # Rsync standard options + # Rsync standard options (archive will be added after is-up-check) # - set -- "$@" "--archive" "--delete" "--numeric-ids" "--relative" \ + set -- "$@" "--delete" "--numeric-ids" "--relative" \ "--delete-excluded" "--sparse" # @@ -437,6 +437,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _exit_err "Source ${source} is not readable. Skipping." fi + # + # Add --archive for real backup (looks nice in front) + # + set -- "--archive" "$@" + # # Check: destination exists? # From c10b46111b83104035e83dc128bd792c9d78a9af Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 01:40:30 +0100 Subject: [PATCH 084/177] more changes Signed-off-by: Nico Schottelius --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/changes/next b/doc/changes/next index a8f9d3c..a63104b 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -10,3 +10,4 @@ * Documentate pre_exec error handling (Nico Schottelius) * Added start script (Thorsten Elle) * Documentate autofs hint (Nico Schottelius) + * Speedup source-is-up check and remove --archive (Nico Schottelius) From 9cb8b99353591bc061d380331965f952efcf1200 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 01:54:55 +0100 Subject: [PATCH 085/177] simply check for incomplete backups Signed-off-by: Nico Schottelius --- ccollect | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/ccollect b/ccollect index a7ce55b..62b3a05 100755 --- a/ccollect +++ b/ccollect @@ -448,18 +448,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do ( pcmd cd "${ddir}" ) || _exit_err "Cannot change to ${ddir}. Skipping." # - # Check: incomplete backups? (needs echo to remove newlines) + # Check incomplete backups (needs echo to remove newlines) # - # *.marker: not possible, creates an error, if no *.marker exists - # -> catch return value + pcmd ls -d1 "${ddir}" | grep "${CMARKER}\$" > "${TMP}" ||\ + _exit_err "Cannot list incomplete files" - pcmd ls -d1 "${ddir}/"*"${CMARKER}" > "${TMP}" 2>/dev/null; ret=$? - - if [ "${ret}" -eq 0 ]; then - _techo "Incomplete backups: $(echo $(cat "${TMP}"))" - if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" "${CMARKER}" - fi + _techo "Incomplete backups: $(echo $(cat "${TMP}"))" + if [ -f "${c_delete_incomplete}" ]; then + delete_from_file "${TMP}" "${CMARKER}" fi # From e39e53d0fbed1c2b1e56c539c8a661e70d164db7 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:04:10 +0100 Subject: [PATCH 086/177] simply check for incomplete backups Signed-off-by: Nico Schottelius --- ccollect | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 62b3a05..cd8cf99 100755 --- a/ccollect +++ b/ccollect @@ -450,7 +450,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check incomplete backups (needs echo to remove newlines) # - pcmd ls -d1 "${ddir}" | grep "${CMARKER}\$" > "${TMP}" ||\ + pcmd ls -d1 "${ddir}" | grep "${CMARKER}\$" > "${TMP}" || \ _exit_err "Cannot list incomplete files" _techo "Incomplete backups: $(echo $(cat "${TMP}"))" @@ -461,6 +461,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: maximum number of backups is reached? # + # FIXME: simplify, grep, cd? => no star, does not get expanded on failure count="$(pcmd ls -d1 "${ddir}/${INTERVAL}."*"/" | wc -l | sed 's/^ *//g')" \ || _exit_err "Counting backups failed" From d2cd0c48f34fa0249bc4a20deb354cf4f8c6fe8a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:04:31 +0100 Subject: [PATCH 087/177] begin to remove support for backing up to remote hosts Signed-off-by: Nico Schottelius --- ccollect | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/ccollect b/ccollect index cd8cf99..f376ca0 100755 --- a/ccollect +++ b/ccollect @@ -88,16 +88,6 @@ add_name() awk "{ print \"[${name}] \" \$0 }" } -# -# Prepend "ssh ${remote_host}", if backing up to a remote host -# -pcmd() -{ - [ "${remote_host}" ] && set -- "ssh" "${remote_host}" "$@" - - "$@" -} - # # ssh-"feature": we cannot do '... read ...; ssh ...; < file', # because ssh reads stdin! -n does not work -> does not ask for password From 0d5b2992c0009cde8910c75512b7f8d824c67984 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:06:20 +0100 Subject: [PATCH 088/177] Remove first pcmd use and make cd ddir default for further actions Signed-off-by: Nico Schottelius --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index f376ca0..8673128 100755 --- a/ccollect +++ b/ccollect @@ -435,7 +435,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: destination exists? # - ( pcmd cd "${ddir}" ) || _exit_err "Cannot change to ${ddir}. Skipping." + cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping." # # Check incomplete backups (needs echo to remove newlines) From 093de8b0a16048b753191790591574df70608602 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:09:27 +0100 Subject: [PATCH 089/177] simplify backup count code Signed-off-by: Nico Schottelius --- ccollect | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/ccollect b/ccollect index 8673128..01ef9b1 100755 --- a/ccollect +++ b/ccollect @@ -440,7 +440,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check incomplete backups (needs echo to remove newlines) # - pcmd ls -d1 "${ddir}" | grep "${CMARKER}\$" > "${TMP}" || \ + ls -1 | grep "${CMARKER}\$" > "${TMP}" || \ _exit_err "Cannot list incomplete files" _techo "Incomplete backups: $(echo $(cat "${TMP}"))" @@ -451,8 +451,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: maximum number of backups is reached? # - # FIXME: simplify, grep, cd? => no star, does not get expanded on failure - count="$(pcmd ls -d1 "${ddir}/${INTERVAL}."*"/" | wc -l | sed 's/^ *//g')" \ + count="$(ls -1 | grep "^${INTERVAL}\\." | wc -l | sed 's/^ *//g')" \ || _exit_err "Counting backups failed" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" From 5066f417a937c4c9e384f821ce2ec0951c8c78ca Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:11:28 +0100 Subject: [PATCH 090/177] remove pcmd from backup listing code Signed-off-by: Nico Schottelius --- ccollect | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 01ef9b1..a3b14bb 100755 --- a/ccollect +++ b/ccollect @@ -461,8 +461,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do remove="$((${count} - ${substract}))" _techo "Removing ${remove} backup(s)..." - pcmd ls -${TSORT}d1r "${ddir}/${INTERVAL}."*"/" | - head -n "${remove}" > "${TMP}" || \ + ls -${TSORT}d1r | grep "${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ _exit_err "Listing old backups failed" delete_from_file "${TMP}" From a949a9e8e747b776c92aa3cd6c2f161e02f8cb54 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:17:00 +0100 Subject: [PATCH 091/177] remove all other pcmd calls Signed-off-by: Nico Schottelius --- ccollect | 12 ++++++------ doc/changes/next | 1 + 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ccollect b/ccollect index a3b14bb..b3254e1 100755 --- a/ccollect +++ b/ccollect @@ -107,7 +107,7 @@ delete_from_file() done < "${file}" _techo "Removing $@ ..." [ "${VVERBOSE}" ] && echo rm "$@" - pcmd rm -rf "$@" || _exit_err "Removing $@ failed." + rm -rf "$@" || _exit_err "Removing $@ failed." } display_version() @@ -470,7 +470,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check for backup directory to clone from: Always clone from the latest one! # - last_dir="$(pcmd ls -${TSORT}p1 "${ddir}" | grep '/$' | head -n 1)" || \ + last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." # @@ -490,13 +490,13 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Creating directory ${destination_dir} ..." [ "${VVERBOSE}" ] && echo "mkdir ${destination_dir}" - pcmd mkdir "${destination_dir}" || \ + mkdir "${destination_dir}" || \ _exit_err "Creating directory ${destination_dir} failed. Skipping." # # added marking in 0.6 (and remove it, if successful later) # - pcmd touch "${destination_dir}${CMARKER}" + touch "${destination_dir}${CMARKER}" # # the rsync part @@ -508,7 +508,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Set modification time (mtime) to current time, if sorting by mtime is enabled # - [ -f "$c_mtime" ] && pcmd touch "${destination_dir}" + [ -f "$c_mtime" ] && touch "${destination_dir}" # # Check if rsync exit code indicates failure. @@ -526,7 +526,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Remove marking here unless rsync failed. # if [ -z "$fail" ]; then - pcmd rm "${destination_dir}${CMARKER}" || \ + rm "${destination_dir}${CMARKER}" || \ _exit_err "Removing ${destination_dir}${CMARKER} failed." if [ "${ret}" -ne 0 ]; then _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." diff --git a/doc/changes/next b/doc/changes/next index a63104b..e865dff 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -11,3 +11,4 @@ * Added start script (Thorsten Elle) * Documentate autofs hint (Nico Schottelius) * Speedup source-is-up check and remove --archive (Nico Schottelius) + * Removed support for remote backup (see doc) (Nico Schottelius) From 410cf58067d07a30fd48aedb9dc79bb297c133f3 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:17:48 +0100 Subject: [PATCH 092/177] update version info Signed-off-by: Nico Schottelius --- ccollect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index b3254e1..bb4c200 100755 --- a/ccollect +++ b/ccollect @@ -41,8 +41,8 @@ CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" -VERSION="0.8.1" -RELEASE="2010-03-26" +VERSION="0.9" +RELEASE="2011-01-XX" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" From 758e5a9059b32c408b049d791f6321ac7e39b03c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:20:10 +0100 Subject: [PATCH 093/177] begin to remove remote_host option Signed-off-by: Nico Schottelius --- ccollect | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/ccollect b/ccollect index bb4c200..bc8015a 100755 --- a/ccollect +++ b/ccollect @@ -303,7 +303,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Read source configuration # for opt in verbose very_verbose summary exclude rsync_options \ - delete_incomplete remote_host rsync_failure_codes \ + delete_incomplete rsync_failure_codes \ mtime quiet_if_down ; do if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}" ]; then eval c_$opt=\"${backup}/$opt\" @@ -358,20 +358,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do fi fi - # - # Set pre-cmd, if we backup to a remote host. - # - if [ -f "${c_remote_host}" ]; then - remote_host="$(cat "${c_remote_host}")"; ret="$?" - if [ "${ret}" -ne 0 ]; then - _exit_err "Remote host file ${c_remote_host} is unreadable. Skipping." - fi - destination="${remote_host}:${ddir}" - else - remote_host="" - destination="${ddir}" - fi - export remote_host + destination="${ddir}" # # Parameters: ccollect defaults, configuration options, user options From c7d35464aeadb7c32701a5de9e6fe52772b05a30 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:24:21 +0100 Subject: [PATCH 094/177] remove redundant destination variable Signed-off-by: Nico Schottelius --- ccollect | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/ccollect b/ccollect index bc8015a..bb98691 100755 --- a/ccollect +++ b/ccollect @@ -358,8 +358,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do fi fi - destination="${ddir}" - # # Parameters: ccollect defaults, configuration options, user options # @@ -473,7 +471,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" export destination_dir="${ddir}/${destination_name}" - export destination_full="${destination}/${destination_name}" _techo "Creating directory ${destination_dir} ..." [ "${VVERBOSE}" ] && echo "mkdir ${destination_dir}" @@ -489,7 +486,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # the rsync part # _techo "Transferring files..." - rsync "$@" "${source}" "${destination_full}"; ret=$? + rsync "$@" "${source}" "${destination_dir}"; ret=$? _techo "Finished backup (rsync return code: $ret)." # From 8c376a31f54d9db5457f42718f72d893b93c869d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:25:24 +0100 Subject: [PATCH 095/177] remove to-remote test source Signed-off-by: Nico Schottelius --- conf/sources/to-remote/destination | 1 - conf/sources/to-remote/exclude | 1 - conf/sources/to-remote/remote_host | 1 - conf/sources/to-remote/source | 1 - 4 files changed, 4 deletions(-) delete mode 100644 conf/sources/to-remote/destination delete mode 100644 conf/sources/to-remote/exclude delete mode 100644 conf/sources/to-remote/remote_host delete mode 100644 conf/sources/to-remote/source diff --git a/conf/sources/to-remote/destination b/conf/sources/to-remote/destination deleted file mode 100644 index 8cac69d..0000000 --- a/conf/sources/to-remote/destination +++ /dev/null @@ -1 +0,0 @@ -/tmp/ccollect diff --git a/conf/sources/to-remote/exclude b/conf/sources/to-remote/exclude deleted file mode 100644 index 6b8710a..0000000 --- a/conf/sources/to-remote/exclude +++ /dev/null @@ -1 +0,0 @@ -.git diff --git a/conf/sources/to-remote/remote_host b/conf/sources/to-remote/remote_host deleted file mode 100644 index 2fbb50c..0000000 --- a/conf/sources/to-remote/remote_host +++ /dev/null @@ -1 +0,0 @@ -localhost diff --git a/conf/sources/to-remote/source b/conf/sources/to-remote/source deleted file mode 100644 index e64611b..0000000 --- a/conf/sources/to-remote/source +++ /dev/null @@ -1 +0,0 @@ -/home/users/nico/bin From 51db5e120444606c6b3b429d009dff76ef1d9863 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:31:29 +0100 Subject: [PATCH 096/177] begin makefile cleanup Signed-off-by: Nico Schottelius --- Makefile | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index bd5869d..becd519 100644 --- a/Makefile +++ b/Makefile @@ -22,8 +22,8 @@ # INSTALL=install -CCOLLECT_SOURCE=ccollect.sh -CCOLLECT_DEST=ccollect.sh +CCOLLECT_SOURCE=ccollect +CCOLLECT_DEST=ccollect LN=ln -sf ASCIIDOC=asciidoc DOCBOOKTOTEXI=docbook2x-texi @@ -116,20 +116,28 @@ install-manlink: install-man # # Tools # -TOOLS=ccollect_add_source.sh \ - ccollect_analyse_logs.sh \ - ccollect_delete_source.sh \ - ccollect_list_intervals.sh \ - ccollect_logwrapper.sh \ - ccollect_list_intervals.sh +TOOLS2=ccollect_add_source +TOOLS2 += ccollect_analyse_logs -TOOLSMAN1 = $(subst ccollect,doc/man/ccollect,$(TOOLS)) -TOOLSMAN = $(subst .sh,.text,$(TOOLSMAN1)) +TOOLS=ccollect_add_source \ + ccollect_analyse_logs \ + ccollect_delete_source \ + ccollect_list_intervals \ + ccollect_logwrapper \ + ccollect_list_intervals + +# Stick to posix +TOOLSMAN1 = $(TOOLS:ccollect=doc/man/ccollect) +TOOLSMAN = $(TOOLSMAN1:=.text) TOOLSFP = $(subst ccollect,tools/ccollect,$(TOOLS)) +## FIXME: posix make: shell? => + t2: - echo $(TOOLS) - $(TOOLSMAN) - $(TOOLSFP) + echo $(TOOLS) - $(TOOLSFP) + echo $(TOOLSMAN) + echo $(TOOLSFP) # docbook gets .htm, asciidoc directly .html @@ -191,9 +199,9 @@ dist: distclean documentation /tmp/ccollect: mkdir -p /tmp/ccollect -test: ccollect.sh /tmp/ccollect - cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect.sh daily "$$s"; done +test: $(CCOLLECT_SOURCE) /tmp/ccollect + cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect daily "$$s"; done touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker - CCOLLECT_CONF=./conf ./ccollect.sh -a daily + CCOLLECT_CONF=./conf ./ccollect -a daily touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker - CCOLLECT_CONF=./conf ./ccollect.sh -a -p daily + CCOLLECT_CONF=./conf ./ccollect -a -p daily From 05de81e0f01ebe374ba5bd530e8c0f8b856381c1 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:31:46 +0100 Subject: [PATCH 097/177] simplify backup counting code Signed-off-by: Nico Schottelius --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index bb98691..9d544ba 100755 --- a/ccollect +++ b/ccollect @@ -436,7 +436,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: maximum number of backups is reached? # - count="$(ls -1 | grep "^${INTERVAL}\\." | wc -l | sed 's/^ *//g')" \ + count="$(ls -1 | grep -c "^${INTERVAL}\\.")" \ || _exit_err "Counting backups failed" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" From 4675bf864c99d51bb199a21e8f06a0896fccaafb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:35:49 +0100 Subject: [PATCH 098/177] rewrite get incomplete backups code Signed-off-by: Nico Schottelius --- ccollect | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/ccollect b/ccollect index 9d544ba..b2ff34f 100755 --- a/ccollect +++ b/ccollect @@ -425,12 +425,13 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check incomplete backups (needs echo to remove newlines) # - ls -1 | grep "${CMARKER}\$" > "${TMP}" || \ - _exit_err "Cannot list incomplete files" + ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? - _techo "Incomplete backups: $(echo $(cat "${TMP}"))" - if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" "${CMARKER}" + if [ "$ret" -eq 0 ]; then + _techo "Incomplete backups: $(echo $(cat "${TMP}"))" + if [ -f "${c_delete_incomplete}" ]; then + delete_from_file "${TMP}" "${CMARKER}" + fi fi # From aaf43af0d9712aea32de8e63f296afea8a24f8e5 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:37:14 +0100 Subject: [PATCH 099/177] fix count of backups Signed-off-by: Nico Schottelius --- ccollect | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ccollect b/ccollect index b2ff34f..1cd295d 100755 --- a/ccollect +++ b/ccollect @@ -437,8 +437,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: maximum number of backups is reached? # - count="$(ls -1 | grep -c "^${INTERVAL}\\.")" \ - || _exit_err "Counting backups failed" + count="$(ls -1 | grep -c "^${INTERVAL}\\.")" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" From 7b65687da57df2e1e1ce62988f4a10cae250489d Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:41:22 +0100 Subject: [PATCH 100/177] simplify backup removal code Signed-off-by: Nico Schottelius --- ccollect | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 1cd295d..5a254da 100755 --- a/ccollect +++ b/ccollect @@ -442,8 +442,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then - substract="$((${c_interval} - 1))" - remove="$((${count} - ${substract}))" + remove="$((${count} - ${c_interval} + 1))" _techo "Removing ${remove} backup(s)..." ls -${TSORT}d1r | grep "${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ From d04972026f3a34cdf966d0d256dac5f52431dcf0 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:44:37 +0100 Subject: [PATCH 101/177] fix selecting removal backups code Signed-off-by: Nico Schottelius --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 5a254da..138a809 100755 --- a/ccollect +++ b/ccollect @@ -445,7 +445,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do remove="$((${count} - ${c_interval} + 1))" _techo "Removing ${remove} backup(s)..." - ls -${TSORT}d1r | grep "${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ _exit_err "Listing old backups failed" delete_from_file "${TMP}" From 9bd09f24a2d1ad4aa3272f2258f8c44330c1a7bb Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 02:47:25 +0100 Subject: [PATCH 102/177] remove directory create code: rsync does that itself Signed-off-by: Nico Schottelius --- ccollect | 5 ----- 1 file changed, 5 deletions(-) diff --git a/ccollect b/ccollect index 138a809..82e4daf 100755 --- a/ccollect +++ b/ccollect @@ -471,11 +471,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" export destination_dir="${ddir}/${destination_name}" - _techo "Creating directory ${destination_dir} ..." - [ "${VVERBOSE}" ] && echo "mkdir ${destination_dir}" - mkdir "${destination_dir}" || \ - _exit_err "Creating directory ${destination_dir} failed. Skipping." - # # added marking in 0.6 (and remove it, if successful later) # From dcb2b60c41d85e92b949f744122186c3ef997212 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 13:51:02 +0100 Subject: [PATCH 103/177] add temporary todo file Signed-off-by: Nico Schottelius --- TODO | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 TODO diff --git a/TODO b/TODO new file mode 100644 index 0000000..b80db08 --- /dev/null +++ b/TODO @@ -0,0 +1,20 @@ +Hinweis: + Zwei Quellen auf ein Ziel funktioniert nicht, da die beiden + Quellen von den unterschiedlichen Verzeichnissen linken. + +Listing: + .../* funktioniert nicht teilweise (abhängig von der Shell?) + +Isup-check: optional! + +line 318/check no_xxx => correct test? + +REMOVE ALL PCMD code + +Backup to remote can be done via ssh tunnel! + + +% remote host: Allow backup host to access our sshd + ssh -R4242:localhost:22 backupserver "ccollect interval backupremotehost" + + remove $destination (==ddir) From bdd6b1539739ebced7b4101ebb88d039c26c9dea Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 20:14:59 +0100 Subject: [PATCH 104/177] add version information Signed-off-by: Nico Schottelius --- ccollect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 82e4daf..80ad20a 100755 --- a/ccollect +++ b/ccollect @@ -41,8 +41,8 @@ CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" -VERSION="0.9" -RELEASE="2011-01-XX" +VERSION="0.9rc1" +RELEASE="2011-01-15" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" From f7f6b4d88556b79db637a68207921de5a0068004 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 15 Jan 2011 21:19:27 +0100 Subject: [PATCH 105/177] date++ Signed-off-by: Nico Schottelius --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 82e4daf..c6c39c7 100755 --- a/ccollect +++ b/ccollect @@ -1,6 +1,6 @@ #!/bin/sh # -# 2005-2009 Nico Schottelius (nico-ccollect at schottelius.org) +# 2005-2011 Nico Schottelius (nico-ccollect at schottelius.org) # # This file is part of ccollect. # From ceb2f31e984c326351e2a1ceeb3e17952318606a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 17 Jan 2011 16:49:26 +0100 Subject: [PATCH 106/177] add templates for remote backup replacement Signed-off-by: Nico Schottelius --- tools/called_from_remote_pre_exec | 30 ++++++++++++++++++++++++++++++ tools/ccollect_push | 30 ++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100755 tools/called_from_remote_pre_exec create mode 100755 tools/ccollect_push diff --git a/tools/called_from_remote_pre_exec b/tools/called_from_remote_pre_exec new file mode 100755 index 0000000..f458114 --- /dev/null +++ b/tools/called_from_remote_pre_exec @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-ccollect at schottelius.org) +# +# This file is part of ccollect. +# +# ccollect is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ccollect is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ccollect. If not, see . +# +# Support backup triggering and actual backup through +# ssh tunnel +# +# Use the supplied called_from_remote script as pre…exec +# +# + +export ssh_remote_port="$1"; shift +export backup_host="$1"; shift + +ssh -R$ssh_remote_port:127.0.0.1:22 "$backup_host" "ccollect" "$@" diff --git a/tools/ccollect_push b/tools/ccollect_push new file mode 100755 index 0000000..f458114 --- /dev/null +++ b/tools/ccollect_push @@ -0,0 +1,30 @@ +#!/bin/sh +# +# 2011 Nico Schottelius (nico-ccollect at schottelius.org) +# +# This file is part of ccollect. +# +# ccollect is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# ccollect is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with ccollect. If not, see . +# +# Support backup triggering and actual backup through +# ssh tunnel +# +# Use the supplied called_from_remote script as pre…exec +# +# + +export ssh_remote_port="$1"; shift +export backup_host="$1"; shift + +ssh -R$ssh_remote_port:127.0.0.1:22 "$backup_host" "ccollect" "$@" From d67b35da2cca9a9aca43a40411e52ebe7bbd588e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 2 Feb 2011 08:52:49 +0100 Subject: [PATCH 107/177] go back to sourcedir, so destination can be unmounted Signed-off-by: Nico Schottelius --- ccollect | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 80ad20a..02d9e5f 100755 --- a/ccollect +++ b/ccollect @@ -472,9 +472,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do export destination_dir="${ddir}/${destination_name}" # - # added marking in 0.6 (and remove it, if successful later) + # Mark backup running and go back to original directory # touch "${destination_dir}${CMARKER}" + cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}." # # the rsync part From 1ee71a9dfb862ad3334dad09b1450084ee92cc4c Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 2 Feb 2011 08:55:15 +0100 Subject: [PATCH 108/177] rename update to pub target for consistency Signed-off-by: Nico Schottelius --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index becd519..f2867d7 100644 --- a/Makefile +++ b/Makefile @@ -173,7 +173,7 @@ t2: # # Developer targets # -update: +pub: @git push publish-doc: documentation From a729e05132c55c8e9e45a5238e8cdc500f43469b Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Wed, 2 Feb 2011 09:00:51 +0100 Subject: [PATCH 109/177] adjust version Signed-off-by: Nico Schottelius --- ccollect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 2d557d9..9a6ebcf 100755 --- a/ccollect +++ b/ccollect @@ -41,8 +41,8 @@ CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" -VERSION="0.9rc1" -RELEASE="2011-01-15" +VERSION="0.9rc2" +RELEASE="2011-02-02" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" From e392792e1e5dc25b5e9ad49621b0c29e33fabc32 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 20 Aug 2011 23:31:52 +0200 Subject: [PATCH 110/177] also push to github by default Signed-off-by: Nico Schottelius --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f2867d7..f44a849 100644 --- a/Makefile +++ b/Makefile @@ -174,7 +174,8 @@ t2: # Developer targets # pub: - @git push + git push + git push github publish-doc: documentation @echo "Transferring files to ${host}" From e0d39084c6dcbc372d21c568034e4efe88e11f63 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 20 Aug 2011 23:38:38 +0200 Subject: [PATCH 111/177] begin 0.9 chapter Signed-off-by: Nico Schottelius --- doc/ccollect.text | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index a8b83f2..cacaa6d 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -0.8.1, for ccollect 0.8.1, Initial Version from 2006-01-13 +0.9, for ccollect 0.9, Initial Version from 2006-01-13 :Author Initials: NS @@ -72,8 +72,11 @@ Incompatibilities and changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Versions 0.8 and 0.9 +^^^^^^^^^^^^^^^^^^^^ + Versions 0.7 and 0.8 -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^ .The argument order changed: - Old: " [args] " @@ -91,7 +94,7 @@ change for you. - New: Options in $CCOLLECT_CONF/defaults are used as defaults (see below) Versions 0.6 and 0.7 -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^ .The format of `destination` changed: - Before 0.7 it was a (link to a) directory - As of 0.7 it is a textfile containing the destination @@ -103,7 +106,7 @@ You can update your configuration using `tools/config-pre-0.7-to-0.7.sh`. Versions 0.5 and 0.6 -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^ .The format of `rsync_options` changed: - Before 0.6 it was whitespace delimeted - As of 0.6 it is newline seperated (so you can pass whitespaces to `rsync`) @@ -120,7 +123,7 @@ XXXXX (- comes before digit). Versions 0.4 and 0.5 -^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^ Not a real incompatibilty, but seems to fit in this section: .0.5 does *NOT* require From dc67c929cfdb440b86f191b286a7e43ebb7a4748 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 20 Aug 2011 23:43:41 +0200 Subject: [PATCH 112/177] begin doc and version modifications for 0.9 Signed-off-by: Nico Schottelius --- ccollect | 4 ++-- doc/ccollect.text | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 9a6ebcf..27dbf42 100755 --- a/ccollect +++ b/ccollect @@ -41,8 +41,8 @@ CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" -VERSION="0.9rc2" -RELEASE="2011-02-02" +VERSION="0.9" +RELEASE="2011-09-XX" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index cacaa6d..851bf2f 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -74,6 +74,10 @@ Incompatibilities and changes Versions 0.8 and 0.9 ^^^^^^^^^^^^^^^^^^^^ +- Renamed script to ccollect (.sh is not needed) +- Removed feature to backup to a host via ccollect, added new tool + (FIXME: insert name here) that takes care of this via tunnel +- Perhaps creating subdirectory of source name (idea from Stefan Schlörholz) Versions 0.7 and 0.8 ^^^^^^^^^^^^^^^^^^^^ From b5eede90c6ab86325ba7ffcdc6d40d2af2dfd16e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 5 Nov 2011 22:44:08 +0100 Subject: [PATCH 113/177] ++todo Signed-off-by: Nico Schottelius --- TODO | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/TODO b/TODO index b80db08..ce63297 100644 --- a/TODO +++ b/TODO @@ -18,3 +18,9 @@ Backup to remote can be done via ssh tunnel! ssh -R4242:localhost:22 backupserver "ccollect interval backupremotehost" remove $destination (==ddir) + +-------------------------------------------------------------------------------- + +Remote backups: + +ccollect_backup_to $host $remote_port $source $interval From 10d4942912f2a875fbf2bb6a45f24a5cddf9b0ae Mon Sep 17 00:00:00 2001 From: Jiri Pinkava Date: Sat, 5 Nov 2011 15:58:26 +0100 Subject: [PATCH 114/177] target all in Makefile should be first of all targets --- Makefile | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index f44a849..908bd6c 100644 --- a/Makefile +++ b/Makefile @@ -79,13 +79,6 @@ DOCBDOCS = ${DOCS:.text=.docbook} DOC_ALL = ${HTMLDOCS} ${DBHTMLDOCS} ${TEXIDOCS} ${MANPDOCS} ${PDFDOCS} -html: ${HTMLDOCS} -htm: ${DBHTMLDOCS} -info: ${TEXIDOCS} -man: ${MANPDOCS} -pdf: ${PDFDOCS} -documentation: ${DOC_ALL} - # # End user targets # @@ -97,6 +90,13 @@ all: @echo "man: only generate manpage{s}" @echo "install: install ccollect to ${prefix}" +html: ${HTMLDOCS} +htm: ${DBHTMLDOCS} +info: ${TEXIDOCS} +man: ${MANPDOCS} +pdf: ${PDFDOCS} +documentation: ${DOC_ALL} + install: install-link install-manlink install-link: install-script From 20abe4f86b6873711d1bf9223160a62efdf2bd6a Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Fri, 4 Jan 2013 10:49:53 +0100 Subject: [PATCH 115/177] add standard exclude list for debian Signed-off-by: Nico Schottelius --- contrib/exclude_lists/debian | 1 + 1 file changed, 1 insertion(+) create mode 100644 contrib/exclude_lists/debian diff --git a/contrib/exclude_lists/debian b/contrib/exclude_lists/debian new file mode 100644 index 0000000..5f43d88 --- /dev/null +++ b/contrib/exclude_lists/debian @@ -0,0 +1 @@ +/var/cache/apt/archives/* From ca45e8429b94f07b64a7f900e236cee931cd4bfe Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 28 Apr 2013 19:03:51 +0200 Subject: [PATCH 116/177] minor cleanups/year change Signed-off-by: Nico Schottelius --- ccollect | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ccollect b/ccollect index 27dbf42..f59f4c7 100755 --- a/ccollect +++ b/ccollect @@ -1,6 +1,6 @@ #!/bin/sh # -# 2005-2011 Nico Schottelius (nico-ccollect at schottelius.org) +# 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org) # # This file is part of ccollect. # @@ -42,7 +42,7 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" VERSION="0.9" -RELEASE="2011-09-XX" +RELEASE="2013-09-XX" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" @@ -129,8 +129,7 @@ ${__myname}: [args] -v, --verbose: Be very verbose (uses set -x) -V, --version: Print version information - This is version ${VERSION}, released on ${RELEASE} - (the first version was written on 2005-12-05 by Nico Schottelius). + This is version ${VERSION} released on ${RELEASE}. Retrieve latest ccollect at http://www.nico.schottelius.org/software/ccollect/ eof From e2ca223432ca449647c7deb87592a50863e4543e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Mon, 29 Apr 2013 10:17:21 +0200 Subject: [PATCH 117/177] indent with 4 spaces Signed-off-by: Nico Schottelius --- ccollect | 684 +++++++++++++++++++++++++++---------------------------- 1 file changed, 342 insertions(+), 342 deletions(-) diff --git a/ccollect b/ccollect index f59f4c7..3c6d77a 100755 --- a/ccollect +++ b/ccollect @@ -72,20 +72,20 @@ trap "rm -f \"${TMP}\"" 1 2 15 # time displaying echo _techo() { - echo "$(${DDATE}): $@" + echo "$(${DDATE}): $@" } # exit on error _exit_err() { - _techo "$@" - rm -f "${TMP}" - exit 1 + _techo "$@" + rm -f "${TMP}" + exit 1 } add_name() { - awk "{ print \"[${name}] \" \$0 }" + awk "{ print \"[${name}] \" \$0 }" } # @@ -95,94 +95,94 @@ add_name() # delete_from_file() { - file="$1"; shift - suffix="" # It will be set, if deleting incomplete backups. - [ $# -eq 1 ] && suffix="$1" && shift - while read to_remove; do - set -- "$@" "${to_remove}" - if [ "${suffix}" ]; then - to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" - set -- "$@" "${to_remove_no_suffix}" - fi - done < "${file}" - _techo "Removing $@ ..." - [ "${VVERBOSE}" ] && echo rm "$@" - rm -rf "$@" || _exit_err "Removing $@ failed." + file="$1"; shift + suffix="" # It will be set, if deleting incomplete backups. + [ $# -eq 1 ] && suffix="$1" && shift + while read to_remove; do + set -- "$@" "${to_remove}" + if [ "${suffix}" ]; then + to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" + set -- "$@" "${to_remove_no_suffix}" + fi + done < "${file}" + _techo "Removing $@ ..." + [ "${VVERBOSE}" ] && echo rm "$@" + rm -rf "$@" || _exit_err "Removing $@ failed." } display_version() { - echo "${FULL_VERSION}" - exit 0 + echo "${FULL_VERSION}" + exit 0 } usage() { - cat << eof + cat << eof ${__myname}: [args] - ccollect creates (pseudo) incremental backups + ccollect creates (pseudo) incremental backups - -h, --help: Show this help screen - -a, --all: Backup all sources specified in ${CSOURCES} - -p, --parallel: Parallelise backup processes - -v, --verbose: Be very verbose (uses set -x) - -V, --version: Print version information + -h, --help: Show this help screen + -a, --all: Backup all sources specified in ${CSOURCES} + -p, --parallel: Parallelise backup processes + -v, --verbose: Be very verbose (uses set -x) + -V, --version: Print version information - This is version ${VERSION} released on ${RELEASE}. + This is version ${VERSION} released on ${RELEASE}. - Retrieve latest ccollect at http://www.nico.schottelius.org/software/ccollect/ + Retrieve latest ccollect at http://www.nico.schottelius.org/software/ccollect/ eof - exit 0 + exit 0 } # # Parse options # while [ "$#" -ge 1 ]; do - case "$1" in - -a|--all) - USE_ALL=1 - ;; - -p|--parallel) - PARALLEL=1 - ;; - -v|--verbose) - set -x - ;; - -V|--version) - display_version - ;; - --) - # ignore the -- itself - shift - break - ;; - -h|--help|-*) - usage - ;; - *) - break - ;; - esac - shift + case "$1" in + -a|--all) + USE_ALL=1 + ;; + -p|--parallel) + PARALLEL=1 + ;; + -v|--verbose) + set -x + ;; + -V|--version) + display_version + ;; + --) + # ignore the -- itself + shift + break + ;; + -h|--help|-*) + usage + ;; + *) + break + ;; + esac + shift done # # Setup interval # if [ $# -ge 1 ]; then - export INTERVAL="$1" - shift + export INTERVAL="$1" + shift else - usage + usage fi # # Check for configuraton directory # [ -d "${CCOLLECT_CONF}" ] || _exit_err "No configuration found in " \ - "\"${CCOLLECT_CONF}\" (is \$CCOLLECT_CONF properly set?)" + "\"${CCOLLECT_CONF}\" (is \$CCOLLECT_CONF properly set?)" # # Create (portable!) source "array" @@ -190,46 +190,46 @@ fi export no_sources=0 if [ "${USE_ALL}" = 1 ]; then - # - # Get sources from source configuration - # - ( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \ - _exit_err "Listing of sources failed. Aborting." + # + # Get sources from source configuration + # + ( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \ + _exit_err "Listing of sources failed. Aborting." - while read tmp; do - eval export source_${no_sources}=\"${tmp}\" - no_sources=$((${no_sources}+1)) - done < "${TMP}" + while read tmp; do + eval export source_${no_sources}=\"${tmp}\" + no_sources=$((${no_sources}+1)) + done < "${TMP}" else - # - # Get sources from command line - # - while [ "$#" -ge 1 ]; do - eval arg=\"\$1\"; shift + # + # Get sources from command line + # + while [ "$#" -ge 1 ]; do + eval arg=\"\$1\"; shift - eval export source_${no_sources}=\"${arg}\" - no_sources="$((${no_sources}+1))" - done + eval export source_${no_sources}=\"${arg}\" + no_sources="$((${no_sources}+1))" + done fi # # Need at least ONE source to backup # if [ "${no_sources}" -lt 1 ]; then - usage + usage else - _techo "${HALF_VERSION}: Beginning backup using interval ${INTERVAL}" + _techo "${HALF_VERSION}: Beginning backup using interval ${INTERVAL}" fi # # Look for pre-exec command (general) # if [ -x "${CPREEXEC}" ]; then - _techo "Executing ${CPREEXEC} ..." - "${CPREEXEC}"; ret=$? - _techo "Finished ${CPREEXEC} (return code: ${ret})." + _techo "Executing ${CPREEXEC} ..." + "${CPREEXEC}"; ret=$? + _techo "Finished ${CPREEXEC} (return code: ${ret})." - [ "${ret}" -eq 0 ] || _exit_err "${CPREEXEC} failed. Aborting" + [ "${ret}" -eq 0 ] || _exit_err "${CPREEXEC} failed. Aborting" fi ################################################################################ @@ -238,304 +238,304 @@ fi # source_no=0 while [ "${source_no}" -lt "${no_sources}" ]; do - # - # Get current source - # - eval export name=\"\$source_${source_no}\" - source_no=$((${source_no}+1)) + # + # Get current source + # + eval export name=\"\$source_${source_no}\" + source_no=$((${source_no}+1)) - # - # Start ourself, if we want parallel execution - # - if [ "${PARALLEL}" ]; then - "$0" "${INTERVAL}" "${name}" & - continue - fi + # + # Start ourself, if we want parallel execution + # + if [ "${PARALLEL}" ]; then + "$0" "${INTERVAL}" "${name}" & + continue + fi # # Start subshell for easy log editing # ( - backup="${CSOURCES}/${name}" - c_source="${backup}/source" - c_dest="${backup}/destination" - c_pre_exec="${backup}/pre_exec" - c_post_exec="${backup}/post_exec" + backup="${CSOURCES}/${name}" + c_source="${backup}/source" + c_dest="${backup}/destination" + c_pre_exec="${backup}/pre_exec" + c_post_exec="${backup}/post_exec" - # - # Stderr to stdout, so we can produce nice logs - # - exec 2>&1 + # + # Stderr to stdout, so we can produce nice logs + # + exec 2>&1 - # - # Record start of backup: internal and for the user - # - begin_s="$(${SDATE})" - _techo "Beginning to backup" + # + # Record start of backup: internal and for the user + # + begin_s="$(${SDATE})" + _techo "Beginning to backup" - # - # Standard configuration checks - # - if [ ! -e "${backup}" ]; then - _exit_err "Source does not exist." - fi + # + # Standard configuration checks + # + if [ ! -e "${backup}" ]; then + _exit_err "Source does not exist." + fi - # - # Configuration _must_ be a directory (cconfig style) - # - if [ ! -d "${backup}" ]; then - _exit_err "\"${backup}\" is not a cconfig-directory. Skipping." - fi + # + # Configuration _must_ be a directory (cconfig style) + # + if [ ! -d "${backup}" ]; then + _exit_err "\"${backup}\" is not a cconfig-directory. Skipping." + fi - # - # First execute pre_exec, which may generate destination or other parameters - # - if [ -x "${c_pre_exec}" ]; then - _techo "Executing ${c_pre_exec} ..." - "${c_pre_exec}"; ret="$?" - _techo "Finished ${c_pre_exec} (return code ${ret})." + # + # First execute pre_exec, which may generate destination or other parameters + # + if [ -x "${c_pre_exec}" ]; then + _techo "Executing ${c_pre_exec} ..." + "${c_pre_exec}"; ret="$?" + _techo "Finished ${c_pre_exec} (return code ${ret})." - [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." - fi + [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." + fi - # - # Read source configuration - # - for opt in verbose very_verbose summary exclude rsync_options \ - delete_incomplete rsync_failure_codes \ - mtime quiet_if_down ; do - if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}" ]; then - eval c_$opt=\"${backup}/$opt\" - else - eval c_$opt=\"${CDEFAULTS}/$opt\" - fi - done + # + # Read source configuration + # + for opt in verbose very_verbose summary exclude rsync_options \ + delete_incomplete rsync_failure_codes \ + mtime quiet_if_down ; do + if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}" ]; then + eval c_$opt=\"${backup}/$opt\" + else + eval c_$opt=\"${CDEFAULTS}/$opt\" + fi + done - # - # Interval definition: First try source specific, fallback to default - # - c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)" + # + # Interval definition: First try source specific, fallback to default + # + c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)" - if [ -z "${c_interval}" ]; then - c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" + if [ -z "${c_interval}" ]; then + c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" - if [ -z "${c_interval}" ]; then - _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." - fi - fi + if [ -z "${c_interval}" ]; then + _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." + fi + fi - # - # Sort by ctime (default) or mtime (configuration option) - # - if [ -f "${c_mtime}" ] ; then - TSORT="t" - else - TSORT="tc" - fi + # + # Sort by ctime (default) or mtime (configuration option) + # + if [ -f "${c_mtime}" ] ; then + TSORT="t" + else + TSORT="tc" + fi - # - # Source configuration checks - # - if [ ! -f "${c_source}" ]; then - _exit_err "Source description \"${c_source}\" is not a file. Skipping." - else - source=$(cat "${c_source}"); ret="$?" - if [ "${ret}" -ne 0 ]; then - _exit_err "Source ${c_source} is not readable. Skipping." - fi - fi + # + # Source configuration checks + # + if [ ! -f "${c_source}" ]; then + _exit_err "Source description \"${c_source}\" is not a file. Skipping." + else + source=$(cat "${c_source}"); ret="$?" + if [ "${ret}" -ne 0 ]; then + _exit_err "Source ${c_source} is not readable. Skipping." + fi + fi - # - # Destination is a path - # - if [ ! -f "${c_dest}" ]; then - _exit_err "Destination ${c_dest} is not a file. Skipping." - else - ddir="$(cat "${c_dest}")"; ret="$?" - if [ "${ret}" -ne 0 ]; then - _exit_err "Destination ${c_dest} is not readable. Skipping." - fi - fi + # + # Destination is a path + # + if [ ! -f "${c_dest}" ]; then + _exit_err "Destination ${c_dest} is not a file. Skipping." + else + ddir="$(cat "${c_dest}")"; ret="$?" + if [ "${ret}" -ne 0 ]; then + _exit_err "Destination ${c_dest} is not readable. Skipping." + fi + fi - # - # Parameters: ccollect defaults, configuration options, user options - # + # + # Parameters: ccollect defaults, configuration options, user options + # - # - # Rsync standard options (archive will be added after is-up-check) - # - set -- "$@" "--delete" "--numeric-ids" "--relative" \ - "--delete-excluded" "--sparse" + # + # Rsync standard options (archive will be added after is-up-check) + # + set -- "$@" "--delete" "--numeric-ids" "--relative" \ + "--delete-excluded" "--sparse" - # - # Exclude list - # - if [ -f "${c_exclude}" ]; then - set -- "$@" "--exclude-from=${c_exclude}" - fi + # + # Exclude list + # + if [ -f "${c_exclude}" ]; then + set -- "$@" "--exclude-from=${c_exclude}" + fi - # - # Output a summary - # - if [ -f "${c_summary}" ]; then - set -- "$@" "--stats" - fi + # + # Output a summary + # + if [ -f "${c_summary}" ]; then + set -- "$@" "--stats" + fi - # - # Verbosity for rsync, rm, and mkdir - # - VVERBOSE="" - if [ -f "${c_very_verbose}" ]; then - set -- "$@" "-vv" - VVERBOSE="-v" - elif [ -f "${c_verbose}" ]; then - set -- "$@" "-v" - fi + # + # Verbosity for rsync, rm, and mkdir + # + VVERBOSE="" + if [ -f "${c_very_verbose}" ]; then + set -- "$@" "-vv" + VVERBOSE="-v" + elif [ -f "${c_verbose}" ]; then + set -- "$@" "-v" + fi - # - # Extra options for rsync provided by the user - # - if [ -f "${c_rsync_options}" ]; then - while read line; do - set -- "$@" "${line}" - done < "${c_rsync_options}" - fi + # + # Extra options for rsync provided by the user + # + if [ -f "${c_rsync_options}" ]; then + while read line; do + set -- "$@" "${line}" + done < "${c_rsync_options}" + fi - # - # Check: source is up and accepting connections (before deleting old backups!) - # - if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then - if [ ! -f "${c_quiet_if_down}" ]; then - cat "${TMP}" - fi - _exit_err "Source ${source} is not readable. Skipping." - fi + # + # Check: source is up and accepting connections (before deleting old backups!) + # + if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then + if [ ! -f "${c_quiet_if_down}" ]; then + cat "${TMP}" + fi + _exit_err "Source ${source} is not readable. Skipping." + fi - # - # Add --archive for real backup (looks nice in front) - # - set -- "--archive" "$@" + # + # Add --archive for real backup (looks nice in front) + # + set -- "--archive" "$@" - # - # Check: destination exists? - # - cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping." + # + # Check: destination exists? + # + cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping." - # - # Check incomplete backups (needs echo to remove newlines) - # - ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? + # + # Check incomplete backups (needs echo to remove newlines) + # + ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? - if [ "$ret" -eq 0 ]; then - _techo "Incomplete backups: $(echo $(cat "${TMP}"))" - if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" "${CMARKER}" - fi - fi + if [ "$ret" -eq 0 ]; then + _techo "Incomplete backups: $(echo $(cat "${TMP}"))" + if [ -f "${c_delete_incomplete}" ]; then + delete_from_file "${TMP}" "${CMARKER}" + fi + fi - # - # Check: maximum number of backups is reached? - # - count="$(ls -1 | grep -c "^${INTERVAL}\\.")" + # + # Check: maximum number of backups is reached? + # + count="$(ls -1 | grep -c "^${INTERVAL}\\.")" - _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" + _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" - if [ "${count}" -ge "${c_interval}" ]; then - remove="$((${count} - ${c_interval} + 1))" - _techo "Removing ${remove} backup(s)..." + if [ "${count}" -ge "${c_interval}" ]; then + remove="$((${count} - ${c_interval} + 1))" + _techo "Removing ${remove} backup(s)..." - ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" - delete_from_file "${TMP}" - fi + delete_from_file "${TMP}" + fi - # - # Check for backup directory to clone from: Always clone from the latest one! - # - last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ - _exit_err "Failed to list contents of ${ddir}." + # + # Check for backup directory to clone from: Always clone from the latest one! + # + last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ + _exit_err "Failed to list contents of ${ddir}." - # - # Clone from old backup, if existing - # - if [ "${last_dir}" ]; then - set -- "$@" "--link-dest=${ddir}/${last_dir}" - _techo "Hard linking from ${last_dir}" - fi + # + # Clone from old backup, if existing + # + if [ "${last_dir}" ]; then + set -- "$@" "--link-dest=${ddir}/${last_dir}" + _techo "Hard linking from ${last_dir}" + fi - # - # Include current time in name, not the time when we began to remove above - # - export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" - export destination_dir="${ddir}/${destination_name}" + # + # Include current time in name, not the time when we began to remove above + # + export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" + export destination_dir="${ddir}/${destination_name}" - # - # Mark backup running and go back to original directory - # - touch "${destination_dir}${CMARKER}" - cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}." + # + # Mark backup running and go back to original directory + # + touch "${destination_dir}${CMARKER}" + cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}." - # - # the rsync part - # - _techo "Transferring files..." - rsync "$@" "${source}" "${destination_dir}"; ret=$? - _techo "Finished backup (rsync return code: $ret)." + # + # the rsync part + # + _techo "Transferring files..." + rsync "$@" "${source}" "${destination_dir}"; ret=$? + _techo "Finished backup (rsync return code: $ret)." - # - # Set modification time (mtime) to current time, if sorting by mtime is enabled - # - [ -f "$c_mtime" ] && touch "${destination_dir}" + # + # Set modification time (mtime) to current time, if sorting by mtime is enabled + # + [ -f "$c_mtime" ] && touch "${destination_dir}" - # - # Check if rsync exit code indicates failure. - # - fail="" - if [ -f "$c_rsync_failure_codes" ]; then - while read code ; do - if [ "$ret" = "$code" ]; then - fail=1 - fi - done <"${c_rsync_failure_codes}" - fi + # + # Check if rsync exit code indicates failure. + # + fail="" + if [ -f "$c_rsync_failure_codes" ]; then + while read code ; do + if [ "$ret" = "$code" ]; then + fail=1 + fi + done <"${c_rsync_failure_codes}" + fi - # - # Remove marking here unless rsync failed. - # - if [ -z "$fail" ]; then - rm "${destination_dir}${CMARKER}" || \ - _exit_err "Removing ${destination_dir}${CMARKER} failed." - if [ "${ret}" -ne 0 ]; then - _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." - fi - else - _techo "Warning: rsync failed with return code $ret." - fi + # + # Remove marking here unless rsync failed. + # + if [ -z "$fail" ]; then + rm "${destination_dir}${CMARKER}" || \ + _exit_err "Removing ${destination_dir}${CMARKER} failed." + if [ "${ret}" -ne 0 ]; then + _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." + fi + else + _techo "Warning: rsync failed with return code $ret." + fi - # - # post_exec - # - if [ -x "${c_post_exec}" ]; then - _techo "Executing ${c_post_exec} ..." - "${c_post_exec}"; ret=$? - _techo "Finished ${c_post_exec}." + # + # post_exec + # + if [ -x "${c_post_exec}" ]; then + _techo "Executing ${c_post_exec} ..." + "${c_post_exec}"; ret=$? + _techo "Finished ${c_post_exec}." - if [ "${ret}" -ne 0 ]; then - _exit_err "${c_post_exec} failed." - fi - fi + if [ "${ret}" -ne 0 ]; then + _exit_err "${c_post_exec} failed." + fi + fi - # - # Time calculation - # - end_s="$(${SDATE})" - full_seconds="$((${end_s} - ${begin_s}))" - hours="$((${full_seconds} / 3600))" - minutes="$(((${full_seconds} % 3600) / 60))" - seconds="$((${full_seconds} % 60))" + # + # Time calculation + # + end_s="$(${SDATE})" + full_seconds="$((${end_s} - ${begin_s}))" + hours="$((${full_seconds} / 3600))" + minutes="$(((${full_seconds} % 3600) / 60))" + seconds="$((${full_seconds} % 60))" - _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" + _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" ) | add_name done @@ -543,21 +543,21 @@ done # Be a good parent and wait for our children, if they are running wild parallel # if [ "${PARALLEL}" ]; then - _techo "Waiting for children to complete..." - wait + _techo "Waiting for children to complete..." + wait fi # # Look for post-exec command (general) # if [ -x "${CPOSTEXEC}" ]; then - _techo "Executing ${CPOSTEXEC} ..." - "${CPOSTEXEC}"; ret=$? - _techo "Finished ${CPOSTEXEC} (return code: ${ret})." + _techo "Executing ${CPOSTEXEC} ..." + "${CPOSTEXEC}"; ret=$? + _techo "Finished ${CPOSTEXEC} (return code: ${ret})." - if [ "${ret}" -ne 0 ]; then - _techo "${CPOSTEXEC} failed." - fi + if [ "${ret}" -ne 0 ]; then + _techo "${CPOSTEXEC} failed." + fi fi rm -f "${TMP}" From 977c7e9c1fb4f765bf30bb2da1fc6d572bdcf123 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Sun, 6 Sep 2015 16:35:53 +0200 Subject: [PATCH 118/177] Added "Error: " prefix in _exit_err() All stdout generated with the _exit_err() function gets a prefix: "Error: " for better usability. --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 3c6d77a..c9496b4 100755 --- a/ccollect +++ b/ccollect @@ -78,7 +78,7 @@ _techo() # exit on error _exit_err() { - _techo "$@" + _techo "Error: $@" rm -f "${TMP}" exit 1 } From 5356370233e0883b5c6cc62b12c84ef058f8c239 Mon Sep 17 00:00:00 2001 From: Dominique Roux Date: Sat, 10 Oct 2015 19:05:39 +0200 Subject: [PATCH 119/177] Creating version 1.0 --- ccollect | 4 ++-- doc/ccollect.text | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index c9496b4..5454f51 100755 --- a/ccollect +++ b/ccollect @@ -41,8 +41,8 @@ CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" -VERSION="0.9" -RELEASE="2013-09-XX" +VERSION="1.0" +RELEASE="2015-10-10" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 851bf2f..4c54191 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -72,6 +72,10 @@ Incompatibilities and changes ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Versions 0.9 and 1.0 +^^^^^^^^^^^^^^^^^^^^ +- Added "Error: " prefix in _exit_err() + Versions 0.8 and 0.9 ^^^^^^^^^^^^^^^^^^^^ - Renamed script to ccollect (.sh is not needed) From 86d562857781aa80e0323b6aae67f65cc63be61f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 13 Jun 2016 13:40:32 +0200 Subject: [PATCH 120/177] Implement per source locking. --- ccollect | 87 +++++++++++++++++++++++++++++++++++++++++++++++- doc/changes/next | 1 + 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 5454f51..20498b2 100755 --- a/ccollect +++ b/ccollect @@ -54,6 +54,67 @@ CDATE="date +%Y%m%d-%H%M" DDATE="date +%Y-%m-%d-%H:%M:%S" SDATE="date +%s" +# +# LOCKING: use flock if available, otherwise mkdir +# Locking is done for each source so that only one instance per source +# can run. +# +LOCKDIR="${CSOURCES}" +# printf pattern: ccollect_.lock +LOCKFILE_PATTERN="ccollect_%s.lock" +LOCKFD=4 + +# +# locking functions using flock +# +lock_flock() +{ + # $1 = source to backup + lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" + eval "exec ${LOCKFD}> ${lockfile}" + + flock -n ${LOCKFD} && return 0 || return 1 +} + +unlock_flock() +{ + # $1 = source to backup + lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" + eval "exec ${LOCKFD}>&-" + rm -f "${lockfile}" +} + +# +# locking functions using mkdir (mkdir is atomic) +# +lock_mkdir() +{ + # $1 = source to backup + lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" + + mkdir "${lockfile}" && return 0 || return 1 +} + +unlock_mkdir() +{ + # $1 = source to backup + lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" + + rmdir "${lockfile}" +} + +# +# determine locking tool: flock or mkdir +# +if $(which flock > /dev/null 2>&1) +then + lockf="lock_flock" + unlockf="unlock_flock" +else + lockf="lock_mkdir" + unlockf="unlock_mkdir" +fi + # # unset values # @@ -63,7 +124,8 @@ USE_ALL="" # # catch signals # -trap "rm -f \"${TMP}\"" 1 2 15 +TRAPFUNC="rm -f \"${TMP}\"" +trap "${TRAPFUNC}" 1 2 15 # # Functions @@ -136,6 +198,18 @@ eof exit 0 } +# locking functions +lock() +{ + "${lockf}" "$@" || _exit_err \ + "Only one instance of ${__myname} for source \"$1\" can run at one time." +} + +unlock() +{ + "${unlockf}" "$@" +} + # # Parse options # @@ -287,6 +361,16 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _exit_err "\"${backup}\" is not a cconfig-directory. Skipping." fi + # + # Acquire lock for source. If lock cannot be acquired, lock will exit + # with error message. + # + lock "${name}" + + # redefine trap to also unlock (rm lockfile) + TRAPFUNC="${TRAPFUNC} && unlock \"${name}\"" + trap "${TRAPFUNC}" 1 2 15 + # # First execute pre_exec, which may generate destination or other parameters # @@ -561,4 +645,5 @@ if [ -x "${CPOSTEXEC}" ]; then fi rm -f "${TMP}" +unlock "${name}" _techo "Finished" diff --git a/doc/changes/next b/doc/changes/next index e865dff..05c1a37 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1,3 +1,4 @@ + * Add locking (Darko Poljak) * Fix source-is-up check (Nikita Koshikov) * Fix some minor command line parsing issues (Nico Schottelius) * Correct output, if configuration is not in cconfig format (Nico Schottelius) From 902a7d667ed41c2142083a0a64a6c0fe17c36d22 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 21 Jul 2016 12:15:37 +0200 Subject: [PATCH 121/177] Introduce -j option and deprecate -p option. --- ccollect | 115 +++++++++++++++++++++++++++++++++++++++--- doc/ccollect.text | 4 +- doc/changes/next | 1 + doc/man/ccollect.text | 12 +++-- 4 files changed, 118 insertions(+), 14 deletions(-) diff --git a/ccollect b/ccollect index 20498b2..de62aa6 100755 --- a/ccollect +++ b/ccollect @@ -20,6 +20,11 @@ # Initially written for SyGroup (www.sygroup.ch) # Date: Mon Nov 14 11:45:11 CET 2005 +# Simulate ccollect without actually performing any backup; +# conf directory need to be specifed. +# Usually used for debugging/testing locking and parallel execution. +SIMULATE="1" + # Error upon expanding unset variables: set -u @@ -41,6 +46,8 @@ CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" +CONTROL_PIPE="/tmp/${__myname}-control-pipe" + VERSION="1.0" RELEASE="2015-10-10" HALF_VERSION="ccollect ${VERSION}" @@ -59,7 +66,10 @@ SDATE="date +%s" # Locking is done for each source so that only one instance per source # can run. # -LOCKDIR="${CSOURCES}" +# Use CCOLLECT_CONF directory for lock files. +# This directory can be set arbitrary so it is writable for user +# executing ccollect. +LOCKDIR="${CCOLLECT_CONF}" # printf pattern: ccollect_.lock LOCKFILE_PATTERN="ccollect_%s.lock" LOCKFD=4 @@ -119,6 +129,7 @@ fi # unset values # PARALLEL="" +MAX_JOBS="" USE_ALL="" # @@ -185,9 +196,11 @@ ${__myname}: [args] ccollect creates (pseudo) incremental backups - -h, --help: Show this help screen - -a, --all: Backup all sources specified in ${CSOURCES} - -p, --parallel: Parallelise backup processes + -h, --help: Show this help screen + -a, --all: Backup all sources specified in ${CSOURCES} + -j [max], --jobs [max] Specifies the number of jobs to run simultaneously. + If max is not specified then parallelise all jobs. + -p, --parallel: Parallelise backup processes (deprecated from 2.0) -v, --verbose: Be very verbose (uses set -x) -V, --version: Print version information @@ -219,7 +232,24 @@ while [ "$#" -ge 1 ]; do USE_ALL=1 ;; -p|--parallel) + _techo "Warning: -p, --parallel option is deprecated," \ + "use -j, --jobs instead." PARALLEL=1 + MAX_JOBS="" + ;; + -j|--jobs) + PARALLEL=1 + if [ "$#" -ge 2 ] + then + case "$2" in + -*) + ;; + *) + MAX_JOBS=$2 + shift + ;; + esac + fi ;; -v|--verbose) set -x @@ -242,6 +272,14 @@ while [ "$#" -ge 1 ]; do shift done +# check that MAX_JOBS is natural number > 0 +# empty string means run all in parallel +echo "${MAX_JOBS}" | awk '/^$/ { exit 0 } /^[1-9][0-9]*$/ { exit 0 } { exit 1 }' +if [ "$?" -ne 0 ] +then + _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" +fi + # # Setup interval # @@ -310,6 +348,26 @@ fi # # Let's do the backup - here begins the real stuff # + +# in PARALLEL mode: +# * create control pipe +# * determine number of jobs to start at once +if [ "${PARALLEL}" ]; then + mkfifo "${CONTROL_PIPE}" + # fd 5 is tied to control pipe + eval "exec 5<>${CONTROL_PIPE}" + TRAPFUNC="${TRAPFUNC}; rm -f \"${CONTROL_PIPE}\"" + trap "${TRAPFUNC}" 0 1 2 15 + + # determine how much parallel jobs to prestart + if [ "${MAX_JOBS}" ] && [ "${MAX_JOBS}" -le "${no_sources}" ] + then + prestart="${MAX_JOBS}" + else + prestart=0 + fi +fi + source_no=0 while [ "${source_no}" -lt "${no_sources}" ]; do # @@ -322,8 +380,35 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Start ourself, if we want parallel execution # if [ "${PARALLEL}" ]; then - "$0" "${INTERVAL}" "${name}" & - continue + if [ "${SIMULATE}" ] + then + # give some time to awk's srand initialized by curr time + sleep 1 + fi + + if [ ! "${MAX_JOBS}" ] + then + # run all in parallel + "$0" "${INTERVAL}" "${name}" & + continue + elif [ "${prestart}" -gt 0 ] + then + # run prestart child if pending + { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & + prestart=$((${prestart} - 1)) + continue + else + # each time a child finishes we get a line from the pipe + # and then launch another child + while read line + do + { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & + # get out of loop so we can contnue with main loop + # for next source + break + done <&5 + continue + fi fi # @@ -366,9 +451,20 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # with error message. # lock "${name}" + if [ "${SIMULATE}" ] + then + rand_low=3 + rand_high=10 + sleep_time=$(echo '' | awk "{srand(); print int(rand()*($rand_high - $rand_low)) + $rand_low;}") + _techo "simulating backup for ${name}: ${sleep_time} secs ..." + sleep $sleep_time + unlock "${name}" + _techo "Finished backup." + break + fi # redefine trap to also unlock (rm lockfile) - TRAPFUNC="${TRAPFUNC} && unlock \"${name}\"" + TRAPFUNC="${TRAPFUNC}; unlock \"${name}\"" trap "${TRAPFUNC}" 1 2 15 # @@ -620,15 +716,19 @@ while [ "${source_no}" -lt "${no_sources}" ]; do seconds="$((${full_seconds} % 60))" _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" + + unlock "${name}" ) | add_name done # # Be a good parent and wait for our children, if they are running wild parallel +# After all children are finished then remove control pipe. # if [ "${PARALLEL}" ]; then _techo "Waiting for children to complete..." wait + rm -f "${CONTROL_PIPE}" fi # @@ -645,5 +745,4 @@ if [ -x "${CPOSTEXEC}" ]; then fi rm -f "${TMP}" -unlock "${name}" _techo "Finished" diff --git a/doc/ccollect.text b/doc/ccollect.text index 4c54191..474c491 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1189,12 +1189,12 @@ rsync -av -H --delete /mnt/archiv/ "$DDIR/archiv/" ------------------------------------------------------------------------- -Processes running when doing ccollect -p +Processes running when doing ccollect -j ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Truncated output from `ps axuwwwf`: ------------------------------------------------------------------------- - S+ 11:40 0:00 | | | \_ /bin/sh /usr/local/bin/ccollect.sh daily -p ddba034 ddba045 ddba046 ddba047 ddba049 ddna010 ddna011 + S+ 11:40 0:00 | | | \_ /bin/sh /usr/local/bin/ccollect.sh daily -j ddba034 ddba045 ddba046 ddba047 ddba049 ddna010 ddna011 S+ 11:40 0:00 | | | \_ /bin/sh /usr/local/bin/ccollect.sh daily ddba034 S+ 11:40 0:00 | | | | \_ /bin/sh /usr/local/bin/ccollect.sh daily ddba034 R+ 11:40 23:40 | | | | | \_ rsync -a --delete --numeric-ids --relative --delete-excluded --link-dest=/home/server/backup/ddba034 diff --git a/doc/changes/next b/doc/changes/next index 05c1a37..2fee139 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1,3 +1,4 @@ + * Introduce -j option for max parallel jobs, deprecate -p (Darko Poljak) * Add locking (Darko Poljak) * Fix source-is-up check (Nikita Koshikov) * Fix some minor command line parsing issues (Nico Schottelius) diff --git a/doc/man/ccollect.text b/doc/man/ccollect.text index 84c1e37..9fc9086 100644 --- a/doc/man/ccollect.text +++ b/doc/man/ccollect.text @@ -26,14 +26,18 @@ texinfo or html). OPTIONS ------- +-a, --all:: + Backup all sources specified in /etc/ccollect/sources + -h, --help:: Show the help screen --p, --parallel:: - Parallelise backup processes +-j [max], --jobs [max]:: + Specifies the number of jobs to run simultaneously. + If max is not specified then parallelise all jobs. --a, --all:: - Backup all sources specified in /etc/ccollect/sources +-p, --parallel:: + Parallelise backup processes (deprecated from 2.0) -v, --verbose:: Be very verbose (uses set -x) From 1df57c815415f1f542e6ff280be8a6621955d8a3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 21 Jul 2016 12:16:34 +0200 Subject: [PATCH 122/177] Turn simulation off. --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index de62aa6..7b5b0ca 100755 --- a/ccollect +++ b/ccollect @@ -23,7 +23,7 @@ # Simulate ccollect without actually performing any backup; # conf directory need to be specifed. # Usually used for debugging/testing locking and parallel execution. -SIMULATE="1" +SIMULATE="" # Error upon expanding unset variables: set -u From 01c36fc699f59225a9d10ab1a262d18b80039d9a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 8 Aug 2016 15:56:19 +0200 Subject: [PATCH 123/177] Check that MAX_JOBS is positive integer or empty using grep. --- ccollect | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 7b5b0ca..2929ac3 100755 --- a/ccollect +++ b/ccollect @@ -274,11 +274,13 @@ done # check that MAX_JOBS is natural number > 0 # empty string means run all in parallel -echo "${MAX_JOBS}" | awk '/^$/ { exit 0 } /^[1-9][0-9]*$/ { exit 0 } { exit 1 }' +echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' if [ "$?" -ne 0 ] then _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" fi +printf "REMOVE ME\n" +exit 0 # # Setup interval From a18a00e773d7e982138ab6db655158d2840ce63d Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Aug 2016 17:12:18 +0200 Subject: [PATCH 124/177] Drop SIMULATE stuff. --- ccollect | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/ccollect b/ccollect index 2929ac3..80b7066 100755 --- a/ccollect +++ b/ccollect @@ -20,11 +20,6 @@ # Initially written for SyGroup (www.sygroup.ch) # Date: Mon Nov 14 11:45:11 CET 2005 -# Simulate ccollect without actually performing any backup; -# conf directory need to be specifed. -# Usually used for debugging/testing locking and parallel execution. -SIMULATE="" - # Error upon expanding unset variables: set -u @@ -382,12 +377,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Start ourself, if we want parallel execution # if [ "${PARALLEL}" ]; then - if [ "${SIMULATE}" ] - then - # give some time to awk's srand initialized by curr time - sleep 1 - fi - if [ ! "${MAX_JOBS}" ] then # run all in parallel @@ -453,17 +442,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # with error message. # lock "${name}" - if [ "${SIMULATE}" ] - then - rand_low=3 - rand_high=10 - sleep_time=$(echo '' | awk "{srand(); print int(rand()*($rand_high - $rand_low)) + $rand_low;}") - _techo "simulating backup for ${name}: ${sleep_time} secs ..." - sleep $sleep_time - unlock "${name}" - _techo "Finished backup." - break - fi # redefine trap to also unlock (rm lockfile) TRAPFUNC="${TRAPFUNC}; unlock \"${name}\"" From 3049849ea698fed16ee78c11733b5fdb29cd27c4 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 10 Aug 2016 17:14:31 +0200 Subject: [PATCH 125/177] Remove testing stuff. --- ccollect | 2 -- 1 file changed, 2 deletions(-) diff --git a/ccollect b/ccollect index 80b7066..82bd8a7 100755 --- a/ccollect +++ b/ccollect @@ -274,8 +274,6 @@ if [ "$?" -ne 0 ] then _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" fi -printf "REMOVE ME\n" -exit 0 # # Setup interval From 6dca5c638d4cbc2d42fbc5a229d7cb13385d5d0f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 26 Sep 2016 17:27:33 +0200 Subject: [PATCH 126/177] Release 2.0. --- ccollect | 5 +++-- doc/ccollect.text | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ccollect b/ccollect index 82bd8a7..e1074b0 100755 --- a/ccollect +++ b/ccollect @@ -1,6 +1,7 @@ #!/bin/sh # # 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org) +# 2016 Darko Poljak (darko.poljak at gmail.com) # # This file is part of ccollect. # @@ -43,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="1.0" -RELEASE="2015-10-10" +VERSION="2.0" +RELEASE="2016-09-26" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 474c491..b1c7dba 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -0.9, for ccollect 0.9, Initial Version from 2006-01-13 +2.0, for ccollect 2.0, Initial Version from 2006-01-13 :Author Initials: NS From fbe17cae44a9b1236cc18c458e4ae16d6fcd8813 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 23 Feb 2017 20:04:53 +0100 Subject: [PATCH 127/177] Improve logging: stdout, file, syslog options. --- ccollect | 135 ++++++++++++++++++++++++++++++++++++------ doc/changes/2.0 | 16 +++++ doc/changes/next | 17 +----- doc/man/ccollect.text | 34 ++++++++++- 4 files changed, 166 insertions(+), 36 deletions(-) create mode 100644 doc/changes/2.0 diff --git a/ccollect b/ccollect index e1074b0..6369e84 100755 --- a/ccollect +++ b/ccollect @@ -127,6 +127,11 @@ fi PARALLEL="" MAX_JOBS="" USE_ALL="" +LOGFILE="" +SYSLOG="" +# e - only errors, a - all output +LOGLEVEL="a" +LOGONLYERRORS="" # # catch signals @@ -138,18 +143,11 @@ trap "${TRAPFUNC}" 1 2 15 # Functions # -# time displaying echo -_techo() +# check if we are running interactive or non-interactive +# see: http://www.tldp.org/LDP/abs/html/intandnonint.html +_is_interactive() { - echo "$(${DDATE}): $@" -} - -# exit on error -_exit_err() -{ - _techo "Error: $@" - rm -f "${TMP}" - exit 1 + [ -t 0 -o -p /dev/stdin ] } add_name() @@ -192,13 +190,16 @@ ${__myname}: [args] ccollect creates (pseudo) incremental backups - -h, --help: Show this help screen - -a, --all: Backup all sources specified in ${CSOURCES} - -j [max], --jobs [max] Specifies the number of jobs to run simultaneously. - If max is not specified then parallelise all jobs. - -p, --parallel: Parallelise backup processes (deprecated from 2.0) - -v, --verbose: Be very verbose (uses set -x) - -V, --version: Print version information + -h, --help: Show this help screen + -a, --all: Backup all sources specified in ${CSOURCES} + -e, --errors: Log only errors + -j [max], --jobs [max] Specifies the number of jobs to run simultaneously. + If max is not specified then parallelise all jobs. + -l FILE, --logfile FILE Log to specified file + -p, --parallel: Parallelise backup processes (deprecated from 2.0) + -s, --syslog: Log to syslog with tag ccollect + -v, --verbose: Be very verbose (uses set -x) + -V, --version: Print version information This is version ${VERSION} released on ${RELEASE}. @@ -219,6 +220,54 @@ unlock() "${unlockf}" "$@" } +# time displaying echo +# stdout version +_techo_stdout() +{ + echo "$(${DDATE}): $@" +} + +# syslog version +_techo_syslog() +{ + logger -t ccollect "$@" +} + +# specified file version +_techo_file() +{ + _techo_stdout "$@" >> "${LOGFILE}" +} + +# determine _techo version before parsing options +if _is_interactive +then + _techof="_techo_stdout" +else + _techof="_techo_syslog" +fi + +# _techo with determined _techo version +_techo() +{ + if [ "${LOGLEVEL}" = "a" ] + then + "${_techof}" "$@" + fi +} + +_techo_err() +{ + _techo "Error: $@" +} + +_exit_err() +{ + _techo_err "$@" + rm -f "${TMP}" + exit 1 +} + # # Parse options # @@ -247,6 +296,28 @@ while [ "$#" -ge 1 ]; do esac fi ;; + -e|--errors) + LOGONLYERRORS="1" + ;; + -l|--logfile) + if [ "$#" -ge 2 ] + then + case "$2" in + -*) + _exit_err "Missing log file" + ;; + *) + LOGFILE="$2" + shift + ;; + esac + else + _exit_err "Missing log file" + fi + ;; + -s|--syslog) + SYSLOG="1" + ;; -v|--verbose) set -x ;; @@ -268,6 +339,34 @@ while [ "$#" -ge 1 ]; do shift done +# determine _techo version and logging level after parsing options +if [ "${LOGFILE}" ] +then + _techof="_techo_file" + LOGLEVEL="a" +elif _is_interactive +then + if [ "${SYSLOG}" ] + then + _techof="_techo_syslog" + LOGLEVEL="a" + else + _techof="_techo_stdout" + LOGLEVEL="e" + fi +else + _techof="_techo_syslog" + LOGLEVEL="a" +fi + +if [ "${LOGFILE}" -o "${SYSLOG}" ] +then + if [ "${LOGONLYERRORS}" ] + then + LOGLEVEL="e" + fi +fi + # check that MAX_JOBS is natural number > 0 # empty string means run all in parallel echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' diff --git a/doc/changes/2.0 b/doc/changes/2.0 new file mode 100644 index 0000000..2fee139 --- /dev/null +++ b/doc/changes/2.0 @@ -0,0 +1,16 @@ + * Introduce -j option for max parallel jobs, deprecate -p (Darko Poljak) + * Add locking (Darko Poljak) + * Fix source-is-up check (Nikita Koshikov) + * Fix some minor command line parsing issues (Nico Schottelius) + * Correct output, if configuration is not in cconfig format (Nico Schottelius) + * Minor code cleanups and optimisations (Nico Schottelius) + * ccollect_analyse_logs.sh traps more errors and warnings (Patrick Drolet) + * Remove -v for mkdir and rm, as they are not POSIX (Patrick Drolet) + * Export destination_* to source specific post_exec (Nico Schottelius) + * Update documentation regarding exported variables (Nico Schottelius) + * Simplify time calculation (Nico Schottelius) + * Documentate pre_exec error handling (Nico Schottelius) + * Added start script (Thorsten Elle) + * Documentate autofs hint (Nico Schottelius) + * Speedup source-is-up check and remove --archive (Nico Schottelius) + * Removed support for remote backup (see doc) (Nico Schottelius) diff --git a/doc/changes/next b/doc/changes/next index 2fee139..ec42d4b 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1,16 +1 @@ - * Introduce -j option for max parallel jobs, deprecate -p (Darko Poljak) - * Add locking (Darko Poljak) - * Fix source-is-up check (Nikita Koshikov) - * Fix some minor command line parsing issues (Nico Schottelius) - * Correct output, if configuration is not in cconfig format (Nico Schottelius) - * Minor code cleanups and optimisations (Nico Schottelius) - * ccollect_analyse_logs.sh traps more errors and warnings (Patrick Drolet) - * Remove -v for mkdir and rm, as they are not POSIX (Patrick Drolet) - * Export destination_* to source specific post_exec (Nico Schottelius) - * Update documentation regarding exported variables (Nico Schottelius) - * Simplify time calculation (Nico Schottelius) - * Documentate pre_exec error handling (Nico Schottelius) - * Added start script (Thorsten Elle) - * Documentate autofs hint (Nico Schottelius) - * Speedup source-is-up check and remove --archive (Nico Schottelius) - * Removed support for remote backup (see doc) (Nico Schottelius) +* Add options for stdout, file and syslog logging (Darko Poljak) diff --git a/doc/man/ccollect.text b/doc/man/ccollect.text index 9fc9086..67f8f47 100644 --- a/doc/man/ccollect.text +++ b/doc/man/ccollect.text @@ -29,6 +29,9 @@ OPTIONS -a, --all:: Backup all sources specified in /etc/ccollect/sources +-e, --errors:: + Log only errors + -h, --help:: Show the help screen @@ -36,14 +39,41 @@ OPTIONS Specifies the number of jobs to run simultaneously. If max is not specified then parallelise all jobs. +-l FILE, --logfile FILE:: + Log to specified file + -p, --parallel:: Parallelise backup processes (deprecated from 2.0) +-s, --syslog:: + Log to syslog with tag ccollect + +-V, --version:: + Show version and exit + -v, --verbose:: Be very verbose (uses set -x) --V, --version:: - Show version and exit + +LOGGING MECHANISM +----------------- +ccollect logging depends on running in non-interactive/interactive mode +and on specified optins. The mechanism behaves as the following: + +non-interactive mode:: + + * standard output goes to syslog + * optional: specify logging into file + * log all output by default + * optional: log only errors + +interactive mode:: + + * standard output goes to stdout + * log only errors + * optional: log into syslog or file + - log all output by default + - optional: log only errors SEE ALSO From 89a82ba55ea72d988eae424c6c711143902b44cc Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 22 Mar 2017 10:24:18 +0100 Subject: [PATCH 128/177] Release 2.1 --- ccollect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 6369e84..47526ad 100755 --- a/ccollect +++ b/ccollect @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.0" -RELEASE="2016-09-26" +VERSION="2.1" +RELEASE="2017-03-22" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" From 07c925de5d174b59fd64a9a15f26a0db2caeedf2 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 22 Mar 2017 10:46:57 +0100 Subject: [PATCH 129/177] Release 2.1 --- doc/ccollect.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index b1c7dba..74fcab6 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.0, for ccollect 2.0, Initial Version from 2006-01-13 +2.1, for ccollect 2.1, Initial Version from 2006-01-13 :Author Initials: NS From 04bf9aff3987c8f2543ff887cbb9f4bc6737e871 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 3 Sep 2017 20:00:54 +0200 Subject: [PATCH 130/177] Bugfix: empty rsync_options line causes destroying source. (#8) Bugfix: empty rsync_options line causes destroying source. --- ccollect | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 47526ad..edae187 100755 --- a/ccollect +++ b/ccollect @@ -655,7 +655,17 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # if [ -f "${c_rsync_options}" ]; then while read line; do - set -- "$@" "${line}" + # Trim line. + ln=$(echo "${line}" | awk '{$1=$1;print;}') + # Only if ln is non zero length string. + # + # If ln is empty then rsync '' DEST evaluates + # to transfer current directory to DEST which would + # with specific options destroy DEST content. + if [ -n "${ln}" ] + then + set -- "$@" "${ln}" + fi done < "${c_rsync_options}" fi From b0f131771387cf8eb46d44eb108c891527f5ee68 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 3 Sep 2017 20:02:42 +0200 Subject: [PATCH 131/177] changelog++ --- doc/changes/2.1 | 1 + doc/changes/next | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 100644 doc/changes/2.1 diff --git a/doc/changes/2.1 b/doc/changes/2.1 new file mode 100644 index 0000000..ec42d4b --- /dev/null +++ b/doc/changes/2.1 @@ -0,0 +1 @@ +* Add options for stdout, file and syslog logging (Darko Poljak) diff --git a/doc/changes/next b/doc/changes/next index ec42d4b..0ee09d6 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1 @@ -* Add options for stdout, file and syslog logging (Darko Poljak) +* Bugfix: empty rsync_options line causes destroying source (Darko Poljak) From e504d1f42b4e4ce73e47e7cdb4d1ae24842e6f85 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 3 Sep 2017 22:29:13 +0200 Subject: [PATCH 132/177] Release 2.2 --- ccollect | 4 ++-- doc/changes/2.2 | 1 + doc/changes/next | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 doc/changes/2.2 diff --git a/ccollect b/ccollect index edae187..35af419 100755 --- a/ccollect +++ b/ccollect @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.1" -RELEASE="2017-03-22" +VERSION="2.2" +RELEASE="2017-09-03" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/changes/2.2 b/doc/changes/2.2 new file mode 100644 index 0000000..0ee09d6 --- /dev/null +++ b/doc/changes/2.2 @@ -0,0 +1 @@ +* Bugfix: empty rsync_options line causes destroying source (Darko Poljak) diff --git a/doc/changes/next b/doc/changes/next index 0ee09d6..a767caf 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1 @@ -* Bugfix: empty rsync_options line causes destroying source (Darko Poljak) +* From 890b166a43c0824d66dfdefd0a38e4643401ec13 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 3 Sep 2017 23:11:13 +0200 Subject: [PATCH 133/177] Release 2.2 --- doc/ccollect.text | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/ccollect.text b/doc/ccollect.text index 74fcab6..b97409f 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.1, for ccollect 2.1, Initial Version from 2006-01-13 +2.2, for ccollect 2.2, Initial Version from 2006-01-13 :Author Initials: NS From bd0fe050038e3137d2ede6db5126a7fb92cdc64a Mon Sep 17 00:00:00 2001 From: Jun Futagawa Date: Tue, 19 Dec 2017 17:42:49 +0900 Subject: [PATCH 134/177] Update rpm spec file to version 2.2 (#9) --- contrib/ccollect.spec | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/contrib/ccollect.spec b/contrib/ccollect.spec index 0e35bb0..91bb83b 100644 --- a/contrib/ccollect.spec +++ b/contrib/ccollect.spec @@ -1,6 +1,6 @@ Summary: (pseudo) incremental backup with different exclude lists using hardlinks and rsync Name: ccollect -Version: 0.8.1 +Version: 2.2 Release: 0 URL: http://www.nico.schottelius.org/software/ccollect Source0: http://www.nico.schottelius.org/software/ccollect/%{name}-%{version}.tar.bz2 @@ -23,10 +23,10 @@ Only the inodes used by the hardlinks and the changed files need additional spac %install rm -rf $RPM_BUILD_ROOT -#Installing main ccollect.sh and /etc directory +#Installing main ccollect and /etc directory %__install -d 755 %buildroot%_bindir %__install -d 755 %buildroot%_sysconfdir/%name -%__install -m 755 ccollect.sh %buildroot%_bindir/ +%__install -m 755 ccollect %buildroot%_bindir/ #bin files from tools directory for t in $(ls tools/ccollect_*) ; do @@ -45,19 +45,13 @@ done #Addition documentation and some config tools %__install -d 755 %buildroot%_datadir/%name/tools +%__install -m 755 tools/called_from_remote_pre_exec %buildroot%_datadir/%name/tools %__cp -pr tools/config-pre-* %buildroot%_datadir/%name/tools -%__install -m 755 tools/gnu-du-backup-size-compare.sh %buildroot%_datadir/%name/tools -%__install -m 755 tools/report_success.sh %buildroot%_datadir/%name/tools +%__install -m 755 tools/report_success %buildroot%_datadir/%name/tools %clean rm -rf $RPM_BUILD_ROOT -%post -%__ln_s %_bindir/ccollect.sh %_bindir/ccollect - -%preun -unlink %_bindir/ccollect - %files %defattr(-,root,root) %_bindir/ccollect* From fc0b86005c6231eac06204351e9601393aff2e7a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 30 Jan 2018 09:48:43 +0100 Subject: [PATCH 135/177] Fix parallel mode deadlock when MAX_JOBS > number of sources. --- ccollect | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 35af419..6b48606 100755 --- a/ccollect +++ b/ccollect @@ -455,9 +455,14 @@ if [ "${PARALLEL}" ]; then trap "${TRAPFUNC}" 0 1 2 15 # determine how much parallel jobs to prestart - if [ "${MAX_JOBS}" ] && [ "${MAX_JOBS}" -le "${no_sources}" ] + if [ "${MAX_JOBS}" ] then - prestart="${MAX_JOBS}" + if [ "${MAX_JOBS}" -le "${no_sources}" ] + then + prestart="${MAX_JOBS}" + else + prestart="${no_sources}" + fi else prestart=0 fi From eeccc0b26019f3d9d5f4a93e97522f5190b1827f Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 2 Feb 2018 07:57:41 +0100 Subject: [PATCH 136/177] Release 2.3 --- ccollect | 4 ++-- contrib/ccollect.spec | 2 +- doc/ccollect.text | 2 +- doc/changes/2.3 | 1 + doc/changes/next | 1 - 5 files changed, 5 insertions(+), 5 deletions(-) create mode 100644 doc/changes/2.3 diff --git a/ccollect b/ccollect index 6b48606..70a998e 100755 --- a/ccollect +++ b/ccollect @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.2" -RELEASE="2017-09-03" +VERSION="2.3" +RELEASE="2018-02-02" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/contrib/ccollect.spec b/contrib/ccollect.spec index 91bb83b..8916b7e 100644 --- a/contrib/ccollect.spec +++ b/contrib/ccollect.spec @@ -1,6 +1,6 @@ Summary: (pseudo) incremental backup with different exclude lists using hardlinks and rsync Name: ccollect -Version: 2.2 +Version: 2.3 Release: 0 URL: http://www.nico.schottelius.org/software/ccollect Source0: http://www.nico.schottelius.org/software/ccollect/%{name}-%{version}.tar.bz2 diff --git a/doc/ccollect.text b/doc/ccollect.text index b97409f..b267cdb 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.2, for ccollect 2.2, Initial Version from 2006-01-13 +2.3, for ccollect 2.3, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/2.3 b/doc/changes/2.3 new file mode 100644 index 0000000..960110c --- /dev/null +++ b/doc/changes/2.3 @@ -0,0 +1 @@ +* Bugfix: Fix parallel mode deadlock when max jobs > number of sources (Darko Poljak) diff --git a/doc/changes/next b/doc/changes/next index a767caf..e69de29 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +0,0 @@ -* From 51f468182f5e68147c7c0bbbfb8611750b8cc90c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 20 Apr 2018 17:27:56 +0200 Subject: [PATCH 137/177] Document Windows(Cygwin) as supported OS. --- doc/ccollect.text | 1 + doc/changes/next | 1 + 2 files changed, 2 insertions(+) diff --git a/doc/ccollect.text b/doc/ccollect.text index b267cdb..5585a97 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -26,6 +26,7 @@ Supported and tested operating systems and architectures - Mac OS X 10.5 - NetBSD on alpha/amd64/i386/sparc/sparc64 - OpenBSD on amd64 +- Windows by installing Cygwin, OpenSSH and rsync It *should* run on any Unix that supports `rsync` and has a POSIX-compatible bourne shell. If your platform is not listed above and you have it successfully diff --git a/doc/changes/next b/doc/changes/next index e69de29..784eac6 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -0,0 +1 @@ +* Add Windows(Cygwin) as supported OS From 420dc3fe7fcf5bfb879e70675cb4d412b969f337 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 20 Sep 2018 16:08:00 +0200 Subject: [PATCH 138/177] Add source name tag in log line --- ccollect | 53 ++++++++++++++++++++++++------------------------ doc/changes/next | 1 + 2 files changed, 28 insertions(+), 26 deletions(-) diff --git a/ccollect b/ccollect index 70a998e..04454a1 100755 --- a/ccollect +++ b/ccollect @@ -514,6 +514,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do c_dest="${backup}/destination" c_pre_exec="${backup}/pre_exec" c_post_exec="${backup}/post_exec" + tag="[${name}]" # # Stderr to stdout, so we can produce nice logs @@ -524,13 +525,13 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Record start of backup: internal and for the user # begin_s="$(${SDATE})" - _techo "Beginning to backup" + _techo "${tag}" "Beginning to backup" # # Standard configuration checks # if [ ! -e "${backup}" ]; then - _exit_err "Source does not exist." + _exit_err "Source \"${backup}\" does not exist." fi # @@ -554,11 +555,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # First execute pre_exec, which may generate destination or other parameters # if [ -x "${c_pre_exec}" ]; then - _techo "Executing ${c_pre_exec} ..." + _techo "${tag}" "Executing ${c_pre_exec} ..." "${c_pre_exec}"; ret="$?" - _techo "Finished ${c_pre_exec} (return code ${ret})." + _techo "${tag}" "Finished ${c_pre_exec} (return code ${ret})." - [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." + [ "${ret}" -eq 0 ] || _exit_err "${tag}" "${c_pre_exec} failed. Skipping." fi # @@ -583,7 +584,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" if [ -z "${c_interval}" ]; then - _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." + _exit_err "${tag}" "No definition for interval \"${INTERVAL}\" found. Skipping." fi fi @@ -612,11 +613,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Destination is a path # if [ ! -f "${c_dest}" ]; then - _exit_err "Destination ${c_dest} is not a file. Skipping." + _exit_err "${tag}" "Destination ${c_dest} is not a file. Skipping." else ddir="$(cat "${c_dest}")"; ret="$?" if [ "${ret}" -ne 0 ]; then - _exit_err "Destination ${c_dest} is not readable. Skipping." + _exit_err "${tag}" "Destination ${c_dest} is not readable. Skipping." fi fi @@ -681,7 +682,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi - _exit_err "Source ${source} is not readable. Skipping." + _exit_err "${tag}" "Source ${source} is not readable. Skipping." fi # @@ -692,7 +693,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: destination exists? # - cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping." + cd "${ddir}" || _exit_err "${tag}" "Cannot change to ${ddir}. Skipping." # # Check incomplete backups (needs echo to remove newlines) @@ -700,7 +701,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? if [ "$ret" -eq 0 ]; then - _techo "Incomplete backups: $(echo $(cat "${TMP}"))" + _techo "${tag}" "Incomplete backups: $(echo $(cat "${TMP}"))" if [ -f "${c_delete_incomplete}" ]; then delete_from_file "${TMP}" "${CMARKER}" fi @@ -711,14 +712,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # count="$(ls -1 | grep -c "^${INTERVAL}\\.")" - _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" + _techo "${tag}" "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then remove="$((${count} - ${c_interval} + 1))" - _techo "Removing ${remove} backup(s)..." + _techo "${tag}" "Removing ${remove} backup(s)..." ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + _exit_err "${tag}" "Listing old backups failed" delete_from_file "${TMP}" fi @@ -727,14 +728,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check for backup directory to clone from: Always clone from the latest one! # last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ - _exit_err "Failed to list contents of ${ddir}." + _exit_err "${tag}" "Failed to list contents of ${ddir}." # # Clone from old backup, if existing # if [ "${last_dir}" ]; then set -- "$@" "--link-dest=${ddir}/${last_dir}" - _techo "Hard linking from ${last_dir}" + _techo "${tag}" "Hard linking from ${last_dir}" fi # @@ -747,14 +748,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Mark backup running and go back to original directory # touch "${destination_dir}${CMARKER}" - cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}." + cd "${__abs_mydir}" || _exit_err "${tag}" "Cannot go back to ${__abs_mydir}." # # the rsync part # - _techo "Transferring files..." + _techo "${tag}" "Transferring files..." rsync "$@" "${source}" "${destination_dir}"; ret=$? - _techo "Finished backup (rsync return code: $ret)." + _techo "${tag}" "Finished backup (rsync return code: $ret)." # # Set modification time (mtime) to current time, if sorting by mtime is enabled @@ -778,24 +779,24 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # if [ -z "$fail" ]; then rm "${destination_dir}${CMARKER}" || \ - _exit_err "Removing ${destination_dir}${CMARKER} failed." + _exit_err "${tag}" "Removing ${destination_dir}${CMARKER} failed." if [ "${ret}" -ne 0 ]; then - _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." + _techo "${tag}" "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." fi else - _techo "Warning: rsync failed with return code $ret." + _techo "${tag}" "Warning: rsync failed with return code $ret." fi # # post_exec # if [ -x "${c_post_exec}" ]; then - _techo "Executing ${c_post_exec} ..." + _techo "${tag}" "Executing ${c_post_exec} ..." "${c_post_exec}"; ret=$? - _techo "Finished ${c_post_exec}." + _techo "${tag}" "Finished ${c_post_exec}." if [ "${ret}" -ne 0 ]; then - _exit_err "${c_post_exec} failed." + _exit_err "${tag}" "${c_post_exec} failed." fi fi @@ -808,7 +809,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do minutes="$(((${full_seconds} % 3600) / 60))" seconds="$((${full_seconds} % 60))" - _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" + _techo "${tag}" "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" ) | add_name diff --git a/doc/changes/next b/doc/changes/next index 784eac6..07ffe9d 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1,2 @@ * Add Windows(Cygwin) as supported OS +* Add source name tag in log line From b47a828af0e5e6b3b230aa7338054799767dde2a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 20 Sep 2018 16:08:00 +0200 Subject: [PATCH 139/177] Add source name tag in log line --- ccollect | 66 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/ccollect b/ccollect index 04454a1..b565355 100755 --- a/ccollect +++ b/ccollect @@ -150,11 +150,6 @@ _is_interactive() [ -t 0 -o -p /dev/stdin ] } -add_name() -{ - awk "{ print \"[${name}] \" \$0 }" -} - # # ssh-"feature": we cannot do '... read ...; ssh ...; < file', # because ssh reads stdin! -n does not work -> does not ask for password @@ -252,7 +247,13 @@ _techo() { if [ "${LOGLEVEL}" = "a" ] then - "${_techof}" "$@" + tag="${name:-}" + if [ "${tag}" ]; then + tag="[${tag}]" + else + tag="" + fi + "${_techof}" ${tag} "$@" fi } @@ -514,7 +515,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do c_dest="${backup}/destination" c_pre_exec="${backup}/pre_exec" c_post_exec="${backup}/post_exec" - tag="[${name}]" # # Stderr to stdout, so we can produce nice logs @@ -525,7 +525,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Record start of backup: internal and for the user # begin_s="$(${SDATE})" - _techo "${tag}" "Beginning to backup" + _techo "Beginning to backup" # # Standard configuration checks @@ -555,11 +555,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # First execute pre_exec, which may generate destination or other parameters # if [ -x "${c_pre_exec}" ]; then - _techo "${tag}" "Executing ${c_pre_exec} ..." + _techo "Executing ${c_pre_exec} ..." "${c_pre_exec}"; ret="$?" - _techo "${tag}" "Finished ${c_pre_exec} (return code ${ret})." + _techo "Finished ${c_pre_exec} (return code ${ret})." - [ "${ret}" -eq 0 ] || _exit_err "${tag}" "${c_pre_exec} failed. Skipping." + [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." fi # @@ -584,7 +584,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" if [ -z "${c_interval}" ]; then - _exit_err "${tag}" "No definition for interval \"${INTERVAL}\" found. Skipping." + _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." fi fi @@ -613,11 +613,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Destination is a path # if [ ! -f "${c_dest}" ]; then - _exit_err "${tag}" "Destination ${c_dest} is not a file. Skipping." + _exit_err "Destination ${c_dest} is not a file. Skipping." else ddir="$(cat "${c_dest}")"; ret="$?" if [ "${ret}" -ne 0 ]; then - _exit_err "${tag}" "Destination ${c_dest} is not readable. Skipping." + _exit_err "Destination ${c_dest} is not readable. Skipping." fi fi @@ -682,7 +682,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi - _exit_err "${tag}" "Source ${source} is not readable. Skipping." + _exit_err "Source ${source} is not readable. Skipping." fi # @@ -693,7 +693,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check: destination exists? # - cd "${ddir}" || _exit_err "${tag}" "Cannot change to ${ddir}. Skipping." + cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping." # # Check incomplete backups (needs echo to remove newlines) @@ -701,7 +701,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? if [ "$ret" -eq 0 ]; then - _techo "${tag}" "Incomplete backups: $(echo $(cat "${TMP}"))" + _techo "Incomplete backups: $(echo $(cat "${TMP}"))" if [ -f "${c_delete_incomplete}" ]; then delete_from_file "${TMP}" "${CMARKER}" fi @@ -712,14 +712,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # count="$(ls -1 | grep -c "^${INTERVAL}\\.")" - _techo "${tag}" "Existing backups: ${count} Total keeping backups: ${c_interval}" + _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then remove="$((${count} - ${c_interval} + 1))" - _techo "${tag}" "Removing ${remove} backup(s)..." + _techo "Removing ${remove} backup(s)..." ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "${tag}" "Listing old backups failed" + _exit_err "Listing old backups failed" delete_from_file "${TMP}" fi @@ -728,14 +728,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check for backup directory to clone from: Always clone from the latest one! # last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ - _exit_err "${tag}" "Failed to list contents of ${ddir}." + _exit_err "Failed to list contents of ${ddir}." # # Clone from old backup, if existing # if [ "${last_dir}" ]; then set -- "$@" "--link-dest=${ddir}/${last_dir}" - _techo "${tag}" "Hard linking from ${last_dir}" + _techo "Hard linking from ${last_dir}" fi # @@ -748,14 +748,14 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Mark backup running and go back to original directory # touch "${destination_dir}${CMARKER}" - cd "${__abs_mydir}" || _exit_err "${tag}" "Cannot go back to ${__abs_mydir}." + cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}." # # the rsync part # - _techo "${tag}" "Transferring files..." + _techo "Transferring files..." rsync "$@" "${source}" "${destination_dir}"; ret=$? - _techo "${tag}" "Finished backup (rsync return code: $ret)." + _techo "Finished backup (rsync return code: $ret)." # # Set modification time (mtime) to current time, if sorting by mtime is enabled @@ -779,24 +779,24 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # if [ -z "$fail" ]; then rm "${destination_dir}${CMARKER}" || \ - _exit_err "${tag}" "Removing ${destination_dir}${CMARKER} failed." + _exit_err "Removing ${destination_dir}${CMARKER} failed." if [ "${ret}" -ne 0 ]; then - _techo "${tag}" "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." + _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." fi else - _techo "${tag}" "Warning: rsync failed with return code $ret." + _techo "Warning: rsync failed with return code $ret." fi # # post_exec # if [ -x "${c_post_exec}" ]; then - _techo "${tag}" "Executing ${c_post_exec} ..." + _techo "Executing ${c_post_exec} ..." "${c_post_exec}"; ret=$? - _techo "${tag}" "Finished ${c_post_exec}." + _techo "Finished ${c_post_exec}." if [ "${ret}" -ne 0 ]; then - _exit_err "${tag}" "${c_post_exec} failed." + _exit_err "${c_post_exec} failed." fi fi @@ -809,10 +809,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do minutes="$(((${full_seconds} % 3600) / 60))" seconds="$((${full_seconds} % 60))" - _techo "${tag}" "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" + _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" -) | add_name +) done # From a63e16efc5a4db733b5505067468f702e48b3e1e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 23 Sep 2018 11:42:09 +0200 Subject: [PATCH 140/177] Use better expansion and parameter setting --- ccollect | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/ccollect b/ccollect index b565355..c3f6328 100755 --- a/ccollect +++ b/ccollect @@ -247,13 +247,8 @@ _techo() { if [ "${LOGLEVEL}" = "a" ] then - tag="${name:-}" - if [ "${tag}" ]; then - tag="[${tag}]" - else - tag="" - fi - "${_techof}" ${tag} "$@" + set -- ${name:+"[${name}]"} $@ + "${_techof}" "$@" fi } From 71eabe2f23f9a9c5fcd3f4d9a94a3d82d2f5cb2a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 25 Sep 2018 21:14:55 +0200 Subject: [PATCH 141/177] Release 2.4 --- ccollect | 6 +++--- doc/ccollect.text | 2 +- doc/changes/{next => 2.4} | 0 3 files changed, 4 insertions(+), 4 deletions(-) rename doc/changes/{next => 2.4} (100%) diff --git a/ccollect b/ccollect index c3f6328..3c39d9c 100755 --- a/ccollect +++ b/ccollect @@ -1,7 +1,7 @@ #!/bin/sh # # 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org) -# 2016 Darko Poljak (darko.poljak at gmail.com) +# 2016-2018 Darko Poljak (darko.poljak at gmail.com) # # This file is part of ccollect. # @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.3" -RELEASE="2018-02-02" +VERSION="2.4" +RELEASE="2018-09-25" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 5585a97..9851d08 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.3, for ccollect 2.3, Initial Version from 2006-01-13 +2.4, for ccollect 2.4, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.4 similarity index 100% rename from doc/changes/next rename to doc/changes/2.4 From 835e21c56c6580debb2d661e3b5145601280294c Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 26 Apr 2019 14:03:01 +0200 Subject: [PATCH 142/177] exit in case of subshell error --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 3c39d9c..335b51d 100755 --- a/ccollect +++ b/ccollect @@ -807,7 +807,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" -) +) || exit done # From 2725a1ced49d8af47abde2618bc53a08f08499f6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 26 Apr 2019 14:47:21 +0200 Subject: [PATCH 143/177] github -> code.ungleich.ch --- Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/Makefile b/Makefile index 908bd6c..dc98502 100644 --- a/Makefile +++ b/Makefile @@ -175,7 +175,6 @@ t2: # pub: git push - git push github publish-doc: documentation @echo "Transferring files to ${host}" From 10dcf076a9f9ccd7e8cca6a514835c1ee0c955ea Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 1 May 2019 14:38:35 +0200 Subject: [PATCH 144/177] Release 2.5 --- ccollect | 6 +++--- doc/ccollect.text | 2 +- doc/changes/2.5 | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 doc/changes/2.5 diff --git a/ccollect b/ccollect index 335b51d..bb44ed6 100755 --- a/ccollect +++ b/ccollect @@ -1,7 +1,7 @@ #!/bin/sh # # 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org) -# 2016-2018 Darko Poljak (darko.poljak at gmail.com) +# 2016-2019 Darko Poljak (darko.poljak at gmail.com) # # This file is part of ccollect. # @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.4" -RELEASE="2018-09-25" +VERSION="2.5" +RELEASE="2019-05-01" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 9851d08..f252b25 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.4, for ccollect 2.4, Initial Version from 2006-01-13 +2.5, for ccollect 2.5, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/2.5 b/doc/changes/2.5 new file mode 100644 index 0000000..a6b319c --- /dev/null +++ b/doc/changes/2.5 @@ -0,0 +1 @@ +* Bugfix: exit in case of subshell error From 1628ce58c768211052b9d8d02cd0c4d3c7ea3a17 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sat, 5 Oct 2019 10:48:19 +0200 Subject: [PATCH 145/177] Replace rm with faster rsync --delete with empty src dir --- ccollect | 18 +++++++++++++++--- doc/changes/next | 1 + 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 doc/changes/next diff --git a/ccollect b/ccollect index bb44ed6..05b7014 100755 --- a/ccollect +++ b/ccollect @@ -160,16 +160,28 @@ delete_from_file() file="$1"; shift suffix="" # It will be set, if deleting incomplete backups. [ $# -eq 1 ] && suffix="$1" && shift + # dirs for deletion will be moved to this trash dir inside destination dir + # - for fast mv operation + trash="$(mktemp -d "trash.XXXXXX")" while read to_remove; do + mv "${to_remove}" "${trash}" || + _exit_err "Moving ${to_remove} to ${trash} failed." set -- "$@" "${to_remove}" if [ "${suffix}" ]; then to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" + mv "${to_remove_no_suffix}" "${trash}" || + _exit_err "Moving ${to_remove_no_suffix} to ${trash} failed." set -- "$@" "${to_remove_no_suffix}" fi done < "${file}" - _techo "Removing $@ ..." - [ "${VVERBOSE}" ] && echo rm "$@" - rm -rf "$@" || _exit_err "Removing $@ failed." + _techo "Removing $@ in ${trash}..." + empty_dir="empty-dir" + mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." + [ "${VVERBOSE}" ] && echo "rsync -a --delete ${empty_dir} ${trash}" + # rsync needs ending slash for directory content + rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." + rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" + rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" } display_version() diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..df06edc --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Performance: replace rm -rf with faster rsync --delete empty src dir From ca6d06c2c32c0296dfc8dcb107d469f863fd67e6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 10 Oct 2019 10:54:56 +0200 Subject: [PATCH 146/177] Add more verbose logging --- ccollect | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 05b7014..8e02524 100755 --- a/ccollect +++ b/ccollect @@ -177,11 +177,12 @@ delete_from_file() _techo "Removing $@ in ${trash}..." empty_dir="empty-dir" mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." - [ "${VVERBOSE}" ] && echo "rsync -a --delete ${empty_dir} ${trash}" + [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}" # rsync needs ending slash for directory content rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" + [ "${VVERBOSE}" ] && echo "Finished: rsync -a --delete ${empty_dir} ${trash}" } display_version() From 30abef474dea75705ee005a0453ea536e73d2fb8 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 16 Oct 2019 14:03:13 +0200 Subject: [PATCH 147/177] Delete in background and finally wait for children --- ccollect | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 8e02524..7ede4dc 100755 --- a/ccollect +++ b/ccollect @@ -711,7 +711,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ "$ret" -eq 0 ]; then _techo "Incomplete backups: $(echo $(cat "${TMP}"))" if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" "${CMARKER}" + delete_from_file "${TMP}" "${CMARKER}" & fi fi @@ -729,7 +729,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ _exit_err "Listing old backups failed" - delete_from_file "${TMP}" + delete_from_file "${TMP}" & fi # @@ -820,6 +820,9 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" + + # wait for children (doing delete_from_file) if any still running + wait ) || exit done From bfb3c6338c3e7a5bb1f4dcf8c3e1aa9b1117fad0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 16 Oct 2019 15:50:54 +0200 Subject: [PATCH 148/177] _techo instead of very verbose --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 7ede4dc..f424f11 100755 --- a/ccollect +++ b/ccollect @@ -182,7 +182,7 @@ delete_from_file() rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" - [ "${VVERBOSE}" ] && echo "Finished: rsync -a --delete ${empty_dir} ${trash}" + _techo "Removing $@ in ${trash} finished." } display_version() From 702cdf931e6b76be8aaad5c8cbfaaf4f9e2dc168 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 06:52:51 +0200 Subject: [PATCH 149/177] Use hidden directory for deletion --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index f424f11..41f23c5 100755 --- a/ccollect +++ b/ccollect @@ -162,7 +162,7 @@ delete_from_file() [ $# -eq 1 ] && suffix="$1" && shift # dirs for deletion will be moved to this trash dir inside destination dir # - for fast mv operation - trash="$(mktemp -d "trash.XXXXXX")" + trash="$(mktemp -d ".trash.XXXXXX")" while read to_remove; do mv "${to_remove}" "${trash}" || _exit_err "Moving ${to_remove} to ${trash} failed." From 51dcf4a02f3b57bc94c1bc15b955ef097bfbf3a6 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 06:52:51 +0200 Subject: [PATCH 150/177] Use hidden empty directory for deletion --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 41f23c5..c744e27 100755 --- a/ccollect +++ b/ccollect @@ -175,7 +175,7 @@ delete_from_file() fi done < "${file}" _techo "Removing $@ in ${trash}..." - empty_dir="empty-dir" + empty_dir=".empty-dir" mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}" # rsync needs ending slash for directory content From 1e18e71b9de892b255516cdd9fbfdfa4304f7c57 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 07:42:31 +0200 Subject: [PATCH 151/177] Use oldest backup as destination dir without deletion --- ccollect | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/ccollect b/ccollect index c744e27..cf7abe9 100755 --- a/ccollect +++ b/ccollect @@ -715,6 +715,12 @@ while [ "${source_no}" -lt "${no_sources}" ]; do fi fi + # + # Include current time in name, not the time when we began to remove above + # + export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" + export destination_dir="${ddir}/${destination_name}" + # # Check: maximum number of backups is reached? # @@ -723,13 +729,26 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then - remove="$((${count} - ${c_interval} + 1))" - _techo "Removing ${remove} backup(s)..." + # Use oldest directory as new backup destination directory. + # It need not to be deleted, rsync will sync its content. + oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ + _exit_err "Listing oldest backup failed") + _techo "Using ${oldest_bak} for destination dir" + mv "${oldest_bak}" "${destination_dir}" || + _exit_err "Moving oldest backup ${oldest_bak} to ${destination_dir} failed." + # Touch dest dir so it is not sorted wrong in listings below. + touch "${destination_dir}" - ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + # We have something to remove only if count > interval. + if [ "${count}" -gt "${c_interval}" ]; then + remove="$((${count} - ${c_interval}))" + _techo "Removing ${remove} backup(s)..." - delete_from_file "${TMP}" & + ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + + delete_from_file "${TMP}" & + fi fi # @@ -746,12 +765,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Hard linking from ${last_dir}" fi - # - # Include current time in name, not the time when we began to remove above - # - export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" - export destination_dir="${ddir}/${destination_name}" - # # Mark backup running and go back to original directory # From 2788de47b8a7f5a23192b48f421ed37a059f8bb0 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 07:52:53 +0200 Subject: [PATCH 152/177] Improve log line --- ccollect | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccollect b/ccollect index cf7abe9..af7a2c1 100755 --- a/ccollect +++ b/ccollect @@ -733,7 +733,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # It need not to be deleted, rsync will sync its content. oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ _exit_err "Listing oldest backup failed") - _techo "Using ${oldest_bak} for destination dir" + _techo "Using ${oldest_bak} for destination dir ${destination_dir}" mv "${oldest_bak}" "${destination_dir}" || _exit_err "Moving oldest backup ${oldest_bak} to ${destination_dir} failed." # Touch dest dir so it is not sorted wrong in listings below. From c39205d30898110b40040c6fad2d6a026013b977 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 08:03:12 +0200 Subject: [PATCH 153/177] Exclude destintion dir from listing for last dir --- ccollect | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index af7a2c1..35d2071 100755 --- a/ccollect +++ b/ccollect @@ -753,8 +753,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check for backup directory to clone from: Always clone from the latest one! + # Exclude destination_dir from listing, it can be touched reused and renamed + # oldest existing destination directory. # - last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ + last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${destination_dir}" | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." # From 7701bdb0a87413bc756ea408656e32d65d7dfb20 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 08:06:13 +0200 Subject: [PATCH 154/177] Use destination dir basename --- ccollect | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ccollect b/ccollect index 35d2071..f5aa145 100755 --- a/ccollect +++ b/ccollect @@ -756,7 +756,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Exclude destination_dir from listing, it can be touched reused and renamed # oldest existing destination directory. # - last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${destination_dir}" | head -n 1)" || \ + dest_dir_name=$(basename "${destination_dir}") + last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${dest_dir_name}" | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." # From e44dede92f9114db1ffec24a46f215fb270ed08a Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 09:05:55 +0200 Subject: [PATCH 155/177] ++changelog --- doc/changes/next | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/next b/doc/changes/next index df06edc..a68cbbc 100644 --- a/doc/changes/next +++ b/doc/changes/next @@ -1 +1 @@ -* Performance: replace rm -rf with faster rsync --delete empty src dir +* Improve performance, improve process of deletion of old backups From de720ecfe9a71bd2fad3e5ef429629007e8ea6ed Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 17 Oct 2019 11:53:46 +0200 Subject: [PATCH 156/177] If renaming oldest bak dir fails then fallback to removing it --- ccollect | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/ccollect b/ccollect index f5aa145..b8011ba 100755 --- a/ccollect +++ b/ccollect @@ -734,14 +734,17 @@ while [ "${source_no}" -lt "${no_sources}" ]; do oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ _exit_err "Listing oldest backup failed") _techo "Using ${oldest_bak} for destination dir ${destination_dir}" - mv "${oldest_bak}" "${destination_dir}" || - _exit_err "Moving oldest backup ${oldest_bak} to ${destination_dir} failed." - # Touch dest dir so it is not sorted wrong in listings below. - touch "${destination_dir}" + if mv "${oldest_bak}" "${destination_dir}"; then + # Touch dest dir so it is not sorted wrong in listings below. + touch "${destination_dir}" - # We have something to remove only if count > interval. - if [ "${count}" -gt "${c_interval}" ]; then + # We have something to remove only if count > interval. remove="$((${count} - ${c_interval}))" + else + _techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it." + remove="$((${count} - ${c_interval} + 1))" + fi + if [ "${remove}" -gt 0 ]; then _techo "Removing ${remove} backup(s)..." ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ From 74e3b26790e4b77f2761d62bd6811aab9bfb9bbd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 12 Nov 2019 17:55:15 +0100 Subject: [PATCH 157/177] Release 2.6 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/{next => 2.6} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename doc/changes/{next => 2.6} (100%) diff --git a/ccollect b/ccollect index b8011ba..4fa6da3 100755 --- a/ccollect +++ b/ccollect @@ -44,8 +44,8 @@ CMARKER=".ccollect-marker" export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.5" -RELEASE="2019-05-01" +VERSION="2.6" +RELEASE="2019-11-12" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index f252b25..37e4eaf 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.5, for ccollect 2.5, Initial Version from 2006-01-13 +2.6, for ccollect 2.6, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.6 similarity index 100% rename from doc/changes/next rename to doc/changes/2.6 From 2cefdaa1a5f6adac7240534e558ba9196872cbfa Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 18 Oct 2019 16:24:22 +0200 Subject: [PATCH 158/177] Fix shellcheck reported issues --- ccollect | 115 ++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 75 insertions(+), 40 deletions(-) diff --git a/ccollect b/ccollect index 4fa6da3..6dd3fc0 100755 --- a/ccollect +++ b/ccollect @@ -27,9 +27,9 @@ set -u # # Standard variables (stolen from cconf) # -__pwd="$(pwd -P)" -__mydir="${0%/*}"; __abs_mydir="$(cd "$__mydir" && pwd -P)" -__myname=${0##*/}; __abs_myname="$__abs_mydir/$__myname" +__mydir="${0%/*}" +__abs_mydir="$(cd "$__mydir" && pwd -P)" +__myname=${0##*/} # # where to find our configuration and temporary file @@ -41,7 +41,8 @@ CPREEXEC="${CDEFAULTS}/pre_exec" CPOSTEXEC="${CDEFAULTS}/post_exec" CMARKER=".ccollect-marker" -export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" +TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" +export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" VERSION="2.6" @@ -76,6 +77,7 @@ LOCKFD=4 lock_flock() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" eval "exec ${LOCKFD}> ${lockfile}" @@ -85,6 +87,7 @@ lock_flock() unlock_flock() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" eval "exec ${LOCKFD}>&-" rm -f "${lockfile}" @@ -96,6 +99,7 @@ unlock_flock() lock_mkdir() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" mkdir "${lockfile}" && return 0 || return 1 @@ -104,6 +108,7 @@ lock_mkdir() unlock_mkdir() { # $1 = source to backup + # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" rmdir "${lockfile}" @@ -112,7 +117,7 @@ unlock_mkdir() # # determine locking tool: flock or mkdir # -if $(which flock > /dev/null 2>&1) +if command -v flock > /dev/null 2>&1 then lockf="lock_flock" unlockf="unlock_flock" @@ -137,6 +142,7 @@ LOGONLYERRORS="" # catch signals # TRAPFUNC="rm -f \"${TMP}\"" +# shellcheck disable=SC2064 trap "${TRAPFUNC}" 1 2 15 # @@ -147,7 +153,7 @@ trap "${TRAPFUNC}" 1 2 15 # see: http://www.tldp.org/LDP/abs/html/intandnonint.html _is_interactive() { - [ -t 0 -o -p /dev/stdin ] + [ -t 0 ] || [ -p /dev/stdin ] } # @@ -163,26 +169,26 @@ delete_from_file() # dirs for deletion will be moved to this trash dir inside destination dir # - for fast mv operation trash="$(mktemp -d ".trash.XXXXXX")" - while read to_remove; do + while read -r to_remove; do mv "${to_remove}" "${trash}" || _exit_err "Moving ${to_remove} to ${trash} failed." set -- "$@" "${to_remove}" if [ "${suffix}" ]; then - to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" + to_remove_no_suffix="$(echo "${to_remove}" | sed "s/$suffix\$//")" mv "${to_remove_no_suffix}" "${trash}" || _exit_err "Moving ${to_remove_no_suffix} to ${trash} failed." set -- "$@" "${to_remove_no_suffix}" fi done < "${file}" - _techo "Removing $@ in ${trash}..." + _techo "Removing $* in ${trash}..." empty_dir=".empty-dir" mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}" # rsync needs ending slash for directory content - rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." + rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $* failed." rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" - _techo "Removing $@ in ${trash} finished." + _techo "Removing $* in ${trash} finished." } display_version() @@ -232,7 +238,7 @@ unlock() # stdout version _techo_stdout() { - echo "$(${DDATE}): $@" + echo "$(${DDATE}): $*" } # syslog version @@ -260,14 +266,16 @@ _techo() { if [ "${LOGLEVEL}" = "a" ] then - set -- ${name:+"[${name}]"} $@ + # name is exported before calling this function + # shellcheck disable=SC2154 + set -- ${name:+"[${name}]"} "$@" "${_techof}" "$@" fi } _techo_err() { - _techo "Error: $@" + _techo "Error: $*" } _exit_err() @@ -368,7 +376,7 @@ else LOGLEVEL="a" fi -if [ "${LOGFILE}" -o "${SYSLOG}" ] +if [ "${LOGFILE}" ] || [ "${SYSLOG}" ] then if [ "${LOGONLYERRORS}" ] then @@ -378,8 +386,7 @@ fi # check that MAX_JOBS is natural number > 0 # empty string means run all in parallel -echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' -if [ "$?" -ne 0 ] +if ! echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$' then _exit_err "Invalid max jobs value \"${MAX_JOBS}\"" fi @@ -412,19 +419,22 @@ if [ "${USE_ALL}" = 1 ]; then ( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \ _exit_err "Listing of sources failed. Aborting." - while read tmp; do - eval export source_${no_sources}=\"${tmp}\" - no_sources=$((${no_sources}+1)) + while read -r tmp; do + eval export "source_${no_sources}=\"${tmp}\"" + no_sources=$((no_sources + 1)) done < "${TMP}" else # # Get sources from command line # while [ "$#" -ge 1 ]; do - eval arg=\"\$1\"; shift + eval "arg=\"\$1\"" + shift - eval export source_${no_sources}=\"${arg}\" - no_sources="$((${no_sources}+1))" + # arg is assigned in the eval above + # shellcheck disable=SC2154 + eval export "source_${no_sources}=\"${arg}\"" + no_sources="$((no_sources + 1))" done fi @@ -461,6 +471,7 @@ if [ "${PARALLEL}" ]; then # fd 5 is tied to control pipe eval "exec 5<>${CONTROL_PIPE}" TRAPFUNC="${TRAPFUNC}; rm -f \"${CONTROL_PIPE}\"" + # shellcheck disable=SC2064 trap "${TRAPFUNC}" 0 1 2 15 # determine how much parallel jobs to prestart @@ -483,7 +494,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Get current source # eval export name=\"\$source_${source_no}\" - source_no=$((${source_no}+1)) + source_no=$((source_no + 1)) # # Start ourself, if we want parallel execution @@ -498,12 +509,12 @@ while [ "${source_no}" -lt "${no_sources}" ]; do then # run prestart child if pending { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & - prestart=$((${prestart} - 1)) + prestart=$((prestart - 1)) continue else # each time a child finishes we get a line from the pipe # and then launch another child - while read line + while read -r line do { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } & # get out of loop so we can contnue with main loop @@ -557,6 +568,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # redefine trap to also unlock (rm lockfile) TRAPFUNC="${TRAPFUNC}; unlock \"${name}\"" + # shellcheck disable=SC2064 trap "${TRAPFUNC}" 1 2 15 # @@ -576,10 +588,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do for opt in verbose very_verbose summary exclude rsync_options \ delete_incomplete rsync_failure_codes \ mtime quiet_if_down ; do - if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}" ]; then - eval c_$opt=\"${backup}/$opt\" + if [ -f "${backup}/${opt}" ] || [ -f "${backup}/no_${opt}" ]; then + eval "c_$opt=\"${backup}/$opt\"" else - eval c_$opt=\"${CDEFAULTS}/$opt\" + eval "c_$opt=\"${CDEFAULTS}/$opt\"" fi done @@ -599,6 +611,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Sort by ctime (default) or mtime (configuration option) # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_mtime}" ] ; then TSORT="t" else @@ -642,6 +656,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Exclude list # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_exclude}" ]; then set -- "$@" "--exclude-from=${c_exclude}" fi @@ -649,6 +665,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Output a summary # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_summary}" ]; then set -- "$@" "--stats" fi @@ -657,6 +675,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Verbosity for rsync, rm, and mkdir # VVERBOSE="" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_very_verbose}" ]; then set -- "$@" "-vv" VVERBOSE="-v" @@ -667,8 +687,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Extra options for rsync provided by the user # + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_rsync_options}" ]; then - while read line; do + while read -r line; do # Trim line. ln=$(echo "${line}" | awk '{$1=$1;print;}') # Only if ln is non zero length string. @@ -687,6 +709,8 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check: source is up and accepting connections (before deleting old backups!) # if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then + # variable is assigned using eval + # shellcheck disable=SC2154 if [ ! -f "${c_quiet_if_down}" ]; then cat "${TMP}" fi @@ -706,10 +730,13 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Check incomplete backups (needs echo to remove newlines) # + # shellcheck disable=SC2010 ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? if [ "$ret" -eq 0 ]; then - _techo "Incomplete backups: $(echo $(cat "${TMP}"))" + _techo "Incomplete backups: $(cat "${TMP}")" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "${c_delete_incomplete}" ]; then delete_from_file "${TMP}" "${CMARKER}" & fi @@ -718,12 +745,15 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # # Include current time in name, not the time when we began to remove above # - export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" - export destination_dir="${ddir}/${destination_name}" + destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" + export destination_name + destination_dir="${ddir}/${destination_name}" + export destination_dir # # Check: maximum number of backups is reached? # + # shellcheck disable=SC2010 count="$(ls -1 | grep -c "^${INTERVAL}\\.")" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" @@ -731,6 +761,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ "${count}" -ge "${c_interval}" ]; then # Use oldest directory as new backup destination directory. # It need not to be deleted, rsync will sync its content. + # shellcheck disable=SC2010 oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ _exit_err "Listing oldest backup failed") _techo "Using ${oldest_bak} for destination dir ${destination_dir}" @@ -739,14 +770,15 @@ while [ "${source_no}" -lt "${no_sources}" ]; do touch "${destination_dir}" # We have something to remove only if count > interval. - remove="$((${count} - ${c_interval}))" + remove="$((count - c_interval))" else _techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it." - remove="$((${count} - ${c_interval} + 1))" + remove="$((count - c_interval + 1))" fi if [ "${remove}" -gt 0 ]; then _techo "Removing ${remove} backup(s)..." + # shellcheck disable=SC2010 ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ _exit_err "Listing old backups failed" @@ -760,6 +792,7 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # oldest existing destination directory. # dest_dir_name=$(basename "${destination_dir}") + # shellcheck disable=SC2010 last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${dest_dir_name}" | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." @@ -793,8 +826,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Check if rsync exit code indicates failure. # fail="" + # variable is assigned using eval + # shellcheck disable=SC2154 if [ -f "$c_rsync_failure_codes" ]; then - while read code ; do + while read -r code ; do if [ "$ret" = "$code" ]; then fail=1 fi @@ -831,10 +866,10 @@ while [ "${source_no}" -lt "${no_sources}" ]; do # Time calculation # end_s="$(${SDATE})" - full_seconds="$((${end_s} - ${begin_s}))" - hours="$((${full_seconds} / 3600))" - minutes="$(((${full_seconds} % 3600) / 60))" - seconds="$((${full_seconds} % 60))" + full_seconds="$((end_s - begin_s))" + hours="$((full_seconds / 3600))" + minutes="$(((full_seconds % 3600) / 60))" + seconds="$((full_seconds % 60))" _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" From a5e565b5d69106c11c8090825038b8af51c6fdb5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 14 Nov 2019 19:16:11 +0100 Subject: [PATCH 159/177] Add shellcheck makefile target --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index dc98502..9b07e09 100644 --- a/Makefile +++ b/Makefile @@ -199,6 +199,9 @@ dist: distclean documentation /tmp/ccollect: mkdir -p /tmp/ccollect +shellcheck: ./ccollect + shellcheck -s sh -f gcc -x ./ccollect + test: $(CCOLLECT_SOURCE) /tmp/ccollect cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect daily "$$s"; done touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker From f818f011e3cdd5aa556ee140913f607381555cf5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Thu, 14 Nov 2019 19:20:16 +0100 Subject: [PATCH 160/177] Release 2.7 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/2.7 | 1 + 3 files changed, 4 insertions(+), 3 deletions(-) create mode 100644 doc/changes/2.7 diff --git a/ccollect b/ccollect index 6dd3fc0..9077e0f 100755 --- a/ccollect +++ b/ccollect @@ -45,8 +45,8 @@ TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.6" -RELEASE="2019-11-12" +VERSION="2.7" +RELEASE="2019-11-14" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 37e4eaf..2478265 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.6, for ccollect 2.6, Initial Version from 2006-01-13 +2.7, for ccollect 2.7, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/2.7 b/doc/changes/2.7 new file mode 100644 index 0000000..bafbb01 --- /dev/null +++ b/doc/changes/2.7 @@ -0,0 +1 @@ +* Fix shellcheck reported issues From 401dd4fa8e4a50e7f8a706c35bf5c505b874a205 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 15 Nov 2019 08:29:21 +0100 Subject: [PATCH 161/177] Fix path with spaces in eval --- ccollect | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ccollect b/ccollect index 9077e0f..d89aec1 100755 --- a/ccollect +++ b/ccollect @@ -79,7 +79,7 @@ lock_flock() # $1 = source to backup # shellcheck disable=SC2059 lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")" - eval "exec ${LOCKFD}> ${lockfile}" + eval "exec ${LOCKFD}> '${lockfile}'" flock -n ${LOCKFD} && return 0 || return 1 } @@ -469,7 +469,7 @@ fi if [ "${PARALLEL}" ]; then mkfifo "${CONTROL_PIPE}" # fd 5 is tied to control pipe - eval "exec 5<>${CONTROL_PIPE}" + eval "exec 5<>'${CONTROL_PIPE}'" TRAPFUNC="${TRAPFUNC}; rm -f \"${CONTROL_PIPE}\"" # shellcheck disable=SC2064 trap "${TRAPFUNC}" 0 1 2 15 From 8f5d9b2c978916d3093ab7c2ce7bb0a94e580018 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Fri, 15 Nov 2019 08:28:55 +0100 Subject: [PATCH 162/177] Add unit testing --- .gitignore | 1 - Makefile | 44 ++++++++++++++++++- test/conf/ccollect_local-with | 0 test/conf/ccollect_source | 0 test/conf/defaults/intervals/daily | 1 + test/conf/defaults/intervals/monthly | 1 + test/conf/defaults/intervals/normal | 1 + test/conf/defaults/intervals/weekly | 1 + test/conf/defaults/post_exec | 5 +++ test/conf/defaults/pre_exec | 5 +++ test/conf/defaults/sources/exclude | 1 + test/conf/defaults/sources/rsync_options | 0 test/conf/defaults/sources/verbose | 0 test/conf/defaults/verbose | 0 .../delete_incomplete/delete_incomplete | 0 .../sources/delete_incomplete/destination | 1 + test/conf/sources/delete_incomplete/exclude | 1 + test/conf/sources/delete_incomplete/source | 1 + .../sources/local-with&ersand/destination | 1 + .../conf/sources/local-with&ersand/exclude | 1 + test/conf/sources/local-with&ersand/source | 1 + .../local-with-interval/delete_incomplete | 0 .../sources/local-with-interval/destination | 1 + test/conf/sources/local-with-interval/exclude | 1 + .../local-with-interval/intervals/daily | 1 + test/conf/sources/local-with-interval/source | 1 + test/conf/sources/local-with-interval/verbose | 0 test/conf/sources/local/destination | 1 + test/conf/sources/local/exclude | 1 + test/conf/sources/local/no_verbose | 0 test/conf/sources/local/source | 1 + .../delete_incomplete | 0 .../destination | 1 + .../source with spaces and interval/exclude | 1 + .../source with spaces and interval/source | 1 + .../source with spaces and interval/verbose | 0 test/conf/sources/very_verbose/destination | 1 + test/conf/sources/very_verbose/exclude | 1 + test/conf/sources/very_verbose/source | 1 + test/conf/sources/very_verbose/summary | 0 test/conf/sources/very_verbose/verbose | 0 test/conf/sources/very_verbose/very_verbose | 0 test/conf/sources/with_exec/destination | 1 + test/conf/sources/with_exec/post_exec | 5 +++ test/conf/sources/with_exec/pre_exec | 5 +++ test/conf/sources/with_exec/source | 1 + test/exec.sh | 18 -------- test/local.sh | 1 - test/remote.sh | 1 - test/return-value.sh | 23 ---------- test/test-ccollect-tools.sh | 29 ------------ test/test-ccollect1.sh | 44 ------------------- 52 files changed, 89 insertions(+), 118 deletions(-) create mode 100644 test/conf/ccollect_local-with create mode 100644 test/conf/ccollect_source create mode 100644 test/conf/defaults/intervals/daily create mode 100644 test/conf/defaults/intervals/monthly create mode 100644 test/conf/defaults/intervals/normal create mode 100644 test/conf/defaults/intervals/weekly create mode 100755 test/conf/defaults/post_exec create mode 100755 test/conf/defaults/pre_exec create mode 100644 test/conf/defaults/sources/exclude create mode 100644 test/conf/defaults/sources/rsync_options create mode 100644 test/conf/defaults/sources/verbose create mode 100644 test/conf/defaults/verbose create mode 100644 test/conf/sources/delete_incomplete/delete_incomplete create mode 100644 test/conf/sources/delete_incomplete/destination create mode 100644 test/conf/sources/delete_incomplete/exclude create mode 100644 test/conf/sources/delete_incomplete/source create mode 100644 test/conf/sources/local-with&ersand/destination create mode 100644 test/conf/sources/local-with&ersand/exclude create mode 100644 test/conf/sources/local-with&ersand/source create mode 100644 test/conf/sources/local-with-interval/delete_incomplete create mode 100644 test/conf/sources/local-with-interval/destination create mode 100644 test/conf/sources/local-with-interval/exclude create mode 100644 test/conf/sources/local-with-interval/intervals/daily create mode 100644 test/conf/sources/local-with-interval/source create mode 100644 test/conf/sources/local-with-interval/verbose create mode 100644 test/conf/sources/local/destination create mode 100644 test/conf/sources/local/exclude create mode 100644 test/conf/sources/local/no_verbose create mode 100644 test/conf/sources/local/source create mode 100644 test/conf/sources/source with spaces and interval/delete_incomplete create mode 100644 test/conf/sources/source with spaces and interval/destination create mode 100644 test/conf/sources/source with spaces and interval/exclude create mode 100644 test/conf/sources/source with spaces and interval/source create mode 100644 test/conf/sources/source with spaces and interval/verbose create mode 100644 test/conf/sources/very_verbose/destination create mode 100644 test/conf/sources/very_verbose/exclude create mode 100644 test/conf/sources/very_verbose/source create mode 100644 test/conf/sources/very_verbose/summary create mode 100644 test/conf/sources/very_verbose/verbose create mode 100644 test/conf/sources/very_verbose/very_verbose create mode 100644 test/conf/sources/with_exec/destination create mode 100755 test/conf/sources/with_exec/post_exec create mode 100755 test/conf/sources/with_exec/pre_exec create mode 100644 test/conf/sources/with_exec/source delete mode 100755 test/exec.sh delete mode 100755 test/local.sh delete mode 100755 test/remote.sh delete mode 100755 test/return-value.sh delete mode 100644 test/test-ccollect-tools.sh delete mode 100755 test/test-ccollect1.sh diff --git a/.gitignore b/.gitignore index 4e9c74d..e5c18f6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ doc/man/*.html doc/man/*.htm doc/man/*.texi doc/man/*.man -test/* .*.swp doc/man/*.[0-9] doc/*.xml diff --git a/Makefile b/Makefile index 9b07e09..9b9f3e5 100644 --- a/Makefile +++ b/Makefile @@ -79,6 +79,8 @@ DOCBDOCS = ${DOCS:.text=.docbook} DOC_ALL = ${HTMLDOCS} ${DBHTMLDOCS} ${TEXIDOCS} ${MANPDOCS} ${PDFDOCS} +TEST_LOG_FILE = /tmp/ccollect/ccollect.log + # # End user targets # @@ -202,9 +204,49 @@ dist: distclean documentation shellcheck: ./ccollect shellcheck -s sh -f gcc -x ./ccollect -test: $(CCOLLECT_SOURCE) /tmp/ccollect +test-nico: $(CCOLLECT_SOURCE) /tmp/ccollect cd ./conf/sources/; for s in *; do CCOLLECT_CONF=../ ../../ccollect daily "$$s"; done touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect -a daily touch /tmp/ccollect/$$(ls /tmp/ccollect | head -n1).ccollect-marker CCOLLECT_CONF=./conf ./ccollect -a -p daily + +test-dir-source: + mkdir -p /tmp/ccollect/source + cp -R -f ./* /tmp/ccollect/source + +test-dir-destination: + mkdir -p /tmp/ccollect/backup + +test-dir-destination-chint: + mkdir -p /tmp/ccollect/backup-chint + +test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint + for s in ./test/conf/sources/*; do \ + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename $$s)"; \ + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0"; \ + done + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -p daily + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" + @printf "\nFixed intervals test ended successfully\n" + +test-interval-changing: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination-chint + rm -rf /tmp/ccollect/backup-chint/* + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "0" + printf "3" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "3" + printf "5" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5 6 7; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "5" + printf "4" > ./test/conf/sources/local-with-interval/intervals/daily + for x in 1 2 3 4 5 6; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "4" + printf "3" > ./test/conf/sources/local-with-interval/intervals/daily + @printf "\nInterval changing test ended successfully\n" + +test: test-fixed-intervals test-interval-changing + test -f "${TEST_LOG_FILE}" + @printf "\nTests ended successfully\n" diff --git a/test/conf/ccollect_local-with b/test/conf/ccollect_local-with new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/ccollect_source b/test/conf/ccollect_source new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/intervals/daily b/test/conf/defaults/intervals/daily new file mode 100644 index 0000000..7ed6ff8 --- /dev/null +++ b/test/conf/defaults/intervals/daily @@ -0,0 +1 @@ +5 diff --git a/test/conf/defaults/intervals/monthly b/test/conf/defaults/intervals/monthly new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/test/conf/defaults/intervals/monthly @@ -0,0 +1 @@ +4 diff --git a/test/conf/defaults/intervals/normal b/test/conf/defaults/intervals/normal new file mode 100644 index 0000000..b8626c4 --- /dev/null +++ b/test/conf/defaults/intervals/normal @@ -0,0 +1 @@ +4 diff --git a/test/conf/defaults/intervals/weekly b/test/conf/defaults/intervals/weekly new file mode 100644 index 0000000..0cfbf08 --- /dev/null +++ b/test/conf/defaults/intervals/weekly @@ -0,0 +1 @@ +2 diff --git a/test/conf/defaults/post_exec b/test/conf/defaults/post_exec new file mode 100755 index 0000000..8aee58f --- /dev/null +++ b/test/conf/defaults/post_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +General post_exec executed. +###################################################################### diff --git a/test/conf/defaults/pre_exec b/test/conf/defaults/pre_exec new file mode 100755 index 0000000..ecd2857 --- /dev/null +++ b/test/conf/defaults/pre_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +General pre__exec executed. +###################################################################### diff --git a/test/conf/defaults/sources/exclude b/test/conf/defaults/sources/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/defaults/sources/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/defaults/sources/rsync_options b/test/conf/defaults/sources/rsync_options new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/sources/verbose b/test/conf/defaults/sources/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/defaults/verbose b/test/conf/defaults/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/delete_incomplete/delete_incomplete b/test/conf/sources/delete_incomplete/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/delete_incomplete/destination b/test/conf/sources/delete_incomplete/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/delete_incomplete/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/delete_incomplete/exclude b/test/conf/sources/delete_incomplete/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/delete_incomplete/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/delete_incomplete/source b/test/conf/sources/delete_incomplete/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/delete_incomplete/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with&ersand/destination b/test/conf/sources/local-with&ersand/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/local-with&ersand/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/local-with&ersand/exclude b/test/conf/sources/local-with&ersand/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local-with&ersand/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local-with&ersand/source b/test/conf/sources/local-with&ersand/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local-with&ersand/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with-interval/delete_incomplete b/test/conf/sources/local-with-interval/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local-with-interval/destination b/test/conf/sources/local-with-interval/destination new file mode 100644 index 0000000..4de7e06 --- /dev/null +++ b/test/conf/sources/local-with-interval/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup-chint diff --git a/test/conf/sources/local-with-interval/exclude b/test/conf/sources/local-with-interval/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local-with-interval/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local-with-interval/intervals/daily b/test/conf/sources/local-with-interval/intervals/daily new file mode 100644 index 0000000..e440e5c --- /dev/null +++ b/test/conf/sources/local-with-interval/intervals/daily @@ -0,0 +1 @@ +3 \ No newline at end of file diff --git a/test/conf/sources/local-with-interval/source b/test/conf/sources/local-with-interval/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local-with-interval/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/local-with-interval/verbose b/test/conf/sources/local-with-interval/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local/destination b/test/conf/sources/local/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/local/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/local/exclude b/test/conf/sources/local/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/local/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/local/no_verbose b/test/conf/sources/local/no_verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/local/source b/test/conf/sources/local/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/local/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/source with spaces and interval/delete_incomplete b/test/conf/sources/source with spaces and interval/delete_incomplete new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/source with spaces and interval/destination b/test/conf/sources/source with spaces and interval/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/source with spaces and interval/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/source with spaces and interval/exclude b/test/conf/sources/source with spaces and interval/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/source with spaces and interval/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/source with spaces and interval/source b/test/conf/sources/source with spaces and interval/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/source with spaces and interval/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/source with spaces and interval/verbose b/test/conf/sources/source with spaces and interval/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/destination b/test/conf/sources/very_verbose/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/very_verbose/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/very_verbose/exclude b/test/conf/sources/very_verbose/exclude new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/test/conf/sources/very_verbose/exclude @@ -0,0 +1 @@ +.git diff --git a/test/conf/sources/very_verbose/source b/test/conf/sources/very_verbose/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/very_verbose/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/conf/sources/very_verbose/summary b/test/conf/sources/very_verbose/summary new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/verbose b/test/conf/sources/very_verbose/verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/very_verbose/very_verbose b/test/conf/sources/very_verbose/very_verbose new file mode 100644 index 0000000..e69de29 diff --git a/test/conf/sources/with_exec/destination b/test/conf/sources/with_exec/destination new file mode 100644 index 0000000..c2a7c55 --- /dev/null +++ b/test/conf/sources/with_exec/destination @@ -0,0 +1 @@ +/tmp/ccollect/backup diff --git a/test/conf/sources/with_exec/post_exec b/test/conf/sources/with_exec/post_exec new file mode 100755 index 0000000..abc0a40 --- /dev/null +++ b/test/conf/sources/with_exec/post_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +Source post_exec executed. +###################################################################### diff --git a/test/conf/sources/with_exec/pre_exec b/test/conf/sources/with_exec/pre_exec new file mode 100755 index 0000000..ba7b2af --- /dev/null +++ b/test/conf/sources/with_exec/pre_exec @@ -0,0 +1,5 @@ +#!/bin/cat + +###################################################################### +Source pre_exec executed. +###################################################################### diff --git a/test/conf/sources/with_exec/source b/test/conf/sources/with_exec/source new file mode 100644 index 0000000..9e90576 --- /dev/null +++ b/test/conf/sources/with_exec/source @@ -0,0 +1 @@ +/tmp/ccollect/source diff --git a/test/exec.sh b/test/exec.sh deleted file mode 100755 index bdf601d..0000000 --- a/test/exec.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/sh - -host="home.schottelius.org" -host="" -set -x -pcmd() -{ - echo "$#", "$@" - if [ "$host" ]; then - ssh "$host" "$@" - else - $@ - fi -} - -#pcmd ls / -#pcmd cd /; ls "/is not there" -pcmd cd / && ls diff --git a/test/local.sh b/test/local.sh deleted file mode 100755 index c2430fd..0000000 --- a/test/local.sh +++ /dev/null @@ -1 +0,0 @@ -CCOLLECT_CONF=./conf ./ccollect.sh daily -v local1 diff --git a/test/remote.sh b/test/remote.sh deleted file mode 100755 index 2af364e..0000000 --- a/test/remote.sh +++ /dev/null @@ -1 +0,0 @@ -CCOLLECT_CONF=./conf ./ccollect.sh daily -v remote1 diff --git a/test/return-value.sh b/test/return-value.sh deleted file mode 100755 index 554def0..0000000 --- a/test/return-value.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/bin/sh - -ls /surely-not-existent$$ 2>/dev/null - -if [ "$?" -ne 0 ]; then - echo "$?" -fi - -ls /surely-not-existent$$ 2>/dev/null - -ret=$? - -if [ "$ret" -ne 0 ]; then - echo "$ret" -fi - -# if is true, ls is fales -if [ "foo" = "foo" ]; then - ls /surely-not-existent$$ 2>/dev/null -fi - -# but that's still the return of ls and not of fi -echo $? diff --git a/test/test-ccollect-tools.sh b/test/test-ccollect-tools.sh deleted file mode 100644 index 5980d04..0000000 --- a/test/test-ccollect-tools.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# 2009 Nico Schottelius (nico-ccollect at schottelius.org) -# -# This file is part of ccollect. -# -# ccollect is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# ccollect is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with ccollect. If not, see . -# -# -# Test the ccollect tools suite -# - -set -x - -tmp="$(mktemp /tmp/ccollect-tools.XXXXXXXXXXX)" - - -rm -rf "${tmp}" diff --git a/test/test-ccollect1.sh b/test/test-ccollect1.sh deleted file mode 100755 index c5acf54..0000000 --- a/test/test-ccollect1.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/bin/sh -# -# Nico Schottelius -# Date: 27-Jan-2007 -# Last Modified: - -# Description: -# - -ccollect=../ccollect.sh -testdir="$(dirname $0)/test-backups" -confdir="$(dirname $0)/test-config" -source="$(hostname)" -source_source="/tmp" -interval="taeglich" - - -# backup destination -mkdir -p "$testdir" -source_dest="$(cd "$testdir"; pwd -P)" - -# configuration -mkdir -p "${confdir}/sources/${source}" -ln -s "$source_dest" "${confdir}/sources/${source}/destination" -echo "$source_source" > "${confdir}/sources/${source}/source" -touch "${confdir}/sources/${source}/summary" -touch "${confdir}/sources/${source}/verbose" - -mkdir -p "${confdir}/defaults/intervals/" -echo 3 > "${confdir}/defaults/intervals/$interval" - -# create backups - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a -touch "${source_source}/$(date +%s)-$$.1982" - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a -touch "${source_source}/$(date +%s)-$$.42" - -CCOLLECT_CONF="$confdir" "$ccollect" "$interval" -p -a - -du -sh "$testdir" -du -shl "$testdir" - -echo "Delete $testdir and $confdir after test" From 5ce3fddf6272cb443f37cde21512bce5eedf9d21 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 19 Nov 2019 18:25:10 +0100 Subject: [PATCH 163/177] Define gitlab CI --- .gitlab-ci.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..b499ddc --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,20 @@ +stages: + - test + +unit_tests: + stage: test + before_script: + - 'apk update' + - 'apk add make rsync' + script: + - make test + +shellcheck: + stage: test + before_script: + - 'apk update' + - 'apk add make' + - 'wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz' + - 'tar xf shellcheck-stable.linux.x86_64.tar.xz && mv shellcheck-stable/shellcheck /usr/bin/' + script: + - make shellcheck From 9ed5912461dcd9b56836ad643d1cfc8b9ff95983 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 08:42:42 +0100 Subject: [PATCH 164/177] Improve unit tests --- Makefile | 14 +++++++------- test/conf/defaults/post_exec | 6 ++---- test/conf/defaults/pre_exec | 6 ++---- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/Makefile b/Makefile index 9b9f3e5..ced0d28 100644 --- a/Makefile +++ b/Makefile @@ -224,26 +224,26 @@ test-dir-destination-chint: test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint for s in ./test/conf/sources/*; do \ CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename $$s)"; \ - test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0"; \ + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; }; \ done CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily - test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; } CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -p daily - test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" + test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; } @printf "\nFixed intervals test ended successfully\n" test-interval-changing: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination-chint rm -rf /tmp/ccollect/backup-chint/* - test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "0" + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "0" || { cat ${TEST_LOG_FILE}; exit 1; } printf "3" > ./test/conf/sources/local-with-interval/intervals/daily for x in 1 2 3 4 5; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done - test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "3" + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "3" || { cat ${TEST_LOG_FILE}; exit 1; } printf "5" > ./test/conf/sources/local-with-interval/intervals/daily for x in 1 2 3 4 5 6 7; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done - test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "5" + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "5" || { cat ${TEST_LOG_FILE}; exit 1; } printf "4" > ./test/conf/sources/local-with-interval/intervals/daily for x in 1 2 3 4 5 6; do CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily local-with-interval; done - test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "4" + test "$$(ls -1 /tmp/ccollect/backup-chint | wc -l)" -eq "4" || { cat ${TEST_LOG_FILE}; exit 1; } printf "3" > ./test/conf/sources/local-with-interval/intervals/daily @printf "\nInterval changing test ended successfully\n" diff --git a/test/conf/defaults/post_exec b/test/conf/defaults/post_exec index 8aee58f..0dac0ed 100755 --- a/test/conf/defaults/post_exec +++ b/test/conf/defaults/post_exec @@ -1,5 +1,3 @@ -#!/bin/cat +#!/bin/sh -###################################################################### -General post_exec executed. -###################################################################### +echo 'General post_exec executed.' diff --git a/test/conf/defaults/pre_exec b/test/conf/defaults/pre_exec index ecd2857..451fdad 100755 --- a/test/conf/defaults/pre_exec +++ b/test/conf/defaults/pre_exec @@ -1,5 +1,3 @@ -#!/bin/cat +#!/bin/sh -###################################################################### -General pre__exec executed. -###################################################################### +echo 'General pre__exec executed.' From 42bd1afb09ce17d84520ae2649c8af1dd91ced8b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 13:35:22 +0100 Subject: [PATCH 165/177] Fix quoting in tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ced0d28..52b7a56 100644 --- a/Makefile +++ b/Makefile @@ -223,7 +223,7 @@ test-dir-destination-chint: test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint for s in ./test/conf/sources/*; do \ - CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename $$s)"; \ + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename '$$s')"; \ test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; }; \ done CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily From 6c24e8a7d35fd7132d2f2b6bbe3977159cf3054b Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 13:35:22 +0100 Subject: [PATCH 166/177] Fix quoting in tests --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 52b7a56..1e783fc 100644 --- a/Makefile +++ b/Makefile @@ -223,7 +223,7 @@ test-dir-destination-chint: test-fixed-intervals: $(CCOLLECT_SOURCE) test-dir-source test-dir-destination test-dir-destination-chint for s in ./test/conf/sources/*; do \ - CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename '$$s')"; \ + CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} daily "$$(basename "$$s")"; \ test "$$(ls -1 /tmp/ccollect/backup | wc -l)" -gt "0" || { cat ${TEST_LOG_FILE}; exit 1; }; \ done CCOLLECT_CONF=./test/conf ./ccollect -l ${TEST_LOG_FILE} -a -v daily From 61ab45fc651b9fafb6da4de208fe0a7d5769c447 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 13:50:30 +0100 Subject: [PATCH 167/177] Fix excluding destination dir from removal Touch can lead to wrong ls order, and destination dir gets selected for removal. Use grep -v to exclude, instead of touch. --- ccollect | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/ccollect b/ccollect index d89aec1..61e4ebd 100755 --- a/ccollect +++ b/ccollect @@ -767,20 +767,27 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Using ${oldest_bak} for destination dir ${destination_dir}" if mv "${oldest_bak}" "${destination_dir}"; then # Touch dest dir so it is not sorted wrong in listings below. - touch "${destination_dir}" + ls_rm_exclude=$(basename "${destination_dir}") # We have something to remove only if count > interval. remove="$((count - c_interval))" else _techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it." remove="$((count - c_interval + 1))" + ls_rm_exclude="" fi if [ "${remove}" -gt 0 ]; then _techo "Removing ${remove} backup(s)..." - # shellcheck disable=SC2010 - ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + if [ -z "${ls_rm_exclude}" ]; then + # shellcheck disable=SC2010 + ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + else + # shellcheck disable=SC2010 + ls -${TSORT}1r | grep -v "${ls_rm_exclude}" | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + fi delete_from_file "${TMP}" & fi From 589fed6107e4cd0c719e523b81f1ae7ff5b55677 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 14:41:41 +0100 Subject: [PATCH 168/177] ++changelog --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/next diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..563b438 --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Fix excluding destination dir from removal From 987277f1cf2d02f2c2e527c423f90becd0e0b4f9 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 Nov 2019 20:52:48 +0100 Subject: [PATCH 169/177] Update Makefile Simplify and generalize. --- Makefile | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 1e783fc..789e099 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ ASCIIDOC=asciidoc DOCBOOKTOTEXI=docbook2x-texi DOCBOOKTOMAN=docbook2x-man XSLTPROC=xsltproc -XSL=/usr/share/xml/docbook/stylesheet/nwalsh/html/docbook.xsl +XSL=/usr/local/share/xsl/docbook/html/docbook.xsl A2X=a2x prefix=/usr/packages/ccollect-git @@ -41,11 +41,7 @@ manlink=/usr/local/man/man1 path_dir=/usr/local/bin path_destination=${path_dir}/${CCOLLECT_DEST} - -# where to publish -host=localhost -dir=/home/users/nico/privat/rechner/netz/seiten/www.nico.schottelius.org/src/software/ccollect -docdir=${dir}/documentation +docs_archive_name=docs.tar # # Asciidoc will be used to generate other formats later @@ -91,6 +87,8 @@ all: @echo "info: only generate Texinfo" @echo "man: only generate manpage{s}" @echo "install: install ccollect to ${prefix}" + @echo "shellcheck: shellcheck ccollect script" + @echo "test: run unit tests" html: ${HTMLDOCS} htm: ${DBHTMLDOCS} @@ -179,9 +177,9 @@ pub: git push publish-doc: documentation - @echo "Transferring files to ${host}" @chmod a+r ${DOCS} ${DOC_ALL} - @tar c ${DOCS} ${DOC_ALL} | ssh ${host} "cd ${dir}; tar xv" + @tar cf ${docs_archive_name} ${DOCS} ${DOC_ALL} + @echo "Documentation files are in ${docs_archive_name}" # # Distribution From 5341de86fb331512dd442bbbcc8d68d20e6b06a5 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Tue, 26 Nov 2019 06:10:17 +0100 Subject: [PATCH 170/177] Release 2.8 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/{next => 2.8} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename doc/changes/{next => 2.8} (100%) diff --git a/ccollect b/ccollect index 61e4ebd..39799cd 100755 --- a/ccollect +++ b/ccollect @@ -45,8 +45,8 @@ TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.7" -RELEASE="2019-11-14" +VERSION="2.8" +RELEASE="2019-11-26" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 2478265..84c326e 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.7, for ccollect 2.7, Initial Version from 2006-01-13 +2.8, for ccollect 2.8, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.8 similarity index 100% rename from doc/changes/next rename to doc/changes/2.8 From 109b70ea769755fbb576e3ee83af399fb5735e9e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 2 Dec 2019 09:26:47 +0100 Subject: [PATCH 171/177] gitlab runner should have necessary tools --- .gitlab-ci.yml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b499ddc..5391f4d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -3,18 +3,10 @@ stages: unit_tests: stage: test - before_script: - - 'apk update' - - 'apk add make rsync' script: - make test shellcheck: stage: test - before_script: - - 'apk update' - - 'apk add make' - - 'wget https://storage.googleapis.com/shellcheck/shellcheck-stable.linux.x86_64.tar.xz' - - 'tar xf shellcheck-stable.linux.x86_64.tar.xz && mv shellcheck-stable/shellcheck /usr/bin/' script: - make shellcheck From a261ef841eb76595d169eef334b8f004a2d3737c Mon Sep 17 00:00:00 2001 From: Steffen Zieger Date: Sun, 24 May 2020 16:40:04 +0200 Subject: [PATCH 172/177] make rsync return code available in post_exec --- ccollect | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ccollect b/ccollect index 39799cd..679e3ca 100755 --- a/ccollect +++ b/ccollect @@ -824,6 +824,11 @@ while [ "${source_no}" -lt "${no_sources}" ]; do rsync "$@" "${source}" "${destination_dir}"; ret=$? _techo "Finished backup (rsync return code: $ret)." + # + # export rsync return code, might be useful in post_exec + # + export rsync_return_code=$ret + # # Set modification time (mtime) to current time, if sorting by mtime is enabled # From 28dec3694ae11b3924ba76c7de830183857d08bd Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Sun, 24 May 2020 17:29:29 +0200 Subject: [PATCH 173/177] ++changelog --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/next diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..45097c7 --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Make rsync return code available in post_exec (Steffen Zieger) From 7a7dec7751862ca1eec30f2041c80e40746766b3 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 May 2020 12:05:35 +0200 Subject: [PATCH 174/177] Release 2.9 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/{next => 2.9} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename doc/changes/{next => 2.9} (100%) diff --git a/ccollect b/ccollect index 679e3ca..01c5d05 100755 --- a/ccollect +++ b/ccollect @@ -45,8 +45,8 @@ TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.8" -RELEASE="2019-11-26" +VERSION="2.9" +RELEASE="2020-05-25" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index 84c326e..bc8326a 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.8, for ccollect 2.8, Initial Version from 2006-01-13 +2.9, for ccollect 2.9, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.9 similarity index 100% rename from doc/changes/next rename to doc/changes/2.9 From 616b1d9e3e9cf996e1f90bfc8ede58143ed0e7e7 Mon Sep 17 00:00:00 2001 From: Steffen Zieger Date: Mon, 25 May 2020 16:16:32 +0200 Subject: [PATCH 175/177] Add 'current' symlink to backup destinations --- ccollect | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ccollect b/ccollect index 01c5d05..b4af2f1 100755 --- a/ccollect +++ b/ccollect @@ -861,6 +861,16 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Warning: rsync failed with return code $ret." fi + # + # Create symlink to newest backup + # + # shellcheck disable=SC2010 + latest_dir="$(ls -${TSORT}p1 "${ddir}" | grep '/$' | head -n 1)" || \ + _exit_err "Failed to list content of ${ddir}." + + ln -snf "${ddir}${latest_dir}" "${ddir}current" || \ + _exit_err "Failed to create 'current' symlink." + # # post_exec # From 309d8dc773182a053619a93bc099f88f087b4c0e Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Mon, 25 May 2020 17:52:23 +0200 Subject: [PATCH 176/177] ++changelog --- doc/changes/next | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/changes/next diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..1d1e85d --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Add 'current' symlink to backup destinations (Steffen Zieger) From 08cb857664a6dbcbacaffc020c747db554ebbf15 Mon Sep 17 00:00:00 2001 From: Darko Poljak Date: Wed, 26 Aug 2020 08:36:43 +0200 Subject: [PATCH 177/177] Release 2.10 --- ccollect | 4 ++-- doc/ccollect.text | 2 +- doc/changes/{next => 2.10} | 0 3 files changed, 3 insertions(+), 3 deletions(-) rename doc/changes/{next => 2.10} (100%) diff --git a/ccollect b/ccollect index b4af2f1..2deaa2d 100755 --- a/ccollect +++ b/ccollect @@ -45,8 +45,8 @@ TMP="$(mktemp "/tmp/${__myname}.XXXXXX")" export TMP CONTROL_PIPE="/tmp/${__myname}-control-pipe" -VERSION="2.9" -RELEASE="2020-05-25" +VERSION="2.10" +RELEASE="2020-08-26" HALF_VERSION="ccollect ${VERSION}" FULL_VERSION="ccollect ${VERSION} (${RELEASE})" diff --git a/doc/ccollect.text b/doc/ccollect.text index bc8326a..f075b9a 100644 --- a/doc/ccollect.text +++ b/doc/ccollect.text @@ -1,7 +1,7 @@ ccollect - Installing, Configuring and Using ============================================ Nico Schottelius -2.9, for ccollect 2.9, Initial Version from 2006-01-13 +2.10, for ccollect 2.10, Initial Version from 2006-01-13 :Author Initials: NS diff --git a/doc/changes/next b/doc/changes/2.10 similarity index 100% rename from doc/changes/next rename to doc/changes/2.10