From 122982b0b98fe13a7f5d26fd9c18bed9e8e3d397 Mon Sep 17 00:00:00 2001 From: jll2 Date: Sun, 14 Jun 2009 17:54:48 -0700 Subject: [PATCH 01/10] Modifies ccollect.sh's interpretation of options: 1). If an option doesn't exist in a source directory, check the defaults directory. 2). For every option, create a corresponding "no_" option so that a source directory can override an option set in defaults. (i.patch) --- ccollect.sh | 34 +++++++++++----------------------- 1 file changed, 11 insertions(+), 23 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index cd13ac8..f1bd00c 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -283,16 +283,15 @@ while [ "${i}" -lt "${no_sources}" ]; do backup="${CSOURCES}/${name}" c_source="${backup}/source" c_dest="${backup}/destination" - c_exclude="${backup}/exclude" - c_verbose="${backup}/verbose" - c_vverbose="${backup}/very_verbose" - c_rsync_extra="${backup}/rsync_options" - c_summary="${backup}/summary" c_pre_exec="${backup}/pre_exec" c_post_exec="${backup}/post_exec" - f_incomplete="delete_incomplete" - c_incomplete="${backup}/${f_incomplete}" - c_remote_host="${backup}/remote_host" + for opt in exclude verbose very_verbose rsync_options summary delete_incomplete remote_host ; do + if [ -f "${backup}/$opt" -o -f "${backup}/no_$opt" ]; then + eval c_$opt=\"${backup}/$opt\" + else + eval c_$opt=\"${CDEFAULTS}/$opt\" + fi + done # # Marking backups: If we abort it's not removed => Backup is broken @@ -307,12 +306,8 @@ while [ "${i}" -lt "${no_sources}" ]; do # # unset possible options # - EXCLUDE="" - RSYNC_EXTRA="" - SUMMARY="" VERBOSE="" VVERBOSE="" - DELETE_INCOMPLETE="" _techo "Beginning to backup" @@ -408,13 +403,6 @@ while [ "${i}" -lt "${no_sources}" ]; do ( pcmd cd "$ddir" ) || _exit_err "Cannot change to ${ddir}. Skipping." - # - # Check whether to delete incomplete backups - # - if [ -f "${c_incomplete}" -o -f "${CDEFAULTS}/${f_incomplete}" ]; then - DELETE_INCOMPLETE="yes" - fi - # NEW method as of 0.6: # - insert ccollect default parameters # - insert options @@ -444,7 +432,7 @@ while [ "${i}" -lt "${no_sources}" ]; do # # Verbosity for rsync # - if [ -f "${c_vverbose}" ]; then + if [ -f "${c_very_verbose}" ]; then set -- "$@" "-vv" elif [ -f "${c_verbose}" ]; then set -- "$@" "-v" @@ -453,10 +441,10 @@ while [ "${i}" -lt "${no_sources}" ]; do # # extra options for rsync provided by the user # - if [ -f "${c_rsync_extra}" ]; then + if [ -f "${c_rsync_options}" ]; then while read line; do set -- "$@" "$line" - done < "${c_rsync_extra}" + done < "${c_rsync_options}" fi # @@ -474,7 +462,7 @@ while [ "${i}" -lt "${no_sources}" ]; do while [ "$j" -lt "$i" ]; do eval realincomplete=\"\$incomplete_$j\" _techo "Incomplete backup: ${realincomplete}" - if [ "${DELETE_INCOMPLETE}" = "yes" ]; then + if [ -f "${c_delete_incomplete}" ]; then _techo "Deleting ${realincomplete} ..." pcmd rm $VVERBOSE -rf "${ddir}/${realincomplete}" || \ _exit_err "Removing ${realincomplete} failed." From 192b55b98d0df8ec3b235b7d61bb919a2e19c190 Mon Sep 17 00:00:00 2001 From: jll2 Date: Sun, 14 Jun 2009 19:15:12 -0700 Subject: [PATCH 02/10] Initialize four variables to prevent unwanted interaction with user's environment. --- ccollect.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ccollect.sh b/ccollect.sh index f1bd00c..f158a19 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -149,8 +149,12 @@ i=1 no_sources=0 # -# Create source "array" +# Capture options and create source "array" # +WE="" +ALL="" +VERBOSE="" +NO_MORE_ARGS="" while [ "$#" -ge 1 ]; do eval arg=\"\$1\"; shift From a4c61e7b6870b0f50a03d8f95298434f1fe8dcc1 Mon Sep 17 00:00:00 2001 From: jll2 Date: Mon, 15 Jun 2009 17:54:03 -0700 Subject: [PATCH 03/10] Bug fix for very_verbose: According to the documentation, "if [the very_verbose] file exists in the source specification -v will be passed to rsync, rm and mkdir." Previously, the -v option was passed only to rsync. This patch passes it to rm and mkdir as well. Actually, as per the behavior of the previous version, it is verbose that sends the -v option to rsync while very_verbose sends it the -vv option. I left it this way because this behavior seems reasonable. Maybe the documentation should be corrected on this point. --- ccollect.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index f158a19..3f2eda2 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -311,7 +311,6 @@ while [ "${i}" -lt "${no_sources}" ]; do # unset possible options # VERBOSE="" - VVERBOSE="" _techo "Beginning to backup" @@ -434,10 +433,12 @@ while [ "${i}" -lt "${no_sources}" ]; do fi # - # Verbosity for rsync + # Verbosity for rsync, rm, and mkdir # + VVERBOSE="" if [ -f "${c_very_verbose}" ]; then set -- "$@" "-vv" + VVERBOSE="-v" elif [ -f "${c_verbose}" ]; then set -- "$@" "-v" fi From ea16af51b28d048c2daf2341ae9faacfbb06bba7 Mon Sep 17 00:00:00 2001 From: jll2 Date: Mon, 15 Jun 2009 18:08:26 -0700 Subject: [PATCH 04/10] Simplify handling of command-line verbose option: Previously, there was a $VERBOSE script variable that was set according to the "-v" command line option and then reset to null within each source sub-shell. With no loss of functionality, this patch removes all references to that variable. This makes the script 13 lines shorter. --- ccollect.sh | 15 +-------------- 1 file changed, 1 insertion(+), 14 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index 3f2eda2..e93956d 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -153,7 +153,6 @@ no_sources=0 # WE="" ALL="" -VERBOSE="" NO_MORE_ARGS="" while [ "$#" -ge 1 ]; do eval arg=\"\$1\"; shift @@ -170,7 +169,7 @@ while [ "$#" -ge 1 ]; do ALL=1 ;; -v|--verbose) - VERBOSE=1 + set -x ;; -p|--parallel) PARALLEL=1 @@ -194,13 +193,6 @@ done # also export number of sources export no_sources -# -# be really, really, really verbose -# -if [ "${VERBOSE}" = 1 ]; then - set -x -fi - # # Look, if we should take ALL sources # @@ -307,11 +299,6 @@ while [ "${i}" -lt "${no_sources}" ]; do # begin_s=$(date +%s) - # - # unset possible options - # - VERBOSE="" - _techo "Beginning to backup" # From 10d420614c473f0f728b340f9172aa89c5a8e065 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 20 Jun 2009 21:00:21 +0200 Subject: [PATCH 05/10] [DOC] give some hints on how to hack ccollect Signed-off-by: Nico Schottelius --- README | 2 ++ doc/HACKING | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 doc/HACKING diff --git a/README b/README index e74464d..401d778 100644 --- a/README +++ b/README @@ -17,6 +17,8 @@ ccollect was inspired by rsnapshot [1], which has some problems: Please use tools/report_success.sh to report success, if you are successfully using ccollect. +Have a look at doc/HACKING, if you plan to change ccollect. + A small try to visualize the differences in a table: +---------------+-------------------------------------------------------------+ diff --git a/doc/HACKING b/doc/HACKING new file mode 100644 index 0000000..00db4d1 --- /dev/null +++ b/doc/HACKING @@ -0,0 +1,37 @@ +Hello Hacker, + +I really appreciate your interest in hacking this software, but +I am kind of critical when seeing patches. Thus I created this +file to give you some hints of my thinking quirks. + + +Submitting patches +------------------ +Make my life easier, make your life easier, use a version control system (vcs). +For this software the preferred vcs is git. Clone the latest repo, create +a new local branch (git checkout -b ) write down your ideas. + +When you're done, push all your stuff out to some public repo and drop a +mail to the mailinglist, what you did and where to get it. + + +Introduce a feature or change behaviour +--------------------------------------- +Uhh, fancy! You have had a great idea, then it's time to change +the major version, so others know that something changed. + +If the configuration format is changed, add a script to tools/ +to allow users upgrade their configuration to this major version. + +And now comes the most difficult part: Add documentation. Nobody +benefits from your cool feature, if it is not known. I know, writing +documentation is not so much fun, but you also expect good documentation +for this software, don't you? + + +If you think my thinking quirks must be corrected +------------------------------------------------- +See above ("Submitting patches") and submit a patch to this file. + + +Thanks for reading. From 4db6b78a13bcb671bab61c3b0f88c2187927e35e Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sat, 20 Jun 2009 21:05:49 +0200 Subject: [PATCH 06/10] Correctly sort CREDITS by alphabet Signed-off-by: Nico Schottelius --- CREDITS | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CREDITS b/CREDITS index 72d59cc..5c73034 100644 --- a/CREDITS +++ b/CREDITS @@ -1,9 +1,9 @@ -Thanks go to the following people: +Thanks go to the following people (sorted by alphabet): -* the whole #cLinux channel - - for testing and debugging (those I mean should know ;-) * Alexey Maximov - for finding return-value and shell limitation bugs +* #cLinux IRC channel on irc.freenode.org + - for testing and debugging (those I mean should know ;-) * Daniel Aubry - for reporting many hints * Markus Meier From d6ea94c6dcc86e3ade57895a3c1a5c50cb05358b Mon Sep 17 00:00:00 2001 From: jll2 Date: Mon, 15 Jun 2009 20:16:25 -0700 Subject: [PATCH 07/10] Fix delete_incomplete bugs: 1). On systems I tried, delete_incomplete failed because the line: < pcmd rm $VVERBOSE -rf "${ddir}/${realincomplete}" || \ should read: > pcmd rm $VVERBOSE -rf "${realincomplete}" || \ (Is this true of all systems?) 2). The marker file was not deleted. Code was added to delete it. 3). The delete_incomplete code was simplified. --- ccollect.sh | 24 ++++++++---------------- 1 file changed, 8 insertions(+), 16 deletions(-) diff --git a/ccollect.sh b/ccollect.sh index e93956d..d6831bc 100755 --- a/ccollect.sh +++ b/ccollect.sh @@ -442,24 +442,16 @@ while [ "${i}" -lt "${no_sources}" ]; do # # Check for incomplete backups # - pcmd ls -1 "$ddir/${INTERVAL}"*".${c_marker}" > "${TMP}" 2>/dev/null - - i=0 - while read incomplete; do - eval incomplete_$i=\"$(echo ${incomplete} | sed "s/\\.${c_marker}\$//")\" - i=$(($i+1)) - done < "${TMP}" - - j=0 - while [ "$j" -lt "$i" ]; do - eval realincomplete=\"\$incomplete_$j\" - _techo "Incomplete backup: ${realincomplete}" + pcmd ls -1 "$ddir/${INTERVAL}"*".${c_marker}" 2>/dev/null | while read marker; do + incomplete="$(echo ${marker} | sed "s/\\.${c_marker}\$//")" + _techo "Incomplete backup: ${incomplete}" if [ -f "${c_delete_incomplete}" ]; then - _techo "Deleting ${realincomplete} ..." - pcmd rm $VVERBOSE -rf "${ddir}/${realincomplete}" || \ - _exit_err "Removing ${realincomplete} failed." + _techo "Deleting ${incomplete} ..." + pcmd rm $VVERBOSE -rf "${incomplete}" || \ + _exit_err "Removing ${incomplete} failed." + pcmd rm $VVERBOSE -f "${marker}" || \ + _exit_err "Removing ${marker} failed." fi - j=$(($j+1)) done # From cbff479c65b7c0c5e9a325179b6f4d1dfba82697 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 21 Jun 2009 00:15:04 +0200 Subject: [PATCH 08/10] Add John to the CREDITS list Signed-off-by: Nico Schottelius --- CREDITS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CREDITS b/CREDITS index 5c73034..2b315a5 100644 --- a/CREDITS +++ b/CREDITS @@ -6,6 +6,8 @@ Thanks go to the following people (sorted by alphabet): - for testing and debugging (those I mean should know ;-) * Daniel Aubry - for reporting many hints +* John Lawless + - A lot of patches and some very interesting discussions. * Markus Meier - for finding a really simple solution for choosing the right backup to clone from: Make it independent of the interval, simply choose the last From 2b285675888bf1f986e642c40476be38c0efd428 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 21 Jun 2009 00:15:21 +0200 Subject: [PATCH 09/10] Add README for contrib Signed-off-by: Nico Schottelius --- contrib/README | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 contrib/README diff --git a/contrib/README b/contrib/README new file mode 100644 index 0000000..ab30b7a --- /dev/null +++ b/contrib/README @@ -0,0 +1,3 @@ +This directory contains patches or programs contributed by others +which are either not yet integrated into ccollect or may be kept +seperated generally. From c9439be432e791d7e0389ca3bdfdcf214bfdf958 Mon Sep 17 00:00:00 2001 From: Nico Schottelius Date: Sun, 21 Jun 2009 00:50:04 +0200 Subject: [PATCH 10/10] add automatic backup manager CREDITS Signed-off-by: Nico Schottelius --- CREDITS | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CREDITS b/CREDITS index 2b315a5..aac73e2 100644 --- a/CREDITS +++ b/CREDITS @@ -6,6 +6,8 @@ Thanks go to the following people (sorted by alphabet): - for testing and debugging (those I mean should know ;-) * Daniel Aubry - for reporting many hints +* Jens-Christoph Brendel + - Added automatic backup manager (contrib/jbrendel-autobackup) * John Lawless - A lot of patches and some very interesting discussions. * Markus Meier