Merge commit 'john/updates'

Fixed alot of conflicts due to parallel work, but it looks very good now!

Conflicts:
	ccollect.sh

ccollect.sh

@@ -1,19 +1,19 @@
 #!/bin/sh
-# 
+#
 # 2005-2009 Nico Schottelius (nico-ccollect at schottelius.org)
-# 
+#
 # This file is part of ccollect.
 #
 # ccollect is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
 # the Free Software Foundation, either version 3 of the License, or
 # (at your option) any later version.
-# 
+#
 # ccollect is distributed in the hope that it will be useful,
 # but WITHOUT ANY WARRANTY; without even the implied warranty of
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU General Public License for more details.
-# 
+#
 # You should have received a copy of the GNU General Public License
 # along with ccollect. If not, see <http://www.gnu.org/licenses/>.
 #
@@ -178,7 +178,6 @@ fi
 [ -d "${CCOLLECT_CONF}" ] || _exit_err "No configuration found in " \
    "\"${CCOLLECT_CONF}\" (is \$CCOLLECT_CONF properly set?)"
 
-
 #
 # Create (portable!) source "array"
 #
@@ -231,14 +230,6 @@ if [ -x "${CPREEXEC}" ]; then
    [ "${ret}" -eq 0 ] || _exit_err "${CPREEXEC} failed. Aborting"
 fi
 
-#
-# check default configuration
-#
-
-D_FILE_INTERVAL="${CDEFAULTS}/intervals/${INTERVAL}"
-D_INTERVAL="$(cat "${D_FILE_INTERVAL}" 2>/dev/null)"
-
-
 #
 # Let's do the backup
 #
@@ -277,6 +268,21 @@ while [ "${i}" -lt "${no_sources}" ]; do
    c_dest="${backup}/destination"
    c_pre_exec="${backup}/pre_exec"
    c_post_exec="${backup}/post_exec"
+   for opt in exclude verbose very_verbose rsync_options summary delete_incomplete \
+         remote_host rsync_failure_codes mtime quiet_if_down ; do
+      if [ -f "${backup}/$opt" -o -f "${backup}/no_$opt"  ]; then
+         eval c_$opt=\"${backup}/$opt\"
+      else
+         eval c_$opt=\"${CDEFAULTS}/$opt\"
+      fi
+   done
+
+   #
+   # With mtime option, sort backup directories with mtime (default is ctime)
+   #
+   if [ -f "$c_mtime" ] ; then
+      TSORT="t"
+   fi
 
    #
    # Marking backups: If we abort it's not removed => Backup is broken
@@ -328,19 +334,6 @@ while [ "${i}" -lt "${no_sources}" ]; do
       fi
    fi
 
-   #
-   # interval definition: First try source specific, fallback to default
-   #
-   c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)"
-
-   if [ -z "${c_interval}" ]; then
-      c_interval="${D_INTERVAL}"
-
-      if [ -z "${c_interval}" ]; then
-         _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping."
-      fi
-   fi
-
    #
    # Source checks
    #
@@ -356,7 +349,14 @@ while [ "${i}" -lt "${no_sources}" ]; do
    #
    # Verify source is up and accepting connections before deleting any old backups
    #
-   rsync "${source}" >/dev/null || _exit_err "Source ${source} is not readable. Skipping."
+   if ! rsync "${source}" >/dev/null 2>"${TMP}" ; then
+      if [ -f "${c_quiet_if_down}" ]; then
+         _exit_err "Source ${source} is not readable. Skipping."
+      else
+         cat "${TMP}"
+         _exit_err "Error: source ${source} is not readable. Skipping."
+      fi
+   fi
 
    #
    # Destination is a path
@@ -394,12 +394,12 @@ while [ "${i}" -lt "${no_sources}" ]; do
    # - insert ccollect default parameters
    # - insert options
    # - insert user options
-   
+
    #
    # rsync standard options
    #
    set -- "$@" "--archive" "--delete" "--numeric-ids" "--relative"   \
-               "--delete-excluded" "--sparse" 
+               "--delete-excluded" "--sparse"
 
    #
    # exclude list
@@ -438,7 +438,7 @@ while [ "${i}" -lt "${no_sources}" ]; do
    #
    # Check for incomplete backups
    #
-   pcmd ls -1 "${ddir}/${INTERVAL}"*".${c_marker}" 2>/dev/null | while read marker; do
+   pcmd ls -1 "${ddir}/"*".${c_marker}" 2>/dev/null | while read marker; do
       incomplete="$(echo ${marker} | sed "s/\\.${c_marker}\$//")"
       _techo "Incomplete backup: ${incomplete}"
       if [ -f "${c_delete_incomplete}" ]; then
@@ -450,6 +450,19 @@ while [ "${i}" -lt "${no_sources}" ]; do
       fi
    done
 
+   #
+   # interval definition: First try source specific, fallback to default
+   #
+   c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)"
+
+   if [ -z "${c_interval}" ]; then
+      c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)"
+
+      if [ -z "${c_interval}" ]; then
+         _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping."
+      fi
+   fi
+
    #
    # check if maximum number of backups is reached, if so remove
    # use grep and ls -p so we only look at directories
@@ -458,7 +471,7 @@ while [ "${i}" -lt "${no_sources}" ]; do
       | sed 's/^ *//g')"  || _exit_err "Counting backups failed"
 
    _techo "Existing backups: ${count} Total keeping backups: ${c_interval}"
-   
+
    if [ "${count}" -ge "${c_interval}" ]; then
       substract="$((${c_interval} - 1))"
       remove="$((${count} - ${substract}))"
@@ -487,12 +500,9 @@ while [ "${i}" -lt "${no_sources}" ]; do
    #
    # Check for backup directory to clone from: Always clone from the latest one!
    #
-   # Use ls -1c instead of -1t, because last modification maybe the same on all
-   # and metadate update (-c) is updated by rsync locally.
-   #
    last_dir="$(pcmd ls -${TSORT}p1 "${ddir}" | grep '/$' | head -n 1)" || \
       _exit_err "Failed to list contents of ${ddir}."
-   
+
    #
    # clone from old backup, if existing
    #
@@ -500,7 +510,7 @@ while [ "${i}" -lt "${no_sources}" ]; do
       set -- "$@" "--link-dest=${ddir}/${last_dir}"
       _techo "Hard linking from ${last_dir}"
    fi
-      
+
    # set time when we really begin to backup, not when we began to remove above
    destination_date="$(${CDATE})"
    destination_dir="${ddir}/${INTERVAL}.${destination_date}.$$"
@@ -523,16 +533,36 @@ while [ "${i}" -lt "${no_sources}" ]; do
    #
    _techo "Transferring files..."
    rsync "$@" "${source}" "${destination_full}"; ret=$?
-
-   #
-   # remove marking here
-   #
-   pcmd rm "${destination_dir}.${c_marker}" || \
-      _exit_err "Removing ${destination_dir}/${c_marker} failed."
-
    _techo "Finished backup (rsync return code: $ret)."
-   if [ "${ret}" -ne 0 ]; then
+
-      _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)."
+   #
+   # Set modification time (mtime) to current time
+   #
+   pcmd touch "${destination_dir}"
+
+   #
+   # Check if rsync exit code indicates failure.
+   #
+   fail=""
+   if [ -f "$c_rsync_failure_codes" ]; then
+      while read code ; do
+         if [ "$ret" = "$code" ]; then
+            fail=1
+         fi
+      done <"$c_rsync_failure_codes"
+   fi
+
+   #
+   # Remove marking here unless rsync failed.
+   #
+   if [ -z "$fail" ]; then
+      pcmd rm "${destination_dir}.${c_marker}" || \
+         _exit_err "Removing ${destination_dir}/${c_marker} failed."
+      if [ "${ret}" -ne 0 ]; then
+         _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)."
+      fi
+   else
+      _techo "Warning: rsync failed with return code $ret."
    fi
 
    #

doc/ccollect.text

@@ -21,7 +21,7 @@ Supported and tested operating systems and architectures
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 `ccollect` was successfully tested on the following platforms:
 
-- GNU/Linux on amd64/hppa/i386/ppc
+- GNU/Linux on amd64/hppa/i386/ppc/ARM
 - FreeBSD   on amd64/i386
 - Mac OS X 10.5
 - NetBSD    on alpha/amd64/i386/sparc/sparc64
@@ -359,6 +359,9 @@ Additionally a source may have the following files:
    
    - `delete_incomplete`   delete incomplete backups
    - `remote_host`         host to backup to
+   - `rsync_failure_codes` list of rsync exit codes that indicate complete failure
+   - `mtime`               Sort backup directories based on their modification time
+   - `quiet_if_down`       Suppress error messages if source is not connectable
 
 
 Example:
@@ -574,6 +577,35 @@ If you create the file `delete_incomplete` in a source specification directory,
 was interrupted) and remove them. Without this file `ccollect` will only warn
 the user.
 
+Detailed description of "rsync_failure_codes"
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+If you have the file `rsync_failure_codes` in your source configuration
+directory, it should contain a newline-separated list of numbers representing
+rsync exit codes.  If rsync exits with any code in this list, a marker will
+be left in the destination directory indicating failure of this backup.  If
+you have enabled delete_incomplete, then this backup will be deleted during
+the next ccollect run on the same interval.
+
+Detailed description of "mtime"
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+By default, ccollect.sh chooses the most recent backup directory for cloning or
+the oldest for deletion based on the directory's last change time (ctime).
+With this option, the sorting is done based on modification time (mtime).  With
+this version of ccollect.sh, the ctime and mtime of your backups will normally
+be the same and this option has no effect.  However, if you, for example, move
+your backups to another hard disk using cp -a or rsync -a, you should use this
+option because the ctimes are not preserved during such operations.
+
+If you have any backups in your repository made with ccollect version 0.7.1 or
+earlier, do not use this option.
+
+Detailed description of "quiet_if_down"
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+By default, ccollect.sh emits a series of error messages if a source is not
+connectable.  With this option enabled, ccollect.sh still reports that the
+source is not connectable but the associated error messages generated by
+rsync or ssh are suppressed.  You may want to use this option for sources,
+like notebook PCs, that are often disconnected.
 
 Hints
 -----