indent with 4 spaces

Signed-off-by: Nico Schottelius <nico@bento.schottelius.org>
This commit is contained in:
Nico Schottelius 2013-04-29 10:17:21 +02:00
parent ca45e8429b
commit e2ca223432

684
ccollect
View file

@ -72,20 +72,20 @@ trap "rm -f \"${TMP}\"" 1 2 15
# time displaying echo # time displaying echo
_techo() _techo()
{ {
echo "$(${DDATE}): $@" echo "$(${DDATE}): $@"
} }
# exit on error # exit on error
_exit_err() _exit_err()
{ {
_techo "$@" _techo "$@"
rm -f "${TMP}" rm -f "${TMP}"
exit 1 exit 1
} }
add_name() add_name()
{ {
awk "{ print \"[${name}] \" \$0 }" awk "{ print \"[${name}] \" \$0 }"
} }
# #
@ -95,94 +95,94 @@ add_name()
# #
delete_from_file() delete_from_file()
{ {
file="$1"; shift file="$1"; shift
suffix="" # It will be set, if deleting incomplete backups. suffix="" # It will be set, if deleting incomplete backups.
[ $# -eq 1 ] && suffix="$1" && shift [ $# -eq 1 ] && suffix="$1" && shift
while read to_remove; do while read to_remove; do
set -- "$@" "${to_remove}" set -- "$@" "${to_remove}"
if [ "${suffix}" ]; then if [ "${suffix}" ]; then
to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")"
set -- "$@" "${to_remove_no_suffix}" set -- "$@" "${to_remove_no_suffix}"
fi fi
done < "${file}" done < "${file}"
_techo "Removing $@ ..." _techo "Removing $@ ..."
[ "${VVERBOSE}" ] && echo rm "$@" [ "${VVERBOSE}" ] && echo rm "$@"
rm -rf "$@" || _exit_err "Removing $@ failed." rm -rf "$@" || _exit_err "Removing $@ failed."
} }
display_version() display_version()
{ {
echo "${FULL_VERSION}" echo "${FULL_VERSION}"
exit 0 exit 0
} }
usage() usage()
{ {
cat << eof cat << eof
${__myname}: [args] <interval name> <sources to backup> ${__myname}: [args] <interval name> <sources to backup>
ccollect creates (pseudo) incremental backups ccollect creates (pseudo) incremental backups
-h, --help: Show this help screen -h, --help: Show this help screen
-a, --all: Backup all sources specified in ${CSOURCES} -a, --all: Backup all sources specified in ${CSOURCES}
-p, --parallel: Parallelise backup processes -p, --parallel: Parallelise backup processes
-v, --verbose: Be very verbose (uses set -x) -v, --verbose: Be very verbose (uses set -x)
-V, --version: Print version information -V, --version: Print version information
This is version ${VERSION} released on ${RELEASE}. This is version ${VERSION} released on ${RELEASE}.
Retrieve latest ccollect at http://www.nico.schottelius.org/software/ccollect/ Retrieve latest ccollect at http://www.nico.schottelius.org/software/ccollect/
eof eof
exit 0 exit 0
} }
# #
# Parse options # Parse options
# #
while [ "$#" -ge 1 ]; do while [ "$#" -ge 1 ]; do
case "$1" in case "$1" in
-a|--all) -a|--all)
USE_ALL=1 USE_ALL=1
;; ;;
-p|--parallel) -p|--parallel)
PARALLEL=1 PARALLEL=1
;; ;;
-v|--verbose) -v|--verbose)
set -x set -x
;; ;;
-V|--version) -V|--version)
display_version display_version
;; ;;
--) --)
# ignore the -- itself # ignore the -- itself
shift shift
break break
;; ;;
-h|--help|-*) -h|--help|-*)
usage usage
;; ;;
*) *)
break break
;; ;;
esac esac
shift shift
done done
# #
# Setup interval # Setup interval
# #
if [ $# -ge 1 ]; then if [ $# -ge 1 ]; then
export INTERVAL="$1" export INTERVAL="$1"
shift shift
else else
usage usage
fi fi
# #
# Check for configuraton directory # Check for configuraton directory
# #
[ -d "${CCOLLECT_CONF}" ] || _exit_err "No configuration found in " \ [ -d "${CCOLLECT_CONF}" ] || _exit_err "No configuration found in " \
"\"${CCOLLECT_CONF}\" (is \$CCOLLECT_CONF properly set?)" "\"${CCOLLECT_CONF}\" (is \$CCOLLECT_CONF properly set?)"
# #
# Create (portable!) source "array" # Create (portable!) source "array"
@ -190,46 +190,46 @@ fi
export no_sources=0 export no_sources=0
if [ "${USE_ALL}" = 1 ]; then if [ "${USE_ALL}" = 1 ]; then
# #
# Get sources from source configuration # Get sources from source configuration
# #
( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \ ( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \
_exit_err "Listing of sources failed. Aborting." _exit_err "Listing of sources failed. Aborting."
while read tmp; do while read tmp; do
eval export source_${no_sources}=\"${tmp}\" eval export source_${no_sources}=\"${tmp}\"
no_sources=$((${no_sources}+1)) no_sources=$((${no_sources}+1))
done < "${TMP}" done < "${TMP}"
else else
# #
# Get sources from command line # Get sources from command line
# #
while [ "$#" -ge 1 ]; do while [ "$#" -ge 1 ]; do
eval arg=\"\$1\"; shift eval arg=\"\$1\"; shift
eval export source_${no_sources}=\"${arg}\" eval export source_${no_sources}=\"${arg}\"
no_sources="$((${no_sources}+1))" no_sources="$((${no_sources}+1))"
done done
fi fi
# #
# Need at least ONE source to backup # Need at least ONE source to backup
# #
if [ "${no_sources}" -lt 1 ]; then if [ "${no_sources}" -lt 1 ]; then
usage usage
else else
_techo "${HALF_VERSION}: Beginning backup using interval ${INTERVAL}" _techo "${HALF_VERSION}: Beginning backup using interval ${INTERVAL}"
fi fi
# #
# Look for pre-exec command (general) # Look for pre-exec command (general)
# #
if [ -x "${CPREEXEC}" ]; then if [ -x "${CPREEXEC}" ]; then
_techo "Executing ${CPREEXEC} ..." _techo "Executing ${CPREEXEC} ..."
"${CPREEXEC}"; ret=$? "${CPREEXEC}"; ret=$?
_techo "Finished ${CPREEXEC} (return code: ${ret})." _techo "Finished ${CPREEXEC} (return code: ${ret})."
[ "${ret}" -eq 0 ] || _exit_err "${CPREEXEC} failed. Aborting" [ "${ret}" -eq 0 ] || _exit_err "${CPREEXEC} failed. Aborting"
fi fi
################################################################################ ################################################################################
@ -238,304 +238,304 @@ fi
# #
source_no=0 source_no=0
while [ "${source_no}" -lt "${no_sources}" ]; do while [ "${source_no}" -lt "${no_sources}" ]; do
# #
# Get current source # Get current source
# #
eval export name=\"\$source_${source_no}\" eval export name=\"\$source_${source_no}\"
source_no=$((${source_no}+1)) source_no=$((${source_no}+1))
# #
# Start ourself, if we want parallel execution # Start ourself, if we want parallel execution
# #
if [ "${PARALLEL}" ]; then if [ "${PARALLEL}" ]; then
"$0" "${INTERVAL}" "${name}" & "$0" "${INTERVAL}" "${name}" &
continue continue
fi fi
# #
# Start subshell for easy log editing # Start subshell for easy log editing
# #
( (
backup="${CSOURCES}/${name}" backup="${CSOURCES}/${name}"
c_source="${backup}/source" c_source="${backup}/source"
c_dest="${backup}/destination" c_dest="${backup}/destination"
c_pre_exec="${backup}/pre_exec" c_pre_exec="${backup}/pre_exec"
c_post_exec="${backup}/post_exec" c_post_exec="${backup}/post_exec"
# #
# Stderr to stdout, so we can produce nice logs # Stderr to stdout, so we can produce nice logs
# #
exec 2>&1 exec 2>&1
# #
# Record start of backup: internal and for the user # Record start of backup: internal and for the user
# #
begin_s="$(${SDATE})" begin_s="$(${SDATE})"
_techo "Beginning to backup" _techo "Beginning to backup"
# #
# Standard configuration checks # Standard configuration checks
# #
if [ ! -e "${backup}" ]; then if [ ! -e "${backup}" ]; then
_exit_err "Source does not exist." _exit_err "Source does not exist."
fi fi
# #
# Configuration _must_ be a directory (cconfig style) # Configuration _must_ be a directory (cconfig style)
# #
if [ ! -d "${backup}" ]; then if [ ! -d "${backup}" ]; then
_exit_err "\"${backup}\" is not a cconfig-directory. Skipping." _exit_err "\"${backup}\" is not a cconfig-directory. Skipping."
fi fi
# #
# First execute pre_exec, which may generate destination or other parameters # First execute pre_exec, which may generate destination or other parameters
# #
if [ -x "${c_pre_exec}" ]; then if [ -x "${c_pre_exec}" ]; then
_techo "Executing ${c_pre_exec} ..." _techo "Executing ${c_pre_exec} ..."
"${c_pre_exec}"; ret="$?" "${c_pre_exec}"; ret="$?"
_techo "Finished ${c_pre_exec} (return code ${ret})." _techo "Finished ${c_pre_exec} (return code ${ret})."
[ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping." [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping."
fi fi
# #
# Read source configuration # Read source configuration
# #
for opt in verbose very_verbose summary exclude rsync_options \ for opt in verbose very_verbose summary exclude rsync_options \
delete_incomplete rsync_failure_codes \ delete_incomplete rsync_failure_codes \
mtime quiet_if_down ; do mtime quiet_if_down ; do
if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}" ]; then if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}" ]; then
eval c_$opt=\"${backup}/$opt\" eval c_$opt=\"${backup}/$opt\"
else else
eval c_$opt=\"${CDEFAULTS}/$opt\" eval c_$opt=\"${CDEFAULTS}/$opt\"
fi fi
done done
# #
# Interval definition: First try source specific, fallback to default # Interval definition: First try source specific, fallback to default
# #
c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)" c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)"
if [ -z "${c_interval}" ]; then if [ -z "${c_interval}" ]; then
c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)" c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)"
if [ -z "${c_interval}" ]; then if [ -z "${c_interval}" ]; then
_exit_err "No definition for interval \"${INTERVAL}\" found. Skipping." _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping."
fi fi
fi fi
# #
# Sort by ctime (default) or mtime (configuration option) # Sort by ctime (default) or mtime (configuration option)
# #
if [ -f "${c_mtime}" ] ; then if [ -f "${c_mtime}" ] ; then
TSORT="t" TSORT="t"
else else
TSORT="tc" TSORT="tc"
fi fi
# #
# Source configuration checks # Source configuration checks
# #
if [ ! -f "${c_source}" ]; then if [ ! -f "${c_source}" ]; then
_exit_err "Source description \"${c_source}\" is not a file. Skipping." _exit_err "Source description \"${c_source}\" is not a file. Skipping."
else else
source=$(cat "${c_source}"); ret="$?" source=$(cat "${c_source}"); ret="$?"
if [ "${ret}" -ne 0 ]; then if [ "${ret}" -ne 0 ]; then
_exit_err "Source ${c_source} is not readable. Skipping." _exit_err "Source ${c_source} is not readable. Skipping."
fi fi
fi fi
# #
# Destination is a path # Destination is a path
# #
if [ ! -f "${c_dest}" ]; then if [ ! -f "${c_dest}" ]; then
_exit_err "Destination ${c_dest} is not a file. Skipping." _exit_err "Destination ${c_dest} is not a file. Skipping."
else else
ddir="$(cat "${c_dest}")"; ret="$?" ddir="$(cat "${c_dest}")"; ret="$?"
if [ "${ret}" -ne 0 ]; then if [ "${ret}" -ne 0 ]; then
_exit_err "Destination ${c_dest} is not readable. Skipping." _exit_err "Destination ${c_dest} is not readable. Skipping."
fi fi
fi fi
# #
# Parameters: ccollect defaults, configuration options, user options # Parameters: ccollect defaults, configuration options, user options
# #
# #
# Rsync standard options (archive will be added after is-up-check) # Rsync standard options (archive will be added after is-up-check)
# #
set -- "$@" "--delete" "--numeric-ids" "--relative" \ set -- "$@" "--delete" "--numeric-ids" "--relative" \
"--delete-excluded" "--sparse" "--delete-excluded" "--sparse"
# #
# Exclude list # Exclude list
# #
if [ -f "${c_exclude}" ]; then if [ -f "${c_exclude}" ]; then
set -- "$@" "--exclude-from=${c_exclude}" set -- "$@" "--exclude-from=${c_exclude}"
fi fi
# #
# Output a summary # Output a summary
# #
if [ -f "${c_summary}" ]; then if [ -f "${c_summary}" ]; then
set -- "$@" "--stats" set -- "$@" "--stats"
fi fi
# #
# Verbosity for rsync, rm, and mkdir # Verbosity for rsync, rm, and mkdir
# #
VVERBOSE="" VVERBOSE=""
if [ -f "${c_very_verbose}" ]; then if [ -f "${c_very_verbose}" ]; then
set -- "$@" "-vv" set -- "$@" "-vv"
VVERBOSE="-v" VVERBOSE="-v"
elif [ -f "${c_verbose}" ]; then elif [ -f "${c_verbose}" ]; then
set -- "$@" "-v" set -- "$@" "-v"
fi fi
# #
# Extra options for rsync provided by the user # Extra options for rsync provided by the user
# #
if [ -f "${c_rsync_options}" ]; then if [ -f "${c_rsync_options}" ]; then
while read line; do while read line; do
set -- "$@" "${line}" set -- "$@" "${line}"
done < "${c_rsync_options}" done < "${c_rsync_options}"
fi fi
# #
# Check: source is up and accepting connections (before deleting old backups!) # Check: source is up and accepting connections (before deleting old backups!)
# #
if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then
if [ ! -f "${c_quiet_if_down}" ]; then if [ ! -f "${c_quiet_if_down}" ]; then
cat "${TMP}" cat "${TMP}"
fi fi
_exit_err "Source ${source} is not readable. Skipping." _exit_err "Source ${source} is not readable. Skipping."
fi fi
# #
# Add --archive for real backup (looks nice in front) # Add --archive for real backup (looks nice in front)
# #
set -- "--archive" "$@" set -- "--archive" "$@"
# #
# Check: destination exists? # Check: destination exists?
# #
cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping." cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping."
# #
# Check incomplete backups (needs echo to remove newlines) # Check incomplete backups (needs echo to remove newlines)
# #
ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$? ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$?
if [ "$ret" -eq 0 ]; then if [ "$ret" -eq 0 ]; then
_techo "Incomplete backups: $(echo $(cat "${TMP}"))" _techo "Incomplete backups: $(echo $(cat "${TMP}"))"
if [ -f "${c_delete_incomplete}" ]; then if [ -f "${c_delete_incomplete}" ]; then
delete_from_file "${TMP}" "${CMARKER}" delete_from_file "${TMP}" "${CMARKER}"
fi fi
fi fi
# #
# Check: maximum number of backups is reached? # Check: maximum number of backups is reached?
# #
count="$(ls -1 | grep -c "^${INTERVAL}\\.")" count="$(ls -1 | grep -c "^${INTERVAL}\\.")"
_techo "Existing backups: ${count} Total keeping backups: ${c_interval}" _techo "Existing backups: ${count} Total keeping backups: ${c_interval}"
if [ "${count}" -ge "${c_interval}" ]; then if [ "${count}" -ge "${c_interval}" ]; then
remove="$((${count} - ${c_interval} + 1))" remove="$((${count} - ${c_interval} + 1))"
_techo "Removing ${remove} backup(s)..." _techo "Removing ${remove} backup(s)..."
ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \
_exit_err "Listing old backups failed" _exit_err "Listing old backups failed"
delete_from_file "${TMP}" delete_from_file "${TMP}"
fi fi
# #
# Check for backup directory to clone from: Always clone from the latest one! # Check for backup directory to clone from: Always clone from the latest one!
# #
last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \
_exit_err "Failed to list contents of ${ddir}." _exit_err "Failed to list contents of ${ddir}."
# #
# Clone from old backup, if existing # Clone from old backup, if existing
# #
if [ "${last_dir}" ]; then if [ "${last_dir}" ]; then
set -- "$@" "--link-dest=${ddir}/${last_dir}" set -- "$@" "--link-dest=${ddir}/${last_dir}"
_techo "Hard linking from ${last_dir}" _techo "Hard linking from ${last_dir}"
fi fi
# #
# Include current time in name, not the time when we began to remove above # Include current time in name, not the time when we began to remove above
# #
export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}"
export destination_dir="${ddir}/${destination_name}" export destination_dir="${ddir}/${destination_name}"
# #
# Mark backup running and go back to original directory # Mark backup running and go back to original directory
# #
touch "${destination_dir}${CMARKER}" touch "${destination_dir}${CMARKER}"
cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}." cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}."
# #
# the rsync part # the rsync part
# #
_techo "Transferring files..." _techo "Transferring files..."
rsync "$@" "${source}" "${destination_dir}"; ret=$? rsync "$@" "${source}" "${destination_dir}"; ret=$?
_techo "Finished backup (rsync return code: $ret)." _techo "Finished backup (rsync return code: $ret)."
# #
# Set modification time (mtime) to current time, if sorting by mtime is enabled # Set modification time (mtime) to current time, if sorting by mtime is enabled
# #
[ -f "$c_mtime" ] && touch "${destination_dir}" [ -f "$c_mtime" ] && touch "${destination_dir}"
# #
# Check if rsync exit code indicates failure. # Check if rsync exit code indicates failure.
# #
fail="" fail=""
if [ -f "$c_rsync_failure_codes" ]; then if [ -f "$c_rsync_failure_codes" ]; then
while read code ; do while read code ; do
if [ "$ret" = "$code" ]; then if [ "$ret" = "$code" ]; then
fail=1 fail=1
fi fi
done <"${c_rsync_failure_codes}" done <"${c_rsync_failure_codes}"
fi fi
# #
# Remove marking here unless rsync failed. # Remove marking here unless rsync failed.
# #
if [ -z "$fail" ]; then if [ -z "$fail" ]; then
rm "${destination_dir}${CMARKER}" || \ rm "${destination_dir}${CMARKER}" || \
_exit_err "Removing ${destination_dir}${CMARKER} failed." _exit_err "Removing ${destination_dir}${CMARKER} failed."
if [ "${ret}" -ne 0 ]; then if [ "${ret}" -ne 0 ]; then
_techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)." _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)."
fi fi
else else
_techo "Warning: rsync failed with return code $ret." _techo "Warning: rsync failed with return code $ret."
fi fi
# #
# post_exec # post_exec
# #
if [ -x "${c_post_exec}" ]; then if [ -x "${c_post_exec}" ]; then
_techo "Executing ${c_post_exec} ..." _techo "Executing ${c_post_exec} ..."
"${c_post_exec}"; ret=$? "${c_post_exec}"; ret=$?
_techo "Finished ${c_post_exec}." _techo "Finished ${c_post_exec}."
if [ "${ret}" -ne 0 ]; then if [ "${ret}" -ne 0 ]; then
_exit_err "${c_post_exec} failed." _exit_err "${c_post_exec} failed."
fi fi
fi fi
# #
# Time calculation # Time calculation
# #
end_s="$(${SDATE})" end_s="$(${SDATE})"
full_seconds="$((${end_s} - ${begin_s}))" full_seconds="$((${end_s} - ${begin_s}))"
hours="$((${full_seconds} / 3600))" hours="$((${full_seconds} / 3600))"
minutes="$(((${full_seconds} % 3600) / 60))" minutes="$(((${full_seconds} % 3600) / 60))"
seconds="$((${full_seconds} % 60))" seconds="$((${full_seconds} % 60))"
_techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)"
) | add_name ) | add_name
done done
@ -543,21 +543,21 @@ done
# Be a good parent and wait for our children, if they are running wild parallel # Be a good parent and wait for our children, if they are running wild parallel
# #
if [ "${PARALLEL}" ]; then if [ "${PARALLEL}" ]; then
_techo "Waiting for children to complete..." _techo "Waiting for children to complete..."
wait wait
fi fi
# #
# Look for post-exec command (general) # Look for post-exec command (general)
# #
if [ -x "${CPOSTEXEC}" ]; then if [ -x "${CPOSTEXEC}" ]; then
_techo "Executing ${CPOSTEXEC} ..." _techo "Executing ${CPOSTEXEC} ..."
"${CPOSTEXEC}"; ret=$? "${CPOSTEXEC}"; ret=$?
_techo "Finished ${CPOSTEXEC} (return code: ${ret})." _techo "Finished ${CPOSTEXEC} (return code: ${ret})."
if [ "${ret}" -ne 0 ]; then if [ "${ret}" -ne 0 ]; then
_techo "${CPOSTEXEC} failed." _techo "${CPOSTEXEC} failed."
fi fi
fi fi
rm -f "${TMP}" rm -f "${TMP}"