ccollect 21.5 KB
Newer Older
Nico Schottelius's avatar
Nico Schottelius committed
1
#!/bin/sh
jll2's avatar
jll2 committed
2
#
Nico Schottelius's avatar
Nico Schottelius committed
3
# 2005-2013 Nico Schottelius (nico-ccollect at schottelius.org)
Darko Poljak's avatar
Darko Poljak committed
4
# 2016-2019 Darko Poljak (darko.poljak at gmail.com)
jll2's avatar
jll2 committed
5
#
Nico Schottelius's avatar
Nico Schottelius committed
6 7 8 9 10 11
# This file is part of ccollect.
#
# ccollect is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
jll2's avatar
jll2 committed
12
#
Nico Schottelius's avatar
Nico Schottelius committed
13 14 15 16
# ccollect is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
jll2's avatar
jll2 committed
17
#
Nico Schottelius's avatar
Nico Schottelius committed
18 19 20
# You should have received a copy of the GNU General Public License
# along with ccollect. If not, see <http://www.gnu.org/licenses/>.
#
Nico Schottelius's avatar
Nico Schottelius committed
21
# Initially written for SyGroup (www.sygroup.ch)
Nico Schottelius's avatar
Nico Schottelius committed
22 23
# Date: Mon Nov 14 11:45:11 CET 2005

jll2's avatar
jll2 committed
24 25 26
# Error upon expanding unset variables:
set -u

27 28 29 30 31 32 33
#
# Standard variables (stolen from cconf)
#
__pwd="$(pwd -P)"
__mydir="${0%/*}"; __abs_mydir="$(cd "$__mydir" && pwd -P)"
__myname=${0##*/}; __abs_myname="$__abs_mydir/$__myname"

34
#
35
# where to find our configuration and temporary file
36
#
37 38 39
CCOLLECT_CONF="${CCOLLECT_CONF:-/etc/ccollect}"
CSOURCES="${CCOLLECT_CONF}/sources"
CDEFAULTS="${CCOLLECT_CONF}/defaults"
Nico Schottelius's avatar
Nico Schottelius committed
40 41
CPREEXEC="${CDEFAULTS}/pre_exec"
CPOSTEXEC="${CDEFAULTS}/post_exec"
42
CMARKER=".ccollect-marker"
43

Nico Schottelius's avatar
Nico Schottelius committed
44
export TMP="$(mktemp "/tmp/${__myname}.XXXXXX")"
45 46
CONTROL_PIPE="/tmp/${__myname}-control-pipe"

Darko Poljak's avatar
Darko Poljak committed
47 48
VERSION="2.5"
RELEASE="2019-05-01"
Nico Schottelius's avatar
Nico Schottelius committed
49 50
HALF_VERSION="ccollect ${VERSION}"
FULL_VERSION="ccollect ${VERSION} (${RELEASE})"
51 52 53

#
# CDATE: how we use it for naming of the archives
54
# DDATE: how the user should see it in our output (DISPLAY)
55
#
Nico Schottelius's avatar
Nico Schottelius committed
56
CDATE="date +%Y%m%d-%H%M"
Nico Schottelius's avatar
Nico Schottelius committed
57
DDATE="date +%Y-%m-%d-%H:%M:%S"
58
SDATE="date +%s"
59

Darko Poljak's avatar
Darko Poljak committed
60 61 62 63 64
#
# LOCKING: use flock if available, otherwise mkdir
# Locking is done for each source so that only one instance per source
# can run.
#
65 66 67 68
# Use CCOLLECT_CONF directory for lock files.
# This directory can be set arbitrary so it is writable for user
# executing ccollect.
LOCKDIR="${CCOLLECT_CONF}"
Darko Poljak's avatar
Darko Poljak committed
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123
# printf pattern: ccollect_<source>.lock
LOCKFILE_PATTERN="ccollect_%s.lock"
LOCKFD=4

#
# locking functions using flock
#
lock_flock()
{
    # $1 = source to backup
    lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")"
    eval "exec ${LOCKFD}> ${lockfile}"

    flock -n ${LOCKFD} && return 0 || return 1
}

unlock_flock()
{
    # $1 = source to backup
    lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")"
    eval "exec ${LOCKFD}>&-"
    rm -f "${lockfile}"
}

#
# locking functions using mkdir (mkdir is atomic)
#
lock_mkdir()
{
    # $1 = source to backup
    lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")"

    mkdir "${lockfile}" && return 0 || return 1
}

unlock_mkdir()
{
    # $1 = source to backup
    lockfile="${LOCKDIR}/$(printf "${LOCKFILE_PATTERN}" "$1")"

    rmdir "${lockfile}"
}

#
# determine locking tool: flock or mkdir
#
if $(which flock > /dev/null 2>&1)
then
    lockf="lock_flock"
    unlockf="unlock_flock"
else
    lockf="lock_mkdir"
    unlockf="unlock_mkdir"
fi

124
#
125
# unset values
126 127
#
PARALLEL=""
128
MAX_JOBS=""
129
USE_ALL=""
130 131 132 133 134
LOGFILE=""
SYSLOG=""
# e - only errors, a - all output
LOGLEVEL="a"
LOGONLYERRORS=""
135

136 137 138
#
# catch signals
#
Darko Poljak's avatar
Darko Poljak committed
139 140
TRAPFUNC="rm -f \"${TMP}\""
trap "${TRAPFUNC}" 1 2 15
Nico Schottelius's avatar
Nico Schottelius committed
141

Nico Schottelius's avatar
Nico Schottelius committed
142 143 144 145
#
# Functions
#

146 147 148
# check if we are running interactive or non-interactive
# see: http://www.tldp.org/LDP/abs/html/intandnonint.html
_is_interactive()
149
{
150
    [ -t 0 -o -p /dev/stdin ]
Nico Schottelius's avatar
Nico Schottelius committed
151 152
}

153 154 155
#
# ssh-"feature": we cannot do '... read ...; ssh  ...; < file',
# because ssh reads stdin! -n does not work -> does not ask for password
Nico Schottelius's avatar
cleanup  
Nico Schottelius committed
156
# Also allow deletion for files without the given suffix
157
#
158 159
delete_from_file()
{
Nico Schottelius's avatar
Nico Schottelius committed
160 161 162
    file="$1"; shift
    suffix="" # It will be set, if deleting incomplete backups.
    [ $# -eq 1 ] && suffix="$1" && shift
163 164
    # dirs for deletion will be moved to this trash dir inside destination dir
    # - for fast mv operation
165
    trash="$(mktemp -d ".trash.XXXXXX")"
Nico Schottelius's avatar
Nico Schottelius committed
166
    while read to_remove; do
167 168
        mv "${to_remove}" "${trash}" ||
            _exit_err "Moving ${to_remove} to ${trash} failed."
Nico Schottelius's avatar
Nico Schottelius committed
169 170 171
        set -- "$@" "${to_remove}"
        if [ "${suffix}" ]; then
            to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")"
172 173
            mv "${to_remove_no_suffix}" "${trash}" ||
                _exit_err "Moving ${to_remove_no_suffix} to ${trash} failed."
Nico Schottelius's avatar
Nico Schottelius committed
174 175 176
            set -- "$@" "${to_remove_no_suffix}"
        fi
    done < "${file}"
177
    _techo "Removing $@ in ${trash}..."
178
    empty_dir=".empty-dir"
179
    mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created."
Darko Poljak's avatar
Darko Poljak committed
180
    [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}"
181 182 183 184
    # rsync needs ending slash for directory content
    rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed."
    rmdir "${trash}" || _exit_err "Removing ${trash} directory failed"
    rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed"
Darko Poljak's avatar
Darko Poljak committed
185
    _techo "Removing $@ in ${trash} finished."
186 187
}

Nico Schottelius's avatar
Nico Schottelius committed
188 189
display_version()
{
Nico Schottelius's avatar
Nico Schottelius committed
190 191
    echo "${FULL_VERSION}"
    exit 0
Nico Schottelius's avatar
Nico Schottelius committed
192 193
}

Nico Schottelius's avatar
Nico Schottelius committed
194 195
usage()
{
Nico Schottelius's avatar
Nico Schottelius committed
196
    cat << eof
197 198
${__myname}: [args] <interval name> <sources to backup>

Nico Schottelius's avatar
Nico Schottelius committed
199
    ccollect creates (pseudo) incremental backups
200

201 202 203 204 205 206 207 208 209 210
    -h, --help:              Show this help screen
    -a, --all:               Backup all sources specified in ${CSOURCES}
    -e, --errors:            Log only errors
    -j [max], --jobs [max]   Specifies the number of jobs to run simultaneously.
                             If max is not specified then parallelise all jobs.
    -l FILE, --logfile FILE  Log to specified file
    -p, --parallel:          Parallelise backup processes (deprecated from 2.0)
    -s, --syslog:            Log to syslog with tag ccollect
    -v, --verbose:           Be very verbose (uses set -x)
    -V, --version:           Print version information
211

Nico Schottelius's avatar
Nico Schottelius committed
212
    This is version ${VERSION} released on ${RELEASE}.
213

Nico Schottelius's avatar
Nico Schottelius committed
214
    Retrieve latest ccollect at http://www.nico.schottelius.org/software/ccollect/
215
eof
Nico Schottelius's avatar
Nico Schottelius committed
216
    exit 0
Nico Schottelius's avatar
Nico Schottelius committed
217 218
}

Darko Poljak's avatar
Darko Poljak committed
219 220 221 222 223 224 225 226 227 228 229 230
# locking functions
lock()
{
    "${lockf}" "$@" || _exit_err \
        "Only one instance of ${__myname} for source \"$1\" can run at one time."
}

unlock()
{
    "${unlockf}" "$@"
}

231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262
# time displaying echo
# stdout version
_techo_stdout()
{
    echo "$(${DDATE}): $@"
}

# syslog version
_techo_syslog()
{
    logger -t ccollect "$@"
}

# specified file version
_techo_file()
{
    _techo_stdout "$@" >> "${LOGFILE}"
}

# determine _techo version before parsing options
if _is_interactive
then
    _techof="_techo_stdout"
else
    _techof="_techo_syslog"
fi

# _techo with determined _techo version
_techo()
{
    if [ "${LOGLEVEL}" = "a" ]
    then
263 264
        set -- ${name:+"[${name}]"} $@
        "${_techof}" "$@"
265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
    fi
}

_techo_err()
{
    _techo "Error: $@"
}

_exit_err()
{
    _techo_err "$@"
    rm -f "${TMP}"
    exit 1
}

280
#
281
# Parse options
282
#
283
while [ "$#" -ge 1 ]; do
Nico Schottelius's avatar
Nico Schottelius committed
284 285 286 287 288
    case "$1" in
        -a|--all)
            USE_ALL=1
            ;;
        -p|--parallel)
289 290
            _techo "Warning: -p, --parallel option is deprecated," \
                   "use -j, --jobs instead."
Nico Schottelius's avatar
Nico Schottelius committed
291
            PARALLEL=1
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306
            MAX_JOBS=""
            ;;
        -j|--jobs)
            PARALLEL=1
            if [ "$#" -ge 2 ]
            then 
                case "$2" in
                    -*)
                        ;;
                    *)
                        MAX_JOBS=$2
                        shift
                        ;;
                esac
            fi
Nico Schottelius's avatar
Nico Schottelius committed
307
            ;;
308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329
        -e|--errors)
            LOGONLYERRORS="1"
            ;;
        -l|--logfile)
            if [ "$#" -ge 2 ]
            then
                case "$2" in
                    -*)
                        _exit_err "Missing log file"
                        ;;
                    *)
                        LOGFILE="$2"
                        shift
                        ;;
                esac
            else
                _exit_err "Missing log file"
            fi
            ;;
        -s|--syslog)
            SYSLOG="1"
            ;;
Nico Schottelius's avatar
Nico Schottelius committed
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348
        -v|--verbose)
            set -x
            ;;
        -V|--version)
            display_version
            ;;
        --)
            # ignore the -- itself
            shift
            break
            ;;
        -h|--help|-*)
            usage
            ;;
        *)
            break
            ;;
    esac
    shift
349
done
Nico Schottelius's avatar
Nico Schottelius committed
350

351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
# determine _techo version and logging level after parsing options
if [ "${LOGFILE}" ]
then
    _techof="_techo_file"
    LOGLEVEL="a"
elif _is_interactive
then
    if [ "${SYSLOG}" ]
    then
        _techof="_techo_syslog"
        LOGLEVEL="a"
    else
        _techof="_techo_stdout"
        LOGLEVEL="e"
    fi
else
    _techof="_techo_syslog"
    LOGLEVEL="a"
fi

if [ "${LOGFILE}" -o "${SYSLOG}" ]
then
    if [ "${LOGONLYERRORS}" ]
    then
        LOGLEVEL="e"
    fi
fi

379 380
# check that MAX_JOBS is natural number > 0
# empty string means run all in parallel
381
echo "${MAX_JOBS}" | grep -q -E '^[1-9][0-9]*$|^$'
382 383 384 385 386
if [ "$?" -ne 0 ]
then
    _exit_err "Invalid max jobs value \"${MAX_JOBS}\""
fi

387 388 389 390
#
# Setup interval
#
if [ $# -ge 1 ]; then
Nico Schottelius's avatar
Nico Schottelius committed
391 392
    export INTERVAL="$1"
    shift
393
else
Nico Schottelius's avatar
Nico Schottelius committed
394
    usage
395
fi
396

397
#
Nico Schottelius's avatar
Nico Schottelius committed
398
# Check for configuraton directory
399
#
400
[ -d "${CCOLLECT_CONF}" ] || _exit_err "No configuration found in " \
Nico Schottelius's avatar
Nico Schottelius committed
401
    "\"${CCOLLECT_CONF}\" (is \$CCOLLECT_CONF properly set?)"
402

Nico Schottelius's avatar
Nico Schottelius committed
403
#
Nico Schottelius's avatar
Nico Schottelius committed
404
# Create (portable!) source "array"
Nico Schottelius's avatar
Nico Schottelius committed
405
#
Nico Schottelius's avatar
Nico Schottelius committed
406
export no_sources=0
Nico Schottelius's avatar
Nico Schottelius committed
407

408
if [ "${USE_ALL}" = 1 ]; then
Nico Schottelius's avatar
Nico Schottelius committed
409 410 411 412 413 414 415 416 417 418
    #
    # Get sources from source configuration
    #
    ( cd "${CSOURCES}" && ls -1 > "${TMP}" ) || \
        _exit_err "Listing of sources failed. Aborting."

    while read tmp; do
        eval export source_${no_sources}=\"${tmp}\"
        no_sources=$((${no_sources}+1))
    done < "${TMP}"
419
else
Nico Schottelius's avatar
Nico Schottelius committed
420 421 422 423 424 425 426 427 428
    #
    # Get sources from command line
    #
    while [ "$#" -ge 1 ]; do
        eval arg=\"\$1\"; shift

        eval export source_${no_sources}=\"${arg}\"
        no_sources="$((${no_sources}+1))"
    done
Nico Schottelius's avatar
Nico Schottelius committed
429 430
fi

431 432 433
#
# Need at least ONE source to backup
#
Nico Schottelius's avatar
Nico Schottelius committed
434
if [ "${no_sources}" -lt 1 ]; then
Nico Schottelius's avatar
Nico Schottelius committed
435
    usage
436
else
Nico Schottelius's avatar
Nico Schottelius committed
437
    _techo "${HALF_VERSION}: Beginning backup using interval ${INTERVAL}"
438 439
fi

440 441 442 443
#
# Look for pre-exec command (general)
#
if [ -x "${CPREEXEC}" ]; then
Nico Schottelius's avatar
Nico Schottelius committed
444 445 446
    _techo "Executing ${CPREEXEC} ..."
    "${CPREEXEC}"; ret=$?
    _techo "Finished ${CPREEXEC} (return code: ${ret})."
447

Nico Schottelius's avatar
Nico Schottelius committed
448
    [ "${ret}" -eq 0 ] || _exit_err "${CPREEXEC} failed. Aborting"
449 450
fi

Nico Schottelius's avatar
Nico Schottelius committed
451
################################################################################
452
#
Nico Schottelius's avatar
Nico Schottelius committed
453
# Let's do the backup - here begins the real stuff
454
#
455 456 457 458 459 460 461 462 463 464 465 466

# in PARALLEL mode:
# * create control pipe
# * determine number of jobs to start at once
if [ "${PARALLEL}" ]; then
    mkfifo "${CONTROL_PIPE}"
    # fd 5 is tied to control pipe
    eval "exec 5<>${CONTROL_PIPE}"
    TRAPFUNC="${TRAPFUNC}; rm -f \"${CONTROL_PIPE}\""
    trap "${TRAPFUNC}" 0 1 2 15

    # determine how much parallel jobs to prestart
467
    if [ "${MAX_JOBS}" ]
468
    then
469 470 471 472 473 474
        if [ "${MAX_JOBS}" -le "${no_sources}" ]
        then
            prestart="${MAX_JOBS}"
        else
            prestart="${no_sources}"
        fi
475 476 477 478 479
    else
        prestart=0
    fi
fi

480 481
source_no=0
while [ "${source_no}" -lt "${no_sources}" ]; do
Nico Schottelius's avatar
Nico Schottelius committed
482 483 484 485 486
    #
    # Get current source
    #
    eval export name=\"\$source_${source_no}\"
    source_no=$((${source_no}+1))
487

Nico Schottelius's avatar
Nico Schottelius committed
488 489 490 491
    #
    # Start ourself, if we want parallel execution
    #
    if [ "${PARALLEL}" ]; then
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514
        if [ ! "${MAX_JOBS}" ]
        then
            # run all in parallel
            "$0" "${INTERVAL}" "${name}" &
            continue
        elif [ "${prestart}" -gt 0 ]
        then
            # run prestart child if pending
            { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } &
            prestart=$((${prestart} - 1))
            continue
        else
            # each time a child finishes we get a line from the pipe
            # and then launch another child
            while read line
            do
                { "$0" "${INTERVAL}" "${name}"; printf '\n' >&5; } &
                # get out of loop so we can contnue with main loop
                # for next source
                break
            done <&5
            continue
        fi
Nico Schottelius's avatar
Nico Schottelius committed
515
    fi
516

517 518 519 520
#
# Start subshell for easy log editing
#
(
Nico Schottelius's avatar
Nico Schottelius committed
521 522 523 524 525 526 527 528 529 530 531 532 533 534 535
    backup="${CSOURCES}/${name}"
    c_source="${backup}/source"
    c_dest="${backup}/destination"
    c_pre_exec="${backup}/pre_exec"
    c_post_exec="${backup}/post_exec"

    #
    # Stderr to stdout, so we can produce nice logs
    #
    exec 2>&1

    #
    # Record start of backup: internal and for the user
    #
    begin_s="$(${SDATE})"
Darko Poljak's avatar
Darko Poljak committed
536
    _techo "Beginning to backup"
Nico Schottelius's avatar
Nico Schottelius committed
537 538 539 540 541

    #
    # Standard configuration checks
    #
    if [ ! -e "${backup}" ]; then
Darko Poljak's avatar
Darko Poljak committed
542
        _exit_err "Source \"${backup}\" does not exist."
Nico Schottelius's avatar
Nico Schottelius committed
543 544 545 546 547 548 549 550 551
    fi

    #
    # Configuration _must_ be a directory (cconfig style)
    #
    if [ ! -d "${backup}" ]; then
        _exit_err "\"${backup}\" is not a cconfig-directory. Skipping."
    fi

Darko Poljak's avatar
Darko Poljak committed
552 553 554 555 556 557 558
    #
    # Acquire lock for source. If lock cannot be acquired, lock will exit
    # with error message.
    #
    lock "${name}"

    # redefine trap to also unlock (rm lockfile)
559
    TRAPFUNC="${TRAPFUNC}; unlock \"${name}\""
Darko Poljak's avatar
Darko Poljak committed
560 561
    trap "${TRAPFUNC}" 1 2 15

Nico Schottelius's avatar
Nico Schottelius committed
562 563 564 565
    #
    # First execute pre_exec, which may generate destination or other parameters
    #
    if [ -x "${c_pre_exec}" ]; then
Darko Poljak's avatar
Darko Poljak committed
566
        _techo "Executing ${c_pre_exec} ..."
Nico Schottelius's avatar
Nico Schottelius committed
567
        "${c_pre_exec}"; ret="$?"
Darko Poljak's avatar
Darko Poljak committed
568
        _techo "Finished ${c_pre_exec} (return code ${ret})."
Nico Schottelius's avatar
Nico Schottelius committed
569

Darko Poljak's avatar
Darko Poljak committed
570
        [ "${ret}" -eq 0 ] || _exit_err "${c_pre_exec} failed. Skipping."
Nico Schottelius's avatar
Nico Schottelius committed
571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594
    fi

    #
    # Read source configuration
    #
    for opt in verbose very_verbose summary exclude rsync_options \
                  delete_incomplete rsync_failure_codes  \
                  mtime quiet_if_down ; do
        if [ -f "${backup}/${opt}" -o -f "${backup}/no_${opt}"  ]; then
            eval c_$opt=\"${backup}/$opt\"
        else
            eval c_$opt=\"${CDEFAULTS}/$opt\"
        fi
    done

    #
    # Interval definition: First try source specific, fallback to default
    #
    c_interval="$(cat "${backup}/intervals/${INTERVAL}" 2>/dev/null)"

    if [ -z "${c_interval}" ]; then
        c_interval="$(cat "${CDEFAULTS}/intervals/${INTERVAL}" 2>/dev/null)"

        if [ -z "${c_interval}" ]; then
Darko Poljak's avatar
Darko Poljak committed
595
            _exit_err "No definition for interval \"${INTERVAL}\" found. Skipping."
Nico Schottelius's avatar
Nico Schottelius committed
596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623
        fi
    fi

    #
    # Sort by ctime (default) or mtime (configuration option)
    #
    if [ -f "${c_mtime}" ] ; then
        TSORT="t"
    else
        TSORT="tc"
    fi

    #
    # Source configuration checks
    #
    if [ ! -f "${c_source}" ]; then
        _exit_err "Source description \"${c_source}\" is not a file. Skipping."
    else
        source=$(cat "${c_source}"); ret="$?"
        if [ "${ret}" -ne 0 ]; then
            _exit_err "Source ${c_source} is not readable. Skipping."
        fi
    fi

    #
    # Destination is a path
    #
    if [ ! -f "${c_dest}" ]; then
Darko Poljak's avatar
Darko Poljak committed
624
        _exit_err "Destination ${c_dest} is not a file. Skipping."
Nico Schottelius's avatar
Nico Schottelius committed
625 626 627
    else
        ddir="$(cat "${c_dest}")"; ret="$?"
        if [ "${ret}" -ne 0 ]; then
Darko Poljak's avatar
Darko Poljak committed
628
            _exit_err "Destination ${c_dest} is not readable. Skipping."
Nico Schottelius's avatar
Nico Schottelius committed
629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671
        fi
    fi

    #
    # Parameters: ccollect defaults, configuration options, user options
    #

    #
    # Rsync standard options (archive will be added after is-up-check)
    #
    set -- "$@" "--delete" "--numeric-ids" "--relative"    \
                    "--delete-excluded" "--sparse"

    #
    # Exclude list
    #
    if [ -f "${c_exclude}" ]; then
        set -- "$@" "--exclude-from=${c_exclude}"
    fi

    #
    # Output a summary
    #
    if [ -f "${c_summary}" ]; then
        set -- "$@" "--stats"
    fi

    #
    # Verbosity for rsync, rm, and mkdir
    #
    VVERBOSE=""
    if [ -f "${c_very_verbose}" ]; then
        set -- "$@" "-vv"
        VVERBOSE="-v"
    elif [ -f "${c_verbose}" ]; then
        set -- "$@" "-v"
    fi

    #
    # Extra options for rsync provided by the user
    #
    if [ -f "${c_rsync_options}" ]; then
        while read line; do
672 673 674 675 676 677 678 679 680 681 682
            # Trim line.
            ln=$(echo "${line}" | awk '{$1=$1;print;}')
            # Only if ln is non zero length string.
            #
            # If ln is empty then rsync '' DEST evaluates
            # to transfer current directory to DEST which would
            # with specific options destroy DEST content.
            if [ -n "${ln}" ]
            then
                set -- "$@" "${ln}"
            fi
Nico Schottelius's avatar
Nico Schottelius committed
683 684 685 686 687 688 689 690 691 692
        done < "${c_rsync_options}"
    fi

    #
    # Check: source is up and accepting connections (before deleting old backups!)
    #
    if ! rsync "$@" "${source}" >/dev/null 2>"${TMP}" ; then
        if [ ! -f "${c_quiet_if_down}" ]; then
            cat "${TMP}"
        fi
Darko Poljak's avatar
Darko Poljak committed
693
        _exit_err "Source ${source} is not readable. Skipping."
Nico Schottelius's avatar
Nico Schottelius committed
694 695 696 697 698 699 700 701 702 703
    fi

    #
    # Add --archive for real backup (looks nice in front)
    #
    set -- "--archive" "$@"

    #
    # Check: destination exists?
    #
Darko Poljak's avatar
Darko Poljak committed
704
    cd "${ddir}" || _exit_err "Cannot change to ${ddir}. Skipping."
Nico Schottelius's avatar
Nico Schottelius committed
705 706 707 708 709 710 711

    #
    # Check incomplete backups (needs echo to remove newlines)
    #
    ls -1 | grep "${CMARKER}\$" > "${TMP}"; ret=$?

    if [ "$ret" -eq 0 ]; then
Darko Poljak's avatar
Darko Poljak committed
712
        _techo "Incomplete backups: $(echo $(cat "${TMP}"))"
Nico Schottelius's avatar
Nico Schottelius committed
713
        if [ -f "${c_delete_incomplete}" ]; then
714
            delete_from_file "${TMP}" "${CMARKER}" &
Nico Schottelius's avatar
Nico Schottelius committed
715 716 717
        fi
    fi

718 719 720 721 722 723
    #
    # Include current time in name, not the time when we began to remove above
    #
    export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}"
    export destination_dir="${ddir}/${destination_name}"

Nico Schottelius's avatar
Nico Schottelius committed
724 725 726 727 728
    #
    # Check: maximum number of backups is reached?
    #
    count="$(ls -1 | grep -c "^${INTERVAL}\\.")"

Darko Poljak's avatar
Darko Poljak committed
729
    _techo "Existing backups: ${count} Total keeping backups: ${c_interval}"
Nico Schottelius's avatar
Nico Schottelius committed
730 731

    if [ "${count}" -ge "${c_interval}" ]; then
732 733 734 735
        # Use oldest directory as new backup destination directory.
        # It need not to be deleted, rsync will sync its content.
        oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \
            _exit_err "Listing oldest backup failed")
Darko Poljak's avatar
Darko Poljak committed
736
        _techo "Using ${oldest_bak} for destination dir ${destination_dir}"
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
        mv "${oldest_bak}" "${destination_dir}" ||
            _exit_err "Moving oldest backup ${oldest_bak} to ${destination_dir} failed."
        # Touch dest dir so it is not sorted wrong in listings below.
        touch "${destination_dir}"

        # We have something to remove only if count > interval.
        if [ "${count}" -gt "${c_interval}" ]; then
            remove="$((${count} - ${c_interval}))"
            _techo "Removing ${remove} backup(s)..."

            ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \
                    _exit_err "Listing old backups failed"

            delete_from_file "${TMP}" &
        fi
Nico Schottelius's avatar
Nico Schottelius committed
752 753 754 755
    fi

    #
    # Check for backup directory to clone from: Always clone from the latest one!
756 757
    # Exclude destination_dir from listing, it can be touched reused and renamed
    # oldest existing destination directory.
Nico Schottelius's avatar
Nico Schottelius committed
758
    #
759
    last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${destination_dir}" | head -n 1)" || \
Darko Poljak's avatar
Darko Poljak committed
760
        _exit_err "Failed to list contents of ${ddir}."
Nico Schottelius's avatar
Nico Schottelius committed
761 762 763 764 765 766

    #
    # Clone from old backup, if existing
    #
    if [ "${last_dir}" ]; then
        set -- "$@" "--link-dest=${ddir}/${last_dir}"
Darko Poljak's avatar
Darko Poljak committed
767
        _techo "Hard linking from ${last_dir}"
Nico Schottelius's avatar
Nico Schottelius committed
768 769 770 771 772 773
    fi

    #
    # Mark backup running and go back to original directory
    #
    touch "${destination_dir}${CMARKER}"
Darko Poljak's avatar
Darko Poljak committed
774
    cd "${__abs_mydir}" || _exit_err "Cannot go back to ${__abs_mydir}."
Nico Schottelius's avatar
Nico Schottelius committed
775 776 777 778

    #
    # the rsync part
    #
Darko Poljak's avatar
Darko Poljak committed
779
    _techo "Transferring files..."
Nico Schottelius's avatar
Nico Schottelius committed
780
    rsync "$@" "${source}" "${destination_dir}"; ret=$?
Darko Poljak's avatar
Darko Poljak committed
781
    _techo "Finished backup (rsync return code: $ret)."
Nico Schottelius's avatar
Nico Schottelius committed
782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804

    #
    # Set modification time (mtime) to current time, if sorting by mtime is enabled
    #
    [ -f "$c_mtime" ] && touch "${destination_dir}"

    #
    # Check if rsync exit code indicates failure.
    #
    fail=""
    if [ -f "$c_rsync_failure_codes" ]; then
        while read code ; do
            if [ "$ret" = "$code" ]; then
                fail=1
            fi
        done <"${c_rsync_failure_codes}"
    fi

    #
    # Remove marking here unless rsync failed.
    #
    if [ -z "$fail" ]; then
        rm "${destination_dir}${CMARKER}" || \
Darko Poljak's avatar
Darko Poljak committed
805
            _exit_err "Removing ${destination_dir}${CMARKER} failed."
Nico Schottelius's avatar
Nico Schottelius committed
806
        if [ "${ret}" -ne 0 ]; then
Darko Poljak's avatar
Darko Poljak committed
807
            _techo "Warning: rsync exited non-zero, the backup may be broken (see rsync errors)."
Nico Schottelius's avatar
Nico Schottelius committed
808 809
        fi
    else
Darko Poljak's avatar
Darko Poljak committed
810
        _techo "Warning: rsync failed with return code $ret."
Nico Schottelius's avatar
Nico Schottelius committed
811 812 813 814 815 816
    fi

    #
    # post_exec
    #
    if [ -x "${c_post_exec}" ]; then
Darko Poljak's avatar
Darko Poljak committed
817
        _techo "Executing ${c_post_exec} ..."
Nico Schottelius's avatar
Nico Schottelius committed
818
        "${c_post_exec}"; ret=$?
Darko Poljak's avatar
Darko Poljak committed
819
        _techo "Finished ${c_post_exec}."
Nico Schottelius's avatar
Nico Schottelius committed
820 821

        if [ "${ret}" -ne 0 ]; then
Darko Poljak's avatar
Darko Poljak committed
822
            _exit_err "${c_post_exec} failed."
Nico Schottelius's avatar
Nico Schottelius committed
823 824 825 826 827 828 829 830 831 832 833 834
        fi
    fi

    #
    # Time calculation
    #
    end_s="$(${SDATE})"
    full_seconds="$((${end_s} - ${begin_s}))"
    hours="$((${full_seconds} / 3600))"
    minutes="$(((${full_seconds} % 3600) / 60))"
    seconds="$((${full_seconds} % 60))"

Darko Poljak's avatar
Darko Poljak committed
835
    _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)"
836 837

    unlock "${name}"
838 839 840

    # wait for children (doing delete_from_file) if any still running
    wait
Darko Poljak's avatar
Darko Poljak committed
841
) || exit
842
done
843

Nico Schottelius's avatar
Nico Schottelius committed
844 845
#
# Be a good parent and wait for our children, if they are running wild parallel
846
# After all children are finished then remove control pipe.
Nico Schottelius's avatar
Nico Schottelius committed
847
#
Nico Schottelius's avatar
Nico Schottelius committed
848
if [ "${PARALLEL}" ]; then
Nico Schottelius's avatar
Nico Schottelius committed
849 850
    _techo "Waiting for children to complete..."
    wait
851
    rm -f "${CONTROL_PIPE}"
Nico Schottelius's avatar
Nico Schottelius committed
852 853
fi

854 855 856
#
# Look for post-exec command (general)
#
Nico Schottelius's avatar
Nico Schottelius committed
857
if [ -x "${CPOSTEXEC}" ]; then
Nico Schottelius's avatar
Nico Schottelius committed
858 859 860
    _techo "Executing ${CPOSTEXEC} ..."
    "${CPOSTEXEC}"; ret=$?
    _techo "Finished ${CPOSTEXEC} (return code: ${ret})."
Nico Schottelius's avatar
Nico Schottelius committed
861

Nico Schottelius's avatar
Nico Schottelius committed
862 863 864
    if [ "${ret}" -ne 0 ]; then
        _techo "${CPOSTEXEC} failed."
    fi
865 866
fi

Nico Schottelius's avatar
Nico Schottelius committed
867
rm -f "${TMP}"
868
_techo "Finished"