forked from ungleich-public/ccollect
Merge branch 'performance/speed-up-removal' into 'master'
Improve performance See merge request ungleich-public/ccollect!12
This commit is contained in:
commit
dcc72aebf7
2 changed files with 52 additions and 16 deletions
67
ccollect
67
ccollect
|
@ -160,16 +160,29 @@ delete_from_file()
|
|||
file="$1"; shift
|
||||
suffix="" # It will be set, if deleting incomplete backups.
|
||||
[ $# -eq 1 ] && suffix="$1" && shift
|
||||
# dirs for deletion will be moved to this trash dir inside destination dir
|
||||
# - for fast mv operation
|
||||
trash="$(mktemp -d ".trash.XXXXXX")"
|
||||
while read to_remove; do
|
||||
mv "${to_remove}" "${trash}" ||
|
||||
_exit_err "Moving ${to_remove} to ${trash} failed."
|
||||
set -- "$@" "${to_remove}"
|
||||
if [ "${suffix}" ]; then
|
||||
to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")"
|
||||
mv "${to_remove_no_suffix}" "${trash}" ||
|
||||
_exit_err "Moving ${to_remove_no_suffix} to ${trash} failed."
|
||||
set -- "$@" "${to_remove_no_suffix}"
|
||||
fi
|
||||
done < "${file}"
|
||||
_techo "Removing $@ ..."
|
||||
[ "${VVERBOSE}" ] && echo rm "$@"
|
||||
rm -rf "$@" || _exit_err "Removing $@ failed."
|
||||
_techo "Removing $@ in ${trash}..."
|
||||
empty_dir=".empty-dir"
|
||||
mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created."
|
||||
[ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}"
|
||||
# rsync needs ending slash for directory content
|
||||
rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed."
|
||||
rmdir "${trash}" || _exit_err "Removing ${trash} directory failed"
|
||||
rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed"
|
||||
_techo "Removing $@ in ${trash} finished."
|
||||
}
|
||||
|
||||
display_version()
|
||||
|
@ -698,10 +711,16 @@ while [ "${source_no}" -lt "${no_sources}" ]; do
|
|||
if [ "$ret" -eq 0 ]; then
|
||||
_techo "Incomplete backups: $(echo $(cat "${TMP}"))"
|
||||
if [ -f "${c_delete_incomplete}" ]; then
|
||||
delete_from_file "${TMP}" "${CMARKER}"
|
||||
delete_from_file "${TMP}" "${CMARKER}" &
|
||||
fi
|
||||
fi
|
||||
|
||||
#
|
||||
# Include current time in name, not the time when we began to remove above
|
||||
#
|
||||
export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}"
|
||||
export destination_dir="${ddir}/${destination_name}"
|
||||
|
||||
#
|
||||
# Check: maximum number of backups is reached?
|
||||
#
|
||||
|
@ -710,19 +729,38 @@ while [ "${source_no}" -lt "${no_sources}" ]; do
|
|||
_techo "Existing backups: ${count} Total keeping backups: ${c_interval}"
|
||||
|
||||
if [ "${count}" -ge "${c_interval}" ]; then
|
||||
remove="$((${count} - ${c_interval} + 1))"
|
||||
_techo "Removing ${remove} backup(s)..."
|
||||
# Use oldest directory as new backup destination directory.
|
||||
# It need not to be deleted, rsync will sync its content.
|
||||
oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \
|
||||
_exit_err "Listing oldest backup failed")
|
||||
_techo "Using ${oldest_bak} for destination dir ${destination_dir}"
|
||||
if mv "${oldest_bak}" "${destination_dir}"; then
|
||||
# Touch dest dir so it is not sorted wrong in listings below.
|
||||
touch "${destination_dir}"
|
||||
|
||||
ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \
|
||||
_exit_err "Listing old backups failed"
|
||||
# We have something to remove only if count > interval.
|
||||
remove="$((${count} - ${c_interval}))"
|
||||
else
|
||||
_techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it."
|
||||
remove="$((${count} - ${c_interval} + 1))"
|
||||
fi
|
||||
if [ "${remove}" -gt 0 ]; then
|
||||
_techo "Removing ${remove} backup(s)..."
|
||||
|
||||
delete_from_file "${TMP}"
|
||||
ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \
|
||||
_exit_err "Listing old backups failed"
|
||||
|
||||
delete_from_file "${TMP}" &
|
||||
fi
|
||||
fi
|
||||
|
||||
#
|
||||
# Check for backup directory to clone from: Always clone from the latest one!
|
||||
# Exclude destination_dir from listing, it can be touched reused and renamed
|
||||
# oldest existing destination directory.
|
||||
#
|
||||
last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \
|
||||
dest_dir_name=$(basename "${destination_dir}")
|
||||
last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${dest_dir_name}" | head -n 1)" || \
|
||||
_exit_err "Failed to list contents of ${ddir}."
|
||||
|
||||
#
|
||||
|
@ -733,12 +771,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do
|
|||
_techo "Hard linking from ${last_dir}"
|
||||
fi
|
||||
|
||||
#
|
||||
# Include current time in name, not the time when we began to remove above
|
||||
#
|
||||
export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}"
|
||||
export destination_dir="${ddir}/${destination_name}"
|
||||
|
||||
#
|
||||
# Mark backup running and go back to original directory
|
||||
#
|
||||
|
@ -807,6 +839,9 @@ while [ "${source_no}" -lt "${no_sources}" ]; do
|
|||
_techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)"
|
||||
|
||||
unlock "${name}"
|
||||
|
||||
# wait for children (doing delete_from_file) if any still running
|
||||
wait
|
||||
) || exit
|
||||
done
|
||||
|
||||
|
|
1
doc/changes/next
Normal file
1
doc/changes/next
Normal file
|
@ -0,0 +1 @@
|
|||
* Improve performance, improve process of deletion of old backups
|
Loading…
Reference in a new issue