Commit dcc72aeb authored by Darko Poljak's avatar Darko Poljak

Merge branch 'performance/speed-up-removal' into 'master'

Improve performance

See merge request ungleich-public/ccollect!12
parents 10dcf076 de720ecf
......@@ -160,16 +160,29 @@ delete_from_file()
file="$1"; shift
suffix="" # It will be set, if deleting incomplete backups.
[ $# -eq 1 ] && suffix="$1" && shift
# dirs for deletion will be moved to this trash dir inside destination dir
# - for fast mv operation
trash="$(mktemp -d ".trash.XXXXXX")"
while read to_remove; do
mv "${to_remove}" "${trash}" ||
_exit_err "Moving ${to_remove} to ${trash} failed."
set -- "$@" "${to_remove}"
if [ "${suffix}" ]; then
to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")"
mv "${to_remove_no_suffix}" "${trash}" ||
_exit_err "Moving ${to_remove_no_suffix} to ${trash} failed."
set -- "$@" "${to_remove_no_suffix}"
fi
done < "${file}"
_techo "Removing $@ ..."
[ "${VVERBOSE}" ] && echo rm "$@"
rm -rf "$@" || _exit_err "Removing $@ failed."
_techo "Removing $@ in ${trash}..."
empty_dir=".empty-dir"
mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created."
[ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}"
# rsync needs ending slash for directory content
rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed."
rmdir "${trash}" || _exit_err "Removing ${trash} directory failed"
rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed"
_techo "Removing $@ in ${trash} finished."
}
display_version()
......@@ -698,10 +711,16 @@ while [ "${source_no}" -lt "${no_sources}" ]; do
if [ "$ret" -eq 0 ]; then
_techo "Incomplete backups: $(echo $(cat "${TMP}"))"
if [ -f "${c_delete_incomplete}" ]; then
delete_from_file "${TMP}" "${CMARKER}"
delete_from_file "${TMP}" "${CMARKER}" &
fi
fi
#
# Include current time in name, not the time when we began to remove above
#
export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}"
export destination_dir="${ddir}/${destination_name}"
#
# Check: maximum number of backups is reached?
#
......@@ -710,19 +729,38 @@ while [ "${source_no}" -lt "${no_sources}" ]; do
_techo "Existing backups: ${count} Total keeping backups: ${c_interval}"
if [ "${count}" -ge "${c_interval}" ]; then
remove="$((${count} - ${c_interval} + 1))"
_techo "Removing ${remove} backup(s)..."
# Use oldest directory as new backup destination directory.
# It need not to be deleted, rsync will sync its content.
oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \
_exit_err "Listing oldest backup failed")
_techo "Using ${oldest_bak} for destination dir ${destination_dir}"
if mv "${oldest_bak}" "${destination_dir}"; then
# Touch dest dir so it is not sorted wrong in listings below.
touch "${destination_dir}"
# We have something to remove only if count > interval.
remove="$((${count} - ${c_interval}))"
else
_techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it."
remove="$((${count} - ${c_interval} + 1))"
fi
if [ "${remove}" -gt 0 ]; then
_techo "Removing ${remove} backup(s)..."
ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \
_exit_err "Listing old backups failed"
ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \
_exit_err "Listing old backups failed"
delete_from_file "${TMP}"
delete_from_file "${TMP}" &
fi
fi
#
# Check for backup directory to clone from: Always clone from the latest one!
# Exclude destination_dir from listing, it can be touched reused and renamed
# oldest existing destination directory.
#
last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \
dest_dir_name=$(basename "${destination_dir}")
last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${dest_dir_name}" | head -n 1)" || \
_exit_err "Failed to list contents of ${ddir}."
#
......@@ -733,12 +771,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do
_techo "Hard linking from ${last_dir}"
fi
#
# Include current time in name, not the time when we began to remove above
#
export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}"
export destination_dir="${ddir}/${destination_name}"
#
# Mark backup running and go back to original directory
#
......@@ -807,6 +839,9 @@ while [ "${source_no}" -lt "${no_sources}" ]; do
_techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)"
unlock "${name}"
# wait for children (doing delete_from_file) if any still running
wait
) || exit
done
......
* Improve performance, improve process of deletion of old backups
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment