diff --git a/ccollect b/ccollect index bb44ed6..b8011ba 100755 --- a/ccollect +++ b/ccollect @@ -160,16 +160,29 @@ delete_from_file() file="$1"; shift suffix="" # It will be set, if deleting incomplete backups. [ $# -eq 1 ] && suffix="$1" && shift + # dirs for deletion will be moved to this trash dir inside destination dir + # - for fast mv operation + trash="$(mktemp -d ".trash.XXXXXX")" while read to_remove; do + mv "${to_remove}" "${trash}" || + _exit_err "Moving ${to_remove} to ${trash} failed." set -- "$@" "${to_remove}" if [ "${suffix}" ]; then to_remove_no_suffix="$(echo ${to_remove} | sed "s/$suffix\$//")" + mv "${to_remove_no_suffix}" "${trash}" || + _exit_err "Moving ${to_remove_no_suffix} to ${trash} failed." set -- "$@" "${to_remove_no_suffix}" fi done < "${file}" - _techo "Removing $@ ..." - [ "${VVERBOSE}" ] && echo rm "$@" - rm -rf "$@" || _exit_err "Removing $@ failed." + _techo "Removing $@ in ${trash}..." + empty_dir=".empty-dir" + mkdir "${empty_dir}" || _exit_err "Empty directory ${empty_dir} cannot be created." + [ "${VVERBOSE}" ] && echo "Starting: rsync -a --delete ${empty_dir} ${trash}" + # rsync needs ending slash for directory content + rsync -a --delete "${empty_dir}/" "${trash}/" || _exit_err "Removing $@ failed." + rmdir "${trash}" || _exit_err "Removing ${trash} directory failed" + rmdir "${empty_dir}" || _exit_err "Removing ${empty_dir} directory failed" + _techo "Removing $@ in ${trash} finished." } display_version() @@ -698,10 +711,16 @@ while [ "${source_no}" -lt "${no_sources}" ]; do if [ "$ret" -eq 0 ]; then _techo "Incomplete backups: $(echo $(cat "${TMP}"))" if [ -f "${c_delete_incomplete}" ]; then - delete_from_file "${TMP}" "${CMARKER}" + delete_from_file "${TMP}" "${CMARKER}" & fi fi + # + # Include current time in name, not the time when we began to remove above + # + export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" + export destination_dir="${ddir}/${destination_name}" + # # Check: maximum number of backups is reached? # @@ -710,19 +729,38 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Existing backups: ${count} Total keeping backups: ${c_interval}" if [ "${count}" -ge "${c_interval}" ]; then - remove="$((${count} - ${c_interval} + 1))" - _techo "Removing ${remove} backup(s)..." + # Use oldest directory as new backup destination directory. + # It need not to be deleted, rsync will sync its content. + oldest_bak=$(ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n 1 || \ + _exit_err "Listing oldest backup failed") + _techo "Using ${oldest_bak} for destination dir ${destination_dir}" + if mv "${oldest_bak}" "${destination_dir}"; then + # Touch dest dir so it is not sorted wrong in listings below. + touch "${destination_dir}" - ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ - _exit_err "Listing old backups failed" + # We have something to remove only if count > interval. + remove="$((${count} - ${c_interval}))" + else + _techo_err "Renaming oldest backup ${oldest_bak} to ${destination_dir} failed, removing it." + remove="$((${count} - ${c_interval} + 1))" + fi + if [ "${remove}" -gt 0 ]; then + _techo "Removing ${remove} backup(s)..." - delete_from_file "${TMP}" + ls -${TSORT}1r | grep "^${INTERVAL}\\." | head -n "${remove}" > "${TMP}" || \ + _exit_err "Listing old backups failed" + + delete_from_file "${TMP}" & + fi fi # # Check for backup directory to clone from: Always clone from the latest one! + # Exclude destination_dir from listing, it can be touched reused and renamed + # oldest existing destination directory. # - last_dir="$(ls -${TSORT}p1 | grep '/$' | head -n 1)" || \ + dest_dir_name=$(basename "${destination_dir}") + last_dir="$(ls -${TSORT}p1 | grep '/$' | grep -v "${dest_dir_name}" | head -n 1)" || \ _exit_err "Failed to list contents of ${ddir}." # @@ -733,12 +771,6 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Hard linking from ${last_dir}" fi - # - # Include current time in name, not the time when we began to remove above - # - export destination_name="${INTERVAL}.$(${CDATE}).$$-${source_no}" - export destination_dir="${ddir}/${destination_name}" - # # Mark backup running and go back to original directory # @@ -807,6 +839,9 @@ while [ "${source_no}" -lt "${no_sources}" ]; do _techo "Backup lasted: ${hours}:${minutes}:${seconds} (h:m:s)" unlock "${name}" + + # wait for children (doing delete_from_file) if any still running + wait ) || exit done diff --git a/doc/changes/next b/doc/changes/next new file mode 100644 index 0000000..a68cbbc --- /dev/null +++ b/doc/changes/next @@ -0,0 +1 @@ +* Improve performance, improve process of deletion of old backups