diff --git a/contrib/patrick/ccollect_mgr.sh b/contrib/patrick/ccollect_mgr.sh index 5678779..fe69286 100644 --- a/contrib/patrick/ccollect_mgr.sh +++ b/contrib/patrick/ccollect_mgr.sh @@ -1,7 +1,7 @@ #!/bin/sh # # ---------------------------------------------------------------------------- -# Last update: 2009-10-29 +# Last update: 2009-12-11 # By : pdrolet (ccollect_mgr@drolet.name) # ---------------------------------------------------------------------------- # Job manager to the ccollect utilities @@ -9,7 +9,8 @@ # # Provides the following features # 1) Determine the interval (daily/weekly/monthly) -# 2) Perform the backups using ccollect +# 2) Check the estimated file transfer size +# 3) Perform the backups using ccollect # 4) Copy the ccollect log to the first backup of the set # 5) Build a periodic report and include the real amount of disk used # 6) Send an email if there has been errors or warnings @@ -48,69 +49,74 @@ # Send warning if the worst case data transfer will be larger than (in MB)... warning_transfer_size=1024 +abort_transfer_size=5120 # Define paths and default file names -ADD_TO_PATH=/opt/bin:/opt/sbin:/usr/local/bin:/usr/local/sbin -CCOLLECT=ccollect.sh -CCOLLECT_CONF=/usr/local/etc/ccollect +ADD_TO_PATH="/opt/bin:/opt/sbin:/usr/local/bin:/usr/local/sbin" +CCOLLECT="ccollect.sh" +CCOLLECT_CONF="/usr/local/etc/ccollect" -per_report=${CCOLLECT_CONF}/periodic_report.log -tmp_report=/tmp/ccollect.$$ -tmp_email=/tmp/email.$$ +PS="/opt/bin/ps" +FIND="/opt/bin/find" + +TEMP_LOG="${CCOLLECT_CONF}"/log.$$ +per_report="${CCOLLECT_CONF}/periodic_report.log" +tmp_report="/tmp/ccollect.$$" +tmp_mgr="/tmp/ccollect_mgr.$$" +tmp_email="/tmp/email.$$" + +backups_not_found="" # Sub routines... -find_interval() +send_email() { - # ---------------------------------------------------- - # Find interval for ccollect backup. - # optional parameters: - # - Day of the week to do weekly backups - # - Do monthly instead of weekly on the Nth week - # ---------------------------------------------------- - - weekly_backup=$1 - monthly_backup=$2 + # Send a simple email using mini-sendmail. - weekday=`date "+%w"` - if [ ${weekday} -eq ${weekly_backup} ]; then - dom=`date "+%d"` - weeknum=$(( ( ${dom} / 7 ) + 1 )) - if [ ${weeknum} -eq ${monthly_backup} ]; then - interval=monthly - else - interval=weekly - fi - else - interval=daily + msg_body_file="$1" + shift + + # ------------------------------ + # Quit if we can't send an email + # ------------------------------ + if [ "${to}" == "" ] || [ "${mail_server}" == "" ]; then + echo "Missing mail server or destination email. No email sent with subject: $@" + exit 1 fi + + echo from: "${from}" > "${tmp_email}" + echo subject: "$@" >> "${tmp_email}" + echo to: "${to}" >> "${tmp_email}" + echo cc: >> "${tmp_email}" + echo bcc: >> "${tmp_email}" + echo "" >> "${tmp_email}" + echo "" >> "${tmp_email}" + cat "${msg_body_file}" >> "${tmp_email}" + echo "" >> "${tmp_email}" + + echo "" + echo Sending email to ${to} to report the following: + echo ----------------------------------------------- + cat "${tmp_email}" + cat "${tmp_email}" | mini_sendmail -f"${from}" -s"${mail_server}" "${to}" + rm "${tmp_email}" } -move_log() +remove_source() { - for backup in $@ ; do - ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" - if [ "${ret}" -ne 0 ]; then - echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping." - backup_dir="" - else - backup_dir=`cat ${TEMP_LOG} | grep "\[${backup}\] .*: Creating ${ddir}" | awk '{ print $4 }'` - fi - if [ "${backup_dir}" != "" ]; then - new_log=${backup_dir}/ccollect.log - mv ${TEMP_LOG} ${new_log} - echo New Log Location: ${new_log} - return 0 - fi - done - echo "WARNING: none of the backup set have been created" - new_log=${TEMP_LOG} + remove_no=$1 + eval echo Removing backup \"\$source_$1\" + + no_sources="$(( ${no_sources} - 1 ))" + while [ "${remove_no}" -lt "${no_sources}" ]; do + eval source_${remove_no}=\"\$source_$(( ${remove_no} + 1))\" + eval ddir_${remove_no}=\"\$ddir_$(( ${remove_no} + 1))\" + remove_no=$(( ${remove_no} + 1 )) + done } compute_rdu() { - # WARNING: Don't pass a directory with a space as parameter (I'm too new at scripting!) - kdivider=1 find_options="" @@ -123,12 +129,17 @@ compute_rdu() kdivider=1048576 ;; *) - find_options="${find_options} $1" + break ;; esac shift done + if [ "$#" == 0 ]; then + rdu=0 + return 1 + fi + # ------------------------------------------------------------------------------------------------------ # Compute the real disk usage (eg: hard links do files outside the backup set don't count) # ------------------------------------------------------------------------------------------------------ @@ -141,103 +152,88 @@ compute_rdu() # 5) Present the result with additional dividers based on command line parameters # - rdu=$(( ( `/opt/bin/find ${find_options} -printf '%n %i %y %k \n' \ + rdu=$(( ( `"${FIND}" "$@" -printf '%n %i %y %k \n' \ | sort -n \ | uniq -c \ | awk '{ if (( $1 == $2 ) || ($4 == "d")) { sum += $5; } } END { printf "%u\n",(sum); }'` \ + ${kdivider} - 1 ) / ${kdivider} )) - echo RDU for ${find_options} is ${rdu} -} - -compute_total_rdu() -{ - real_usage=0 - - # ------------------------------------------ - # Get the real disk usage for the backup set - # ------------------------------------------ - for backup in $@ ; do - ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" - echo ${backup} - Adding ${ddir} to backup list - backup_dir_list="${backup_dir_list} ${ddir}" - if [ "${ret}" -ne 0 ]; then - echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping." - else - backup_dir=`find ${ddir}/${interval}.* -maxdepth 0 -type d -print | sort -r | head -n 1` - compute_rdu -m ${backup_dir} - real_usage=$(( ${real_usage} + ${rdu} )) - fi - done - echo Backup list - ${backup_dir_list} -} - -send_email() -{ - # Send a simple email using mini-sendmail. - - msg_body_file=$1 - shift - - # ------------------------------ - # Quit if we can't send an email - # ------------------------------ - if [ "${to}" == "" ] || [ "${mail_server}" == "" ]; then - echo "Missing mail server or destination email. No email sent with subject: $@" - exit 1 - fi - - echo from: ${from} > ${tmp_email} - echo subject: $@ >> ${tmp_email} - echo to: ${to} >> ${tmp_email} - echo cc: >> ${tmp_email} - echo bcc: >> ${tmp_email} - echo "" >> ${tmp_email} - echo "" >> ${tmp_email} - cat ${msg_body_file} >> ${tmp_email} - echo "" >> ${tmp_email} - - echo Sending email to ${to} to report the following error: - echo ----------------------------------------------------- - cat ${tmp_email} - cat ${tmp_email} | mini_sendmail -f${from} -s${mail_server} ${to} - rm ${tmp_email} } check_running_backups() { # Check if a backup is already ongoing. If so, skip and send email # Don't use the ccollect marker as this is no indication if it is still running - - for backup in ${ccollect_backups} ; do + + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup=\"\$source_${source_no}\" + PID=$$ - /opt/bin/ps -e -o pid,ppid,args 2> /dev/null | grep -v -e grep -e "${PID}.*ccollect.*${backup}" | grep "ccollect.*${backup}" > /tmp/ccollect_mgr.$$ 2> /dev/null - running_proc=`cat /tmp/ccollect_mgr.$$ | wc -l` + "${PS}" -e -o pid,ppid,args 2> /dev/null \ + | grep -v -e grep -e "${PID}.*ccollect.*${backup}" \ + | grep "ccollect.*${backup}" > "${tmp_mgr}" 2> /dev/null + running_proc=`cat "${tmp_mgr}" | wc -l` + if [ ${running_proc} -gt 0 ]; then + # Remove backup from list running_backups="${running_backups}${backup} " - echo "Process:" - cat /tmp/ccollect.$$ + + echo "Process already running:" + cat "${tmp_mgr}" + + remove_source ${source_no} else - backups_to_do="${backups_to_do}${backup} " + source_no=$(( ${source_no} + 1 )) fi - rm /tmp/ccollect_mgr.$$ + rm "${tmp_mgr}" done - ccollect_backups=${backups_to_do} if [ "${running_backups}" != "" ]; then - echo "skipping ccollect backups already running: ${running_backups}" | tee ${tmp_report} - send_email ${tmp_report} "WARNING - skipping ccollect backups already running: ${running_backups}" - rm ${tmp_report} + echo "skipping ccollect backups already running: ${running_backups}" | tee "${tmp_report}" + send_email "${tmp_report}" "WARNING - skipping ccollect backups already running: ${running_backups}" + rm "${tmp_report}" fi } +find_interval() +{ + # ---------------------------------------------------- + # Find interval for ccollect backup. + # optional parameters: + # - Day of the week to do weekly backups + # - Do monthly instead of weekly on the Nth week + # ---------------------------------------------------- + + weekly_backup="$1" + monthly_backup="$2" + + weekday=`date "+%w"` + if [ ${weekday} -eq ${weekly_backup} ]; then + dom=`date "+%e"` + weeknum=$(( ( ${dom} / 7 ) + 1 )) + if [ "${weeknum}" -eq "${monthly_backup}" ]; then + interval=monthly + else + interval=weekly + fi + else + interval=daily + fi +} + precheck_transfer_size() { # Check the estimated (worst case) transfer size and send email if larger than certain size + # Abort backup if total transfer is larger than maximum limit (ex: an error somewhere + # requires to do full backup and not incremental, which could blow the quota with ISP) # # Be nice and add error checking one day... - for backup in ${ccollect_backups} ; do - ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup=\"\$source_${source_no}\" + eval ddir=\"\$ddir_${source_no}\" + last_dir="$(ls -tcp1 "${ddir}" | grep '/$' | head -n 1)" sdir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/source)"; ret="$?" if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/exclude ]; then @@ -245,66 +241,152 @@ precheck_transfer_size() else exclude="" fi + rsync_options="" if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/rsync_options ]; then while read line; do rsync_options="${rsync_options} ${line}" done < ${CCOLLECT_CONF}/sources/${backup}/rsync_options fi - rsync -n -a --delete --stats ${rsync_options} ${exclude} ${sdir} ${ddir}/${last_dir} > ${tmp_report} - tx_rx=`cat ${tmp_report} | grep "Total transferred file size" | \ + + rsync -n -a --delete --stats ${rsync_options} "${exclude}" "${sdir}" "${ddir}/${last_dir}" > "${tmp_report}" + + tx_rx=`cat "${tmp_report}" | grep "Total transferred file size" | \ awk '{ { tx += $5 } } END { printf "%u",(((tx)+1024*1024-1)/1024/1024); }'` total_xfer=$(( ${total_xfer} + ${tx_rx} )) + + source_no=$(( ${source_no} + 1 )) done - echo Transfer estimation for ${ccollect_backups}: ${total_xfer} MB - if [ ${total_xfer} -gt ${warning_transfer_size} ]; then + echo "Transfer estimation for${ccollect_backups}: ${total_xfer} MB" + + if [ ${total_xfer} -gt ${abort_transfer_size} ]; then + # -------------------------------------------------- + # Send an error if transfer is larger than max limit + # -------------------------------------------------- + # Useful to detect potential issues when there is transfer quota (ex: with ISP) + + echo "Data transfer larger than ${abort_transfer_size} MB is expected for${ccollect_backups}" >> "${tmp_report}" + echo "** BACKUP ABORTED **" >> "${tmp_report}" + + send_email "${tmp_report}" "ERROR: aborted ccollect for${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB" + rm "${tmp_report}" + exit 1 + elif [ ${total_xfer} -gt ${warning_transfer_size} ]; then # -------------------------------------------------- # Send a warning if transfer is expected to be large # -------------------------------------------------- # Useful to detect potential issues when there is transfer quota (ex: with ISP) - echo Data transfer larger than ${warning_transfer_size} MB is expected for ${ccollect_backups} > ${tmp_report} + echo "Data transfer larger than ${warning_transfer_size} MB is expected for${ccollect_backups}" > "${tmp_report}" - send_email ${tmp_report} "WARNING ccollect for ${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB" - rm ${tmp_report} + send_email "${tmp_report}" "WARNING ccollect for${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB" + rm "${tmp_report}" + fi +} + +build_backup_dir_list() +{ + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup=\"\$source_${source_no}\" + eval ddir=\"\$ddir_${source_no}\" + + backup_dir="`cat "${TEMP_LOG}" \ + | grep "\[${backup}\] .*: Creating.* ${ddir}" \ + | head -n 1 \ + | sed 's/[^\/]*\//\//; s/ \.\.\.//'`" + + if [ ! -d "${backup_dir}" ]; then + backups_not_found="${backups_not_found}\"${backup}\" " + echo -n "Backup directory for \"${backup}\" not found. " + remove_source "${source_no}" + else + eval export backup_dir_list_${source_no}="${backup_dir}" +# eval echo Backup Dir List: \"\$backup_dir_list_${source_no}\" + source_no=$(( ${source_no} + 1 )) + fi + done +} + +move_log() +{ + if [ "${no_sources}" -gt 0 ]; then + eval log_file=\"\$backup_dir_list_1\"/ccollect.log + mv "${TEMP_LOG}" "${log_file}" + echo New Log Location: "${log_file}" + else + echo "WARNING: none of the backup set have been created" + log_file="${TEMP_LOG}" fi } send_report() { - log=$1 - # Analyze log for periodic report and for error status report - cat ${log} | ccollect_analyse_logs.sh iwe > ${tmp_report} + cat "${log_file}" | ccollect_analyse_logs.sh iwe > "${tmp_report}" # ------------------------- # Build the periodic report # ------------------------- - # Compute the total number of MB sent and received for all the backup sets - tx_rx=`cat ${tmp_report} | \ + tx_rx=`cat "${tmp_report}" | \ grep 'sent [[:digit:]]* bytes received [0-9]* bytes' | \ awk '{ { tx += $3 } { rx += $6} } END \ - { printf "%u",(((tx+rx)+1024*1024-1)/1024/1024); }'` + { printf "%u",(((tx+rx)+(1024*1024)-1)/1024/1024); }'` current_date=`date +'20%y/%m/%d %Hh%M -- '` - + + # ------------------------------------------ + # Get the real disk usage for the backup set + # ------------------------------------------ + total_rdu=0 + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup_dir=\"\$backup_dir_list_${source_no}\" + compute_rdu -m "${backup_dir}" + total_rdu=$(( ${total_rdu} + ${rdu} )) + source_no=$(( ${source_no} + 1 )) + done + # --------------------------------------------------------- # Get the disk usage for all backups of each backup sets... - # ** be patient ** + # ** BE PATIENT!!! ** # --------------------------------------------------------- - compute_rdu -g ${backup_dir_list} + historical_rdu=0 + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup_dir=\"\$ddir_${source_no}\" + compute_rdu -m "${backup_dir}" + historical_rdu=$(( ${historical_rdu} + ${rdu} )) + source_no=$(( ${source_no} + 1 )) + done + + historical_rdu=$(( (${historical_rdu}+1023) / 1024 )) + + if [ "${no_sources}" -gt 0 ]; then + ccollect_backups="" + else + ccollect_backups="(none performed) " + fi + + source_no=0 + while [ "${source_no}" -lt "${no_sources}" ]; do + eval backup=\"\$source_${source_no}\" + ccollect_backups="${ccollect_backups}\"${backup}\" " + source_no=$(( ${source_no} + 1 )) + done echo ${current_date} Tx+Rx: ${tx_rx} MB -- \ - Disk Usage: ${real_usage} MB -- \ - Backup set \(${interval}\): ${ccollect_backups} -- \ - Historical backups usage: ${rdu} GB >> ${per_report} - + Disk Usage: ${total_rdu} MB -- \ + Backup set \(${interval}\):${ccollect_backups} -- \ + Historical backups usage: ${historical_rdu} GB >> "${per_report}" + echo "Total Data Transfer: ${tx_rx} MB -- Total Disk Usage: ${total_rdu} MB -- Total Historical backups usage: ${historical_rdu} GB" + # ---------------------------------------- # Send a status email if there is an error # ---------------------------------------- - ccollect_we=`cat ${log} | ccollect_analyse_logs.sh we | wc -l` + ccollect_we=`cat "${log_file}" | ccollect_analyse_logs.sh we | wc -l` if [ ${ccollect_we} -ge 1 ]; then - send_email ${tmp_report} "ERROR ccollect for ${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" + send_email "${tmp_report}" "ERROR ccollect for${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" fi # -------------------- @@ -313,27 +395,27 @@ send_report() if [ ${report_interval} == ${interval} ] || [ ${interval} == "monthly" ]; then # Make reporting atomic to handle concurrent ccollect_mgr instances - mv ${per_report} ${per_report}.$$ - cat ${per_report}.$$ >> ${per_report}.history + mv "${per_report}" "${per_report}".$$ + cat "${per_report}".$$ >> "${per_report}".history # Calculate total amount of bytes sent and received - tx_rx=`cat ${per_report}.$$ | \ + tx_rx=`cat "${per_report}".$$ | \ awk '{ { transfer += $5 } } END \ { printf "%u",(transfer); }'` # Send email - send_email ${per_report}.$$ "${report_interval} ccollect status for ${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" - rm ${per_report}.$$ + send_email "${per_report}.$$" "${report_interval} ccollect status for${ccollect_backups} -- Tx+Rx: ${tx_rx} MB" + rm "${per_report}.$$" fi - rm ${tmp_report} + rm "${tmp_report}" } # ------------------------------------------------ # Add to PATH in case we're launching from crontab # ------------------------------------------------ -PATH=${ADD_TO_PATH}:${PATH} +PATH="${ADD_TO_PATH}:${PATH}" # -------------- # Default Values @@ -349,59 +431,74 @@ weekly_backup=1 # Set the monthly backup interval. Default is 4th Monday of every month monthly_backup=4 -show_help=0 - # --------------------------------- # Parse command line # --------------------------------- +show_help=0 +export no_sources=0 + while [ "$#" -ge 1 ]; do case "$1" in -help) show_help=1 ;; -from) - from=$2 + from="$2" shift ;; -to) - to=$2 + to="$2" shift ;; -server|mail_server) - mail_server=$2 + mail_server="$2" shift ;; -weekly) - weekly_backup=$2 + weekly_backup="$2" shift ;; -monthly) - monthly_backup=$2 + monthly_backup="$2" shift ;; -warning_size) - warning_transfer_size=$2 + warning_transfer_size="$2" + shift + ;; + -abort_size) + abort_transfer_size="$2" shift ;; -report) - report_interval=$2 + report_interval="$2" shift ;; -*) - ccollect_options="${ccollect_options} $1" + ccollect_options="${ccollect_options}$1 " ;; daily|weekly|monthly) ;; *) - ccollect_backups="${ccollect_backups}$1 " + eval backup=\"\$1\" + ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?" + if [ "${ret}" -ne 0 ]; then + echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping." + else + ccollect_backups="${ccollect_backups} \"$1\"" + eval export source_${no_sources}=\"\$1\" + eval export ddir_${no_sources}="${ddir}" +# eval echo Adding source \"\$source_${no_sources}\" -- \"\$ddir_${no_sources}\" + no_sources="$(( ${no_sources} + 1 ))" + fi ;; esac shift done -if [ "${ccollect_backups}" == "" ] || [ ${show_help} -eq 1 ]; then - echo +if [ "${no_sources}" -lt 1 ] || [ ${show_help} -eq 1 ]; then + echo "" echo "$0: Syntax" echo " -help This help" echo " -from From email address (ex.: -from nas@home.com)" @@ -414,58 +511,32 @@ if [ "${ccollect_backups}" == "" ] || [ ${show_help} -eq 1 ]; then echo " -report Frequency of report email (daily, weekly or monthly)" echo " Default is ${report_interval}" echo " -warning_size Send a warning email if the transfer size exceed this" - echo " Default is ${warning_transfer_size}" + echo " Default is ${warning_transfer_size} MB" + echo " -abort_size Abort and send an error email if the transfer size exceed this" + echo " Default is ${abort_transfer_size} MB" echo "" echo " other parameters are transfered to ccollect" - echo + echo "" exit 0 fi -# ------------------------------------------------------------------ -# Check if ccollect_mgr is already running for the given backup sets -# ------------------------------------------------------------------ - +#echo Backup sets:"${ccollect_backups}" check_running_backups -if [ "${ccollect_backups}" == "" ]; then +if [ "${no_sources}" -lt 1 ]; then echo "No backup sets are reachable" exit 1 fi -# ---------------------------------------------------------- -# Set the interval type -# -# Here, weeklys are Mondays, and Monthlys are the 4th Monday -# ---------------------------------------------------------- - find_interval ${weekly_backup} ${monthly_backup} echo Interval: ${interval} -# -------------- -# Do the backups -# -------------- -TEMP_LOG=${CCOLLECT_CONF}/log.$$ -echo Backup sets: ${ccollect_backups} - -# Check the transfer size (to issue email warning) precheck_transfer_size -${CCOLLECT} ${ccollect_options} ${interval} ${ccollect_backups} | tee ${TEMP_LOG} +"${CCOLLECT}" ${ccollect_options} ${interval} ${ccollect_backups} | tee "${TEMP_LOG}" -# --------------------------------------- -# Move log to the last backup of the set -# --------------------------------------- +build_backup_dir_list +move_log -move_log ${ccollect_backups} +send_report -# ----------------------------------------- -# Compute the physical amount of disk usage -# ----------------------------------------- - -compute_total_rdu ${ccollect_backups} - -# ----------------- -# Send status email -# ----------------- - -send_report ${new_log}