import new ccollect_mgr version from patrick
Signed-off-by: Nico Schottelius <nico@ikn.schottelius.org>
This commit is contained in:
parent
86992b9787
commit
e47fb78603
1 changed files with 269 additions and 198 deletions
|
@ -1,7 +1,7 @@
|
||||||
#!/bin/sh
|
#!/bin/sh
|
||||||
#
|
#
|
||||||
# ----------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------
|
||||||
# Last update: 2009-10-29
|
# Last update: 2009-12-11
|
||||||
# By : pdrolet (ccollect_mgr@drolet.name)
|
# By : pdrolet (ccollect_mgr@drolet.name)
|
||||||
# ----------------------------------------------------------------------------
|
# ----------------------------------------------------------------------------
|
||||||
# Job manager to the ccollect utilities
|
# Job manager to the ccollect utilities
|
||||||
|
@ -9,7 +9,8 @@
|
||||||
#
|
#
|
||||||
# Provides the following features
|
# Provides the following features
|
||||||
# 1) Determine the interval (daily/weekly/monthly)
|
# 1) Determine the interval (daily/weekly/monthly)
|
||||||
# 2) Perform the backups using ccollect
|
# 2) Check the estimated file transfer size
|
||||||
|
# 3) Perform the backups using ccollect
|
||||||
# 4) Copy the ccollect log to the first backup of the set
|
# 4) Copy the ccollect log to the first backup of the set
|
||||||
# 5) Build a periodic report and include the real amount of disk used
|
# 5) Build a periodic report and include the real amount of disk used
|
||||||
# 6) Send an email if there has been errors or warnings
|
# 6) Send an email if there has been errors or warnings
|
||||||
|
@ -48,69 +49,74 @@
|
||||||
|
|
||||||
# Send warning if the worst case data transfer will be larger than (in MB)...
|
# Send warning if the worst case data transfer will be larger than (in MB)...
|
||||||
warning_transfer_size=1024
|
warning_transfer_size=1024
|
||||||
|
abort_transfer_size=5120
|
||||||
|
|
||||||
# Define paths and default file names
|
# Define paths and default file names
|
||||||
ADD_TO_PATH=/opt/bin:/opt/sbin:/usr/local/bin:/usr/local/sbin
|
ADD_TO_PATH="/opt/bin:/opt/sbin:/usr/local/bin:/usr/local/sbin"
|
||||||
CCOLLECT=ccollect.sh
|
CCOLLECT="ccollect.sh"
|
||||||
CCOLLECT_CONF=/usr/local/etc/ccollect
|
CCOLLECT_CONF="/usr/local/etc/ccollect"
|
||||||
|
|
||||||
per_report=${CCOLLECT_CONF}/periodic_report.log
|
PS="/opt/bin/ps"
|
||||||
tmp_report=/tmp/ccollect.$$
|
FIND="/opt/bin/find"
|
||||||
tmp_email=/tmp/email.$$
|
|
||||||
|
TEMP_LOG="${CCOLLECT_CONF}"/log.$$
|
||||||
|
per_report="${CCOLLECT_CONF}/periodic_report.log"
|
||||||
|
tmp_report="/tmp/ccollect.$$"
|
||||||
|
tmp_mgr="/tmp/ccollect_mgr.$$"
|
||||||
|
tmp_email="/tmp/email.$$"
|
||||||
|
|
||||||
|
backups_not_found=""
|
||||||
|
|
||||||
# Sub routines...
|
# Sub routines...
|
||||||
|
|
||||||
find_interval()
|
send_email()
|
||||||
{
|
{
|
||||||
# ----------------------------------------------------
|
# Send a simple email using mini-sendmail.
|
||||||
# Find interval for ccollect backup.
|
|
||||||
# optional parameters:
|
|
||||||
# - Day of the week to do weekly backups
|
|
||||||
# - Do monthly instead of weekly on the Nth week
|
|
||||||
# ----------------------------------------------------
|
|
||||||
|
|
||||||
weekly_backup=$1
|
|
||||||
monthly_backup=$2
|
|
||||||
|
|
||||||
weekday=`date "+%w"`
|
msg_body_file="$1"
|
||||||
if [ ${weekday} -eq ${weekly_backup} ]; then
|
shift
|
||||||
dom=`date "+%d"`
|
|
||||||
weeknum=$(( ( ${dom} / 7 ) + 1 ))
|
# ------------------------------
|
||||||
if [ ${weeknum} -eq ${monthly_backup} ]; then
|
# Quit if we can't send an email
|
||||||
interval=monthly
|
# ------------------------------
|
||||||
else
|
if [ "${to}" == "" ] || [ "${mail_server}" == "" ]; then
|
||||||
interval=weekly
|
echo "Missing mail server or destination email. No email sent with subject: $@"
|
||||||
fi
|
exit 1
|
||||||
else
|
|
||||||
interval=daily
|
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
echo from: "${from}" > "${tmp_email}"
|
||||||
|
echo subject: "$@" >> "${tmp_email}"
|
||||||
|
echo to: "${to}" >> "${tmp_email}"
|
||||||
|
echo cc: >> "${tmp_email}"
|
||||||
|
echo bcc: >> "${tmp_email}"
|
||||||
|
echo "" >> "${tmp_email}"
|
||||||
|
echo "" >> "${tmp_email}"
|
||||||
|
cat "${msg_body_file}" >> "${tmp_email}"
|
||||||
|
echo "" >> "${tmp_email}"
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo Sending email to ${to} to report the following:
|
||||||
|
echo -----------------------------------------------
|
||||||
|
cat "${tmp_email}"
|
||||||
|
cat "${tmp_email}" | mini_sendmail -f"${from}" -s"${mail_server}" "${to}"
|
||||||
|
rm "${tmp_email}"
|
||||||
}
|
}
|
||||||
|
|
||||||
move_log()
|
remove_source()
|
||||||
{
|
{
|
||||||
for backup in $@ ; do
|
remove_no=$1
|
||||||
ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?"
|
eval echo Removing backup \"\$source_$1\"
|
||||||
if [ "${ret}" -ne 0 ]; then
|
|
||||||
echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping."
|
no_sources="$(( ${no_sources} - 1 ))"
|
||||||
backup_dir=""
|
while [ "${remove_no}" -lt "${no_sources}" ]; do
|
||||||
else
|
eval source_${remove_no}=\"\$source_$(( ${remove_no} + 1))\"
|
||||||
backup_dir=`cat ${TEMP_LOG} | grep "\[${backup}\] .*: Creating ${ddir}" | awk '{ print $4 }'`
|
eval ddir_${remove_no}=\"\$ddir_$(( ${remove_no} + 1))\"
|
||||||
fi
|
remove_no=$(( ${remove_no} + 1 ))
|
||||||
if [ "${backup_dir}" != "" ]; then
|
done
|
||||||
new_log=${backup_dir}/ccollect.log
|
|
||||||
mv ${TEMP_LOG} ${new_log}
|
|
||||||
echo New Log Location: ${new_log}
|
|
||||||
return 0
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo "WARNING: none of the backup set have been created"
|
|
||||||
new_log=${TEMP_LOG}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
compute_rdu()
|
compute_rdu()
|
||||||
{
|
{
|
||||||
# WARNING: Don't pass a directory with a space as parameter (I'm too new at scripting!)
|
|
||||||
|
|
||||||
kdivider=1
|
kdivider=1
|
||||||
find_options=""
|
find_options=""
|
||||||
|
|
||||||
|
@ -123,12 +129,17 @@ compute_rdu()
|
||||||
kdivider=1048576
|
kdivider=1048576
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
find_options="${find_options} $1"
|
break
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
shift
|
shift
|
||||||
done
|
done
|
||||||
|
|
||||||
|
if [ "$#" == 0 ]; then
|
||||||
|
rdu=0
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
# ------------------------------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------------------------------
|
||||||
# Compute the real disk usage (eg: hard links do files outside the backup set don't count)
|
# Compute the real disk usage (eg: hard links do files outside the backup set don't count)
|
||||||
# ------------------------------------------------------------------------------------------------------
|
# ------------------------------------------------------------------------------------------------------
|
||||||
|
@ -141,103 +152,88 @@ compute_rdu()
|
||||||
# 5) Present the result with additional dividers based on command line parameters
|
# 5) Present the result with additional dividers based on command line parameters
|
||||||
#
|
#
|
||||||
|
|
||||||
rdu=$(( ( `/opt/bin/find ${find_options} -printf '%n %i %y %k \n' \
|
rdu=$(( ( `"${FIND}" "$@" -printf '%n %i %y %k \n' \
|
||||||
| sort -n \
|
| sort -n \
|
||||||
| uniq -c \
|
| uniq -c \
|
||||||
| awk '{ if (( $1 == $2 ) || ($4 == "d")) { sum += $5; } } END { printf "%u\n",(sum); }'` \
|
| awk '{ if (( $1 == $2 ) || ($4 == "d")) { sum += $5; } } END { printf "%u\n",(sum); }'` \
|
||||||
+ ${kdivider} - 1 ) / ${kdivider} ))
|
+ ${kdivider} - 1 ) / ${kdivider} ))
|
||||||
echo RDU for ${find_options} is ${rdu}
|
|
||||||
}
|
|
||||||
|
|
||||||
compute_total_rdu()
|
|
||||||
{
|
|
||||||
real_usage=0
|
|
||||||
|
|
||||||
# ------------------------------------------
|
|
||||||
# Get the real disk usage for the backup set
|
|
||||||
# ------------------------------------------
|
|
||||||
for backup in $@ ; do
|
|
||||||
ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?"
|
|
||||||
echo ${backup} - Adding ${ddir} to backup list
|
|
||||||
backup_dir_list="${backup_dir_list} ${ddir}"
|
|
||||||
if [ "${ret}" -ne 0 ]; then
|
|
||||||
echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping."
|
|
||||||
else
|
|
||||||
backup_dir=`find ${ddir}/${interval}.* -maxdepth 0 -type d -print | sort -r | head -n 1`
|
|
||||||
compute_rdu -m ${backup_dir}
|
|
||||||
real_usage=$(( ${real_usage} + ${rdu} ))
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
echo Backup list - ${backup_dir_list}
|
|
||||||
}
|
|
||||||
|
|
||||||
send_email()
|
|
||||||
{
|
|
||||||
# Send a simple email using mini-sendmail.
|
|
||||||
|
|
||||||
msg_body_file=$1
|
|
||||||
shift
|
|
||||||
|
|
||||||
# ------------------------------
|
|
||||||
# Quit if we can't send an email
|
|
||||||
# ------------------------------
|
|
||||||
if [ "${to}" == "" ] || [ "${mail_server}" == "" ]; then
|
|
||||||
echo "Missing mail server or destination email. No email sent with subject: $@"
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo from: ${from} > ${tmp_email}
|
|
||||||
echo subject: $@ >> ${tmp_email}
|
|
||||||
echo to: ${to} >> ${tmp_email}
|
|
||||||
echo cc: >> ${tmp_email}
|
|
||||||
echo bcc: >> ${tmp_email}
|
|
||||||
echo "" >> ${tmp_email}
|
|
||||||
echo "" >> ${tmp_email}
|
|
||||||
cat ${msg_body_file} >> ${tmp_email}
|
|
||||||
echo "" >> ${tmp_email}
|
|
||||||
|
|
||||||
echo Sending email to ${to} to report the following error:
|
|
||||||
echo -----------------------------------------------------
|
|
||||||
cat ${tmp_email}
|
|
||||||
cat ${tmp_email} | mini_sendmail -f${from} -s${mail_server} ${to}
|
|
||||||
rm ${tmp_email}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
check_running_backups()
|
check_running_backups()
|
||||||
{
|
{
|
||||||
# Check if a backup is already ongoing. If so, skip and send email
|
# Check if a backup is already ongoing. If so, skip and send email
|
||||||
# Don't use the ccollect marker as this is no indication if it is still running
|
# Don't use the ccollect marker as this is no indication if it is still running
|
||||||
|
|
||||||
for backup in ${ccollect_backups} ; do
|
source_no=0
|
||||||
|
while [ "${source_no}" -lt "${no_sources}" ]; do
|
||||||
|
eval backup=\"\$source_${source_no}\"
|
||||||
|
|
||||||
PID=$$
|
PID=$$
|
||||||
/opt/bin/ps -e -o pid,ppid,args 2> /dev/null | grep -v -e grep -e "${PID}.*ccollect.*${backup}" | grep "ccollect.*${backup}" > /tmp/ccollect_mgr.$$ 2> /dev/null
|
"${PS}" -e -o pid,ppid,args 2> /dev/null \
|
||||||
running_proc=`cat /tmp/ccollect_mgr.$$ | wc -l`
|
| grep -v -e grep -e "${PID}.*ccollect.*${backup}" \
|
||||||
|
| grep "ccollect.*${backup}" > "${tmp_mgr}" 2> /dev/null
|
||||||
|
running_proc=`cat "${tmp_mgr}" | wc -l`
|
||||||
|
|
||||||
if [ ${running_proc} -gt 0 ]; then
|
if [ ${running_proc} -gt 0 ]; then
|
||||||
|
# Remove backup from list
|
||||||
running_backups="${running_backups}${backup} "
|
running_backups="${running_backups}${backup} "
|
||||||
echo "Process:"
|
|
||||||
cat /tmp/ccollect.$$
|
echo "Process already running:"
|
||||||
|
cat "${tmp_mgr}"
|
||||||
|
|
||||||
|
remove_source ${source_no}
|
||||||
else
|
else
|
||||||
backups_to_do="${backups_to_do}${backup} "
|
source_no=$(( ${source_no} + 1 ))
|
||||||
fi
|
fi
|
||||||
rm /tmp/ccollect_mgr.$$
|
rm "${tmp_mgr}"
|
||||||
done
|
done
|
||||||
ccollect_backups=${backups_to_do}
|
|
||||||
|
|
||||||
if [ "${running_backups}" != "" ]; then
|
if [ "${running_backups}" != "" ]; then
|
||||||
echo "skipping ccollect backups already running: ${running_backups}" | tee ${tmp_report}
|
echo "skipping ccollect backups already running: ${running_backups}" | tee "${tmp_report}"
|
||||||
send_email ${tmp_report} "WARNING - skipping ccollect backups already running: ${running_backups}"
|
send_email "${tmp_report}" "WARNING - skipping ccollect backups already running: ${running_backups}"
|
||||||
rm ${tmp_report}
|
rm "${tmp_report}"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
find_interval()
|
||||||
|
{
|
||||||
|
# ----------------------------------------------------
|
||||||
|
# Find interval for ccollect backup.
|
||||||
|
# optional parameters:
|
||||||
|
# - Day of the week to do weekly backups
|
||||||
|
# - Do monthly instead of weekly on the Nth week
|
||||||
|
# ----------------------------------------------------
|
||||||
|
|
||||||
|
weekly_backup="$1"
|
||||||
|
monthly_backup="$2"
|
||||||
|
|
||||||
|
weekday=`date "+%w"`
|
||||||
|
if [ ${weekday} -eq ${weekly_backup} ]; then
|
||||||
|
dom=`date "+%e"`
|
||||||
|
weeknum=$(( ( ${dom} / 7 ) + 1 ))
|
||||||
|
if [ "${weeknum}" -eq "${monthly_backup}" ]; then
|
||||||
|
interval=monthly
|
||||||
|
else
|
||||||
|
interval=weekly
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
interval=daily
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
precheck_transfer_size()
|
precheck_transfer_size()
|
||||||
{
|
{
|
||||||
# Check the estimated (worst case) transfer size and send email if larger than certain size
|
# Check the estimated (worst case) transfer size and send email if larger than certain size
|
||||||
|
# Abort backup if total transfer is larger than maximum limit (ex: an error somewhere
|
||||||
|
# requires to do full backup and not incremental, which could blow the quota with ISP)
|
||||||
#
|
#
|
||||||
# Be nice and add error checking one day...
|
# Be nice and add error checking one day...
|
||||||
|
|
||||||
for backup in ${ccollect_backups} ; do
|
source_no=0
|
||||||
ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?"
|
while [ "${source_no}" -lt "${no_sources}" ]; do
|
||||||
|
eval backup=\"\$source_${source_no}\"
|
||||||
|
eval ddir=\"\$ddir_${source_no}\"
|
||||||
|
|
||||||
last_dir="$(ls -tcp1 "${ddir}" | grep '/$' | head -n 1)"
|
last_dir="$(ls -tcp1 "${ddir}" | grep '/$' | head -n 1)"
|
||||||
sdir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/source)"; ret="$?"
|
sdir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/source)"; ret="$?"
|
||||||
if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/exclude ]; then
|
if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/exclude ]; then
|
||||||
|
@ -245,66 +241,152 @@ precheck_transfer_size()
|
||||||
else
|
else
|
||||||
exclude=""
|
exclude=""
|
||||||
fi
|
fi
|
||||||
|
rsync_options=""
|
||||||
if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/rsync_options ]; then
|
if [ -f "${CCOLLECT_CONF}"/sources/"${backup}"/rsync_options ]; then
|
||||||
while read line; do
|
while read line; do
|
||||||
rsync_options="${rsync_options} ${line}"
|
rsync_options="${rsync_options} ${line}"
|
||||||
done < ${CCOLLECT_CONF}/sources/${backup}/rsync_options
|
done < ${CCOLLECT_CONF}/sources/${backup}/rsync_options
|
||||||
fi
|
fi
|
||||||
rsync -n -a --delete --stats ${rsync_options} ${exclude} ${sdir} ${ddir}/${last_dir} > ${tmp_report}
|
|
||||||
tx_rx=`cat ${tmp_report} | grep "Total transferred file size" | \
|
rsync -n -a --delete --stats ${rsync_options} "${exclude}" "${sdir}" "${ddir}/${last_dir}" > "${tmp_report}"
|
||||||
|
|
||||||
|
tx_rx=`cat "${tmp_report}" | grep "Total transferred file size" | \
|
||||||
awk '{ { tx += $5 } } END { printf "%u",(((tx)+1024*1024-1)/1024/1024); }'`
|
awk '{ { tx += $5 } } END { printf "%u",(((tx)+1024*1024-1)/1024/1024); }'`
|
||||||
total_xfer=$(( ${total_xfer} + ${tx_rx} ))
|
total_xfer=$(( ${total_xfer} + ${tx_rx} ))
|
||||||
|
|
||||||
|
source_no=$(( ${source_no} + 1 ))
|
||||||
done
|
done
|
||||||
echo Transfer estimation for ${ccollect_backups}: ${total_xfer} MB
|
|
||||||
|
|
||||||
if [ ${total_xfer} -gt ${warning_transfer_size} ]; then
|
echo "Transfer estimation for${ccollect_backups}: ${total_xfer} MB"
|
||||||
|
|
||||||
|
if [ ${total_xfer} -gt ${abort_transfer_size} ]; then
|
||||||
|
# --------------------------------------------------
|
||||||
|
# Send an error if transfer is larger than max limit
|
||||||
|
# --------------------------------------------------
|
||||||
|
# Useful to detect potential issues when there is transfer quota (ex: with ISP)
|
||||||
|
|
||||||
|
echo "Data transfer larger than ${abort_transfer_size} MB is expected for${ccollect_backups}" >> "${tmp_report}"
|
||||||
|
echo "** BACKUP ABORTED **" >> "${tmp_report}"
|
||||||
|
|
||||||
|
send_email "${tmp_report}" "ERROR: aborted ccollect for${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB"
|
||||||
|
rm "${tmp_report}"
|
||||||
|
exit 1
|
||||||
|
elif [ ${total_xfer} -gt ${warning_transfer_size} ]; then
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# Send a warning if transfer is expected to be large
|
# Send a warning if transfer is expected to be large
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# Useful to detect potential issues when there is transfer quota (ex: with ISP)
|
# Useful to detect potential issues when there is transfer quota (ex: with ISP)
|
||||||
|
|
||||||
echo Data transfer larger than ${warning_transfer_size} MB is expected for ${ccollect_backups} > ${tmp_report}
|
echo "Data transfer larger than ${warning_transfer_size} MB is expected for${ccollect_backups}" > "${tmp_report}"
|
||||||
|
|
||||||
send_email ${tmp_report} "WARNING ccollect for ${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB"
|
send_email "${tmp_report}" "WARNING ccollect for${ccollect_backups} -- Estimated Tx+Rx: ${total_xfer} MB"
|
||||||
rm ${tmp_report}
|
rm "${tmp_report}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
build_backup_dir_list()
|
||||||
|
{
|
||||||
|
source_no=0
|
||||||
|
while [ "${source_no}" -lt "${no_sources}" ]; do
|
||||||
|
eval backup=\"\$source_${source_no}\"
|
||||||
|
eval ddir=\"\$ddir_${source_no}\"
|
||||||
|
|
||||||
|
backup_dir="`cat "${TEMP_LOG}" \
|
||||||
|
| grep "\[${backup}\] .*: Creating.* ${ddir}" \
|
||||||
|
| head -n 1 \
|
||||||
|
| sed 's/[^\/]*\//\//; s/ \.\.\.//'`"
|
||||||
|
|
||||||
|
if [ ! -d "${backup_dir}" ]; then
|
||||||
|
backups_not_found="${backups_not_found}\"${backup}\" "
|
||||||
|
echo -n "Backup directory for \"${backup}\" not found. "
|
||||||
|
remove_source "${source_no}"
|
||||||
|
else
|
||||||
|
eval export backup_dir_list_${source_no}="${backup_dir}"
|
||||||
|
# eval echo Backup Dir List: \"\$backup_dir_list_${source_no}\"
|
||||||
|
source_no=$(( ${source_no} + 1 ))
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
move_log()
|
||||||
|
{
|
||||||
|
if [ "${no_sources}" -gt 0 ]; then
|
||||||
|
eval log_file=\"\$backup_dir_list_1\"/ccollect.log
|
||||||
|
mv "${TEMP_LOG}" "${log_file}"
|
||||||
|
echo New Log Location: "${log_file}"
|
||||||
|
else
|
||||||
|
echo "WARNING: none of the backup set have been created"
|
||||||
|
log_file="${TEMP_LOG}"
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
send_report()
|
send_report()
|
||||||
{
|
{
|
||||||
log=$1
|
|
||||||
|
|
||||||
# Analyze log for periodic report and for error status report
|
# Analyze log for periodic report and for error status report
|
||||||
cat ${log} | ccollect_analyse_logs.sh iwe > ${tmp_report}
|
cat "${log_file}" | ccollect_analyse_logs.sh iwe > "${tmp_report}"
|
||||||
|
|
||||||
# -------------------------
|
# -------------------------
|
||||||
# Build the periodic report
|
# Build the periodic report
|
||||||
# -------------------------
|
# -------------------------
|
||||||
|
|
||||||
# Compute the total number of MB sent and received for all the backup sets
|
# Compute the total number of MB sent and received for all the backup sets
|
||||||
tx_rx=`cat ${tmp_report} | \
|
tx_rx=`cat "${tmp_report}" | \
|
||||||
grep 'sent [[:digit:]]* bytes received [0-9]* bytes' | \
|
grep 'sent [[:digit:]]* bytes received [0-9]* bytes' | \
|
||||||
awk '{ { tx += $3 } { rx += $6} } END \
|
awk '{ { tx += $3 } { rx += $6} } END \
|
||||||
{ printf "%u",(((tx+rx)+1024*1024-1)/1024/1024); }'`
|
{ printf "%u",(((tx+rx)+(1024*1024)-1)/1024/1024); }'`
|
||||||
current_date=`date +'20%y/%m/%d %Hh%M -- '`
|
current_date=`date +'20%y/%m/%d %Hh%M -- '`
|
||||||
|
|
||||||
|
# ------------------------------------------
|
||||||
|
# Get the real disk usage for the backup set
|
||||||
|
# ------------------------------------------
|
||||||
|
total_rdu=0
|
||||||
|
source_no=0
|
||||||
|
while [ "${source_no}" -lt "${no_sources}" ]; do
|
||||||
|
eval backup_dir=\"\$backup_dir_list_${source_no}\"
|
||||||
|
compute_rdu -m "${backup_dir}"
|
||||||
|
total_rdu=$(( ${total_rdu} + ${rdu} ))
|
||||||
|
source_no=$(( ${source_no} + 1 ))
|
||||||
|
done
|
||||||
|
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
# Get the disk usage for all backups of each backup sets...
|
# Get the disk usage for all backups of each backup sets...
|
||||||
# ** be patient **
|
# ** BE PATIENT!!! **
|
||||||
# ---------------------------------------------------------
|
# ---------------------------------------------------------
|
||||||
compute_rdu -g ${backup_dir_list}
|
historical_rdu=0
|
||||||
|
source_no=0
|
||||||
|
while [ "${source_no}" -lt "${no_sources}" ]; do
|
||||||
|
eval backup_dir=\"\$ddir_${source_no}\"
|
||||||
|
compute_rdu -m "${backup_dir}"
|
||||||
|
historical_rdu=$(( ${historical_rdu} + ${rdu} ))
|
||||||
|
source_no=$(( ${source_no} + 1 ))
|
||||||
|
done
|
||||||
|
|
||||||
|
historical_rdu=$(( (${historical_rdu}+1023) / 1024 ))
|
||||||
|
|
||||||
|
if [ "${no_sources}" -gt 0 ]; then
|
||||||
|
ccollect_backups=""
|
||||||
|
else
|
||||||
|
ccollect_backups="(none performed) "
|
||||||
|
fi
|
||||||
|
|
||||||
|
source_no=0
|
||||||
|
while [ "${source_no}" -lt "${no_sources}" ]; do
|
||||||
|
eval backup=\"\$source_${source_no}\"
|
||||||
|
ccollect_backups="${ccollect_backups}\"${backup}\" "
|
||||||
|
source_no=$(( ${source_no} + 1 ))
|
||||||
|
done
|
||||||
|
|
||||||
echo ${current_date} Tx+Rx: ${tx_rx} MB -- \
|
echo ${current_date} Tx+Rx: ${tx_rx} MB -- \
|
||||||
Disk Usage: ${real_usage} MB -- \
|
Disk Usage: ${total_rdu} MB -- \
|
||||||
Backup set \(${interval}\): ${ccollect_backups} -- \
|
Backup set \(${interval}\):${ccollect_backups} -- \
|
||||||
Historical backups usage: ${rdu} GB >> ${per_report}
|
Historical backups usage: ${historical_rdu} GB >> "${per_report}"
|
||||||
|
echo "Total Data Transfer: ${tx_rx} MB -- Total Disk Usage: ${total_rdu} MB -- Total Historical backups usage: ${historical_rdu} GB"
|
||||||
|
|
||||||
# ----------------------------------------
|
# ----------------------------------------
|
||||||
# Send a status email if there is an error
|
# Send a status email if there is an error
|
||||||
# ----------------------------------------
|
# ----------------------------------------
|
||||||
ccollect_we=`cat ${log} | ccollect_analyse_logs.sh we | wc -l`
|
ccollect_we=`cat "${log_file}" | ccollect_analyse_logs.sh we | wc -l`
|
||||||
if [ ${ccollect_we} -ge 1 ]; then
|
if [ ${ccollect_we} -ge 1 ]; then
|
||||||
send_email ${tmp_report} "ERROR ccollect for ${ccollect_backups} -- Tx+Rx: ${tx_rx} MB"
|
send_email "${tmp_report}" "ERROR ccollect for${ccollect_backups} -- Tx+Rx: ${tx_rx} MB"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# --------------------
|
# --------------------
|
||||||
|
@ -313,27 +395,27 @@ send_report()
|
||||||
if [ ${report_interval} == ${interval} ] || [ ${interval} == "monthly" ]; then
|
if [ ${report_interval} == ${interval} ] || [ ${interval} == "monthly" ]; then
|
||||||
|
|
||||||
# Make reporting atomic to handle concurrent ccollect_mgr instances
|
# Make reporting atomic to handle concurrent ccollect_mgr instances
|
||||||
mv ${per_report} ${per_report}.$$
|
mv "${per_report}" "${per_report}".$$
|
||||||
cat ${per_report}.$$ >> ${per_report}.history
|
cat "${per_report}".$$ >> "${per_report}".history
|
||||||
|
|
||||||
# Calculate total amount of bytes sent and received
|
# Calculate total amount of bytes sent and received
|
||||||
tx_rx=`cat ${per_report}.$$ | \
|
tx_rx=`cat "${per_report}".$$ | \
|
||||||
awk '{ { transfer += $5 } } END \
|
awk '{ { transfer += $5 } } END \
|
||||||
{ printf "%u",(transfer); }'`
|
{ printf "%u",(transfer); }'`
|
||||||
|
|
||||||
# Send email
|
# Send email
|
||||||
send_email ${per_report}.$$ "${report_interval} ccollect status for ${ccollect_backups} -- Tx+Rx: ${tx_rx} MB"
|
send_email "${per_report}.$$" "${report_interval} ccollect status for${ccollect_backups} -- Tx+Rx: ${tx_rx} MB"
|
||||||
rm ${per_report}.$$
|
rm "${per_report}.$$"
|
||||||
fi
|
fi
|
||||||
|
|
||||||
rm ${tmp_report}
|
rm "${tmp_report}"
|
||||||
}
|
}
|
||||||
|
|
||||||
# ------------------------------------------------
|
# ------------------------------------------------
|
||||||
# Add to PATH in case we're launching from crontab
|
# Add to PATH in case we're launching from crontab
|
||||||
# ------------------------------------------------
|
# ------------------------------------------------
|
||||||
|
|
||||||
PATH=${ADD_TO_PATH}:${PATH}
|
PATH="${ADD_TO_PATH}:${PATH}"
|
||||||
|
|
||||||
# --------------
|
# --------------
|
||||||
# Default Values
|
# Default Values
|
||||||
|
@ -349,59 +431,74 @@ weekly_backup=1
|
||||||
# Set the monthly backup interval. Default is 4th Monday of every month
|
# Set the monthly backup interval. Default is 4th Monday of every month
|
||||||
monthly_backup=4
|
monthly_backup=4
|
||||||
|
|
||||||
show_help=0
|
|
||||||
|
|
||||||
# ---------------------------------
|
# ---------------------------------
|
||||||
# Parse command line
|
# Parse command line
|
||||||
# ---------------------------------
|
# ---------------------------------
|
||||||
|
|
||||||
|
show_help=0
|
||||||
|
export no_sources=0
|
||||||
|
|
||||||
while [ "$#" -ge 1 ]; do
|
while [ "$#" -ge 1 ]; do
|
||||||
case "$1" in
|
case "$1" in
|
||||||
-help)
|
-help)
|
||||||
show_help=1
|
show_help=1
|
||||||
;;
|
;;
|
||||||
-from)
|
-from)
|
||||||
from=$2
|
from="$2"
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
-to)
|
-to)
|
||||||
to=$2
|
to="$2"
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
-server|mail_server)
|
-server|mail_server)
|
||||||
mail_server=$2
|
mail_server="$2"
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
-weekly)
|
-weekly)
|
||||||
weekly_backup=$2
|
weekly_backup="$2"
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
-monthly)
|
-monthly)
|
||||||
monthly_backup=$2
|
monthly_backup="$2"
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
-warning_size)
|
-warning_size)
|
||||||
warning_transfer_size=$2
|
warning_transfer_size="$2"
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
-abort_size)
|
||||||
|
abort_transfer_size="$2"
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
-report)
|
-report)
|
||||||
report_interval=$2
|
report_interval="$2"
|
||||||
shift
|
shift
|
||||||
;;
|
;;
|
||||||
-*)
|
-*)
|
||||||
ccollect_options="${ccollect_options} $1"
|
ccollect_options="${ccollect_options}$1 "
|
||||||
;;
|
;;
|
||||||
daily|weekly|monthly)
|
daily|weekly|monthly)
|
||||||
;;
|
;;
|
||||||
*)
|
*)
|
||||||
ccollect_backups="${ccollect_backups}$1 "
|
eval backup=\"\$1\"
|
||||||
|
ddir="$(cat "${CCOLLECT_CONF}"/sources/"${backup}"/destination)"; ret="$?"
|
||||||
|
if [ "${ret}" -ne 0 ]; then
|
||||||
|
echo "Destination ${CCOLLECT_CONF}/sources/${backup}/destination is not readable... Skipping."
|
||||||
|
else
|
||||||
|
ccollect_backups="${ccollect_backups} \"$1\""
|
||||||
|
eval export source_${no_sources}=\"\$1\"
|
||||||
|
eval export ddir_${no_sources}="${ddir}"
|
||||||
|
# eval echo Adding source \"\$source_${no_sources}\" -- \"\$ddir_${no_sources}\"
|
||||||
|
no_sources="$(( ${no_sources} + 1 ))"
|
||||||
|
fi
|
||||||
;;
|
;;
|
||||||
esac
|
esac
|
||||||
shift
|
shift
|
||||||
done
|
done
|
||||||
|
|
||||||
if [ "${ccollect_backups}" == "" ] || [ ${show_help} -eq 1 ]; then
|
if [ "${no_sources}" -lt 1 ] || [ ${show_help} -eq 1 ]; then
|
||||||
echo
|
echo ""
|
||||||
echo "$0: Syntax"
|
echo "$0: Syntax"
|
||||||
echo " -help This help"
|
echo " -help This help"
|
||||||
echo " -from <email> From email address (ex.: -from nas@home.com)"
|
echo " -from <email> From email address (ex.: -from nas@home.com)"
|
||||||
|
@ -414,58 +511,32 @@ if [ "${ccollect_backups}" == "" ] || [ ${show_help} -eq 1 ]; then
|
||||||
echo " -report <interval> Frequency of report email (daily, weekly or monthly)"
|
echo " -report <interval> Frequency of report email (daily, weekly or monthly)"
|
||||||
echo " Default is ${report_interval}"
|
echo " Default is ${report_interval}"
|
||||||
echo " -warning_size <MB> Send a warning email if the transfer size exceed this"
|
echo " -warning_size <MB> Send a warning email if the transfer size exceed this"
|
||||||
echo " Default is ${warning_transfer_size}"
|
echo " Default is ${warning_transfer_size} MB"
|
||||||
|
echo " -abort_size <MB> Abort and send an error email if the transfer size exceed this"
|
||||||
|
echo " Default is ${abort_transfer_size} MB"
|
||||||
echo ""
|
echo ""
|
||||||
echo " other parameters are transfered to ccollect"
|
echo " other parameters are transfered to ccollect"
|
||||||
echo
|
echo ""
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ------------------------------------------------------------------
|
#echo Backup sets:"${ccollect_backups}"
|
||||||
# Check if ccollect_mgr is already running for the given backup sets
|
|
||||||
# ------------------------------------------------------------------
|
|
||||||
|
|
||||||
check_running_backups
|
check_running_backups
|
||||||
|
|
||||||
if [ "${ccollect_backups}" == "" ]; then
|
if [ "${no_sources}" -lt 1 ]; then
|
||||||
echo "No backup sets are reachable"
|
echo "No backup sets are reachable"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# ----------------------------------------------------------
|
|
||||||
# Set the interval type
|
|
||||||
#
|
|
||||||
# Here, weeklys are Mondays, and Monthlys are the 4th Monday
|
|
||||||
# ----------------------------------------------------------
|
|
||||||
|
|
||||||
find_interval ${weekly_backup} ${monthly_backup}
|
find_interval ${weekly_backup} ${monthly_backup}
|
||||||
echo Interval: ${interval}
|
echo Interval: ${interval}
|
||||||
|
|
||||||
# --------------
|
|
||||||
# Do the backups
|
|
||||||
# --------------
|
|
||||||
TEMP_LOG=${CCOLLECT_CONF}/log.$$
|
|
||||||
echo Backup sets: ${ccollect_backups}
|
|
||||||
|
|
||||||
# Check the transfer size (to issue email warning)
|
|
||||||
precheck_transfer_size
|
precheck_transfer_size
|
||||||
|
|
||||||
${CCOLLECT} ${ccollect_options} ${interval} ${ccollect_backups} | tee ${TEMP_LOG}
|
"${CCOLLECT}" ${ccollect_options} ${interval} ${ccollect_backups} | tee "${TEMP_LOG}"
|
||||||
|
|
||||||
# ---------------------------------------
|
build_backup_dir_list
|
||||||
# Move log to the last backup of the set
|
move_log
|
||||||
# ---------------------------------------
|
|
||||||
|
|
||||||
move_log ${ccollect_backups}
|
send_report
|
||||||
|
|
||||||
# -----------------------------------------
|
|
||||||
# Compute the physical amount of disk usage
|
|
||||||
# -----------------------------------------
|
|
||||||
|
|
||||||
compute_total_rdu ${ccollect_backups}
|
|
||||||
|
|
||||||
# -----------------
|
|
||||||
# Send status email
|
|
||||||
# -----------------
|
|
||||||
|
|
||||||
send_report ${new_log}
|
|
||||||
|
|
Loading…
Reference in a new issue