cat ProxyErrorCheck_sshpass.

sh
#!/bin/sh
source /opt/apigee/custom_lib/dec.sh

logDate=$(date +%Y-%m-%d)
#logDate="2019-07-05"
timeStart=$(date +"${logDate}T%H:%M:%S.%N")

username=apigee
server1=a0110papirmp01
server2=a0110papirmp02
password=$( depcrypt )

proxy=$1
logLines=$2

# Directory to store temporarily files

tempDir=/home/${username}/ProxyErrorCheck_v2/data

# Directory to store temporarily logs

logDir=/home/${username}/ProxyErrorCheck_v2/logs

# MessageLogging path
LOG_HOME="/opt/apigee/var/log/edge-message-processor/messagelogging/digi/
production/${proxy}/*/ML-TransactionLogging/*"

# Hours to be reported (TECHNICAL)

technicalHours=6

# Hours to be reported (traffic)

reportHours=24

# Maximum errors for reporting

maxError=1000

# Maximum errors in single attachment

maxAttachment=100

# Last occurrence to be alerted (seconds)

lastLimit=3600

# Email to send report to (if there are errors to report):

email="[email protected] "

# Additional email for specific proxy

if [ "$proxy" == "payments-v2" ]; then
email="$email [email protected]"
echo "boo"
fi

# If there are no errors, only send to the following:

email_empty=""

# Email subject
#subject="ALERT: API proxy [${proxy}] errors on ${logDate}"
#subject="TEST: API proxy [${proxy}] errors on ${logDate}"
subject="INFO: API proxy [${proxy}] errors on ${logDate}"

# Maximum file size (.log) for attachment (5 KB)

maxLog=5120

# Maximum file size (.zip) for attachment (200 KB)

maxZip=204800

debug=false

#==================================================================================
=====#
#==================================================================================
=====#
# DO NO CHANGE ANYTHING BELOW THIS POINT
#==================================================================================
=====#
#==================================================================================
=====#

if [ -z "$proxy" ] || [ -z "logLines" ]; then

echo "ERROR! Please specify the Proxy name and Log Lines"
echo "Example: ./$(basename $0) customer-v1_Internet 4"
echo ""
exit
fi

timeLabel=$(date --date="$timeStart" +"%Y%m%dT%H%M%S")

timeNano=$(date --date="$timeStart" +"%N")
timeStamp=$(date --date="$timeStart" +"%Y-%m-%dT%H:%M:%S")
timeDate=$(date --date="$timeStart" +"%Y%m%d")
timeMonth=$(date --date="$timeStart" +"%Y%m")
timeSeconds=$(date --date="$timeStart" +"%s")

echo "timeStart=$timeStart"
echo "timeStamp=$timeStamp"
echo "timeLabel=$timeLabel"
echo "timeNano=$timeNano"
echo "timeSeconds=$timeSeconds"

logTrimmed=false
declare -a fileTrimmed=()
tempDir=${tempDir}/${timeLabel}.${timeNano}

grepLines=$(( $logLines - 1 ))
logTECHNICAL=${proxy}_TECHNICAL.log
logSTART=${proxy}_START.log
logCOUNT=${proxy}_COUNT.log

LOG_LOCAL1=$tempDir/${proxy}/rmp01
LOG_LOCAL2=$tempDir/${proxy}/rmp02
LOG_TEMP=$tempDir/${proxy}

set -x
# Create the temporarily directory, if it's not there
rm -fr $tempDir
mkdir -p $tempDir
mkdir -p $logDir
mkdir -p $LOG_LOCAL1
mkdir -p $LOG_LOCAL2
set +x
echo "${timeStamp}: [00:00:00] ${proxy} - START" >>
$logDir/ProxyErrorCheck_$timeMonth.log

echo "###########################################"
echo "## START"
echo "###########################################"
echo "## Date: $timeStart"
echo "## Proxy: $proxy"
echo "## Log: $LOG_HOME"
echo "###########################################"

###################################################################################
######

# Calculate the hours to retrieve the logs

# Today's hour + 3 hours, for extra
collectHours=$(($(date +%-H) + 3))
echo "collectHours=$collectHours"

# Copy from RMP01 to local

LOG_SERVER1=$(sshpass -p ${password} ssh -o StrictHostKeyChecking=no ${username}@$
{server1} find $LOG_HOME -mmin -$((60*$collectHours)) -type f)
set -x
sshpass -p ${password} ssh -o StrictHostKeyChecking=no -t ${username}@${server1} rm
-fr $tempDir
sshpass -p ${password} ssh -o StrictHostKeyChecking=no -t ${username}@${server1}
mkdir -p $LOG_TEMP
sshpass -p ${password} ssh -o StrictHostKeyChecking=no -t ${username}@${server1} cp
-p --parents $LOG_SERVER1 $LOG_TEMP/
sshpass -p ${password} scp -r -p $username@$server1:$LOG_TEMP/* $LOG_LOCAL1
sshpass -p ${password} ssh -o StrictHostKeyChecking=no -t ${username}@${server1} rm
-fr $tempDir
set +x

echo "###########################################"
echo "## RUNNING: Completed RMP01 transfer ($(date +"%d-%m-%Y, %r")) [$(( $(date +
%s) - $timeSeconds ))s]"
echo "###########################################"

# Copy from RMP02 to local

LOG_SERVER2=$(sshpass -p ${password} ssh -o StrictHostKeyChecking=no $
{username}@$server2 find $LOG_HOME -mmin -$((60*$collectHours)) -type f)
set -x
sshpass -p ${password} ssh -o StrictHostKeyChecking=no -t ${username}@${server2} rm
-fr $tempDir
sshpass -p ${password} ssh -o StrictHostKeyChecking=no -t ${username}@${server2}
mkdir -p $LOG_TEMP
sshpass -p ${password} ssh -o StrictHostKeyChecking=no -t ${username}@${server2} cp
-p --parents $LOG_SERVER2 $LOG_TEMP/
sshpass -p ${password} scp -r -p $username@$server2:$LOG_TEMP/* $LOG_LOCAL2
sshpass -p ${password} ssh -o StrictHostKeyChecking=no -t ${username}@${server2} rm
-fr $tempDir
set +x

echo "LOG_SERVER1=${LOG_SERVER1}"
echo "LOG_SERVER2=${LOG_SERVER2}"
echo "LOG_LOCAL1=${LOG_LOCAL1}"
echo "LOG_LOCAL2=${LOG_LOCAL2}"
echo "###########################################"
echo "## RUNNING: Completed RMP02 transfer ($(date +"%d-%m-%Y, %r")) [$(( $(date +
%s) - $timeSeconds ))s]"
echo "###########################################"

###################################################################################
######

# Count today OK RMP01+RMP02

set -x
todayOK=$(grep -r "START|${logDate}" $LOG_TEMP/* | grep "|OK" | wc -l)
set +x

# Count today BUSINESS errors RMP01+RMP02

set -x
todayBUSINESS=$(grep -r "START|${logDate}" $LOG_TEMP/* | grep "|BUSINESS" | wc -l)
set +x

# Count today TECHNICAL errors RMP01+RMP02

set -x
todayTECHNICAL=$(grep -r "START|${logDate}" $LOG_TEMP/* | grep "|TECHNICAL" | wc -
l)
set +x

# Count TOTAL errors

todayTOTAL=$(( $todayOK + $todayBUSINESS + $todayTECHNICAL ))

echo "todayOK=$todayOK"
echo "todayBUSINESS=$todayBUSINESS"
echo "todayTECHNICAL=$todayTECHNICAL"
echo "todayTOTAL=$todayTOTAL"

echo "###########################################"
echo "## RUNNING: Finish today counts ($(date +"%d-%m-%Y, %r")) [$(( $(date +%s) -
$timeSeconds ))s]"
echo "###########################################"

###################################################################################
######

for ((h=0; h<=$technicalHours; h++)); do

timeHour=$(date --date="-$h hours $timeStart" +"%Y-%m-%dT%H")
echo "timeHour=$timeHour"

technicalString="${technicalString}START\|${timeHour}"
if [ $h -lt $technicalHours ]; then
technicalString="${technicalString}|"
fi
done

echo "technicalString=$technicalString"

# Compile TECHNICAL errors RMP01+RMP02

set -x
techFiles=$(find $LOG_TEMP/* -type f -print0)
set +x

if [[ -z $techFiles ]]; then

touch $tempDir/$logTECHNICAL
else
set -x
find $LOG_TEMP/* -type f -print0 | xargs -r0 stat -c %y\ %n | sort | awk '{print
$4}' | sed "s|^\./||" | xargs egrep -rh -A $grepLines "$technicalString" --no-
group-separator | grep -A $grepLines "|TECHNICAL" --no-group-separator >
$tempDir/$logTECHNICAL
set +x
fi

# Count TECHNICAL errors RMP01+RMP02

set -x
totalTECHNICAL=$(grep -r "|TECHNICAL" $tempDir/$logTECHNICAL | wc -l)
set +x
echo "totalTECHNICAL=$totalTECHNICAL"

echo "###########################################"
echo "## RUNNING: Finish compiling TECHNICAL ($technicalHours hour(s)) ($(date
+"%d-%m-%Y, %r")) [$(( $(date +%s) - $timeSeconds ))s]"
echo "###########################################"

###################################################################################
######

# Don't need the local logs anymore, delete them!

set -x
if [ "$debug" = false ]; then
rm -fr $LOG_TEMP
fi
set +x

ls -l $tempDir/$logTECHNICAL

echo "###########################################"
echo "## RUNNING: Completed TECHNICAL log ($(date +"%d-%m-%Y, %r")) [$(( $(date +
%s) - $timeSeconds ))s]"
echo "###########################################"

###################################################################################
######

# Check the number of START lines

startError=$(grep "START|" $tempDir/$logTECHNICAL | wc -l)

# Get only the last x lines, we do not want the log to be too huge!
maxLines=$(( $maxError * $logLines ))

# Debug
echo "startError=$startError"
echo "logLines=$logLines"
echo "maxError=$maxError"
echo "maxLines=$maxLines"

if [ $startError -gt $maxError ]; then

logTrimmed=true

# Regenerate the log again, but trimmed

echo "$(tail -${maxLines} $tempDir/$logTECHNICAL)" >
$tempDir/$logTECHNICAL.trimmed
 mv $tempDir/$logTECHNICAL $tempDir/$logTECHNICAL.full
mv $tempDir/$logTECHNICAL.trimmed $tempDir/$logTECHNICAL

echo "###########################################"
echo "## RUNNING: Completed trimming $tempDir/$logTECHNICAL ($(date +"%d-%m-%Y,
%r")) [$(( $(date +%s) - $timeSeconds ))s]"
echo "###########################################"
fi

###################################################################################
######

# Get only the START line, and sort it

grep "START|" $tempDir/$logTECHNICAL | sort > $tempDir/$logSTART

ls -l $tempDir/$logSTART

echo "###########################################"
echo "## RUNNING: Completed START log ($(date +"%d-%m-%Y, %r")) [$(( $(date +%s) -
$timeSeconds ))s]"
echo "###########################################"

# Output only the relevant content

while read temp
do
#First 4 = START|{isoTimestamp}|{request.verb}|{proxy.basepath}|
{proxy.pathsuffix}
isoTimestamp=$(echo "$temp" | cut -d '|' -f 2)
requestVerb=$(echo "$temp" | cut -d '|' -f 3)
basePath=$(echo "$temp" | cut -d '|' -f 4)
pathSuffix=$(echo "$temp" | cut -d '|' -f 5)

#Last 4 = {message.status.code}|{errorCode}|{errorMessage}|{logType}
statusCode=$(echo "$temp" | rev | cut -d '|' -f 4 | rev)
errorCode=$(echo "$temp" | rev | cut -d '|' -f 3 | rev)
errorMessage=$(echo "$temp" | rev | cut -d '|' -f 2 | rev)
logType=$(echo "$temp" | rev | cut -d '|' -f 1 | rev)

# Format the table of contents

echo "$requestVerb|$basePath|$pathSuffix|$statusCode|$errorCode|$errorMessage|
$logType" >> $tempDir/$logTECHNICAL.tmp.line
done < $tempDir/$logSTART

echo "###########################################"
echo "## RUNNING: Completed temporarily LINE log ($(date +"%d-%m-%Y, %r")) [$(( $
(date +%s) - $timeSeconds ))s]"
echo "###########################################"

###################################################################################
######

# Sort the lines

cat $tempDir/$logTECHNICAL.tmp.line | sort > $tempDir/$logTECHNICAL.tmp.sort

echo "###########################################"
echo "## RUNNING: Completed temporarily SORT log ($(date +"%d-%m-%Y, %r")) [$(( $
(date +%s) - $timeSeconds ))s]"
echo "###########################################"
###################################################################################
######

# Count the unique errors

cat $tempDir/$logTECHNICAL.tmp.sort | uniq -c > $tempDir/$logCOUNT

# Remove all the temporarily files

set -x
rm $tempDir/$logTECHNICAL.tmp.*
set +x

echo "##### START: Count #####"

cat $tempDir/$logCOUNT
echo "##### END: Count #####"

echo "###########################################"
echo "## RUNNING: Completed COUNT log ($(date +"%d-%m-%Y, %r")) [$(( $(date +%s) -
$timeSeconds ))s]"
echo "###########################################"

###################################################################################
######

if [ "$logTrimmed" = true ]; then

trimmedNotice="<p><font color=red>This [<b>${proxy}</b>] proxy has generated too
many TECHNICAL errors (<b>$(printf "%'d" ${totalTECHNICAL})</b> errors logged for
the past $technicalHours hours!). This report will now only list the latest $
(printf "%'d" ${maxError}) error occurrences.</font></p>"
fi

html="<html>
<head>
<style>
.tab, .tab th, .tab td {
border: 1px solid black;
padding: 5px;
}
th {
background-color: yellow;
}
.nok {
background-color: orange;
color: white;
}
.h {
background-color: yellow;
font-weight: bold;
}
.l {
background-color: whitesmoke;
}
</style>
</head>
<body>${trimmedNotice}
<p style=margin:0px;padding:0px;>Proxy [<b>$proxy</b>] TECHNICAL errors for the
past <b>${technicalHours}+ hours</b>:</p>
<table border=0 cellpadding=0 cellspacing=0>
<tr><td><b>From</b></td><td><b>:</b> $(date --date="-$technicalHours hours
$timeStart" +"%d-%m-%Y, %I:00:00 %p")</td></tr>
<tr><td><b>To</b></td><td><b>:</b> $(date --date="$timeStart" +"%d-%m-%Y,
%r")</td></tr>
</table>
<br>
<table class=tab>
<tr>
<th>Method</th>
<th>API</th>
<th>Status Code</th>
<th>Error Code</th>
<th>Error Message</th>
<th>Count</th>
<th>Last Occurrence</th>
</tr>"

lastIsAlert=0
lastLatest=0
lastDisplay=0
errorTotal=0
while read line
do
errorCount=$(echo $line | cut -d " " -f 1)
errorTotal=$(($errorTotal+$errorCount))
countLength=${#errorCount}
errorLine=${line:($countLength + 1)}

# Debug
echo "errorCount=$errorCount"
echo "errorTotal=$errorTotal"
echo "errorLine=$errorLine"

#errorLine={request.verb}|{proxy.basepath}|{proxy.pathsuffix}|
{message.status.code}|{errorCode}|{errorMessage}|{logType}
requestVerb=$(echo "$errorLine" | cut -d '|' -f 1)
basePath=$(echo "$errorLine" | cut -d '|' -f 2)
pathSuffix=$(echo "$errorLine" | cut -d '|' -f 3)
statusCode=$(echo "$errorLine" | cut -d '|' -f 4)
errorCode=$(echo "$errorLine" | cut -d '|' -f 5)
errorMessage=$(echo "$errorLine" | cut -d '|' -f 6)
logType=$(echo "$errorLine" | cut -d '|' -f 7)

# Get the last occurrence of the error

# Escape strings with ()
errorMessageEsc=$(echo "${errorMessage}" | sed 's/(/\\\(/g')
errorMessageEsc=$(echo "${errorMessageEsc}" | sed 's/)/\\\)/g')
# Escape strings with []
errorMessageEsc=$(echo "${errorMessageEsc}" | sed 's/\[/\\[/g')
errorMessageEsc=$(echo "${errorMessageEsc}" | sed 's/\]/\\]/g')

echo "errorMessage=$errorMessage"
echo "errorMessageEsc=$errorMessageEsc"

set -x
lastLine=$(grep -P "^(?=.*START\|)(?=.*\|${basePath}\|)(?=.*\|${pathSuffix}\|)(?
=.*\|${errorCode}\|)(?=.*\|${errorMessageEsc}\|)" $tempDir/$logSTART | tail -1)
lastTime=$(echo "$lastLine" | cut -d '|' -f 2)
set +x

# Debug
 echo "lastLine=$lastLine"
echo "lastTime=$lastTime"

# Calculate if last occurrence is within the hour

lastNow=$(date +%s)
lastOccur=$(date -d "$lastTime" +%s)
lastGap=$(($lastNow-$lastOccur))
if [ $lastGap -le $lastLimit ]; then
lastIsActive=1
lastIsAlert=1
else
lastIsActive=0
fi

if [ $lastOccur -ge $lastLatest ]; then

lastLatest=$lastOccur
lastDisplay=$lastTime
fi

# Debug
echo "lastNow=$lastNow"
echo "lastOccur=$lastOccur"
echo "lastGap=$lastGap"
echo "lastIsActive=$lastIsActive"
echo "lastLatest=$lastLatest"
echo "lastDisplay=$lastDisplay"

# For attachments
attachmentList="${attachmentList}${requestVerb}|${basePath}|${pathSuffix}|$
{errorCode}\n"

html="${html}<tr>
<td>${requestVerb}</td>
<td>${basePath}${pathSuffix}</td>
<td>${statusCode}</td>
<td>${errorCode}</td>
<td>${errorMessage}</td>
<td align=right>$(printf "%'d" ${errorCount})</td>
<td$([ $lastIsActive -eq 1 ] && (echo " class=nok"))>${lastTime}</td>
</tr>"

# Debug
echo "----------"
done < $tempDir/$logCOUNT

if [ $errorTotal -eq 0 ]; then

html="${html}<tr><td colspan=7><font color=green>No TECHNICAL error(s) to report!
</font></td></tr>"
else
html="${html}<tr><td colspan=5 align=right class=l><b>Total:</b></td><td
align=right><font color=red><b>$(printf "%'d" ${errorTotal})</b></font></td><td
class=l></td></tr>"
fi

# Debug
echo "attachmentList=$attachmentList"

echo "###########################################"
echo "## RUNNING: Completed content manipulation ($(date +"%d-%m-%Y, %r")) [$(( $
(date +%s) - $timeSeconds ))s]"
echo "###########################################"

# Ensure the attachments are distinct

attachmentDistinct=$(echo -e $attachmentList | sort | uniq)
for attachmentLine in $attachmentDistinct; do
# Debug
echo "attachmentLine=$attachmentLine"

requestVerb=$(echo "$attachmentLine" | cut -d '|' -f 1)

basePath=$(echo "$attachmentLine" | cut -d '|' -f 2)
pathSuffix=$(echo "$attachmentLine" | cut -d '|' -f 3)
errorCode=$(echo "$attachmentLine" | cut -d '|' -f 4)

# Debug
echo "requestVerb=$requestVerb"
echo "basePath=$basePath"
echo "pathSuffix=$pathSuffix"
echo "errorCode=$errorCode"

requestPath=$(echo "${pathSuffix}" | sed 's/\//-/g')

echo "requestPath=$requestPath"

filename=$tempDir/${proxy}${requestPath}_${requestVerb}_${errorCode}_$
{timeDate}.log

set -x
grep -P -A $grepLines --no-group-separator "^(?=.*START\|)(?=.*\|${basePath}\|)(?
=.*\|${pathSuffix}\|)(?=.*\|${errorCode}\|)" $tempDir/$logTECHNICAL | sed
"s/|END/&\n----/" > $filename
set +x

# Check the number of START lines

startError=$(grep "START|" $filename | wc -l)

# Get only the last x lines, we do not want the log to be too huge!
maxLines=$(( $maxAttachment * $(($logLines+1)) ))

# Debug
echo "startError=$startError"
echo "logLines=$logLines"
echo "maxAttachment=$maxAttachment"
echo "maxLines=$maxLines"

if [ $startError -gt $maxAttachment ]; then

fileTrimmed+=($filename)

# Regenerate the log again, but trimmed

echo "$(tail -${maxLines} $filename)" > $filename.trimmed

mv $filename $filename.full
mv $filename.trimmed $filename

echo "###########################################"
echo "## RUNNING: Completed trimming $filename ($(date +"%d-%m-%Y, %r")) [$(( $
(date +%s) - $timeSeconds ))s]"
echo "###########################################"
fi
 # If filesize bigger than x MB, then zip it
filesize=$(stat --printf="%s" "$filename")

# Debug
echo "filename=$filename"
echo "filesize=$filesize"

if [ $filesize -gt $maxLog ]; then

zip -jm ${filename}.zip $filename

# If filesize (after zipped) less than x MB, include as attachment

# Else exclude from email
zipsize=$(stat --printf="%s" $filename.zip)
if [ $zipsize -le $maxZip ]; then
attachment="${attachment} -a ${filename}.zip"
else
fileDisplay=$(basename "$filename")
attachmentExcluded="${attachmentExcluded}<br>- ${fileDisplay} ($(printf "%'d"
${filesize}) KB)"
fi
else
attachment="${attachment} -a ${filename}"
fi
done

echo "fileTrimmed[*]=${fileTrimmed[*]}"

# Debug
echo "attachment=$attachment"

echo "###########################################"
echo "## RUNNING: Completed attachments ($(date +"%d-%m-%Y, %r")) [$(( $(date +%s)
- $timeSeconds ))s]"
echo "###########################################"

# If there are files trimmed, display on the email body

if [ ${#fileTrimmed[@]} -gt 0 ]; then
trimmedHTML="<p>The following file(s) have been trimmed to only include the
latest $(printf "%'d" ${maxAttachment}) errors:"
for f in "${fileTrimmed[@]}"; do
fileDisplay=$(basename "$f")
trimmedHTML="${trimmedHTML}<br>- $fileDisplay"
done
trimmedHTML="${trimmedHTML}</p>"
fi

# If there are files excluded, display on the email body

if [[ ! -z $attachmentExcluded ]]; then
attachmentHTML="<p>The following file(s) have been removed as attachment(s) (size
more $(printf "%'d" ${maxLog}) bytes):${attachmentExcluded}</p>"
fi

set -x
#todayPctOK=$(echo "scale=4;(((${todayOK}/${todayTOTAL})*100))" | bc)
#todayPctBUSINESS=$(echo "scale=4;(((${todayBUSINESS}/${todayTOTAL})*100))" | bc)
#todayPctTECHNICAL=$(echo "scale=4;(((${todayTECHNICAL}/${todayTOTAL})*100))" | bc)
todayPctOK=$(awk -v v1="$todayOK" -v v2="$todayTOTAL" 'BEGIN{printf "%.2f",
v1/v2*100}')
todayPctBUSINESS=$(awk -v v1="$todayBUSINESS" -v v2="$todayTOTAL" 'BEGIN{printf
"%.2f", v1/v2*100}')
todayPctTECHNICAL=$(awk -v v1="$todayTECHNICAL" -v v2="$todayTOTAL" 'BEGIN{printf
"%.2f", v1/v2*100}')
set +x

lastAgo=$(date -d@$(($(date +%s)-$lastLatest)) -u +%H:%M:%S)

# Debug
echo "lastLatest=$lastLatest"
echo "lastDisplay=$lastDisplay"
echo "lastAgo=$lastAgo"

if [ $errorTotal -eq 0 ]; then

lastHTML=""
else
lastHTML="<p style=margin:0px;padding:0px;><b>Notes:</b><br>
- <font color=$([ $lastIsAlert -eq 1 ] && (echo "red") || (echo "black"))>The last
error occurrence was $lastAgo ago ($lastDisplay)</font><br>
- Error(s) older than $technicalHours hours will not be reported</font></p>"
fi

html="${html}</table>${lastHTML}
<br><hr><br>
<table class=tab>
<tr>
<th></th>
<th colspan=2>Today, so far...<br>($(date --date="$timeStart" +"%d-%m-%Y"))</th>
</tr>
<tr>
<td class=h>OK Transactions</td>
<td align=right>$(printf "%'d" $todayOK)</td>
<td align=right>${todayPctOK}%</td>
</tr>
<tr>
<td class=h>BUSINESS Errors</td>
<td align=right>$(printf "%'d" $todayBUSINESS)</td>
<td align=right>${todayPctBUSINESS}%</td>
</tr>
<tr>
<td class=h>TECHNICAL Errors</td>
<td align=right><font color=red>$(printf "%'d" $todayTECHNICAL)</font></td>
<td align=right>${todayPctTECHNICAL}%</td>
</tr>
<tr>
<td class=h>TOTAL Transactions</td>
<td align=right>$(printf "%'d" $todayTOTAL)</td>
<td align=right>-</td>
</tr>
</table>
${trimmedHTML}
${attachmentHTML}
<p>Time taken to generate this report: $(date -d@$(($(date +%s)-$timeSeconds)) -u +
%H:%M:%S)<br>
Script location: $0</p>
</body>"

contentType="set content_type=text/html"

if [ $errorTotal -gt 0 ]; then

email_to=$email
 email_send=true
else
email_to=$email_empty

# AM/PM has the same hour!

case $(date --date="$timeStart" +"%I:%M") in
(12:[012]*);&
(03:[012]*);&
(06:[012]*);&
(09:[012]*)
email_send=true
;;
(*)
email_send=false
;;
esac

if [[ -z $email_to ]]; then

email_send=false
fi
fi

set -x
if [ "$email_send" = true ]; then
if [[ ! -z $attachment ]]; then
echo -e $html | mutt -e "${contentType}" $attachment -s "${subject}" --
$email_to
else
echo -e $html | mutt -e "${contentType}" -s "${subject}" -- $email_to
fi

echo "###########################################"
echo "## Email sent ($(date +"%d-%m-%Y, %r")) [$(( $(date +%s) -
$timeSeconds ))s]"
echo "###########################################"
else
echo "###########################################"
echo "## Email not sent ($(date +"%d-%m-%Y, %r")) [$(( $(date +%s) - $timeSeconds
))s]"
echo "###########################################"
fi
set +x

# Debug
if [ "$debug" = false ]; then
# Cleanup!
set -x
rm -fr $tempDir
set +x
else
ls -l $tempDir
fi

timeTaken=$(date -d@$(($(date +%s)-$timeSeconds)) -u +%H:%M:%S)

echo "$(date +"%Y-%m-%dT%H:%M:%S"): [$timeTaken] ${proxy} - TECHNICAL=$errorTotal,

EMAIL=$email_send" >> $logDir/ProxyErrorCheck_$timeMonth.log

echo "###########################################"
echo "## END ($(date +"%d-%m-%Y, %r")) [$(( $(date +%s) - $timeSeconds ))s]"
echo "###########################################"