Round 2 of enhancements:
log-stats.sh gives you detailed statistics from your current log file:
Code:
#!/bin/sh
dir=/tmp/mnt/USB/adblocking/logs
L=5
LINE=--------------------------------
AL=1
printf "\nThe top $L requested domains were:\n$LINE\n"
awk '/query\[A]/ {print $6}' $dir/dnsmasq.log | awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n%s %s ",a[o],o}' | sort -nr | head -$L
printf "\nThe top $L blocked ad domains were:\n$LINE\n"
awk '/is 0.0.0.0/ {print $6}' $dir/dnsmasq.log | awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n%s %s ",a[o],o}' | sort -nr | head -$L
printf "\nThe top $L blocked custom domains were:\n$LINE\n"
awk '/is 10.0.0.1/ {print $6}' $dir/dnsmasq.log | awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n%s %s ",a[o],o}' | sort -nr | head -$L
rm -f temp*
printf "\nThe top $L noisiest name clients:\n$LINE\n"
awk '/from 192.168.1./ {print $8}' $dir/dnsmasq.log | awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n%s %s ",a[o],o}' | sort -nr | head -$L > temp1
for i in $(awk '{print $2}' temp1); do
grep -e "$i" $dir/dnsmasq.log | awk '{print $6}'| awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n %s %s",a[o],o}' | sort -nr | head -1 >> temp2
CH="$(awk 'END{print $1}' temp2)"
TH="$(awk -v AL="$AL" 'FNR==AL{print $1}' temp1)"
let AL=AL+1
awk -v CH="$CH" -v TH="$TH" 'BEGIN{print ((CH * 100)/TH)"%"}' >> temp3
done
awk 'NR==FNR{a[FNR]=$0 " -";next} {print a[FNR],$0}' temp2 temp3 > temp4
awk 'NR==FNR{a[FNR]=$0 " -";next} {print a[FNR],$0}' temp1 temp4
rm -f temp*
Example output (domains replaced with snbforums.com):
admin@AC88U:/tmp/mnt/USB/adblocking/scripts# sh log-stats.sh
The top 5 requested domains were:
--------------------------------
3914 snbforums.com
1925 snbforums.com
1700 snbforums.com
1027 snbforums.com
909 snbforums.com
The top 5 blocked ad domains were:
--------------------------------
798 snbforums.com
321 snbforums.com
317 snbforums.com
196 snbforums.com
190 snbforums.com
The top 5 blocked custom domains were:
--------------------------------
3914 snbforums.com
1700 snbforums.com
858 snbforums.com
103 snbforums.com
2 snbforums.com
The top 5 noisiest name clients:
--------------------------------
23560 192.168.1.5 - 3914 snbforums.com - 16.6129%
5048 192.168.1.15 - 1027 snbforums.com - 20.3447%
2530 192.168.1.10 - 677 snbforums.com - 26.7589%
2361 192.168.1.13 - 258 snbforums.com - 10.9276%
1651 192.168.1.9 - 190 snbforums.com - 11.5082%
All fairly self explanatory except the last section which shows:
-the total requests for each noisiest client
-the most requested domain from that client
-the number of hits for that domain from that client
-a percentage value showing how much that domain constituted of that client's overall requests
You can also change the verbosity by changing the $L variable to 10 for example to show the top 10 instead of the default top 5 for each section.
update-hosts.sh has been updated to:
Code:
#!/bin/sh
# generated by AB-Solution 1.07
dir=/tmp/mnt/USB/adblocking
ldir=/mnt/USB/adblocking/logs
# removing blank, empty and Windows EOL in white- and blacklist
sed -i '/^[[:blank:]]*$/d;s/\r$//' $dir/whitelist.txt
sed -i '/^[[:blank:]]*$/d;s/\r$//' $dir/blacklist.txt
# backup existing stats
L=5
line=______________________________
aline=***************************************************************************
dline=---------------------------
echo -e "$aline\n\n\n$aline\n$dline\n$(date)\n$dline" >> $ldir/logstats
echo -e "\nThe top $L requested domains were:\n$line\n" >> $ldir/logstats
grep -e "query\[A]" $ldir/dnsmasq.log | awk '{print $6}' | awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n%s %s ",a[o],o}' | sort -nr | head -$L >> $ldir/logstats
echo -e "\nThe top $L blocked ad domains were:\n$line\n" >> $ldir/logstats
grep -e "is 0.0.0.0" $ldir/dnsmasq.log | awk '{print $6}' | awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n%s %s ",a[o],o}' | sort -nr | head -$L >> $ldir/logstats
echo -e "\nThe top $L blocked custom domains were:\n$line\n" >> $ldir/logstats
grep -e "is 10.0.0.1" $ldir/dnsmasq.log | awk '{print $6}' | awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n%s %s ",a[o],o}' | sort -nr | head -$L >> $ldir/logstats
echo -e "\nThe top $L noisiest name clients:\n$line\n" >> $ldir/logstats
grep -e "192.168.1." $ldir/dnsmasq.log | awk '{print $8}' | awk '{for(i=1;i<=NF;i++)a[$i]++}END{for(o in a) printf "\n%s %s ",a[o],o}' | sort -nr | head -$L >> $ldir/logstats
# get hosts files and combine
SQ=1
rm -f $dir/db/names
rm -f $dir/db/*.txt
for i in $(cat $dir/db/db); do
wget -qO $dir/db/$SQ.txt $i
if [ $? -eq 0 ]; then
echo $i >> $dir/db/names
printf "$SQ - $i downloaded\n"
cat $dir/db/$SQ.txt >> $dir/temp1
let SQ=SQ+1
else
printf "$i was NOT downloaded\n"
fi
done
# amalgamate the hosts files, removing duplicates and sort in alphabetical order
printf "Files are being processed. Please wait...\n"
cat $dir/temp1 | sed 's/127.0.0.1/0.0.0.0/g;s/\r$//' | grep -w ^0.0.0.0 | awk '{print $1 " " $2"\n:: " $2}' > $dir/temp2
printf "File processing completed.\nDuplicates are being removed. Please wait...\n"
sort -u $dir/temp2 > $dir/temp3
TOTALLINES=$(wc -l < $dir/temp2)
UNIQUELINES=$(wc -l < $dir/temp3)
printf "Duplicates removed.\n"
# remove localhost and whitelisted entries
cat $dir/temp3 | sed '/\b\(localhost\|local\)\b/d;/localhost.localdomain/d' | fgrep -vf $dir/whitelist.txt > $dir/hosts-adblock
printf "Localhost and whitelisted entries removed.\n"
NUMOFLINES=$(wc -l < "$dir/hosts-adblock")
printf "\nDuplicates removed: %s" "$(($TOTALLINES - $UNIQUELINES))"
printf "\nUnique ad domains blocked: %s" "$NUMOFLINES"
#remove temp files
rm $dir/temp*
#rotate the logs when logging is enabled
if [ -f $dir/logs/dnsmasq.log ];then
mv $dir/logs/dnsmasq.log $dir/logs/dnsmasq.log.old
logger "AB-Solution rotated dnsmasq log file"
fi
#restart dnsmasq to reload the files
service restart_dnsmasq
logger "AB-Solution updated hosts file and restarted dnsmasq"
Differences compared to the last one:
- IPv6 fix added thanks t
o charlie2alpha
- Friendly messages explaining what the script is currently doing
- Cut down version of log-stats.sh that appends the output to /logs/logstats
And finally, the IPv6 fix added to update-blacklist.sh in a transparent manner so you only have to deal with a single blacklist:
Code:
dir=/tmp/mnt/USB/adblocking
sed -i '/^[[:blank:]]*$/d;s/\r$//' $dir/blacklist.txt
NUMOFLINES=$(wc -l < "$dir/hosts-adblock")
NUMOFNAMESTEMP=$NUMOFLINES
printf "Unique ad domains blocked: %s\n" "$NUMOFLINES"
NUMOFLINES=$(cat $dir/blacklist.txt | sort -u | grep -c 10.0.0.1)
printf "Unique custom domains blocked: %s" "$NUMOFLINES"
NUMOFNAMES=$(($NUMOFLINES + $NUMOFNAMESTEMP))
mv $dir/blacklist.txt $dir/tmp
awk '{print $1 " " $2"\n:: " $2}' $dir/tmp > $dir/blacklist.txt
service restart_dnsmasq
mv $dir/tmp $dir/blacklist.txt
logger "EAZ update the ad-blocker blacklist - currently $NUMOFLINES custom and $NUMOFNAMES ad domains blocked"