tyler.durden
/
pi-hole
mirror of https://github.com/pi-hole/pi-hole.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #180 from jacobsalmela/whitelist 

Whitelist/blacklist improvements
This commit is contained in:
Jacob Salmela 2016-01-16 12:24:43 -06:00
parent 9ff1129510 87980c6df8
commit a2a58f7b39
4 changed files with 368 additions and 74 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

182
advanced/Scripts/blacklist.sh Normal file
View File

@ -0,0 +1,182 @@
#!/usr/bin/env bash
# (c) 2015 by Jacob Salmela
# This file is part of Pi-hole.
#
# Pi-hole is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
if [[ $# = 0 ]]; then
echo "Immediately blacklists one or more domains in the hosts file"
echo " "
echo "Usage: blacklist.sh domain1 [domain2 ...]"
echo " "
echo "Options:"
echo " -d, --delmode Remove domains from the blacklist"
echo " -nr, --noreload Update blacklist without refreshing dnsmasq"
echo " -f, --force Force updating of the hosts files, even if there are no changes"
echo " -q, --quiet output is less verbose"
exit 1
fi
#globals
blacklist=/etc/pihole/blacklist.txt
adList=/etc/pihole/gravity.list
reload=true
addmode=true
force=false
versbose=true
domList=()
domToRemoveList=()
piholeIPfile=/tmp/piholeIP
piholeIPv6file=/etc/pihole/.useIPv6
# Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script
IPv4dev=$(ip route get 8.8.8.8 | awk '{for(i=1;i<=NF;i++)if($i~/dev/)print $(i+1)}')
piholeIPCIDR=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}' | awk 'END {print}')
piholeIP=${piholeIPCIDR%/*}
modifyHost=false
if [[ -f $piholeIPv6file ]];then
# If the file exists, then the user previously chose to use IPv6 in the automated installer
piholeIPv6=$(ip -6 route get 2001:4860:4860::8888 | awk -F " " '{ for(i=1;i<=NF;i++) if ($i == "src") print $(i+1) }')
fi
function HandleOther(){
#check validity of domain
validDomain=$(echo $1 | perl -ne'print if /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/')
if [ -z "$validDomain" ]; then
echo $1 is not a valid argument or domain name
else
domList=("${domList[@]}" $validDomain)
fi
}
function PopBlacklistFile(){
#check blacklist file exists, and if not, create it
if [[ ! -f $blacklist ]];then
touch $blacklist
fi
for dom in "${domList[@]}"
do
if $addmode; then
AddDomain $dom
else
RemoveDomain $dom
fi
done
}
function AddDomain(){
#| sed 's/\./\\./g'
bool=false
grep -Ex -q "$1" $blacklist || bool=true
if $bool; then
#domain not found in the blacklist file, add it!
if $versbose; then
echo "** Adding $1 to blacklist file"
fi
echo $1 >> $blacklist
modifyHost=true
else
if $versbose; then
echo "** $1 already blacklisted! No need to add"
fi
fi
}
function RemoveDomain(){
bool=false
grep -Ex -q "$1" $blacklist || bool=true
if $bool; then
#Domain is not in the blacklist file, no need to Remove
if $versbose; then
echo "** $1 is NOT blacklisted! No need to remove"
fi
else
#Domain is in the blacklist file, add to a temporary array
if $versbose; then
echo "** Un-blacklisting $dom..."
fi
domToRemoveList=("${domToRemoveList[@]}" $1)
modifyHost=true
fi
}
function ModifyHostFile(){
if $addmode; then
#add domains to the hosts file
if [[ -r $blacklist ]];then
numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l)
plural=; [[ "$numberOf" != "1" ]] && plural=s
echo "** blacklisting a total of $numberOf domain${plural}..."
if [[ -n $piholeIPv6 ]];then
cat $blacklist | awk -v ipv4addr="$piholeIP" -v ipv6addr="$piholeIPv6" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> $adList
else
cat $blacklist | awk -v ipv4addr="$piholeIP" '{sub(/\r$/,""); print ipv4addr" "$0}' >>$adList
fi
fi
else
for dom in "${domToRemoveList[@]}"
do
#we need to remove the domains from the blacklist file and the host file
echo $dom | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /[^.]'{}'(?!.)/;' $adList
echo $dom | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /'{}'(?!.)/;' $blacklist
done
fi
}
function Reload() {
# Reload hosts file
echo "** Refresh lists in dnsmasq..."
dnsmasqPid=$(pidof dnsmasq)
if [[ $dnsmasqPid ]]; then
# service already running - reload config
sudo kill -HUP $dnsmasqPid
else
# service not running, start it up
sudo service dnsmasq start
fi
}
###################################################
for var in "$@"
do
case "$var" in
"-nr"| "--noreload" ) reload=false;;
"-d" | "--delmode" ) addmode=false;;
"-f" | "--force" ) force=true;;
"-q" | "--quiet" ) versbose=false;;
* ) HandleOther $var;;
esac
done
PopBlacklistFile
if $modifyHost || $force; then
echo "** Modifying Hosts File"
ModifyHostFile
else
if $versbose; then
echo "** No changes need to be made"
fi
exit 1
fi
if $reload; then
Reload
fi

195
advanced/Scripts/whitelist.sh
View File

@ -7,51 +7,174 @@
# the Free Software Foundation, either version 2 of the License, or
# (at your option) any later version.
if [[ $# = 0 ]]; then
echo "Immediately whitelists one or more domains in the hosts file"
echo " "
echo "Usage: whitelist.sh domain1 [domain2 ...]"
echo " "
echo "Options:"
echo " -d, --delmode Remove domains from the whitelist"
echo " -nr, --noreload Update Whitelist without refreshing dnsmasq"
echo " -f, --force Force updating of the hosts files, even if there are no changes"
echo " -q, --quiet output is less verbose"
exit 1
fi
#globals
whitelist=/etc/pihole/whitelist.txt
adList=/etc/pihole/gravity.list
if [[ ! -f $whitelist ]];then
touch $whitelist
reload=true
addmode=true
force=false
versbose=true
domList=()
domToRemoveList=()
piholeIPfile=/tmp/piholeIP
piholeIPv6file=/etc/pihole/.useIPv6
# Otherwise, the IP address can be taken directly from the machine, which will happen when the script is run by the user and not the installation script
IPv4dev=$(ip route get 8.8.8.8 | awk '{for(i=1;i<=NF;i++)if($i~/dev/)print $(i+1)}')
piholeIPCIDR=$(ip -o -f inet addr show dev $IPv4dev | awk '{print $4}' | awk 'END {print}')
piholeIP=${piholeIPCIDR%/*}
modifyHost=false
if [[ -f $piholeIPv6file ]];then
# If the file exists, then the user previously chose to use IPv6 in the automated installer
piholeIPv6=$(ip -6 route get 2001:4860:4860::8888 | awk -F " " '{ for(i=1;i<=NF;i++) if ($i == "src") print $(i+1) }')
fi
if [[ $# = 0 ]]; then
echo "Immediately whitelists one or more domains."
echo "Usage: whitelist.sh domain1 [domain2 ...]"
fi
combopattern=""
function HandleOther(){
#check validity of domain
validDomain=$(echo $1 | perl -ne'print if /\b((?=[a-z0-9-]{1,63}\.)(xn--)?[a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,63}\b/')
if [ -z "$validDomain" ]; then
echo $1 is not a valid argument or domain name
else
domList=("${domList[@]}" $validDomain)
fi
}
function PopWhitelistFile(){
#check whitelist file exists, and if not, create it
if [[ ! -f $whitelist ]];then
touch $whitelist
fi
for dom in "${domList[@]}"
do
if $addmode; then
AddDomain $dom
else
RemoveDomain $dom
fi
done
}
function AddDomain(){
#| sed 's/\./\\./g'
bool=false
grep -Ex -q "$1" $whitelist || bool=true
if $bool; then
#domain not found in the whitelist file, add it!
if $versbose; then
echo "** Adding $1 to whitelist file"
fi
echo $1 >> $whitelist
modifyHost=true
else
if $versbose; then
echo "** $1 already whitelisted! No need to add"
fi
fi
}
function RemoveDomain(){
bool=false
grep -Ex -q "$1" $whitelist || bool=true
if $bool; then
#Domain is not in the whitelist file, no need to Remove
if $versbose; then
echo "** $1 is NOT whitelisted! No need to remove"
fi
else
#Domain is in the whitelist file, add to a temporary array and remove from whitelist file
if $versbose; then
echo "** Un-whitelisting $dom..."
fi
domToRemoveList=("${domToRemoveList[@]}" $1)
modifyHost=true
fi
}
function ModifyHostFile(){
if $addmode; then
#remove domains in from hosts file
if [[ -r $whitelist ]];then
# Remove whitelist entries
numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l)
plural=; [[ "$numberOf" != "1" ]] && plural=s
echo "** Whitelisting a total of $numberOf domain${plural}..."
awk -F':' '{ print $1 }' $whitelist | sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /[^.]'{}'(?!.)/;' $adList
fi
else
#we need to add the removed domains to the hosts file
for rdom in "${domToRemoveList[@]}"
do
if [[ -n $piholeIPv6 ]];then
echo "**Blacklisting $rdom on IPv4 and IPv6"
echo $rdom | awk -v ipv4addr="$piholeIP" -v ipv6addr="$piholeIPv6" '{sub(/\r$/,""); print ipv4addr" "$0"\n"ipv6addr" "$0}' >> $adList
else
echo "**Blacklisting $rdom on IPv4"
echo $rdom | awk -v ipv4addr="$piholeIP" '{sub(/\r$/,""); print ipv4addr" "$0}' >>$adList
fi
echo $rdom| sed 's/\./\\./g' | xargs -I {} perl -i -ne'print unless /'{}'(?!.)/;' $whitelist
done
fi
}
function Reload() {
# Reload hosts file
echo "** Refresh lists in dnsmasq..."
dnsmasqPid=$(pidof dnsmasq)
if [[ $dnsmasqPid ]]; then
# service already running - reload config
sudo kill -HUP $dnsmasqPid
else
# service not running, start it up
sudo service dnsmasq start
fi
}
###################################################
# For each argument passed to this script
for var in "$@"
do
echo "Whitelisting $var..."
# Construct basic pattern to match domain name.
basicpattern=$(echo $var | awk -F '[# \t]' 'NF>0&&$1!="" {print ""$1""}' | sed 's/\./\\./g')
if [[ "$basicpattern" != "" ]]; then
# Add to the combination pattern that will be used below
if [[ "$combopattern" != "" ]]; then combopattern="$combopattern|"; fi
combopattern="$combopattern$basicpattern"
# Also add the domain to the whitelist but only if it's not already present
grep -E -q "^$basicpattern$" $whitelist \
|| echo "$var" >> $whitelist
fi
case "$var" in
"-nr"| "--noreload" ) reload=false;;
"-d" | "--delmode" ) addmode=false;;
"-f" | "--force" ) force=true;;
"-q" | "--quiet" ) versbose=false;;
* ) HandleOther $var;;
esac
done
# Now report on and remove matched domains
if [[ "$combopattern" != "" ]]; then
echo "Modifying hosts file..."
PopWhitelistFile
# Construct pattern to match entry in hosts file.
# This consists of one or more IP addresses followed by the domain name.
pattern=$(echo $combopattern | awk -F '[# \t]' '{printf "%s", "^(([0-9]+\.){3}[0-9]+ +)+("$1")$"}')
# Output what will be removed and then actually remove
sed -r -n 's/'"$pattern"'/ Removed: \3/p' $adList
sed -r -i '/'"$pattern"'/d' $adList
echo "** $# domain(s) whitelisted."
# Force dnsmasq to reload /etc/pihole/gravity.list
kill -HUP $(pidof dnsmasq)
if $modifyHost || $force; then
echo "** Modifying Hosts File"
ModifyHostFile
else
if $versbose; then
echo "** No changes need to be made"
exit 1
fi
fi
if $reload; then
Reload
fi

3
automated install/basic-install.sh
View File

@ -207,8 +207,9 @@ installScripts(){
sudo curl -o /usr/local/bin/gravity.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/gravity.sh
sudo curl -o /usr/local/bin/chronometer.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/chronometer.sh
sudo curl -o /usr/local/bin/whitelist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/whitelist.sh
sudo curl -o /usr/local/bin/blacklist.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/blacklist.sh
sudo curl -o /usr/local/bin/piholeLogFlush.sh https://raw.githubusercontent.com/jacobsalmela/pi-hole/master/advanced/Scripts/piholeLogFlush.sh
sudo chmod 755 /usr/local/bin/{gravity,chronometer,whitelist,piholeLogFlush}.sh
sudo chmod 755 /usr/local/bin/{gravity,chronometer,whitelist,blacklist,piholeLogFlush}.sh
}
installConfigs(){

62
gravity.sh
View File

@ -48,12 +48,11 @@ blacklist=$piholeDir/blacklist.txt
whitelist=$piholeDir/whitelist.txt
latentWhitelist=$piholeDir/latentWhitelist.txt
justDomainsExtension=domains
matter=$basename.0.matter.txt
andLight=$basename.1.andLight.txt
supernova=$basename.2.supernova.txt
eventHorizon=$basename.3.eventHorizon.txt
accretionDisc=$basename.4.accretionDisc.txt
eyeOfTheNeedle=$basename.5.wormhole.txt
matterandlight=$basename.0.matterandlight.txt
supernova=$basename.1.supernova.txt
eventHorizon=$basename.2.eventHorizon.txt
accretionDisc=$basename.3.accretionDisc.txt
eyeOfTheNeedle=$basename.4.wormhole.txt
# After setting defaults, check if there's local overrides
if [[ -r $piholeDir/pihole.conf ]];then
@ -160,48 +159,35 @@ function gravity_Schwarzchild() {
# Find all active domains and compile them into one file and remove CRs
echo "** Aggregating list of domains..."
truncate -s 0 $piholeDir/$matter
truncate -s 0 $piholeDir/$matterandlight
for i in "${activeDomains[@]}"
do
cat $i |tr -d '\r' >> $piholeDir/$matter
cat $i |tr -d '\r' >> $piholeDir/$matterandlight
done
}
# Pulsar - White/blacklist application
function gravity_pulsar() {
function gravity_Blacklist(){
# Append blacklist entries if they exist
if [[ -r $blacklist ]];then
numberOf=$(cat $blacklist | sed '/^\s*$/d' | wc -l)
echo "** Blacklisting $numberOf domain(s)..."
cat $blacklist >> $piholeDir/$matter
fi
blacklist.sh -f -nr -q
}
# Whitelist (if applicable) domains
if [[ -r $whitelist ]];then
# Remove whitelist entries
numberOf=$(cat $whitelist | sed '/^\s*$/d' | wc -l)
plural=; [[ "$numberOf" != "1" ]] && plural=s
echo "** Whitelisting $numberOf domain${plural}..."
# Append a "$" to the end, prepend a "^" to the beginning, and
# replace "." with "\." of each line to turn each entry into a
# regexp so it can be parsed out with grep -x
awk -F '[# \t]' 'NF>0&&$1!="" {print "^"$1"$"}' $whitelist | sed 's/\./\\./g' > $latentWhitelist
else
rm $latentWhitelist 2>/dev/null
fi
function gravity_Whitelist() {
# Prevent our sources from being pulled into the hole
plural=; [[ "${#sources[@]}" != "1" ]] && plural=s
plural=; [[ "${sources[@]}" != "1" ]] && plural=s
echo "** Whitelisting ${#sources[@]} ad list source${plural}..."
urls=()
for url in ${sources[@]}
do
echo "$url" | awk -F '/' '{print "^"$3"$"}' | sed 's/\./\\./g' >> $latentWhitelist
tmp=$(echo "$url" | awk -F '/' '{print $3}')
urls=("${urls[@]}" $tmp)
done
whitelist.sh -f -nr -q ${urls[@]}
# Remove whitelist entries from list
grep -vxf $latentWhitelist $piholeDir/$matter > $piholeDir/$andLight
}
function gravity_unique() {
@ -240,12 +226,13 @@ function gravity_blackbody() {
}
function gravity_advanced() {
# Remove comments and print only the domain name
# Most of the lists downloaded are already in hosts file format but the spacing/formating is not contigious
# This helps with that and makes it easier to read
# It also helps with debugging so each stage of the script can be researched more in depth
awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$andLight | \
sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova
awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' $piholeDir/$matterandlight | sed -nr -e 's/\.{2,}/./g' -e '/\./p' > $piholeDir/$supernova
numberOf=$(wc -l < $piholeDir/$supernova)
echo "** $numberOf domains being pulled in by gravity..."
@ -256,7 +243,6 @@ function gravity_advanced() {
function gravity_reload() {
# Reload hosts file
echo "** Refresh lists in dnsmasq..."
dnsmasqPid=$(pidof dnsmasq)
if [[ $dnsmasqPid ]]; then
@ -268,11 +254,13 @@ function gravity_reload() {
fi
}
gravity_collapse
gravity_spinup
gravity_Schwarzchild
gravity_pulsar
gravity_advanced
gravity_hostFormat
gravity_blackbody
gravity_Whitelist
gravity_Blacklist
gravity_reload