From 3631d1349ebe06ad804df8ca4ba008ca1b917433 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Sat, 15 Jul 2017 20:11:06 +1000 Subject: [PATCH 1/3] Prevent Web Admin from printing restartdns colour codes (#1575) * Prevent Web Admin from printing unnecessary msgs * Make DNS restart behaviour consistent --- advanced/Scripts/webpage.sh | 17 +++++++++-------- pihole | 24 ++++++++++++++++-------- 2 files changed, 25 insertions(+), 16 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 5aae18f7..42272122 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -221,18 +221,19 @@ Reboot() { } RestartDNS() { - local str="Restarting dnsmasq" - echo -ne " ${INFO} ${str}..." - if [[ -x "$(command -v systemctl)" ]]; then - systemctl restart dnsmasq + local str="Restarting DNS service" + [[ -t 1 ]] && echo -ne " ${INFO} ${str}" + if command -v systemctl &> /dev/null; then + output=$( { systemctl restart dnsmasq; } 2>&1 ) else - service dnsmasq restart + output=$( { service dnsmasq restart; } 2>&1 ) fi - if [[ "$?" == 0 ]]; then - echo -e "${OVER} ${TICK} ${str}" + if [[ -z "${output}" ]]; then + [[ -t 1 ]] && echo -e "${OVER} ${TICK} ${str}" else - echo -e "${OVER} ${CROSS} ${str}" + [[ ! -t 1 ]] && OVER="" + echo -e "${OVER} ${CROSS} ${output}" fi } diff --git a/pihole b/pihole index 3c321b93..b4b5e886 100755 --- a/pihole +++ b/pihole @@ -173,24 +173,32 @@ versionFunc() { restartDNS() { dnsmasqPid=$(pidof dnsmasq) + local str="Restarting DNS service" + echo -ne " ${INFO} ${str}" if [[ "${dnsmasqPid}" ]]; then # Service already running - reload config - echo -ne " ${INFO} Restarting dnsmasq" if [[ -x "$(command -v systemctl)" ]]; then - systemctl restart dnsmasq + output=$( { systemctl restart dnsmasq; } 2>&1 ) else - service dnsmasq restart + output=$( { service dnsmasq restart; } 2>&1 ) + fi + if [[ -z "${output}" ]]; then + echo -e "${OVER} ${TICK} ${str}" + else + echo -e "${OVER} ${CROSS} ${output}" fi - [[ "$?" == 0 ]] && echo -e "${OVER} ${TICK} Restarted dnsmasq" || echo -e "${OVER} ${CROSS} Failed to restart dnsmasq" else # Service not running, start it up - echo -ne " ${INFO} Starting dnsmasq" if [[ -x "$(command -v systemctl)" ]]; then - systemctl start dnsmasq + output=$( { systemctl start dnsmasq; } 2>&1 ) else - service dnsmasq start + output=$( { service dnsmasq start; } 2>&1 ) + fi + if [[ -z "${output}" ]]; then + echo -e "${OVER} ${TICK} ${str}" + else + echo -e "${OVER} ${CROSS} ${output}" fi - [[ "$?" == 0 ]] && echo -e "${OVER} ${TICK} Restarted dnsmasq" || echo -e "${OVER} ${CROSS} Failed to restart dnsmasq" fi } From 3a50b91722ed438e7db86e2b0a147b91642116c8 Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Sat, 15 Jul 2017 20:56:40 +1000 Subject: [PATCH 2/3] User-friendly queryFunc() output (#1483) * User-friendly queryFunc() output * Silence grep errors * Provide 'pihole -q -h' help output * Rewrite option handling * Loop through grep stdout to make query output user friendly * Add -adlist option to show block list URL instead of internal file name * Limit general searches to 10 matches per block list * Add -all option to override 10 match limit * Fixed 'pihole -h' wording * Further query optimisations * Optimised scanList() output by switching folder * Re-added processWildcards() function * Added "-bp" exact matching option for use with block page * Standardised query output * Separated wildcard search from blacklist/whitelist search * Optimised sorting by sorting glob output and not scanList() output * Fixed result skipping * Add text for wildcard result on exact query * Fix wildcard result output * Multiple wildcard matches on exact query could cause unexpected output * Remove unnecessary replacement * Make grep only output matching text * HOSTS format lists will also output the IP address * That substitution was necessary * Remove IP address from HOSTS format lists * Filter unwanted content * Add /dev/null to grep, to always print file name (even when searching only one block list) * Use three seds to remove unwanted content from block lists * Merge with development * Simplify queryFunc code --- pihole | 244 ++++++++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 206 insertions(+), 38 deletions(-) diff --git a/pihole b/pihole index b4b5e886..1bd35dbb 100755 --- a/pihole +++ b/pihole @@ -87,10 +87,14 @@ scanList(){ domain="${1}" list="${2}" method="${3}" - if [[ ${method} == "-exact" ]] ; then - grep -i -E "(^|\s)${domain}($|\s)" "${list}" + + # Switch folder, preventing grep from printing file path + cd "/etc/pihole" || return 1 + + if [[ -n "${method}" ]]; then + grep -i -E -l "(^|\s|\/)${domain}($|\s|\/)" ${list} /dev/null 2> /dev/null else - grep -i "${domain}" "${list}" + grep -i "${domain}" ${list} /dev/null 2> /dev/null fi } @@ -110,46 +114,210 @@ processWildcards() { } queryFunc() { - domain="${2}" - - if [[ -z "${domain}" ]]; then - echo -e " ${COL_LIGHT_RED}Invalid option${COL_NC} - Try 'pihole query --help' for more information." + options="$*" + options="${options/-q /}" + + if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then + echo "Usage: pihole -q [option] +Example: 'pihole -q -exact domain.com' +Query the adlists for a specified domain + +Options: + -adlist Print the name of the block list URL + -exact Search the block lists for exact domain matches + -all Return all query matches within a block list + -h, --help Show this help dialog" + exit 0 + fi + + if [[ "${options}" == *"-exact"* ]]; then + method="exact" + exact=true + fi + + if [[ "${options}" == *"-adlist"* ]]; then + adlist=true + fi + + if [[ "${options}" == *"-bp"* ]]; then + method="exact" + blockpage=true + fi + + if [[ "${options}" == *"-all"* ]]; then + all=true + fi + + # Strip valid options, leaving only the domain and invalid options + options=$(sed 's/ \?-\(exact\|adlist\|bp\|all\) \?//g' <<< "$options") + + # Handle errors + if [[ "${options}" == *" "* ]]; then + error=true + str="Unknown option specified" + elif [[ "${options}" == "-q" ]]; then + error=true + str="No domain specified" + fi + + if [[ -n "${error}" ]]; then + echo -e " ${COL_LIGHT_RED}${str}${COL_NC} + Try 'pihole -q --help' for more information." exit 1 fi - - method="${3}" - lists=( /etc/pihole/list.* /etc/pihole/blacklist.txt) - for list in ${lists[@]}; do - if [ -e "${list}" ]; then - result=$(scanList ${domain} ${list} ${method}) - # Remove empty lines before couting number of results - count=$(sed '/^\s*$/d' <<< "$result" | wc -l) - echo "${list} (${count} results)" - if [[ ${count} > 0 ]]; then - echo "${result}" - fi - echo "" - else - echo -e " ${CROSS} List does not exist" - echo "" - fi - done - # Scan for possible wildcard matches - if [ -e "${wildcardlist}" ]; then - local wildcards=($(processWildcards "${domain}")) - for domain in ${wildcards[@]}; do - result=$(scanList "\/${domain}\/" ${wildcardlist}) - # Remove empty lines before couting number of results - count=$(sed '/^\s*$/d' <<< "$result" | wc -l) - if [[ ${count} > 0 ]]; then - echo -e " ${TICK} Wildcard blocking ${domain} (${count} results)" - echo "${result}" - echo "" + # If domain contains non ASCII characters, convert domain to punycode if python is available + # Cr: https://serverfault.com/a/335079 + if [[ "$options" = *[![:ascii:]]* ]]; then + if command -v python &> /dev/null; then + query=$(python -c 'import sys;print sys.argv[1].decode("utf-8").encode("idna")' "${options}") + fi + else + query="${options}" + fi + + # Scan Whitelist and Blacklist + lists="whitelist.txt blacklist.txt" + results=($(scanList "${query}" "${lists}" "${method}")) + + if [[ -n "${results[*]}" ]]; then + # Loop through each scanList line to print appropriate title + for result in "${results[@]}"; do + filename="${result/:*/}" + if [[ -n "$exact" ]]; then + printf " Exact result in %s\n" "${filename}" + elif [[ -n "$blockpage" ]]; then + printf " [i] %s\n" "${filename}" + else + domain="${result/*:/}" + if [[ ! "${filename}" == "${filename_prev:-}" ]]; then + printf " Result from %s\n" "${filename}" + fi + printf " %s\n" "${domain}" + filename_prev="${filename}" fi done fi + + # Scan Wildcards + if [[ -e "${wildcardlist}" ]]; then + wildcards=($(processWildcards "${query}")) + + for match in "${wildcards[@]}"; do + results=($(scanList "\/${match}\/" ${wildcardlist})) + + if [[ -n "${results[*]}" ]]; then + # Remove empty lines before couting number of results + count=$(sed '/^\s*$/d' <<< "${results[@]}" | wc -l) + if [[ "${count}" -ge 0 ]]; then + blResult=true + if [[ -z "${blockpage}" ]]; then + printf " Wildcard result in %s\n" "${wildcardlist/*dnsmasq.d\/}" + fi + + if [[ -n "${blockpage}" ]]; then + echo " ${INFO} ${match}" + else + echo " *.${match}" + fi + fi + fi + done + + [[ -n "${blResult}" ]] && [[ -n "${blockpage}" ]] && exit 0 + fi + + # Glob *.domains file names, remove file paths and sort by list number + lists_raw=(/etc/pihole/*.domains) + IFS_OLD=$IFS + IFS=$'\n' + lists=$(sort -t . -k 2 -g <<< "${lists_raw[*]//\/etc\/pihole\//}") + + # Scan Domains files + results=($(scanList "${query}" "${lists}" "${method}")) + + # Handle notices + if [[ -z "${blResult}" ]] && [[ -z "${results[*]}" ]]; then + notice=true + str="No ${method/t/t }results found for ${query} found within block lists" + elif [[ -z "${all}" ]] && [[ "${#results[*]}" -ge 16000 ]]; then + # 16000 chars is 15 chars X 1000 lines worth of results + notice=true + str="Hundreds of ${method/t/t }results found for ${query} + This can be overriden using the -all option" + fi + + if [[ -n "${notice}" ]]; then + echo -e " ${INFO} ${str}" + exit + fi + + # Remove unwanted content from results + if [[ -z "${method}" ]]; then + results=($(sed "/:#/d" <<< "${results[*]}")) # Lines starting with comments + results=($(sed "s/[ \t]#.*//g" <<< "${results[*]}")) # Comments after domain + results=($(sed "s/:.*[ \t]/:/g" <<< "${results[*]}")) # IP address + fi + IFS=$IFS_OLD + + # Get adlist content as array + if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then + if [[ -f "/etc/pihole/adlists.list" ]]; then + for url in $(< /etc/pihole/adlists.list); do + if [[ "${url:0:4}" == "http" ]] || [[ "${url:0:3}" == "www" ]]; then + adlists+=("$url") + fi + done + else + echo -e " ${COL_LIGHT_RED}The file '/etc/pihole/adlists.list' was not found${COL_NC}" + exit 1 + fi + fi + + if [[ -n "${results[*]}" ]]; then + if [[ -n "${exact}" ]]; then + echo " Exact result(s) for ${query} found in:" + fi + + for result in "${results[@]}"; do + filename="${result/:*/}" + + # Convert file name to URL name for -adlist or -bp options + if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then + filenum=("${filename/list./}") + filenum=("${filenum/.*/}") + filename="${adlists[$filenum]}" + fi + + if [[ -n "${exact}" ]]; then + printf " %s\n" "${filename}" + elif [[ -n "${blockpage}" ]]; then + printf " [%s] %s\n" "${filenum}" "${filename}" + else # Standard query output + + # Print filename heading once per file, not for every match + if [[ ! "${filename}" == "${filename_prev:-}" ]]; then + unset count + printf " Result from %s\n" "${filename}" + else + let count++ + fi + + # Print matching domain if $max_count has not been reached + [[ -z "${all}" ]] && max_count="20" + if [[ -z "${all}" ]] && [[ "${count}" -eq "${max_count}" ]]; then + echo " Over $count results found, skipping rest of file" + elif [[ -z "${all}" ]] && [[ "${count}" -gt "${max_count}" ]]; then + continue + else + domain="${result/*:/}" + printf " %s\n" "${domain}" + fi + filename_prev="${filename}" + fi + done + fi + exit 0 } @@ -438,7 +606,7 @@ Options: -l, logging Specify whether the Pi-hole log should be used Add '-h' for more info on logging usage -q, query Query the adlists for a specified domain - Add '-exact' AFTER a specified domain for exact match + Add '-h' for more info on query usage -up, updatePihole Update Pi-hole subsystems -v, version Show installed versions of Pi-hole, Admin Console & FTL Add '-h' for more info on version usage From c9a98b68c8eff21e997c10f3282686ae820f833f Mon Sep 17 00:00:00 2001 From: WaLLy3K Date: Mon, 17 Jul 2017 01:44:14 +1000 Subject: [PATCH 3/3] Avoid reactivating a deactivated lighttpd service (#1485) * Do not activate disabled lighttpd upon update * Fixes #1362 * Use systemctl when available * Move `finalexports` to the very end of the install script set value of LIGHTTPD_ENABLED to 1 or 0 depending on whether or not lighttpd is enabled or disabled. actually save LIGHTTPD_ENABLED value to setupvars.conf Signed-off-by: Adam Warner * add [[ -z "${LIGHTTPD_ENABLED}" ]] back in! Signed-off-by: Adam Warner * Ensure "Loaded:" is the line being checked * Colourise disabled lighttpd message * Prevent disabled lighttpd triggering error * change of plan, don't need that [[ -z "${LIGHTTPD_ENABLED}" ]] Signed-off-by: Adam Warner --- automated install/basic-install.sh | 35 ++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 74e2a61d..a3c41b11 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1485,8 +1485,7 @@ finalExports() { # If the setup variable file exists, if [ -e "${setupVars}" ]; then - # update the variables in the file - sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1/d;/PIHOLE_DNS_2/d;/QUERY_LOGGING/d;/INSTALL_WEB/d;' "${setupVars}" + sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1/d;/PIHOLE_DNS_2/d;/QUERY_LOGGING/d;/INSTALL_WEB/d;/LIGHTTPD_ENABLED/d;' "${setupVars}" fi # echo the information to the user { @@ -1497,6 +1496,7 @@ finalExports() { echo "PIHOLE_DNS_2=${PIHOLE_DNS_2}" echo "QUERY_LOGGING=${QUERY_LOGGING}" echo "INSTALL_WEB=${INSTALL_WEB}" + echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}" }>> "${setupVars}" # Look for DNS server settings which would have to be reapplied @@ -1585,9 +1585,6 @@ installPihole() { FTLdetect || echo -e " ${CROSS} FTL Engine not installed." # Configure the firewall configureFirewall - # Run the final exports - finalExports - #runGravity } # At some point in the future this list can be pruned, for now we'll need it to ensure updates don't break. @@ -1621,8 +1618,8 @@ updatePihole() { installLogrotate # Detect if FTL is installed FTLdetect || echo -e " ${CROSS} FTL Engine not installed." - finalExports #re-export setupVars.conf to account for any new vars added in new versions - #runGravity + + } @@ -2052,10 +2049,24 @@ main() { enable_service dnsmasq # If the Web server was installed, - if [[ ${INSTALL_WEB} == true ]]; then - # enable it - start_service lighttpd - enable_service lighttpd + if [[ "${INSTALL_WEB}" == true ]]; then + # Check to see if lighttpd was already set to run on reboot + if [[ "${useUpdateVars}" == true ]]; then + if [[ -x "$(command -v systemctl)" ]]; then + # Value will either be 1, if true, or 0 + LIGHTTPD_ENABLED=$(systemctl is-enabled lighttpd | grep -c 'enabled' || true) + else + # Value will either be 1, if true, or 0 + LIGHTTPD_ENABLED=$(service lighttpd status | awk '/Loaded:/ {print $0}' | grep -c 'enabled' || true) + fi + fi + + if [[ "${LIGHTTPD_ENABLED}" == "1" ]]; then + start_service lighttpd + enable_service lighttpd + else + echo -e " ${INFO} Lighttpd is disabled, skipping service restart" + fi fi # Download and compile the aggregated block list @@ -2103,6 +2114,8 @@ main() { # Display where the log file is echo -e "\n ${INFO} The install log is located at: /etc/pihole/install.log ${COL_LIGHT_GREEN}${INSTALL_TYPE} Complete! ${COL_NC}" + #update setupvars.conf with any variables that may or may not have been changed during the install + finalExports } #