Merge branch 'release/v5.4' into development

advanced/06-rfc6761.conf

@@ -0,0 +1,41 @@
+# Pi-hole: A black hole for Internet advertisements
+# (c) 2021 Pi-hole, LLC (https://pi-hole.net)
+# Network-wide ad blocking via your own hardware.
+#
+# RFC 6761 config file for Pi-hole
+#
+# This file is copyright under the latest version of the EUPL.
+# Please see LICENSE file for your rights under this license.
+
+###############################################################################
+#      FILE AUTOMATICALLY POPULATED BY PI-HOLE INSTALL/UPDATE PROCEDURE.      #
+# ANY CHANGES MADE TO THIS FILE AFTER INSTALL WILL BE LOST ON THE NEXT UPDATE #
+#                                                                             #
+#             CHANGES SHOULD BE MADE IN A SEPARATE CONFIG FILE                #
+#                    WITHIN /etc/dnsmasq.d/yourname.conf                      #
+###############################################################################
+
+# RFC 6761: Caching DNS servers SHOULD recognize
+#     test, localhost, invalid
+# names as special and SHOULD NOT attempt to look up NS records for them, or
+# otherwise query authoritative DNS servers in an attempt to resolve these
+# names.
+server=/test/
+server=/localhost/
+server=/invalid/
+
+# The same RFC requests something similar for
+#     16.172.in-addr.arpa.  22.172.in-addr.arpa.  27.172.in-addr.arpa.
+#     17.172.in-addr.arpa.  30.172.in-addr.arpa.  28.172.in-addr.arpa.
+#     18.172.in-addr.arpa.  23.172.in-addr.arpa.  29.172.in-addr.arpa.
+#     19.172.in-addr.arpa.  24.172.in-addr.arpa.  31.172.in-addr.arpa.
+#     20.172.in-addr.arpa.  25.172.in-addr.arpa.  168.192.in-addr.arpa.
+# Pi-hole implements this via the dnsmasq option "bogus-priv" (see
+# 01-pihole.conf) because this also covers IPv6.
+
+# OpenWRT furthermore blocks    bind, local, onion    domains
+# see https://git.openwrt.org/?p=openwrt/openwrt.git;a=blob_plain;f=package/network/services/dnsmasq/files/rfc6761.conf;hb=HEAD
+# and https://www.iana.org/assignments/special-use-domain-names/special-use-domain-names.xhtml
+# We do not include the ".local" rule ourselves, see https://github.com/pi-hole/pi-hole/pull/4282#discussion_r689112972
+server=/bind/
+server=/onion/

advanced/Scripts/piholeDebug.sh

@@ -73,15 +73,12 @@ HTML_DIRECTORY="/var/www/html"
 WEB_GIT_DIRECTORY="${HTML_DIRECTORY}/admin"
 #BLOCK_PAGE_DIRECTORY="${HTML_DIRECTORY}/pihole"
 SHM_DIRECTORY="/dev/shm"
+ETC="/etc"
 
 # Files required by Pi-hole
 # https://discourse.pi-hole.net/t/what-files-does-pi-hole-use/1684
 PIHOLE_CRON_FILE="${CRON_D_DIRECTORY}/pihole"
 
-PIHOLE_DNS_CONFIG_FILE="${DNSMASQ_D_DIRECTORY}/01-pihole.conf"
-PIHOLE_DHCP_CONFIG_FILE="${DNSMASQ_D_DIRECTORY}/02-pihole-dhcp.conf"
-PIHOLE_WILDCARD_CONFIG_FILE="${DNSMASQ_D_DIRECTORY}/03-wildcard.conf"
-
 WEB_SERVER_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/lighttpd.conf"
 WEB_SERVER_CUSTOM_CONFIG_FILE="${WEB_SERVER_CONFIG_DIRECTORY}/external.conf"
 
@@ -136,6 +133,9 @@ PIHOLE_FTL_LOG="$(get_ftl_conf_value "LOGFILE" "${LOG_DIRECTORY}/pihole-FTL.log"
 PIHOLE_WEB_SERVER_ACCESS_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/access.log"
 PIHOLE_WEB_SERVER_ERROR_LOG_FILE="${WEB_SERVER_LOG_DIRECTORY}/error.log"
 
+RESOLVCONF="${ETC}/resolv.conf"
+DNSMASQ_CONF="${ETC}/dnsmasq.conf"
+
 # An array of operating system "pretty names" that we officially support
 # We can loop through the array at any time to see if it matches a value
 #SUPPORTED_OS=("Raspbian" "Ubuntu" "Fedora" "Debian" "CentOS")
@@ -160,9 +160,6 @@ PIHOLE_PROCESSES=( "lighttpd" "pihole-FTL" )
 
 # Store the required directories in an array so it can be parsed through
 REQUIRED_FILES=("${PIHOLE_CRON_FILE}"
-"${PIHOLE_DNS_CONFIG_FILE}"
-"${PIHOLE_DHCP_CONFIG_FILE}"
-"${PIHOLE_WILDCARD_CONFIG_FILE}"
 "${WEB_SERVER_CONFIG_FILE}"
 "${WEB_SERVER_CUSTOM_CONFIG_FILE}"
 "${PIHOLE_INSTALL_LOG_FILE}"
@@ -180,7 +177,9 @@ REQUIRED_FILES=("${PIHOLE_CRON_FILE}"
 "${PIHOLE_DEBUG_LOG}"
 "${PIHOLE_FTL_LOG}"
 "${PIHOLE_WEB_SERVER_ACCESS_LOG_FILE}"
-"${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}")
+"${PIHOLE_WEB_SERVER_ERROR_LOG_FILE}"
+"${RESOLVCONF}"
+"${DNSMASQ_CONF}")
 
 DISCLAIMER="This process collects information from your Pi-hole, and optionally uploads it to a unique and random directory on tricorder.pi-hole.net.
 
@@ -1074,12 +1073,16 @@ list_files_in_dir() {
         elif [[ "${dir_to_parse}" == "${SHM_DIRECTORY}" ]]; then
             # SHM file - we do not want to see the content, but we want to see the files and their sizes
             log_write "$(ls -lhd "${dir_to_parse}"/"${each_file}")"
+        elif [[ "${dir_to_parse}" == "${DNSMASQ_D_DIRECTORY}" ]]; then
+            # in case of the dnsmasq directory inlcuede all files in the debug output
+            log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}"
+            make_array_from_file "${dir_to_parse}/${each_file}"
         else
             # Then, parse the file's content into an array so each line can be analyzed if need be
             for i in "${!REQUIRED_FILES[@]}"; do
                 if [[ "${dir_to_parse}/${each_file}" == "${REQUIRED_FILES[$i]}" ]]; then
                     # display the filename
-                    log_write "\\n${COL_GREEN}$(ls -ld "${dir_to_parse}"/"${each_file}")${COL_NC}"
+                    log_write "\\n${COL_GREEN}$(ls -lhd "${dir_to_parse}"/"${each_file}")${COL_NC}"
                     # Check if the file we want to view has a limit (because sometimes we just need a little bit of info from the file, not the entire thing)
                     case "${dir_to_parse}/${each_file}" in
                         # If it's Web server error log, give the first and last 25 lines
@@ -1118,6 +1121,7 @@ show_content_of_pihole_files() {
     show_content_of_files_in_dir "${WEB_SERVER_LOG_DIRECTORY}"
     show_content_of_files_in_dir "${LOG_DIRECTORY}"
     show_content_of_files_in_dir "${SHM_DIRECTORY}"
+    show_content_of_files_in_dir "${ETC}"
 }
 
 head_tail_log() {
@@ -1333,11 +1337,15 @@ analyze_pihole_log() {
 curl_to_tricorder() {
     # Users can submit their debug logs using curl (encrypted)
     log_write "    * Using ${COL_GREEN}curl${COL_NC} for transmission."
-    # transmit he log via TLS and store the token returned in a variable
+    # transmit the log via TLS and store the token returned in a variable
-    tricorder_token=$(curl --silent --upload-file ${PIHOLE_DEBUG_LOG} https://tricorder.pi-hole.net)
+    tricorder_token=$(curl --silent --fail --show-error --upload-file ${PIHOLE_DEBUG_LOG} https://tricorder.pi-hole.net 2>&1)
-    if [ -z "${tricorder_token}" ]; then
+    if [[ "${tricorder_token}" != "https://tricorder.pi-hole.net/"* ]]; then
-        # curl failed, fallback to nc
         log_write "    * ${COL_GREEN}curl${COL_NC} failed, contact Pi-hole support for assistance."
+        # Log curl error (if available)
+        if [ -n "${tricorder_token}" ]; then
+            log_write "    * Error message: ${COL_RED}${tricorder_token}${COL_NC}\\n"
+            tricorder_token=""
+        fi
     fi
 }
 
@@ -1382,15 +1390,14 @@ upload_to_tricorder() {
         # Again, try to make this visually striking so the user realizes they need to do something with this information
         # Namely, provide the Pi-hole devs with the token
         log_write ""
-        log_write "${COL_PURPLE}***********************************${COL_NC}"
+        log_write "${COL_PURPLE}*****************************************************************${COL_NC}"
-        log_write "${COL_PURPLE}***********************************${COL_NC}"
+        log_write "${COL_PURPLE}*****************************************************************${COL_NC}\\n"
         log_write "${TICK} Your debug token is: ${COL_GREEN}${tricorder_token}${COL_NC}"
-        log_write "${INFO}${COL_RED} Logs are deleted 48 hours after upload.${COL_NC}"
+        log_write "${INFO}${COL_RED} Logs are deleted 48 hours after upload.${COL_NC}\\n"
-        log_write "${COL_PURPLE}***********************************${COL_NC}"
+        log_write "${COL_PURPLE}*****************************************************************${COL_NC}"
-        log_write "${COL_PURPLE}***********************************${COL_NC}"
+        log_write "${COL_PURPLE}*****************************************************************${COL_NC}"
         log_write ""
-        log_write "   * Provide the token above to the Pi-hole team for assistance at"
+        log_write "   * Provide the token above to the Pi-hole team for assistance at ${FORUMS_URL}"
-        log_write "   * ${FORUMS_URL}"
 
     # If no token was generated
     else

advanced/Scripts/webpage.sh

@@ -54,7 +54,7 @@ add_setting() {
 }
 
 delete_setting() {
-    sed -i "/${1}/d" "${setupVars}"
+    sed -i "/^${1}/d" "${setupVars}"
 }
 
 change_setting() {
@@ -67,7 +67,7 @@ addFTLsetting() {
 }
 
 deleteFTLsetting() {
-    sed -i "/${1}/d" "${FTLconf}"
+    sed -i "/^${1}/d" "${FTLconf}"
 }
 
 changeFTLsetting() {
@@ -84,7 +84,7 @@ add_dnsmasq_setting() {
 }
 
 delete_dnsmasq_setting() {
-    sed -i "/${1}/d" "${dnsmasqconfig}"
+    sed -i "/^${1}/d" "${dnsmasqconfig}"
 }
 
 SetTemperatureUnit() {
@@ -267,11 +267,21 @@ trust-anchor=.,20326,8,2,E06D44B80B8F1D39A95C0B0D7C65D08458E880409BBC68345710423
         delete_setting "CONDITIONAL_FORWARDING_IP"
     fi
 
+    delete_dnsmasq_setting "rev-server"
+
     if [[ "${REV_SERVER}" == true ]]; then
         add_dnsmasq_setting "rev-server=${REV_SERVER_CIDR},${REV_SERVER_TARGET}"
         if [ -n "${REV_SERVER_DOMAIN}" ]; then
+            # Forward local domain names to the CF target, too
             add_dnsmasq_setting "server=/${REV_SERVER_DOMAIN}/${REV_SERVER_TARGET}"
         fi
+
+        if [[ "${DNS_FQDN_REQUIRED}" != true ]]; then
+            # Forward unqualified names to the CF target only when the "never
+            # forward non-FQDN" option is unticked
+            add_dnsmasq_setting "server=//${REV_SERVER_TARGET}"
+        fi
+
     fi
 
     # We need to process DHCP settings here as well to account for possible

advanced/Templates/pihole-FTL.service

@@ -27,7 +27,7 @@ start() {
     chmod 0644 /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases /etc/pihole/macvendor.db
     # Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist
     chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db
-    if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE+eip "/usr/bin/pihole-FTL"; then
+    if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then
       su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole
     else
       echo "Warning: Starting pihole-FTL as root because setting capabilities is not supported on this system"

automated install/basic-install.sh

@@ -1247,8 +1247,10 @@ version_check_dnsmasq() {
     local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list"
     local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS"
     local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original"
-    local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf"
+    local dnsmasq_pihole_01_source="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf"
-    local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf"
+    local dnsmasq_pihole_01_target="/etc/dnsmasq.d/01-pihole.conf"
+    local dnsmasq_rfc6761_06_source="${PI_HOLE_LOCAL_REPO}/advanced/06-rfc6761.conf"
+    local dnsmasq_rfc6761_06_target="/etc/dnsmasq.d/06-rfc6761.conf"
 
     # If the dnsmasq config file exists
     if [[ -f "${dnsmasq_conf}" ]]; then
@@ -1277,44 +1279,48 @@ version_check_dnsmasq() {
         printf "%b  %b No dnsmasq.conf found... restoring default dnsmasq.conf...\\n" "${OVER}"  "${TICK}"
     fi
 
-    printf "  %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf..." "${INFO}"
+    printf "  %b Installing %s..." "${INFO}" "${dnsmasq_pihole_01_target}"
     # Check to see if dnsmasq directory exists (it may not due to being a fresh install and dnsmasq no longer being a dependency)
     if [[ ! -d "/etc/dnsmasq.d"  ]];then
         install -d -m 755 "/etc/dnsmasq.d"
     fi
     # Copy the new Pi-hole DNS config file into the dnsmasq.d directory
-    install -D -m 644 -T "${dnsmasq_pihole_01_snippet}" "${dnsmasq_pihole_01_location}"
+    install -D -m 644 -T "${dnsmasq_pihole_01_source}" "${dnsmasq_pihole_01_target}"
-    printf "%b  %b Copying 01-pihole.conf to /etc/dnsmasq.d/01-pihole.conf\\n" "${OVER}"  "${TICK}"
+    printf "%b  %b Installed %s\n" "${OVER}"  "${TICK}" "${dnsmasq_pihole_01_target}"
     # Replace our placeholder values with the GLOBAL DNS variables that we populated earlier
     # First, swap in the interface to listen on,
-    sed -i "s/@INT@/$PIHOLE_INTERFACE/" "${dnsmasq_pihole_01_location}"
+    sed -i "s/@INT@/$PIHOLE_INTERFACE/" "${dnsmasq_pihole_01_target}"
     if [[ "${PIHOLE_DNS_1}" != "" ]]; then
         # then swap in the primary DNS server.
-        sed -i "s/@DNS1@/$PIHOLE_DNS_1/" "${dnsmasq_pihole_01_location}"
+        sed -i "s/@DNS1@/$PIHOLE_DNS_1/" "${dnsmasq_pihole_01_target}"
     else
         # Otherwise, remove the line which sets DNS1.
-        sed -i '/^server=@DNS1@/d' "${dnsmasq_pihole_01_location}"
+        sed -i '/^server=@DNS1@/d' "${dnsmasq_pihole_01_target}"
     fi
     # Ditto if DNS2 is not empty
     if [[ "${PIHOLE_DNS_2}" != "" ]]; then
-        sed -i "s/@DNS2@/$PIHOLE_DNS_2/" "${dnsmasq_pihole_01_location}"
+        sed -i "s/@DNS2@/$PIHOLE_DNS_2/" "${dnsmasq_pihole_01_target}"
     else
-        sed -i '/^server=@DNS2@/d' "${dnsmasq_pihole_01_location}"
+        sed -i '/^server=@DNS2@/d' "${dnsmasq_pihole_01_target}"
     fi
 
 	# Set the cache size
-	sed -i "s/@CACHE_SIZE@/$CACHE_SIZE/" ${dnsmasq_pihole_01_location}
+	sed -i "s/@CACHE_SIZE@/$CACHE_SIZE/" "${dnsmasq_pihole_01_target}"
 
     sed -i 's/^#conf-dir=\/etc\/dnsmasq.d$/conf-dir=\/etc\/dnsmasq.d/' "${dnsmasq_conf}"
 
     # If the user does not want to enable logging,
     if [[ "${QUERY_LOGGING}" == false ]] ; then
         # disable it by commenting out the directive in the DNS config file
-        sed -i 's/^log-queries/#log-queries/' "${dnsmasq_pihole_01_location}"
+        sed -i 's/^log-queries/#log-queries/' "${dnsmasq_pihole_01_target}"
     else
         # Otherwise, enable it by uncommenting the directive in the DNS config file
-        sed -i 's/^#log-queries/log-queries/' "${dnsmasq_pihole_01_location}"
+        sed -i 's/^#log-queries/log-queries/' "${dnsmasq_pihole_01_target}"
     fi
+
+    printf "  %b Installing %s..." "${INFO}" "${dnsmasq_rfc6761_06_source}"
+    install -D -m 644 -T "${dnsmasq_rfc6761_06_source}" "${dnsmasq_rfc6761_06_target}"
+    printf "%b  %b Installed %s\n" "${OVER}"  "${TICK}" "${dnsmasq_rfc6761_06_target}"
 }
 
 # Clean an existing installation to prepare for upgrade/reinstall
@@ -1578,7 +1584,7 @@ update_package_cache() {
     else
         # Otherwise, show an error and exit
         printf "%b  %b %s\\n" "${OVER}" "${CROSS}" "${str}"
-        printf "  %bError: Unable to update package cache. Please try \"%s\"%b" "${COL_LIGHT_RED}" "${UPDATE_PKG_CACHE}" "${COL_NC}"
+        printf "  %bError: Unable to update package cache. Please try \"%s\"%b" "${COL_LIGHT_RED}" "sudo ${UPDATE_PKG_CACHE}" "${COL_NC}"
         return 1
     fi
 }

automated install/uninstall.sh

@@ -145,6 +145,7 @@ removeNoPurge() {
 
     ${SUDO} rm -f /etc/dnsmasq.d/adList.conf &> /dev/null
     ${SUDO} rm -f /etc/dnsmasq.d/01-pihole.conf &> /dev/null
+    ${SUDO} rm -f /etc/dnsmasq.d/06-rfc6761.conf &> /dev/null
     ${SUDO} rm -rf /var/log/*pihole* &> /dev/null
     ${SUDO} rm -rf /etc/pihole/ &> /dev/null
     ${SUDO} rm -rf /etc/.pihole/ &> /dev/null

pihole

@@ -399,34 +399,24 @@ Branches:
 }
 
 tricorderFunc() {
+  local tricorder_token
   if [[ ! -p "/dev/stdin" ]]; then
     echo -e "  ${INFO} Please do not call Tricorder directly"
     exit 1
   fi
 
-  if ! (echo > /dev/tcp/tricorder.pi-hole.net/9998) >/dev/null 2>&1; then
+  tricorder_token=$(curl --silent --fail --show-error --upload-file "-" https://tricorder.pi-hole.net/upload < /dev/stdin 2>&1)
-    echo -e "  ${CROSS} Unable to connect to Pi-hole's Tricorder server"
+  if [[ "${tricorder_token}" != "https://tricorder.pi-hole.net/"* ]]; then
-    exit 1
+      echo -e "${CROSS} uploading failed, contact Pi-hole support for assistance."
-  fi
+      # Log curl error (if available)
-
+      if [ -n "${tricorder_token}" ]; then
-  if command -v openssl &> /dev/null; then
+          echo -e "${INFO} Error message: ${COL_RED}${tricorder_token}${COL_NC}\\n"
-    openssl s_client -quiet -connect tricorder.pi-hole.net:9998 2> /dev/null < /dev/stdin
+          tricorder_token=""
-    exit "$?"
+      fi
-  else
+      exit 1
-    echo -e "  ${INFO} ${COL_YELLOW}Security Notice${COL_NC}: ${COL_WHITE}openssl${COL_NC} is not installed
-       Your debug log will be transmitted unencrypted via plain-text
-       There is a possibility that this could be intercepted by a third party
-       If you wish to cancel, press Ctrl-C to exit within 10 seconds"
-    secs="10"
-    while [[ "$secs" -gt "0" ]]; do
-       echo -ne "."
-       sleep 1
-       : $((secs--))
-    done
-    echo " "
-    nc tricorder.pi-hole.net 9999 < /dev/stdin
-    exit "$?"
   fi
+  echo "Upload successful, your token is: ${COL_GREEN}${tricorder_token}${COL_NC}"
+  exit 0
 }
 
 updateCheckFunc() {