From fd050693a27c62d7845eaf555686ed9a17648b80 Mon Sep 17 00:00:00 2001 From: MichaIng Date: Wed, 7 Apr 2021 21:53:52 +0200 Subject: [PATCH 01/33] Remove obsolete DEB package name checks The installer contains three checks for specific DEB package names, which did change in past Debian/Ubuntu versions. These checks are obsolete with the current set of supported distro versions: iproute vs iproute2: All distro versions down to Debian Jessie and Ubuntu Xenial ship the iproute2 package: - https://packages.debian.org/search?keywords=iproute - https://packages.ubuntu.com/search?suite=all&keywords=iproute php5 vs php: None of the Ubuntu version down to Xenial and only Debian Jessie ships the php5 package: - https://packages.debian.org/search?keywords=php5 - https://packages.ubuntu.com/search?suite=all&keywords=php5 Moreover, installs with PHP5 would fail anyway for a longer time, due to the added php-xml module package, which became a dedicated package with PHP7.0 while being part of the core package with PHP5: - https://packages.debian.org/search?keywords=php5-xml php-sqlite vs php-sqlite3: With PHP7, the SQLite module package name changed to sqlite3 prefix: - https://packages.debian.org/search?keywords=php-sqlite - https://packages.ubuntu.com/search?suite=all&keywords=php-sqlite Additionally the code comment about minimal apt-get call output was moved to the actual apt-get install call section, as if refers to issues with package installs that require interactive action and hence output about it to the console. The package cache update as well requires an interactive confirmation when the underlying suite code name changes, e.g. when "buster" becomes "oldstable" and "bullseye" becomes "stable". But that is not what the referred issue was about. The comments around the installer and Pi-hole package dependencies have been aligned with the current v6 branch as attempt to resolve merge conflicts in the first place. Signed-off-by: MichaIng --- automated install/basic-install.sh | 54 ++++-------------------------- 1 file changed, 6 insertions(+), 48 deletions(-) mode change 100755 => 100644 automated install/basic-install.sh diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh old mode 100755 new mode 100644 index e4c168ea..bf9ab680 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -273,66 +273,25 @@ if is_command apt-get ; then PKG_INSTALL=("${PKG_MANAGER}" -qq --no-install-recommends install) # grep -c will return 1 if there are no matches. This is an acceptable condition, so we OR TRUE to prevent set -e exiting the script. PKG_COUNT="${PKG_MANAGER} -s -o Debug::NoLocking=true upgrade | grep -c ^Inst || true" - # Update package cache. This is required already here to assure apt-cache calls have package lists available. + # Update package cache update_package_cache || exit 1 - # Debian 7 doesn't have iproute2 so check if it's available first - if apt-cache show iproute2 > /dev/null 2>&1; then - iproute_pkg="iproute2" - # Otherwise, check if iproute is available - elif apt-cache show iproute > /dev/null 2>&1; then - iproute_pkg="iproute" - # Else print error and exit - else - printf " %b Aborting installation: iproute2 and iproute packages were not found in APT repository.\\n" "${CROSS}" - exit 1 - fi # Check for and determine version number (major and minor) of current php install + local phpVer="php" if is_command php ; then printf " %b Existing PHP installation detected : PHP version %s\\n" "${INFO}" "$(php <<< "")" printf -v phpInsMajor "%d" "$(php <<< "")" printf -v phpInsMinor "%d" "$(php <<< "")" - # Is installed php version 7.0 or greater - if [ "${phpInsMajor}" -ge 7 ]; then - phpInsNewer=true - fi - fi - # Several other packages depend on the version of PHP. If PHP is not installed, or an insufficient version, - # those packages should fall back to the default (latest?) - if [[ "$phpInsNewer" != true ]]; then - # Prefer the php metapackage if it's there - if apt-cache show php > /dev/null 2>&1; then - phpVer="php" - # Else fall back on the php5 package if it's there - elif apt-cache show php5 > /dev/null 2>&1; then - phpVer="php5" - # Else print error and exit - else - printf " %b Aborting installation: No PHP packages were found in APT repository.\\n" "${CROSS}" - exit 1 - fi - else - # Else, PHP is already installed at a version beyond v7.0, so the additional packages - # should match version with the current PHP version. phpVer="php$phpInsMajor.$phpInsMinor" fi - # We also need the correct version for `php-sqlite` (which differs across distros) - if apt-cache show "${phpVer}-sqlite3" > /dev/null 2>&1; then - phpSqlite="sqlite3" - elif apt-cache show "${phpVer}-sqlite" > /dev/null 2>&1; then - phpSqlite="sqlite" - else - printf " %b Aborting installation: No SQLite PHP module was found in APT repository.\\n" "${CROSS}" - exit 1 - fi # Packages required to perfom the os_check (stored as an array) OS_CHECK_DEPS=(grep dnsutils) # Packages required to run this install script (stored as an array) - INSTALLER_DEPS=(git "${iproute_pkg}" whiptail ca-certificates) + INSTALLER_DEPS=(git iproute2 whiptail ca-certificates) # Packages required to run Pi-hole (stored as an array) PIHOLE_DEPS=(cron curl iputils-ping lsof netcat psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2) # Packages required for the Web admin interface (stored as an array) # It's useful to separate this from Pi-hole, since the two repos are also setup separately - PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-${phpSqlite}" "${phpVer}-xml" "${phpVer}-intl") + PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php if [[ "${phpInsNewer}" != true || "${phpInsMajor}" -lt 8 ]]; then PIHOLE_WEB_DEPS+=("${phpVer}-json") @@ -1555,9 +1514,6 @@ disable_resolved_stublistener() { } update_package_cache() { - # Running apt-get update/upgrade with minimal output can cause some issues with - # requiring user input (e.g password for phpmyadmin see #218) - # Update package cache on apt based OSes. Do this every time since # it's quick and packages can be updated at any time. @@ -1622,6 +1578,8 @@ install_dependent_packages() { # If there's anything to install, install everything in the list. if [[ "${#installArray[@]}" -gt 0 ]]; then test_dpkg_lock + # Running apt-get install with minimal output can cause some issues with + # requiring user input (e.g password for phpmyadmin see #218) printf " %b Processing %s install(s) for: %s, please wait...\\n" "${INFO}" "${PKG_MANAGER}" "${installArray[*]}" printf '%*s\n' "$columns" '' | tr " " -; "${PKG_INSTALL[@]}" "${installArray[@]}" From a9b9718ffad011232b171ef546ec127c9332d7cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 2 Oct 2021 23:29:23 +0200 Subject: [PATCH 02/33] Do not let the user select if they want to blocking via IPv4 and/or IPv6 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 73 ++++++------------------------ 1 file changed, 15 insertions(+), 58 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index bf9ab680..f6eeca8b 100644 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -717,9 +717,8 @@ testIPv6() { fi } -# A dialog for showing the user about IPv6 blocking -useIPv6dialog() { - # Determine the IPv6 address used for blocking +find_IPv6_information() { + # Detects IPv6 address used for communication to WAN addresses. IPV6_ADDRESSES=($(ip -6 address | grep 'scope global' | awk '{print $2}')) # For each address in the array above, determine the type of IPv6 address it is @@ -739,76 +738,34 @@ useIPv6dialog() { # set the IPv6 address to the ULA address IPV6_ADDRESS="${ULA_ADDRESS}" # Show this info to the user - printf " %b Found IPv6 ULA address, using it for blocking IPv6 ads\\n" "${INFO}" + printf " %b Found IPv6 ULA address\\n" "${INFO}" # Otherwise, if the GUA_ADDRESS has a value, elif [[ ! -z "${GUA_ADDRESS}" ]]; then # Let the user know - printf " %b Found IPv6 GUA address, using it for blocking IPv6 ads\\n" "${INFO}" + printf " %b Found IPv6 GUA address\\n" "${INFO}" # And assign it to the global variable IPV6_ADDRESS="${GUA_ADDRESS}" # If none of those work, else - # explain that IPv6 blocking will not be used - printf " %b Unable to find IPv6 ULA/GUA address, IPv6 adblocking will not be enabled\\n" "${INFO}" + printf " %b Unable to find IPv6 ULA/GUA address\\n" "${INFO}" # So set the variable to be empty IPV6_ADDRESS="" fi - - # If the IPV6_ADDRESS contains a value - if [[ ! -z "${IPV6_ADDRESS}" ]]; then - # Display that IPv6 is supported and will be used - whiptail --msgbox --backtitle "IPv6..." --title "IPv6 Supported" "$IPV6_ADDRESS will be used to block ads." "${r}" "${c}" - fi } -# A function to check if we should use IPv4 and/or IPv6 for blocking ads -use4andor6() { - # Named local variables - local useIPv4 - local useIPv6 - # Let user choose IPv4 and/or IPv6 via a checklist - cmd=(whiptail --separate-output --checklist "Select Protocols (press space to toggle selection)" "${r}" "${c}" 2) - # In an array, show the options available: - # IPv4 (on by default) - options=(IPv4 "Block ads over IPv4" on - # or IPv6 (on by default if available) - IPv6 "Block ads over IPv6" on) - # In a variable, show the choices available; exit if Cancel is selected - choices=$("${cmd[@]}" "${options[@]}" 2>&1 >/dev/tty) || { printf " %bCancel was selected, exiting installer%b\\n" "${COL_LIGHT_RED}" "${COL_NC}"; exit 1; } - # For each choice available, - for choice in ${choices} - do - # Set the values to true - case ${choice} in - IPv4 ) useIPv4=true;; - IPv6 ) useIPv6=true;; - esac - done - # If IPv4 is to be used, - if [[ "${useIPv4}" ]]; then - # Run our function to get the information we need - find_IPv4_information - if [[ -f "/etc/dhcpcd.conf" ]]; then +# A function to collect IPv4 and IPv6 information of the device +collect_v4andv6_information() { + find_IPv4_information + # Echo the information to the user + printf " %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" + # if `dhcpcd` is used offer to set this as static IP for the device + if [[ -f "/etc/dhcpcd.conf" ]]; then # configure networking via dhcpcd getStaticIPv4Settings setDHCPCD - fi fi - # If IPv6 is to be used, - if [[ "${useIPv6}" ]]; then - # Run our function to get this information - useIPv6dialog - fi - # Echo the information to the user - printf " %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" + find_IPv6_information printf " %b IPv6 address: %s\\n" "${INFO}" "${IPV6_ADDRESS}" - # If neither protocol is selected, - if [[ ! "${useIPv4}" ]] && [[ ! "${useIPv6}" ]]; then - # Show an error in red - printf " %bError: Neither IPv4 or IPv6 selected%b\\n" "${COL_LIGHT_RED}" "${COL_NC}" - # and exit with an error - exit 1 - fi } getStaticIPv4Settings() { @@ -2544,8 +2501,8 @@ main() { setDNS # Give the user a choice of blocklists to include in their install. Or not. chooseBlocklists - # Let the user decide if they want to block ads over IPv4 and/or IPv6 - use4andor6 + # find IPv4 and IPv6 information of the device + collect_v4andv6_information # Let the user decide if they want the web interface to be installed automatically setAdminFlag # Let the user decide if they want query logging enabled... From 9dbcbdbe666f2ea81f31b9ef77248473b50cb296 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 2 Oct 2021 23:43:08 +0200 Subject: [PATCH 03/33] Adjust tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_automated_install.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index 9938dd99..faf67c92 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -421,10 +421,9 @@ def test_IPv6_only_link_local(Pihole): ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - useIPv6dialog + find_IPv6_information ''') - expected_stdout = ('Unable to find IPv6 ULA/GUA address, ' - 'IPv6 adblocking will not be enabled') + expected_stdout = ('Unable to find IPv6 ULA/GUA address') assert expected_stdout in detectPlatform.stdout @@ -468,9 +467,9 @@ def test_IPv6_only_GUA(Pihole): ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - useIPv6dialog + find_IPv6_information ''') - expected_stdout = 'Found IPv6 GUA address, using it for blocking IPv6 ads' + expected_stdout = 'Found IPv6 GUA address' assert expected_stdout in detectPlatform.stdout From 466520366d2825f6596675003da8468c485f2ea7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sat, 2 Oct 2021 23:47:13 +0200 Subject: [PATCH 04/33] Fogot to save... MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- test/test_automated_install.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test/test_automated_install.py b/test/test_automated_install.py index faf67c92..eb1bfd2c 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -444,9 +444,9 @@ def test_IPv6_only_ULA(Pihole): ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - useIPv6dialog + find_IPv6_information ''') - expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads' + expected_stdout = 'Found IPv6 ULA address' assert expected_stdout in detectPlatform.stdout @@ -491,9 +491,9 @@ def test_IPv6_GUA_ULA_test(Pihole): ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - useIPv6dialog + find_IPv6_information ''') - expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads' + expected_stdout = 'Found IPv6 ULA address' assert expected_stdout in detectPlatform.stdout @@ -515,9 +515,9 @@ def test_IPv6_ULA_GUA_test(Pihole): ) detectPlatform = Pihole.run(''' source /opt/pihole/basic-install.sh - useIPv6dialog + find_IPv6_information ''') - expected_stdout = 'Found IPv6 ULA address, using it for blocking IPv6 ads' + expected_stdout = 'Found IPv6 ULA address' assert expected_stdout in detectPlatform.stdout From 5b03160295103aa38a7b2554fc057fea03b04d97 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Mon, 4 Oct 2021 11:40:53 +0100 Subject: [PATCH 05/33] Install script comment tweaks (#4361) --- automated install/basic-install.sh | 12 ++++++------ test/test_automated_install.py | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index f6eeca8b..c1a1c613 100644 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -262,10 +262,10 @@ os_check() { # Compatibility package_manager_detect() { -# If apt-get is installed, then we know it's part of the Debian family +# First check to see if apt-get is installed. if is_command apt-get ; then # Set some global variables here - # We don't set them earlier since the family might be Red Hat, so these values would be different + # We don't set them earlier since the installed package manager might be rpm, so these values would be different PKG_MANAGER="apt-get" # A variable to store the command used to update the package cache UPDATE_PKG_CACHE="${PKG_MANAGER} update" @@ -319,7 +319,7 @@ if is_command apt-get ; then return 0 } -# If apt-get is not found, check for rpm to see if it's a Red Hat family OS +# If apt-get is not found, check for rpm. elif is_command rpm ; then # Then check if dnf or yum is the package manager if is_command dnf ; then @@ -328,7 +328,7 @@ elif is_command rpm ; then PKG_MANAGER="yum" fi - # These variable names match the ones in the Debian family. See above for an explanation of what they are for. + # These variable names match the ones for apt-get. See above for an explanation of what they are for. PKG_INSTALL=("${PKG_MANAGER}" install -y) PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" OS_CHECK_DEPS=(grep bind-utils) @@ -341,8 +341,8 @@ elif is_command rpm ; then # If neither apt-get or yum/dnf package managers were found else - # it's not an OS we can support, - printf " %b OS distribution not supported\\n" "${CROSS}" + # we cannot install required packages + printf " %b No supported package manager found\\n" "${CROSS}" # so exit the installer exit fi diff --git a/test/test_automated_install.py b/test/test_automated_install.py index eb1bfd2c..37ebdad2 100644 --- a/test/test_automated_install.py +++ b/test/test_automated_install.py @@ -11,18 +11,18 @@ from .conftest import ( ) -def test_supported_operating_system(Pihole): +def test_supported_package_manager(Pihole): ''' - confirm installer exists on unsupported distribution + confirm installer exits when no supported package manager found ''' - # break supported package managers to emulate an unsupported distribution + # break supported package managers Pihole.run('rm -rf /usr/bin/apt-get') Pihole.run('rm -rf /usr/bin/rpm') package_manager_detect = Pihole.run(''' source /opt/pihole/basic-install.sh package_manager_detect ''') - expected_stdout = cross_box + ' OS distribution not supported' + expected_stdout = cross_box + ' No supported package manager found' assert expected_stdout in package_manager_detect.stdout # assert package_manager_detect.rc == 1 From 0f246b8df522ce0eeccfd5ee47b5e714ba250b52 Mon Sep 17 00:00:00 2001 From: xanoni <77220130+xanoni@users.noreply.github.com> Date: Sun, 3 Oct 2021 23:16:19 -0400 Subject: [PATCH 06/33] Update upstream DNS server capability descriptions Mention that the below 3 upstream DNS support DNSSEC: - Cloudflare (see https://developers.cloudflare.com/1.1.1.1/faq#how-does-1111-work-with-dnssec) - DNS.WATCH (see https://dns.watch/index) - Google (see https://developers.google.com/speed/public-dns/faq#dnssec) - Quad9 (see https://www.quad9.net/support/faq/#dnssec) Other providers and capabilities (e.g., ECS) were not checked. Signed-off-by: xanoni <77220130+xanoni@users.noreply.github.com> --- automated install/basic-install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c1a1c613..63b060f2 100644 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -34,15 +34,15 @@ export PATH+=':/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' # List of supported DNS servers DNS_SERVERS=$(cat << EOM -Google (ECS);8.8.8.8;8.8.4.4;2001:4860:4860:0:0:0:0:8888;2001:4860:4860:0:0:0:0:8844 +Google (ECS, DNSSEC);8.8.8.8;8.8.4.4;2001:4860:4860:0:0:0:0:8888;2001:4860:4860:0:0:0:0:8844 OpenDNS (ECS, DNSSEC);208.67.222.222;208.67.220.220;2620:119:35::35;2620:119:53::53 Level3;4.2.2.1;4.2.2.2;; Comodo;8.26.56.26;8.20.247.20;; -DNS.WATCH;84.200.69.80;84.200.70.40;2001:1608:10:25:0:0:1c04:b12f;2001:1608:10:25:0:0:9249:d69b +DNS.WATCH (DNSSEC);84.200.69.80;84.200.70.40;2001:1608:10:25:0:0:1c04:b12f;2001:1608:10:25:0:0:9249:d69b Quad9 (filtered, DNSSEC);9.9.9.9;149.112.112.112;2620:fe::fe;2620:fe::9 Quad9 (unfiltered, no DNSSEC);9.9.9.10;149.112.112.10;2620:fe::10;2620:fe::fe:10 -Quad9 (filtered + ECS);9.9.9.11;149.112.112.11;2620:fe::11;2620:fe::fe:11 -Cloudflare;1.1.1.1;1.0.0.1;2606:4700:4700::1111;2606:4700:4700::1001 +Quad9 (filtered, ECS, DNSSEC);9.9.9.11;149.112.112.11;2620:fe::11;2620:fe::fe:11 +Cloudflare (DNSSEC);1.1.1.1;1.0.0.1;2606:4700:4700::1111;2606:4700:4700::1001 EOM ) From 109340033eec06a89761914527d48a06f14e71cf Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 5 Oct 2021 02:22:27 +0200 Subject: [PATCH 07/33] Do not account for refactor anymore (#4355) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index c1a1c613..5a036db9 100644 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1778,27 +1778,6 @@ installLogrotate() { printf "%b %b %s\\n" "${OVER}" "${TICK}" "${str}" } -# At some point in the future this list can be pruned, for now we'll need it to ensure updates don't break. -# Refactoring of install script has changed the name of a couple of variables. Sort them out here. -accountForRefactor() { - sed -i 's/piholeInterface/PIHOLE_INTERFACE/g' "${setupVars}" - sed -i 's/IPv4_address/IPV4_ADDRESS/g' "${setupVars}" - sed -i 's/IPv4addr/IPV4_ADDRESS/g' "${setupVars}" - sed -i 's/IPv6_address/IPV6_ADDRESS/g' "${setupVars}" - sed -i 's/piholeIPv6/IPV6_ADDRESS/g' "${setupVars}" - sed -i 's/piholeDNS1/PIHOLE_DNS_1/g' "${setupVars}" - sed -i 's/piholeDNS2/PIHOLE_DNS_2/g' "${setupVars}" - sed -i 's/^INSTALL_WEB=/INSTALL_WEB_INTERFACE=/' "${setupVars}" - # Add 'INSTALL_WEB_SERVER', if its not been applied already: https://github.com/pi-hole/pi-hole/pull/2115 - if ! grep -q '^INSTALL_WEB_SERVER=' ${setupVars}; then - local webserver_installed=false - if grep -q '^INSTALL_WEB_INTERFACE=true' ${setupVars}; then - webserver_installed=true - fi - echo -e "INSTALL_WEB_SERVER=$webserver_installed" >> "${setupVars}" - fi -} - # Install base files and web interface installPihole() { # If the user wants to install the Web interface, @@ -1829,10 +1808,6 @@ installPihole() { fi fi fi - # For updates and unattended install. - if [[ "${useUpdateVars}" == true ]]; then - accountForRefactor - fi # Install base files and web interface if ! installScripts; then printf " %b Failure in dependent script copy function.\\n" "${CROSS}" From 2b74b47b4a499f1838f098e95e10574ff80fe3df Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 5 Oct 2021 02:47:18 +0200 Subject: [PATCH 08/33] Remove netcat from dependencies (#4346) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 5a036db9..98296e5e 100644 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -288,7 +288,7 @@ if is_command apt-get ; then # Packages required to run this install script (stored as an array) INSTALLER_DEPS=(git iproute2 whiptail ca-certificates) # Packages required to run Pi-hole (stored as an array) - PIHOLE_DEPS=(cron curl iputils-ping lsof netcat psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2) + PIHOLE_DEPS=(cron curl iputils-ping lsof psmisc sudo unzip idn2 sqlite3 libcap2-bin dns-root-data libcap2) # Packages required for the Web admin interface (stored as an array) # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") @@ -333,7 +333,7 @@ elif is_command rpm ; then PKG_COUNT="${PKG_MANAGER} check-update | egrep '(.i686|.x86|.noarch|.arm|.src)' | wc -l" OS_CHECK_DEPS=(grep bind-utils) INSTALLER_DEPS=(git iproute newt procps-ng which chkconfig ca-certificates) - PIHOLE_DEPS=(cronie curl findutils nmap-ncat sudo unzip libidn2 psmisc sqlite libcap lsof) + PIHOLE_DEPS=(cronie curl findutils sudo unzip libidn2 psmisc sqlite libcap lsof) PIHOLE_WEB_DEPS=(lighttpd lighttpd-fastcgi php-common php-cli php-pdo php-xml php-json php-intl) LIGHTTPD_USER="lighttpd" LIGHTTPD_GROUP="lighttpd" From 38bb4a49088febe3aa749ff197b30d5497f024f5 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 5 Oct 2021 14:09:16 +0200 Subject: [PATCH 09/33] Remove unused wildcard_regex_converter.sh (#4369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove unused wildcard_regex_converter.sh Signed-off-by: Christian König * Remove regexconverter Signed-off-by: Christian König --- advanced/Scripts/wildcard_regex_converter.sh | 28 -------------------- gravity.sh | 2 -- 2 files changed, 30 deletions(-) delete mode 100644 advanced/Scripts/wildcard_regex_converter.sh diff --git a/advanced/Scripts/wildcard_regex_converter.sh b/advanced/Scripts/wildcard_regex_converter.sh deleted file mode 100644 index b4b6b4a1..00000000 --- a/advanced/Scripts/wildcard_regex_converter.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# Pi-hole: A black hole for Internet advertisements -# (c) 2017 Pi-hole, LLC (https://pi-hole.net) -# Network-wide ad blocking via your own hardware. -# -# Provides an automated migration subroutine to convert Pi-hole v3.x wildcard domains to Pi-hole v4.x regex filters -# -# This file is copyright under the latest version of the EUPL. -# Please see LICENSE file for your rights under this license. - -# regexFile set in gravity.sh - -wildcardFile="/etc/dnsmasq.d/03-pihole-wildcard.conf" - -convert_wildcard_to_regex() { - if [ ! -f "${wildcardFile}" ]; then - return - fi - local addrlines domains uniquedomains - # Obtain wildcard domains from old file - addrlines="$(grep -oE "/.*/" ${wildcardFile})" - # Strip "/" from domain names and convert "." to regex-compatible "\." - domains="$(sed 's/\///g;s/\./\\./g' <<< "${addrlines}")" - # Remove repeated domains (may have been inserted two times due to A and AAAA blocking) - uniquedomains="$(uniq <<< "${domains}")" - # Automatically generate regex filters and remove old wildcards file - awk '{print "(^|\\.)"$0"$"}' <<< "${uniquedomains}" >> "${regexFile:?}" && rm "${wildcardFile}" -} diff --git a/gravity.sh b/gravity.sh index 99d1bcda..dfaf4fea 100755 --- a/gravity.sh +++ b/gravity.sh @@ -15,8 +15,6 @@ export LC_ALL=C coltable="/opt/pihole/COL_TABLE" source "${coltable}" -regexconverter="/opt/pihole/wildcard_regex_converter.sh" -source "${regexconverter}" # shellcheck disable=SC1091 source "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" From 80560d4a4ace2db1389c78caab0971e397b4f476 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Tue, 5 Oct 2021 15:36:00 +0200 Subject: [PATCH 10/33] Do not export `DNS_FQDN_REQUIRED` and `DNS_BOGUS_PRIV` unconditionally (#4354) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Do not export unconditionally Signed-off-by: Christian König * Check if variable is unset instead of grep for it Signed-off-by: Christian König * Use bash's buld in word syntax Signed-off-by: Christian König * Move export back to their brothers Signed-off-by: Christian König --- automated install/basic-install.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 98296e5e..42e660a9 100644 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1715,7 +1715,7 @@ finalExports() { # If the setup variable file exists, if [[ -e "${setupVars}" ]]; then # update the variables in the file - sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1\b/d;/PIHOLE_DNS_2\b/d;/QUERY_LOGGING/d;/INSTALL_WEB_SERVER/d;/INSTALL_WEB_INTERFACE/d;/LIGHTTPD_ENABLED/d;/CACHE_SIZE/d;' "${setupVars}" + sed -i.update.bak '/PIHOLE_INTERFACE/d;/IPV4_ADDRESS/d;/IPV6_ADDRESS/d;/PIHOLE_DNS_1\b/d;/PIHOLE_DNS_2\b/d;/QUERY_LOGGING/d;/INSTALL_WEB_SERVER/d;/INSTALL_WEB_INTERFACE/d;/LIGHTTPD_ENABLED/d;/CACHE_SIZE/d;/DNS_FQDN_REQUIRED/d;/DNS_BOGUS_PRIV/d;' "${setupVars}" fi # echo the information to the user { @@ -1729,8 +1729,8 @@ finalExports() { echo "INSTALL_WEB_INTERFACE=${INSTALL_WEB_INTERFACE}" echo "LIGHTTPD_ENABLED=${LIGHTTPD_ENABLED}" echo "CACHE_SIZE=${CACHE_SIZE}" - echo "DNS_FQDN_REQUIRED=true" - echo "DNS_BOGUS_PRIV=true" + echo "DNS_FQDN_REQUIRED=${DNS_FQDN_REQUIRED:-true}" + echo "DNS_BOGUS_PRIV=${DNS_BOGUS_PRIV:-true}" }>> "${setupVars}" chmod 644 "${setupVars}" From 541257849df1955c1de7c43f0ba1a537b17e00b5 Mon Sep 17 00:00:00 2001 From: Andras Tim Date: Tue, 5 Oct 2021 16:52:51 +0200 Subject: [PATCH 11/33] List fix: no reload (#3981) * scripts/list: Fixed --noreload options We should differentiate the ability and the needings. Signed-off-by: Andras Tim * scripts/list: Removed unnecessary tailing whitespaces Signed-off-by: Andras Tim * Update advanced/Scripts/list.sh Signed-off-by: Andras Tim * Merge bash conditions according to MichaIng suggestion Co-authored-by: micha@dietpi.com Signed-off-by: Andras Tim --- advanced/Scripts/list.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index e213b014..5bd42d55 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -23,7 +23,7 @@ fi # have changed gravityDBfile="${GRAVITYDB}" -reload=false +noReloadRequested=false addmode=true verbose=true wildcard=false @@ -35,6 +35,7 @@ typeId="" comment="" declare -i domaincount domaincount=0 +reload=false colfile="/opt/pihole/COL_TABLE" source ${colfile} @@ -242,13 +243,13 @@ Displaylist() { NukeList() { count=$(sqlite3 "${gravityDBfile}" "SELECT COUNT(1) FROM domainlist WHERE type = ${typeId};") - listname="$(GetListnameFromTypeId "${typeId}")" + listname="$(GetListnameFromTypeId "${typeId}")" if [ "$count" -gt 0 ];then sqlite3 "${gravityDBfile}" "DELETE FROM domainlist WHERE type = ${typeId};" echo " ${TICK} Removed ${count} domain(s) from the ${listname}" else echo " ${INFO} ${listname} already empty. Nothing to do!" - fi + fi exit 0; } @@ -268,7 +269,7 @@ while (( "$#" )); do "--white-wild" | "white-wild" ) typeId=2; wildcard=true;; "--wild" | "wildcard" ) typeId=3; wildcard=true;; "--regex" | "regex" ) typeId=3;; - "-nr"| "--noreload" ) reload=false;; + "-nr"| "--noreload" ) noReloadRequested=true;; "-d" | "--delmode" ) addmode=false;; "-q" | "--quiet" ) verbose=false;; "-h" | "--help" ) helpFunc;; @@ -294,6 +295,6 @@ if $web; then echo "DONE" fi -if [[ "${reload}" != false ]]; then +if [[ ${reload} == true && ${noReloadRequested} == false ]]; then pihole restartdns reload-lists fi From 77e322afa644e9c3a75182c263d92aaa722d874d Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 5 Oct 2021 16:25:29 +0100 Subject: [PATCH 12/33] (docs) update README.md (#4371) - correct grammar - correct punctuation - correct pronoun usage Co-authored-by: Vladislav Doster Signed-off-by: Adam Warner Co-authored-by: Vladislav Doster --- README.md | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 06f541f4..b993cfe9 100644 --- a/README.md +++ b/README.md @@ -11,9 +11,9 @@

-The Pi-hole® is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) that protects your devices from unwanted content, without installing any client-side software. +The Pi-hole® is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) that protects your devices from unwanted content without installing any client-side software. -- **Easy-to-install**: our versatile installer walks you through the process, and takes less than ten minutes +- **Easy-to-install**: our versatile installer walks you through the process and takes less than ten minutes - **Resolute**: content is blocked in _non-browser locations_, such as ad-laden mobile apps and smart TVs - **Responsive**: seamlessly speeds up the feel of everyday browsing by caching DNS queries - **Lightweight**: runs smoothly with [minimal hardware and software requirements](https://docs.pi-hole.net/main/prerequisites/) @@ -22,7 +22,7 @@ The Pi-hole® is a [DNS sinkhole](https://en.wikipedia.org/wiki/DNS_Sinkhole) th - **Versatile**: can optionally function as a [DHCP server](https://discourse.pi-hole.net/t/how-do-i-use-pi-holes-built-in-dhcp-server-and-why-would-i-want-to/3026), ensuring *all* your devices are protected automatically - **Scalable**: [capable of handling hundreds of millions of queries](https://pi-hole.net/2017/05/24/how-much-traffic-can-pi-hole-handle/) when installed on server-grade hardware - **Modern**: blocks ads over both IPv4 and IPv6 -- **Free**: open source software which helps ensure _you_ are the sole person in control of your privacy +- **Free**: open source software that helps ensure _you_ are the sole person in control of your privacy ----- @@ -57,21 +57,21 @@ Please refer to the [Pi-hole docker repo](https://github.com/pi-hole/docker-pi-h Once the installer has been run, you will need to [configure your router to have **DHCP clients use Pi-hole as their DNS server**](https://discourse.pi-hole.net/t/how-do-i-configure-my-devices-to-use-pi-hole-as-their-dns-server/245) which ensures that all devices connecting to your network will have content blocked without any further intervention. -If your router does not support setting the DNS server, you can [use Pi-hole's built-in DHCP server](https://discourse.pi-hole.net/t/how-do-i-use-pi-holes-built-in-dhcp-server-and-why-would-i-want-to/3026); just be sure to disable DHCP on your router first (if it has that feature available). +If your router does not support setting the DNS server, you can [use Pi-hole's built-in DHCP server](https://discourse.pi-hole.net/t/how-do-i-use-pi-holes-built-in-dhcp-server-and-why-would-i-want-to/3026); be sure to disable DHCP on your router first (if it has that feature available). -As a last resort, you can always manually set each device to use Pi-hole as their DNS server. +As a last resort, you can manually set each device to use Pi-hole as their DNS server. ----- -## Pi-hole is free, but powered by your support +## Pi-hole is free but powered by your support -There are many reoccurring costs involved with maintaining free, open source, and privacy-respecting software; expenses which [our volunteer developers](https://github.com/orgs/pi-hole/people) pitch in to cover out-of-pocket. This is just one example of how strongly we feel about our software, as well as the importance of keeping it maintained. +There are many reoccurring costs involved with maintaining free, open source, and privacy-respecting software; expenses which [our volunteer developers](https://github.com/orgs/pi-hole/people) pitch in to cover out-of-pocket. This is just one example of how strongly we feel about our software and the importance of keeping it maintained. Make no mistake: **your support is absolutely vital to help keep us innovating!** ### [Donations](https://pi-hole.net/donate) -Sending a donation using our Sponsor Button is **extremely helpful** in offsetting a portion of our monthly expenses and rewarding our dedicated development team: +Donating using our Sponsor Button is **extremely helpful** in offsetting a portion of our monthly expenses: ### Alternative support @@ -83,13 +83,13 @@ If you'd rather not donate (_which is okay!_), there are other ways you can help - [Digital Ocean](https://www.digitalocean.com/?refcode=344d234950e1) _affiliate link_ - [Stickermule](https://www.stickermule.com/unlock?ref_id=9127301701&utm_medium=link&utm_source=invite) _earn a $10 credit after your first purchase_ - [Amazon US](http://www.amazon.com/exec/obidos/redirect-home/pihole09-20) _affiliate link_ -- Spreading the word about our software, and how you have benefited from it +- Spreading the word about our software and how you have benefited from it ### Contributing via GitHub We welcome _everyone_ to contribute to issue reports, suggest new features, and create pull requests. -If you have something to add - anything from a typo through to a whole new feature, we're happy to check it out! Just make sure to fill out our template when submitting your request; the questions that it asks will help the volunteers quickly understand what you're aiming to achieve. +If you have something to add - anything from a typo through to a whole new feature, we're happy to check it out! Just make sure to fill out our template when submitting your request; the questions it asks will help the volunteers quickly understand what you're aiming to achieve. You'll find that the [install script](https://github.com/pi-hole/pi-hole/blob/master/automated%20install/basic-install.sh) and the [debug script](https://github.com/pi-hole/pi-hole/blob/master/advanced/Scripts/piholeDebug.sh) have an abundance of comments, which will help you better understand how Pi-hole works. They're also a valuable resource to those who want to learn how to write scripts or code a program! We encourage anyone who likes to tinker to read through it and submit a pull request for us to review. @@ -97,9 +97,9 @@ You'll find that the [install script](https://github.com/pi-hole/pi-hole/blob/ma ## Getting in touch with us -While we are primarily reachable on our [Discourse User Forum](https://discourse.pi-hole.net/), we can also be found on a variety of social media outlets. +While we are primarily reachable on our [Discourse User Forum](https://discourse.pi-hole.net/), we can also be found on various social media outlets. -**Please be sure to check the FAQ's** before starting a new discussion. Many user questions already have answers and can be solved without any additional assistance. +**Please be sure to check the FAQs** before starting a new discussion, as we do not have the spare time to reply to every request for assistance. - [Frequently Asked Questions](https://discourse.pi-hole.net/c/faqs) - [Feature Requests](https://discourse.pi-hole.net/c/feature-requests?order=votes) @@ -125,15 +125,15 @@ Some of the statistics you can integrate include: - Queries cached - Unique clients -The API can be accessed via [`telnet`](https://github.com/pi-hole/FTL), the Web (`admin/api.php`) and Command Line (`pihole -c -j`). You can find out [more details over here](https://discourse.pi-hole.net/t/pi-hole-api/1863). +Access the API via [`telnet`](https://github.com/pi-hole/FTL), the Web (`admin/api.php`) and Command Line (`pihole -c -j`). You can find out [more details over here](https://discourse.pi-hole.net/t/pi-hole-api/1863). ### The Command Line Interface -The [pihole](https://docs.pi-hole.net/core/pihole-command/) command has all the functionality necessary to be able to fully administer the Pi-hole, without the need of the Web Interface. It's fast, user-friendly, and auditable by anyone with an understanding of `bash`. +The [pihole](https://docs.pi-hole.net/core/pihole-command/) command has all the functionality necessary to fully administer the Pi-hole, without the need of the Web Interface. It's fast, user-friendly, and auditable by anyone with an understanding of `bash`. Some notable features include: -- [Whitelisting, Blacklisting and Regex](https://docs.pi-hole.net/core/pihole-command/#whitelisting-blacklisting-and-regex) +- [Whitelisting, Blacklisting, and Regex](https://docs.pi-hole.net/core/pihole-command/#whitelisting-blacklisting-and-regex) - [Debugging utility](https://docs.pi-hole.net/core/pihole-command/#debugger) - [Viewing the live log file](https://docs.pi-hole.net/core/pihole-command/#tail) - [Updating Ad Lists](https://docs.pi-hole.net/core/pihole-command/#gravity) @@ -149,7 +149,7 @@ This [optional dashboard](https://github.com/pi-hole/AdminLTE) allows you to vie Some notable features include: -- Mobile friendly interface +- Mobile-friendly interface - Password protection - Detailed graphs and doughnut charts - Top lists of domains and clients From c5828df198cb54311ffdc1ecee1e17c3e3da9772 Mon Sep 17 00:00:00 2001 From: MichaIng Date: Tue, 5 Oct 2021 17:40:12 +0200 Subject: [PATCH 13/33] Consequently use defined file path variables (#4105) The script defines variables for the most important file paths which are not always used to call the file paths. "lighttpdConfig" was never used in the script itself, so that a shellcheck exception needed to be used. With this change, the defined variables are consequently used, which makes the shellcheck exception obsolete as well. Additionally the assigned strings are quoted, which is not necessary here but aligns with the coding standard and highlights the strings in most editors and development platforms for developer convenience. Signed-off-by: MichaIng --- automated install/basic-install.sh | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 42e660a9..422c1904 100644 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -47,14 +47,13 @@ EOM ) # Location for final installation log storage -installLogLoc=/etc/pihole/install.log +installLogLoc="/etc/pihole/install.log" # This is an important file as it contains information specific to the machine it's being installed on -setupVars=/etc/pihole/setupVars.conf +setupVars="/etc/pihole/setupVars.conf" # Pi-hole uses lighttpd as a Web server, and this is the config file for it -# shellcheck disable=SC2034 -lighttpdConfig=/etc/lighttpd/lighttpd.conf +lighttpdConfig="/etc/lighttpd/lighttpd.conf" # This is a file used for the colorized output -coltable=/opt/pihole/COL_TABLE +coltable="/opt/pihole/COL_TABLE" # Root of the web server webroot="/var/www/html" @@ -1313,18 +1312,18 @@ installConfigs() { # make it and set the owners install -d -m 755 -o "${USER}" -g root /etc/lighttpd # Otherwise, if the config file already exists - elif [[ -f "/etc/lighttpd/lighttpd.conf" ]]; then + elif [[ -f "${lighttpdConfig}" ]]; then # back up the original - mv /etc/lighttpd/lighttpd.conf /etc/lighttpd/lighttpd.conf.orig + mv "${lighttpdConfig}"{,.orig} fi # and copy in the config file Pi-hole needs - install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} /etc/lighttpd/lighttpd.conf + install -D -m 644 -T ${PI_HOLE_LOCAL_REPO}/advanced/${LIGHTTPD_CFG} "${lighttpdConfig}" # Make sure the external.conf file exists, as lighttpd v1.4.50 crashes without it touch /etc/lighttpd/external.conf chmod 644 /etc/lighttpd/external.conf # If there is a custom block page in the html/pihole directory, replace 404 handler in lighttpd config if [[ -f "${PI_HOLE_BLOCKPAGE_DIR}/custom.php" ]]; then - sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' /etc/lighttpd/lighttpd.conf + sed -i 's/^\(server\.error-handler-404\s*=\s*\).*$/\1"pihole\/custom\.php"/' "${lighttpdConfig}" fi # Make the directories if they do not exist and set the owners mkdir -p /run/lighttpd @@ -1892,7 +1891,7 @@ displayFinalMessage() { if [[ "${#1}" -gt 0 ]] ; then # set the password to the first argument. pwstring="$1" - elif [[ $(grep 'WEBPASSWORD' -c /etc/pihole/setupVars.conf) -gt 0 ]]; then + elif [[ $(grep 'WEBPASSWORD' -c "${setupVars}") -gt 0 ]]; then # Else if the password exists from previous setup, we'll load it later pwstring="unchanged" else @@ -2550,7 +2549,7 @@ main() { # Add password to web UI if there is none pw="" # If no password is set, - if [[ $(grep 'WEBPASSWORD' -c /etc/pihole/setupVars.conf) == 0 ]] ; then + if [[ $(grep 'WEBPASSWORD' -c "${setupVars}") == 0 ]] ; then # generate a random password pw=$(tr -dc _A-Z-a-z-0-9 < /dev/urandom | head -c 8) # shellcheck disable=SC1091 From 5bf35dc687008b7a6aa15330d4bd03329c37282c Mon Sep 17 00:00:00 2001 From: Jauder Ho Date: Tue, 5 Oct 2021 17:22:46 +0000 Subject: [PATCH 14/33] Add Dependabot and CodeQL support (#4286) Signed-off-by: Jauder Ho Co-authored-by: Adam Warner --- .github/dependabot.yml | 17 ++++++++++++ .github/workflows/codeql-analysis.yml | 40 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/codeql-analysis.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..bc08634e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,17 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + # Maintain dependencies for GitHub Actions + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + day: saturday + time: "10:00" + open-pull-requests-limit: 10 + target-branch: development + versioning-strategy: increase diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml new file mode 100644 index 00000000..a4f67b81 --- /dev/null +++ b/.github/workflows/codeql-analysis.yml @@ -0,0 +1,40 @@ +name: "CodeQL" + +on: + push: + branches: + - master + - development + pull_request: + branches: + - master + - development + schedule: + - cron: '32 11 * * 6' + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + + permissions: + actions: read + contents: read + security-events: write + + steps: + - + name: Checkout repository + uses: actions/checkout@v2 + # Initializes the CodeQL tools for scanning. + - + name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: 'python' + - + name: Autobuild + uses: github/codeql-action/autobuild@v1 + - + name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 From 55dce14655a0188998c6ce5603092c387d130565 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Tue, 5 Oct 2021 19:34:51 +0100 Subject: [PATCH 15/33] Add execution bit accidentally dropped in #4106 (#4368) Add some smoke tests for the repository. - Add x bit to piholeCheckout.sh. Possibly not needed, but consistency is no bad thing - Ensure all files in script directorys have executable bit set Signed-off-by: Adam Warner --- .github/workflows/test.yml | 31 ++++++++++++++++--- .../Scripts/database_migration/gravity-db.sh | 0 advanced/Scripts/piholeCheckout.sh | 0 automated install/basic-install.sh | 0 4 files changed, 27 insertions(+), 4 deletions(-) mode change 100644 => 100755 advanced/Scripts/database_migration/gravity-db.sh mode change 100644 => 100755 advanced/Scripts/piholeCheckout.sh mode change 100644 => 100755 automated install/basic-install.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 129caea4..c2b4dbbc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,21 +5,44 @@ on: types: [opened, synchronize, reopened, ready_for_review] jobs: + smoke-test: + if: github.event.pull_request.draft == false + runs-on: ubuntu-latest + steps: + - + name: Checkout repository + uses: actions/checkout@v2 + - + name: Run Smoke Tests + run: | + # Ensure scripts in repository are executable + IFS=$'\n'; + for f in $(find . -name '*.sh'); do if [[ ! -x $f ]]; then echo "$f is not executable" && FAIL=1; fi ;done + unset IFS; + # If FAIL is 1 then we fail. + [[ $FAIL == 1 ]] && exit 1 || echo "Smoke Tests Passed" + distro-test: if: github.event.pull_request.draft == false runs-on: ubuntu-latest + needs: smoke-test strategy: matrix: distro: [debian_9, debian_10, debian_11, ubuntu_16, ubuntu_18, ubuntu_20, ubuntu_21, centos_7, centos_8, fedora_32, fedora_33] env: DISTRO: ${{matrix.distro}} steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.7 + - + name: Checkout repository + uses: actions/checkout@v2 + - + name: Set up Python 3.7 uses: actions/setup-python@v2 with: python-version: 3.7 - - name: Install dependencies + - + name: Install dependencies run: pip install -r test/requirements.txt - - name: Test with tox + - + name: Test with tox run: tox -c test/tox.${DISTRO}.ini diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh old mode 100644 new mode 100755 diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh old mode 100644 new mode 100755 diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh old mode 100644 new mode 100755 From 0ea7344c309a81c64fe11769c0921cd0141b935e Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 6 Oct 2021 01:19:28 +0100 Subject: [PATCH 16/33] add --no-rebase to the git pull command(s) to squelch hint message in newer versions of git (#4226) Signed-off-by: Adam Warner --- automated install/basic-install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 422c1904..9cba9339 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -514,7 +514,7 @@ update_repo() { git stash --all --quiet &> /dev/null || true # Okay for stash failure git clean --quiet --force -d || true # Okay for already clean directory # Pull the latest commits - git pull --quiet &> /dev/null || return $? + git pull --no-rebase --quiet &> /dev/null || return $? # Check current branch. If it is master, then reset to the latest available tag. # In case extra commits have been added after tagging/release (i.e in case of metadata updates/README.MD tweaks) curBranch=$(git rev-parse --abbrev-ref HEAD) @@ -2028,7 +2028,7 @@ checkout_pull_branch() { # Data in the repositories is public anyway so we can make it readable by everyone (+r to keep executable permission if already set by git) chmod -R a+rX "${directory}" - git_pull=$(git pull || return 1) + git_pull=$(git pull --no-rebase || return 1) if [[ "$git_pull" == *"up-to-date"* ]]; then printf " %b %s\\n" "${INFO}" "${git_pull}" From 3cad8e4c5b4d226b9b85251b55647bd5aa1b2cc2 Mon Sep 17 00:00:00 2001 From: Frieder Bluemle Date: Wed, 6 Oct 2021 13:33:13 +0200 Subject: [PATCH 17/33] Remove .idea/ Signed-off-by: Frieder Bluemle --- .gitignore | 67 +--------------------------- .idea/codeStyleSettings.xml | 25 ----------- .idea/codeStyles/Project.xml | 7 --- .idea/codeStyles/codeStyleConfig.xml | 5 --- 4 files changed, 1 insertion(+), 103 deletions(-) delete mode 100644 .idea/codeStyleSettings.xml delete mode 100644 .idea/codeStyles/Project.xml delete mode 100644 .idea/codeStyles/codeStyleConfig.xml diff --git a/.gitignore b/.gitignore index b7ad1e41..c19555ed 100644 --- a/.gitignore +++ b/.gitignore @@ -7,70 +7,5 @@ __pycache__ .tox .eggs *.egg-info - - -# Created by https://www.gitignore.io/api/jetbrains+iml - -### JetBrains+iml ### -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 - -# All idea files, with exceptions -.idea -!.idea/codeStyles/* -!.idea/codeStyleSettings.xml - - -# Sensitive or high-churn files: -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.xml -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml - -# Gradle: -.idea/**/gradle.xml -.idea/**/libraries - -# CMake -cmake-build-debug/ - -# Mongo Explorer plugin: -.idea/**/mongoSettings.xml - -## File-based project format: -*.iws - -## Plugin-specific files: - -# IntelliJ -/out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# Ruby plugin and RubyMine -/.rakeTasks - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -### JetBrains+iml Patch ### -# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 - +.idea/ *.iml -.idea/misc.xml -*.ipr - -# End of https://www.gitignore.io/api/jetbrains+iml diff --git a/.idea/codeStyleSettings.xml b/.idea/codeStyleSettings.xml deleted file mode 100644 index 6ad75d68..00000000 --- a/.idea/codeStyleSettings.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/codeStyles/Project.xml b/.idea/codeStyles/Project.xml deleted file mode 100644 index 79a710fd..00000000 --- a/.idea/codeStyles/Project.xml +++ /dev/null @@ -1,7 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/.idea/codeStyles/codeStyleConfig.xml b/.idea/codeStyles/codeStyleConfig.xml deleted file mode 100644 index 79ee123c..00000000 --- a/.idea/codeStyles/codeStyleConfig.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - \ No newline at end of file From f3acc7c8393b8c038a448e2fff9a0840d67422fa Mon Sep 17 00:00:00 2001 From: yubiuser Date: Thu, 7 Oct 2021 00:19:25 +0200 Subject: [PATCH 18/33] Make debug log file size human readable (#4350) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index d199b4f5..71e5c696 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -1238,10 +1238,10 @@ show_messages() { } analyze_gravity_list() { - echo_current_diagnostic "Gravity List and Database" + echo_current_diagnostic "Gravity Database" local gravity_permissions - gravity_permissions=$(ls -ld "${PIHOLE_GRAVITY_DB_FILE}") + gravity_permissions=$(ls -lhd "${PIHOLE_GRAVITY_DB_FILE}") log_write "${COL_GREEN}${gravity_permissions}${COL_NC}" show_db_entries "Info table" "SELECT property,value FROM info" "20 40" @@ -1320,7 +1320,7 @@ analyze_pihole_log() { OLD_IFS="$IFS" # Get the lines that are in the file(s) and store them in an array for parsing later IFS=$'\r\n' - pihole_log_permissions=$(ls -ld "${PIHOLE_LOG}") + pihole_log_permissions=$(ls -lhd "${PIHOLE_LOG}") log_write "${COL_GREEN}${pihole_log_permissions}${COL_NC}" mapfile -t pihole_log_head < <(head -n 20 ${PIHOLE_LOG}) log_write " ${COL_CYAN}-----head of $(basename ${PIHOLE_LOG})------${COL_NC}" From ab27a3bd452dac07f452f5a98e14c850b06ea63c Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Wed, 6 Oct 2021 23:30:29 +0100 Subject: [PATCH 19/33] Dependabot config tweak Signed-off-by: Adam Warner --- .github/dependabot.yml | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index bc08634e..e10beb30 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,17 +1,10 @@ -# To get started with Dependabot version updates, you'll need to specify which -# package ecosystems to update and where the package manifests are located. -# Please see the documentation for all configuration options: -# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates - version: 2 updates: - # Maintain dependencies for GitHub Actions - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly day: saturday time: "10:00" open-pull-requests-limit: 10 - target-branch: development - versioning-strategy: increase + target-branch: developement \ No newline at end of file From f8af1a1baa5f0edee437e39c9a5fcf9a1ee6e944 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Thu, 7 Oct 2021 00:16:07 +0100 Subject: [PATCH 20/33] Allow iFrame for teleporter.php, see https://github.com/pi-hole/AdminLTE/pull/1231 Signed-off-by: Adam Warner --- advanced/lighttpd.conf.debian | 7 +++++++ advanced/lighttpd.conf.fedora | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/advanced/lighttpd.conf.debian b/advanced/lighttpd.conf.debian index 3ecd7213..a58b5a88 100644 --- a/advanced/lighttpd.conf.debian +++ b/advanced/lighttpd.conf.debian @@ -85,5 +85,12 @@ $HTTP["url"] =~ "^/admin/\.(.*)" { url.access-deny = ("") } +# allow teleporter iframe on settings page +$HTTP["url"] =~ "/teleporter\.php$" { + $HTTP["referer"] =~ "/admin/settings\.php" { + setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) + } +} + # Default expire header expire.url = ( "" => "access plus 0 seconds" ) diff --git a/advanced/lighttpd.conf.fedora b/advanced/lighttpd.conf.fedora index 5a99a9bf..ad336a93 100644 --- a/advanced/lighttpd.conf.fedora +++ b/advanced/lighttpd.conf.fedora @@ -93,5 +93,12 @@ $HTTP["url"] =~ "^/admin/\.(.*)" { url.access-deny = ("") } +# allow teleporter iframe on settings page +$HTTP["url"] =~ "/teleporter\.php$" { + $HTTP["referer"] =~ "/admin/settings\.php" { + setenv.add-response-header = ( "X-Frame-Options" => "SAMEORIGIN" ) + } +} + # Default expire header expire.url = ( "" => "access plus 0 seconds" ) From b30d729aa49e5ecdc260a8a43afeabcb5c164673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 8 Oct 2021 21:03:21 +0200 Subject: [PATCH 21/33] Simplify vw_adlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/database_migration/gravity-db.sh | 6 ++++++ .../database_migration/gravity/13_to_14.sql | 2 +- .../database_migration/gravity/14_to_15.sql | 15 +++++++++++++++ advanced/Templates/gravity.db.sql | 10 ++++------ 4 files changed, 26 insertions(+), 7 deletions(-) create mode 100644 advanced/Scripts/database_migration/gravity/14_to_15.sql diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index 22f241dd..0fecf34a 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -122,4 +122,10 @@ upgrade_gravityDB(){ sqlite3 "${database}" < "${scriptPath}/13_to_14.sql" version=14 fi + if [[ "$version" == "14" ]]; then + # Changes the vw_adlist created in 5_to_6 + echo -e " ${INFO} Upgrading gravity database from version 14 to 15" + sqlite3 "${database}" < "${scriptPath}/14_to_15.sql" + version=15 +fi } diff --git a/advanced/Scripts/database_migration/gravity/13_to_14.sql b/advanced/Scripts/database_migration/gravity/13_to_14.sql index fa230865..0a465d1d 100644 --- a/advanced/Scripts/database_migration/gravity/13_to_14.sql +++ b/advanced/Scripts/database_migration/gravity/13_to_14.sql @@ -10,4 +10,4 @@ ALTER TABLE adlist ADD COLUMN status INTEGER NOT NULL DEFAULT 0; UPDATE info SET value = 14 WHERE property = 'version'; -COMMIT; \ No newline at end of file +COMMIT; diff --git a/advanced/Scripts/database_migration/gravity/14_to_15.sql b/advanced/Scripts/database_migration/gravity/14_to_15.sql new file mode 100644 index 00000000..41cb7517 --- /dev/null +++ b/advanced/Scripts/database_migration/gravity/14_to_15.sql @@ -0,0 +1,15 @@ +.timeout 30000 + +PRAGMA FOREIGN_KEYS=OFF; + +BEGIN TRANSACTION; +DROP VIEW vw_adlist; + +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id + FROM adlist + WHERE enabled = 1 + ORDER BY id; + +UPDATE info SET value = 15 WHERE property = 'version'; + +COMMIT; diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 5d7bafa9..3f696d6d 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -57,7 +57,7 @@ CREATE TABLE info value TEXT NOT NULL ); -INSERT INTO "info" VALUES('version','14'); +INSERT INTO "info" VALUES('version','15'); CREATE TABLE domain_audit ( @@ -143,12 +143,10 @@ CREATE VIEW vw_gravity AS SELECT domain, adlist_by_group.group_id AS group_id LEFT JOIN "group" ON "group".id = adlist_by_group.group_id WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1); -CREATE VIEW vw_adlist AS SELECT DISTINCT address, adlist.id AS id +CREATE VIEW vw_adlist AS SELECT DISTINCT address, id FROM adlist - LEFT JOIN adlist_by_group ON adlist_by_group.adlist_id = adlist.id - LEFT JOIN "group" ON "group".id = adlist_by_group.group_id - WHERE adlist.enabled = 1 AND (adlist_by_group.group_id IS NULL OR "group".enabled = 1) - ORDER BY adlist.id; + WHERE enabled = 1 + ORDER BY id; CREATE TRIGGER tr_domainlist_add AFTER INSERT ON domainlist BEGIN From a0ecfcc1dcb11e541456caeb76fc3c8758f1b785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 8 Oct 2021 21:50:46 +0200 Subject: [PATCH 22/33] Include df -h in debug log MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 71e5c696..d8ef7f7e 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -585,6 +585,13 @@ processor_check() { fi } +disk_usage() { + local df + echo_current_diagnostic "Disk usage" + DF=$(df -h) + log_write "${DF}"; +} + parse_setup_vars() { echo_current_diagnostic "Setup variables" # If the file exists, @@ -1421,6 +1428,7 @@ diagnose_operating_system check_selinux check_firewalld processor_check +disk_usage check_networking check_name_resolution check_dhcp_servers From fdc4cf9869e11df5fed525b09684736c247b7e2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Fri, 8 Oct 2021 21:54:50 +0200 Subject: [PATCH 23/33] Fix stickler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index d8ef7f7e..01daaa9f 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -586,7 +586,7 @@ processor_check() { } disk_usage() { - local df + local DF echo_current_diagnostic "Disk usage" DF=$(df -h) log_write "${DF}"; From 3c41ec08a3dedf18ac8a6004aaa34bff534bc295 Mon Sep 17 00:00:00 2001 From: yubiuser Date: Fri, 8 Oct 2021 23:54:23 +0200 Subject: [PATCH 24/33] Set file permission for querie database in pihole-FTL.service (#4328) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Set file permission for querie database in pihole-FTL.service Signed-off-by: Christian König * Use -f flag for chmod of the macvendor.db Signed-off-by: Christian König * Fix missing space Signed-off-by: Christian König * Fix spelling Signed-off-by: Christian König --- advanced/Templates/pihole-FTL.service | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 55a68b15..865e2cd9 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -24,9 +24,13 @@ start() { touch /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases # Ensure that permissions are set so that pihole-FTL can edit all necessary files chown pihole:pihole /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases /run/pihole /etc/pihole - chmod 0644 /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases /etc/pihole/macvendor.db + chmod 0644 /run/pihole-FTL.pid /run/pihole-FTL.port /var/log/pihole-FTL.log /var/log/pihole.log /etc/pihole/dhcp.leases + # Ensure that permissions are set so that pihole-FTL can edit the files. We ignore errors as the file may not (yet) exist + chmod -f 0644 /etc/pihole/macvendor.db # Chown database files to the user FTL runs as. We ignore errors as the files may not (yet) exist chown -f pihole:pihole /etc/pihole/pihole-FTL.db /etc/pihole/gravity.db /etc/pihole/macvendor.db + # Chown database file permissions so that the pihole group (web interface) can edit the file. We ignore errors as the files may not (yet) exist + chmod -f 0664 /etc/pihole/pihole-FTL.db if setcap CAP_NET_BIND_SERVICE,CAP_NET_RAW,CAP_NET_ADMIN,CAP_SYS_NICE,CAP_IPC_LOCK,CAP_CHOWN+eip "/usr/bin/pihole-FTL"; then su -s /bin/sh -c "/usr/bin/pihole-FTL" pihole else From 99981b5e662c489fb0840f2a0da666bddf43f8d3 Mon Sep 17 00:00:00 2001 From: Adam Warner Date: Sat, 11 Sep 2021 22:43:27 +0100 Subject: [PATCH 25/33] now that whiptail size is fixed, lose a couple of lines from the final whiptail output Signed-off-by: Adam Warner --- automated install/basic-install.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index 2e86f024..a39f7381 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1914,8 +1914,6 @@ IPv6: ${IPV6_ADDRESS:-"Not Configured"} If you have not done so already, the above IP should be set to static. -The install log is in /etc/pihole. - ${additional}" "${r}" "${c}" } From 77a30ac0c25ca1f788e974b72d787e225a4ea82d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 11 Oct 2021 17:31:03 +0200 Subject: [PATCH 26/33] Use mapfile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 01daaa9f..1366c14b 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -586,10 +586,13 @@ processor_check() { } disk_usage() { - local DF + local file_system echo_current_diagnostic "Disk usage" - DF=$(df -h) - log_write "${DF}"; + mapfile -t file_system < <(df -h) + + for line in "${file_system[@]}"; do + log_write " ${line}" + done } parse_setup_vars() { From d84da7131000502a1b2d9792eab6124f6036081d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Mon, 11 Oct 2021 18:02:47 +0200 Subject: [PATCH 27/33] Only show lines not containing sensitive keywords MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/piholeDebug.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index 1366c14b..cd615825 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -587,11 +587,22 @@ processor_check() { disk_usage() { local file_system + local hide + echo_current_diagnostic "Disk usage" mapfile -t file_system < <(df -h) + # Some lines of df might contain sensitive information like usernames and passwords. + # E.g. curlftpfs filesystems (https://www.looklinux.com/mount-ftp-share-on-linux-using-curlftps/) + # We are not interested in those lines so we collect keyword, to remove them from the output + # Additinal keywords can be added, separated by "|" + hide="curlftpfs" + + # only show those lines not containg a sensitive phrase for line in "${file_system[@]}"; do + if [[ ! $line =~ $hide ]]; then log_write " ${line}" + fi done } From 04f9e92bffd316320266293106e1a596f90e4f6e Mon Sep 17 00:00:00 2001 From: MichaIng Date: Mon, 11 Oct 2021 21:43:12 +0200 Subject: [PATCH 28/33] Fix PHP8.0 detection (#4383) The phpInsNewer variable is not set anymore, so that the JSON module is now always tried to be installed. Instead of checking for phpInsNewer to derive whether PHP was installed already, phpInsMajor is now checked. If it is set, PHP is installed already, and only if the major version is lower than 8, the JSON module can be installed. Signed-off-by: MichaIng --- automated install/basic-install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index a39f7381..babb8213 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -292,7 +292,7 @@ if is_command apt-get ; then # It's useful to separate this from Pi-hole, since the two repos are also setup separately PIHOLE_WEB_DEPS=(lighttpd "${phpVer}-common" "${phpVer}-cgi" "${phpVer}-sqlite3" "${phpVer}-xml" "${phpVer}-intl") # Prior to PHP8.0, JSON functionality is provided as dedicated module, required by Pi-hole AdminLTE: https://www.php.net/manual/json.installation.php - if [[ "${phpInsNewer}" != true || "${phpInsMajor}" -lt 8 ]]; then + if [[ -z "${phpInsMajor}" || "${phpInsMajor}" -lt 8 ]]; then PIHOLE_WEB_DEPS+=("${phpVer}-json") fi # The Web server user, From b7bba6a689d135a9de972393fee47a520e6c5406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 12 Oct 2021 18:15:56 +0200 Subject: [PATCH 29/33] Validate when adding not when removing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 44 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 52c388f8..b651bed5 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -709,7 +709,13 @@ AddCustomDNSAddress() { ip="${args[2]}" host="${args[3]}" - echo "${ip} ${host}" >> "${dnscustomfile}" + + if valid_ip "${ip}" || valid_ip6 "${ip}" ; then + echo "${ip} ${host}" >> "${dnscustomfile}" + else + echo -e " ${CROSS} Invalid IP has been passed" + exit 1 + fi # Restart dnsmasq to load new custom DNS entries RestartDNS @@ -721,12 +727,7 @@ RemoveCustomDNSAddress() { ip="${args[2]}" host="${args[3]}" - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi + sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" # Restart dnsmasq to update removed custom DNS entries RestartDNS @@ -738,8 +739,19 @@ AddCustomCNAMERecord() { domain="${args[2]}" target="${args[3]}" - echo "cname=${domain},${target}" >> "${dnscustomcnamefile}" - + validDomain="$(checkDomain "${domain}")" + if [[ -n "${validDomain}" ]]; then + validTarget="$(checkDomain "${target}")" + if [[ -n "${validTarget}" ]]; then + echo "cname=${validDomain},${validTarget}" >> "${dnscustomcnamefile}" + else + echo " ${CROSS} Invalid Target Passed!" + exit 1 + fi + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 + fi # Restart dnsmasq to load new custom CNAME records RestartDNS } @@ -750,19 +762,7 @@ RemoveCustomCNAMERecord() { domain="${args[2]}" target="${args[3]}" - validDomain="$(checkDomain "${domain}")" - if [[ -n "${validDomain}" ]]; then - validTarget="$(checkDomain "${target}")" - if [[ -n "${validDomain}" ]]; then - sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}" - else - echo " ${CROSS} Invalid Target Passed!" - exit 1 - fi - else - echo " ${CROSS} Invalid Domain passed!" - exit 1 - fi + sed -i "/cname=${domain},${target}$/d" "${dnscustomcnamefile}" # Restart dnsmasq to update removed custom CNAME records RestartDNS From bc21a7155de841b5f39ce8c3c3aa51a691d25fdf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 12 Oct 2021 19:49:36 +0200 Subject: [PATCH 30/33] Add option to not reload MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index b651bed5..f382b4d1 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -709,6 +709,7 @@ AddCustomDNSAddress() { ip="${args[2]}" host="${args[3]}" + reload="${args[4]}" if valid_ip "${ip}" || valid_ip6 "${ip}" ; then echo "${ip} ${host}" >> "${dnscustomfile}" @@ -717,8 +718,10 @@ AddCustomDNSAddress() { exit 1 fi - # Restart dnsmasq to load new custom DNS entries - RestartDNS + # Restart dnsmasq to load new custom DNS entries only if $reload not false + if [[ ! $reload == "false" ]]; then + RestartDNS + fi } RemoveCustomDNSAddress() { @@ -726,11 +729,14 @@ RemoveCustomDNSAddress() { ip="${args[2]}" host="${args[3]}" + reload="${args[4]}" sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" - # Restart dnsmasq to update removed custom DNS entries - RestartDNS + # Restart dnsmasq to load new custom DNS entries only if reload is not false + if [[ ! $reload == "false" ]]; then + RestartDNS + fi } AddCustomCNAMERecord() { @@ -738,6 +744,7 @@ AddCustomCNAMERecord() { domain="${args[2]}" target="${args[3]}" + reload="${args[4]}" validDomain="$(checkDomain "${domain}")" if [[ -n "${validDomain}" ]]; then @@ -752,8 +759,10 @@ AddCustomCNAMERecord() { echo " ${CROSS} Invalid Domain passed!" exit 1 fi - # Restart dnsmasq to load new custom CNAME records - RestartDNS + # Restart dnsmasq to load new custom CNAME records only if reload is not false + if [[ ! $reload == "false" ]]; then + RestartDNS + fi } RemoveCustomCNAMERecord() { @@ -761,11 +770,14 @@ RemoveCustomCNAMERecord() { domain="${args[2]}" target="${args[3]}" + reload="${args[4]}" sed -i "/cname=${domain},${target}$/d" "${dnscustomcnamefile}" - # Restart dnsmasq to update removed custom CNAME records - RestartDNS + # Restart dnsmasq to update removed custom CNAME records only if $reload not false + if [[ ! $reload == "false" ]]; then + RestartDNS + fi } main() { From a872fabe7d8518a9d025887da70ad922251e2cf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Sun, 17 Oct 2021 20:51:59 +0200 Subject: [PATCH 31/33] Validate on removal as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index f382b4d1..463b12fe 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -731,7 +731,12 @@ RemoveCustomDNSAddress() { host="${args[3]}" reload="${args[4]}" - sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" + if valid_ip "${ip}" || valid_ip6 "${ip}" ; then + sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" + else + echo -e " ${CROSS} Invalid IP has been passed" + exit 1 + fi # Restart dnsmasq to load new custom DNS entries only if reload is not false if [[ ! $reload == "false" ]]; then @@ -772,7 +777,19 @@ RemoveCustomCNAMERecord() { target="${args[3]}" reload="${args[4]}" - sed -i "/cname=${domain},${target}$/d" "${dnscustomcnamefile}" + validDomain="$(checkDomain "${domain}")" + if [[ -n "${validDomain}" ]]; then + validTarget="$(checkDomain "${target}")" + if [[ -n "${validTarget}" ]]; then + sed -i "/cname=${validDomain},${validTarget}$/d" "${dnscustomcnamefile}" + else + echo " ${CROSS} Invalid Target Passed!" + exit 1 + fi + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 + fi # Restart dnsmasq to update removed custom CNAME records only if $reload not false if [[ ! $reload == "false" ]]; then From 596689b4c99f794eba8ddd14b51e914d0eac5917 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20K=C3=B6nig?= Date: Tue, 19 Oct 2021 21:34:16 +0200 Subject: [PATCH 32/33] Validate host/domain of Local DNS records as well MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Christian König --- advanced/Scripts/webpage.sh | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh index 463b12fe..a739d898 100755 --- a/advanced/Scripts/webpage.sh +++ b/advanced/Scripts/webpage.sh @@ -711,12 +711,18 @@ AddCustomDNSAddress() { host="${args[3]}" reload="${args[4]}" - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - echo "${ip} ${host}" >> "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 - fi + validHost="$(checkDomain "${host}")" + if [[ -n "${validHost}" ]]; then + if valid_ip "${ip}" || valid_ip6 "${ip}" ; then + echo "${ip} ${validHost}" >> "${dnscustomfile}" + else + echo -e " ${CROSS} Invalid IP has been passed" + exit 1 + fi + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 + fi # Restart dnsmasq to load new custom DNS entries only if $reload not false if [[ ! $reload == "false" ]]; then @@ -731,11 +737,17 @@ RemoveCustomDNSAddress() { host="${args[3]}" reload="${args[4]}" - if valid_ip "${ip}" || valid_ip6 "${ip}" ; then - sed -i "/^${ip} ${host}$/d" "${dnscustomfile}" - else - echo -e " ${CROSS} Invalid IP has been passed" - exit 1 + validHost="$(checkDomain "${host}")" + if [[ -n "${validHost}" ]]; then + if valid_ip "${ip}" || valid_ip6 "${ip}" ; then + sed -i "/^${ip} ${validHost}$/d" "${dnscustomfile}" + else + echo -e " ${CROSS} Invalid IP has been passed" + exit 1 + fi + else + echo " ${CROSS} Invalid Domain passed!" + exit 1 fi # Restart dnsmasq to load new custom DNS entries only if reload is not false From 8713135b018fd0464c55dee53393eaae23c195a0 Mon Sep 17 00:00:00 2001 From: Blayne Campbell Date: Sat, 23 Oct 2021 12:43:20 -0600 Subject: [PATCH 33/33] update tests: remove fedora 32, add fedora 34 (#4403) Signed-off-by: bcambl --- .github/workflows/test.yml | 2 +- test/{_fedora_32.Dockerfile => _fedora_34.Dockerfile} | 2 +- test/{tox.fedora_32.ini => tox.fedora_34.ini} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename test/{_fedora_32.Dockerfile => _fedora_34.Dockerfile} (97%) rename test/{tox.fedora_32.ini => tox.fedora_34.ini} (78%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c2b4dbbc..49f139e1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: needs: smoke-test strategy: matrix: - distro: [debian_9, debian_10, debian_11, ubuntu_16, ubuntu_18, ubuntu_20, ubuntu_21, centos_7, centos_8, fedora_32, fedora_33] + distro: [debian_9, debian_10, debian_11, ubuntu_16, ubuntu_18, ubuntu_20, ubuntu_21, centos_7, centos_8, fedora_33, fedora_34] env: DISTRO: ${{matrix.distro}} steps: diff --git a/test/_fedora_32.Dockerfile b/test/_fedora_34.Dockerfile similarity index 97% rename from test/_fedora_32.Dockerfile rename to test/_fedora_34.Dockerfile index e9c2ff2a..96de18da 100644 --- a/test/_fedora_32.Dockerfile +++ b/test/_fedora_34.Dockerfile @@ -1,4 +1,4 @@ -FROM fedora:32 +FROM fedora:34 ENV GITDIR /etc/.pihole ENV SCRIPTDIR /opt/pihole diff --git a/test/tox.fedora_32.ini b/test/tox.fedora_34.ini similarity index 78% rename from test/tox.fedora_32.ini rename to test/tox.fedora_34.ini index c68e0757..154662cf 100644 --- a/test/tox.fedora_32.ini +++ b/test/tox.fedora_34.ini @@ -4,5 +4,5 @@ envlist = py37 [testenv] whitelist_externals = docker deps = -rrequirements.txt -commands = docker build -f _fedora_32.Dockerfile -t pytest_pihole:test_container ../ +commands = docker build -f _fedora_34.Dockerfile -t pytest_pihole:test_container ../ pytest {posargs:-vv -n auto} ./test_automated_install.py ./test_centos_fedora_common_support.py ./test_fedora_support.py