diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 00000000..58192bc1 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,5 @@ +# see https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-syntax + +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +* @pi-hole/core-maintainers diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e140f792..af9b74db 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,8 +8,6 @@ updates: time: "10:00" open-pull-requests-limit: 10 target-branch: development - reviewers: - - "pi-hole/core-maintainers" - package-ecosystem: pip directory: "/test" schedule: @@ -18,5 +16,3 @@ updates: time: "10:00" open-pull-requests-limit: 10 target-branch: development - reviewers: - - "pi-hole/core-maintainers" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43cd8ad4..ac496406 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,6 +19,8 @@ jobs: steps: - name: Checkout repository uses: actions/checkout@v4.2.2 + with: + fetch-depth: 0 # Differential ShellCheck requires full git history - name: Check scripts in repository are executable run: | @@ -28,12 +30,12 @@ jobs: # If FAIL is 1 then we fail. [[ $FAIL == 1 ]] && exit 1 || echo "Scripts are executable!" - - name: Run shellcheck - uses: ludeeus/action-shellcheck@master + - name: Differential ShellCheck + uses: redhat-plumbers-in-action/differential-shellcheck@v5 with: - check_together: 'yes' - format: tty - severity: error + severity: warning + display-engine: sarif-fmt + - name: Spell-Checking uses: codespell-project/actions-codespell@master @@ -67,8 +69,10 @@ jobs: ubuntu_22, ubuntu_24, centos_9, + centos_10, fedora_40, fedora_41, + fedora_42, ] env: DISTRO: ${{matrix.distro}} @@ -77,7 +81,7 @@ jobs: uses: actions/checkout@v4.2.2 - name: Set up Python 3.10 - uses: actions/setup-python@v5.5.0 + uses: actions/setup-python@v5.6.0 with: python-version: "3.10" diff --git a/.gitignore b/.gitignore index 8016472b..6322fd3e 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,6 @@ __pycache__ .idea/ *.iml .vscode/ +.venv/ +.fleet/ +.cache/ diff --git a/.shellcheckrc b/.shellcheckrc new file mode 100644 index 00000000..8e0b8387 --- /dev/null +++ b/.shellcheckrc @@ -0,0 +1,2 @@ +external-sources=true # allow shellcheck to read external sources +disable=SC3043 #disable SC3043: In POSIX sh, local is undefined. diff --git a/README.md b/README.md index f320f8c5..622ff202 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,10 @@ Some of the statistics you can integrate include: - Queries cached - Unique clients -Access the API via [`telnet`](https://github.com/pi-hole/FTL), the Web (`admin/api.php`) and Command Line (`pihole -c -j`). You can find out [more details over here](https://discourse.pi-hole.net/t/pi-hole-api/1863). +Access the API using: +- your browser: http://pi.hole/api/docs +- `curl`: `curl --connect-timeout 2 -ks "https://pi.hole/api/stats/summary" -H "Accept: application/json"`; +- the command line - examples: `pihole api config/webserver/port` or `pihole api stats/summary`. ### The Command-Line Interface diff --git a/advanced/Scripts/api.sh b/advanced/Scripts/api.sh index 79fc90f4..613a8d86 100755 --- a/advanced/Scripts/api.sh +++ b/advanced/Scripts/api.sh @@ -1,5 +1,4 @@ #!/usr/bin/env sh -# shellcheck disable=SC3043 #https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions # Pi-hole: A black hole for Internet advertisements # (c) 2017 Pi-hole, LLC (https://pi-hole.net) @@ -20,13 +19,19 @@ TestAPIAvailability() { + local chaos_api_list authResponse authStatus authData apiAvailable DNSport + # as we are running locally, we can get the port value from FTL directly - local chaos_api_list authResponse authStatus authData + readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" + # shellcheck source=./advanced/Scripts/utils.sh + . "${utilsfile}" + + DNSport=$(getFTLConfigValue dns.port) # Query the API URLs from FTL using CHAOS TXT local.api.ftl # The result is a space-separated enumeration of full URLs # e.g., "http://localhost:80/api/" "https://localhost:443/api/" - chaos_api_list="$(dig +short chaos txt local.api.ftl @127.0.0.1)" + chaos_api_list="$(dig +short -p "${DNSport}" chaos txt local.api.ftl @127.0.0.1)" # If the query was not successful, the variable is empty if [ -z "${chaos_api_list}" ]; then @@ -48,48 +53,50 @@ TestAPIAvailability() { API_URL="${API_URL%\"}" API_URL="${API_URL#\"}" - # Test if the API is available at this URL - authResponse=$(curl --connect-timeout 2 -skS -w "%{http_code}" "${API_URL}auth") + # Test if the API is available at this URL, include delimiter for ease in splitting payload + authResponse=$(curl --connect-timeout 2 -skS -w ">>%{http_code}" "${API_URL}auth") - # authStatus are the last 3 characters - # not using ${authResponse#"${authResponse%???}"}" here because it's extremely slow on big responses - authStatus=$(printf "%s" "${authResponse}" | tail -c 3) - # data is everything from response without the last 3 characters - authData=$(printf %s "${authResponse%???}") + # authStatus is the response http_code, eg. 200, 401. + # Shell parameter expansion, remove everything up to and including the >> delim + authStatus=${authResponse#*>>} + # data is everything from response + # Shell parameter expansion, remove the >> delim and everything after + authData=${authResponse%>>*} # Test if http status code was 200 (OK) or 401 (authentication required) - if [ ! "${authStatus}" = 200 ] && [ ! "${authStatus}" = 401 ]; then - # API is not available at this port/protocol combination - API_PORT="" - else - # API is available at this URL combination - - if [ "${authStatus}" = 200 ]; then - # API is available without authentication - needAuth=false - fi + if [ "${authStatus}" = 200 ]; then + # API is available without authentication + apiAvailable=true + needAuth=false + break + elif [ "${authStatus}" = 401 ]; then + # API is available with authentication + apiAvailable=true + needAuth=true # Check if 2FA is required needTOTP=$(echo "${authData}"| jq --raw-output .session.totp 2>/dev/null) - break - fi - # Remove the first URL from the list - local last_api_list - last_api_list="${chaos_api_list}" - chaos_api_list="${chaos_api_list#* }" + else + # API is not available at this port/protocol combination + apiAvailable=false + # Remove the first URL from the list + local last_api_list + last_api_list="${chaos_api_list}" + chaos_api_list="${chaos_api_list#* }" - # If the list did not change, we are at the last element - if [ "${last_api_list}" = "${chaos_api_list}" ]; then - # Remove the last element - chaos_api_list="" + # If the list did not change, we are at the last element + if [ "${last_api_list}" = "${chaos_api_list}" ]; then + # Remove the last element + chaos_api_list="" + fi fi done - # if API_PORT is empty, no working API port was found - if [ -n "${API_PORT}" ]; then - echo "API not available at: ${API_URL}" + # if apiAvailable is false, no working API was found + if [ "${apiAvailable}" = false ]; then + echo "API not available. Please check FTL.log" echo "Exiting." exit 1 fi @@ -227,7 +234,7 @@ GetFTLData() { # return only the data if [ "${status}" = 200 ]; then # response OK - echo "${data}" + printf %s "${data}" else # connection lost echo "${status}" @@ -301,14 +308,23 @@ secretRead() { } apiFunc() { - local data response status status_col + local data response status status_col verbosity + + # Define if the output will be silent (default) or verbose + verbosity="silent" + if [ "$1" = "verbose" ]; then + verbosity="verbose" + shift + fi # Authenticate with the API - LoginAPI verbose - echo "" + LoginAPI "${verbosity}" - echo "Requesting: ${COL_PURPLE}GET ${COL_CYAN}${API_URL}${COL_YELLOW}$1${COL_NC}" - echo "" + if [ "${verbosity}" = "verbose" ]; then + echo "" + echo "Requesting: ${COL_PURPLE}GET ${COL_CYAN}${API_URL}${COL_YELLOW}$1${COL_NC}" + echo "" + fi # Get the data from the API response=$(GetFTLData "$1" raw) @@ -325,11 +341,18 @@ apiFunc() { else status_col="${COL_RED}" fi - echo "Status: ${status_col}${status}${COL_NC}" + + # Only print the status in verbose mode or if the status is not 200 + if [ "${verbosity}" = "verbose" ] || [ "${status}" != 200 ]; then + echo "Status: ${status_col}${status}${COL_NC}" + fi # Output the data. Format it with jq if available and data is actually JSON. # Otherwise just print it - echo "Data:" + if [ "${verbosity}" = "verbose" ]; then + echo "Data:" + fi + if command -v jq >/dev/null && echo "${data}" | jq . >/dev/null 2>&1; then echo "${data}" | jq . else @@ -337,5 +360,5 @@ apiFunc() { fi # Delete the session - LogoutAPI verbose + LogoutAPI "${verbosity}" } diff --git a/advanced/Scripts/database_migration/gravity-db.sh b/advanced/Scripts/database_migration/gravity-db.sh index b0982bcc..41593368 100755 --- a/advanced/Scripts/database_migration/gravity-db.sh +++ b/advanced/Scripts/database_migration/gravity-db.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# shellcheck disable=SC1090 + # Pi-hole: A black hole for Internet advertisements # (c) 2019 Pi-hole, LLC (https://pi-hole.net) @@ -13,9 +13,8 @@ readonly scriptPath="/etc/.pihole/advanced/Scripts/database_migration/gravity" upgrade_gravityDB(){ - local database piholeDir version + local database version database="${1}" - piholeDir="${2}" # Exit early if the database does not exist (e.g. in CI tests) if [[ ! -f "${database}" ]]; then diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh index 5c57f878..fa356f16 100755 --- a/advanced/Scripts/list.sh +++ b/advanced/Scripts/list.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# shellcheck disable=SC1090 # Pi-hole: A black hole for Internet advertisements # (c) 2017 Pi-hole, LLC (https://pi-hole.net) @@ -12,9 +11,11 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +# shellcheck source="./advanced/Scripts/utils.sh" source "${utilsfile}" readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh" +# shellcheck source="./advanced/Scripts/api.sh" source "${apifile}" # Determine database location @@ -39,6 +40,7 @@ typeId="" comment="" colfile="/opt/pihole/COL_TABLE" +# shellcheck source="./advanced/Scripts/COL_TABLE" source ${colfile} helpFunc() { diff --git a/advanced/Scripts/piholeARPTable.sh b/advanced/Scripts/piholeARPTable.sh index f55b1320..120df5b8 100755 --- a/advanced/Scripts/piholeARPTable.sh +++ b/advanced/Scripts/piholeARPTable.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# shellcheck disable=SC1090 # Pi-hole: A black hole for Internet advertisements # (c) 2019 Pi-hole, LLC (https://pi-hole.net) @@ -12,13 +11,22 @@ coltable="/opt/pihole/COL_TABLE" if [[ -f ${coltable} ]]; then +# shellcheck source="./advanced/Scripts/COL_TABLE" source ${coltable} fi readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +# shellcheck source=./advanced/Scripts/utils.sh source "${utilsfile}" +readonly PI_HOLE_FILES_DIR="/etc/.pihole" +SKIP_INSTALL="true" +# shellcheck source="./automated install/basic-install.sh" +source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" +# stop_service() is defined in basic-install.sh +# restart_service() is defined in basic-install.sh + # Determine database location DBFILE=$(getFTLConfigValue "files.database") if [ -z "$DBFILE" ]; then @@ -32,7 +40,7 @@ flushARP(){ fi # Stop FTL to prevent database access - if ! output=$(service pihole-FTL stop 2>&1); then + if ! output=$(stop_service pihole-FTL 2>&1); then echo -e "${OVER} ${CROSS} Failed to stop FTL" echo " Output: ${output}" return 1 @@ -64,7 +72,7 @@ flushARP(){ fi # Start FTL again - if ! output=$(service pihole-FTL restart 2>&1); then + if ! output=$(restart_service pihole-FTL 2>&1); then echo -e "${OVER} ${CROSS} Failed to restart FTL" echo " Output: ${output}" return 1 diff --git a/advanced/Scripts/piholeCheckout.sh b/advanced/Scripts/piholeCheckout.sh index 84c966df..beaac5f1 100755 --- a/advanced/Scripts/piholeCheckout.sh +++ b/advanced/Scripts/piholeCheckout.sh @@ -10,6 +10,7 @@ readonly PI_HOLE_FILES_DIR="/etc/.pihole" SKIP_INSTALL="true" +# shellcheck source="./automated install/basic-install.sh" source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" # webInterfaceGitUrl set in basic-install.sh @@ -109,7 +110,7 @@ checkout() { echo -e "${OVER} ${CROSS} $str" exit 1 fi - corebranches=($(get_available_branches "${PI_HOLE_FILES_DIR}")) + mapfile -t corebranches < <(get_available_branches "${PI_HOLE_FILES_DIR}") if [[ "${corebranches[*]}" == *"master"* ]]; then echo -e "${OVER} ${TICK} $str" @@ -136,7 +137,7 @@ checkout() { echo -e "${OVER} ${CROSS} $str" exit 1 fi - webbranches=($(get_available_branches "${webInterfaceDir}")) + mapfile -t webbranches < <(get_available_branches "${webInterfaceDir}") if [[ "${webbranches[*]}" == *"master"* ]]; then echo -e "${OVER} ${TICK} $str" @@ -167,7 +168,7 @@ checkout() { # Check if requested branch is available echo -e " ${INFO} Checking for availability of branch ${COL_CYAN}${2}${COL_NC} on GitHub" - ftlbranches=( $(git ls-remote https://github.com/pi-hole/ftl | grep "refs/heads" | cut -d'/' -f3- -) ) + mapfile -t ftlbranches < <(git ls-remote https://github.com/pi-hole/ftl | grep "refs/heads" | cut -d'/' -f3- -) # If returned array is empty -> connectivity issue if [[ ${#ftlbranches[@]} -eq 0 ]]; then echo -e " ${CROSS} Unable to fetch branches from GitHub. Please check your Internet connection and try again later." @@ -209,13 +210,15 @@ checkout() { # Update local and remote versions via updatechecker /opt/pihole/updatecheck.sh else - if [ $? -eq 1 ]; then + local status + status=$? + if [ $status -eq 1 ]; then # Binary for requested branch is not available, may still be # int he process of being built or CI build job failed - printf " %b Binary for requested branch is not available, please try again later.\\n" ${CROSS} + printf " %b Binary for requested branch is not available, please try again later.\\n" "${CROSS}" printf " If the issue persists, please contact Pi-hole Support and ask them to re-generate the binary.\\n" exit 1 - elif [ $? -eq 2 ]; then + elif [ $status -eq 2 ]; then printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" exit 1 else diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh index f4226299..741ff2f4 100755 --- a/advanced/Scripts/piholeDebug.sh +++ b/advanced/Scripts/piholeDebug.sh @@ -8,7 +8,6 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# shellcheck source=/dev/null # -e option instructs bash to immediately exit if any command [1] has a non-zero exit status # -u a reference to any variable you haven't previously defined @@ -27,6 +26,7 @@ PIHOLE_COLTABLE_FILE="${PIHOLE_SCRIPTS_DIRECTORY}/COL_TABLE" # These provide the colors we need for making the log more readable if [[ -f ${PIHOLE_COLTABLE_FILE} ]]; then +# shellcheck source=./advanced/Scripts/COL_TABLE source ${PIHOLE_COLTABLE_FILE} else COL_NC='\e[0m' # No Color @@ -41,7 +41,7 @@ else #OVER="\r\033[K" fi -# shellcheck disable=SC1091 +# shellcheck source=/dev/null . /etc/pihole/versions # Read the value of an FTL config key. The value is printed to stdout. @@ -213,7 +213,7 @@ compare_local_version_to_git_version() { local local_status local_status=$(git status -s) # echo this information out to the user in a nice format - if [ ${local_version} ]; then + if [ "${local_version}" ]; then log_write "${TICK} Version: ${local_version}" elif [ -n "${DOCKER_VERSION}" ]; then log_write "${TICK} Version: Pi-hole Docker Container ${COL_BOLD}${DOCKER_VERSION}${COL_NC}" @@ -296,91 +296,12 @@ check_component_versions() { check_ftl_version } -os_check() { - # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net - # and determines whether or not the script is running on one of those systems - local remote_os_domain valid_os valid_version detected_os detected_version cmdResult digReturnCode response - remote_os_domain=${OS_CHECK_DOMAIN_NAME:-"versions.pi-hole.net"} - - detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"') - detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - - cmdResult="$(dig -4 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" - #Get the return code of the previous command (last line) - digReturnCode="${cmdResult##*$'\n'}" - - # Extract dig response - response="${cmdResult%%$'\n'*}" - - if [ "${digReturnCode}" -ne 0 ]; then - log_write "${INFO} Distro: ${detected_os^}" - log_write "${INFO} Version: ${detected_version}" - log_write "${CROSS} dig IPv4 return code: ${COL_RED}${digReturnCode}${COL_NC}" - log_write "${CROSS} dig response: ${response}" - log_write "${INFO} Retrying via IPv6" - - cmdResult="$(dig -6 +short -t txt "${remote_os_domain}" @ns1.pi-hole.net 2>&1; echo $?)" - #Get the return code of the previous command (last line) - digReturnCode="${cmdResult##*$'\n'}" - - # Extract dig response - response="${cmdResult%%$'\n'*}" - fi - # If also no success via IPv6 - if [ "${digReturnCode}" -ne 0 ]; then - log_write "${CROSS} dig IPv6 return code: ${COL_RED}${digReturnCode}${COL_NC}" - log_write "${CROSS} dig response: ${response}" - log_write "${CROSS} Error: ${COL_RED}dig command failed - Unable to check OS${COL_NC}" - else - IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') - for distro_and_versions in "${supportedOS[@]}" - do - distro_part="${distro_and_versions%%=*}" - versions_part="${distro_and_versions##*=}" - - if [[ "${detected_os^^}" =~ ${distro_part^^} ]]; then - valid_os=true - IFS="," read -r -a supportedVer <<<"${versions_part}" - for version in "${supportedVer[@]}" - do - if [[ "${detected_version}" =~ $version ]]; then - valid_version=true - break - fi - done - break - fi - done - - # If it is a docker container, we can assume the OS is supported - [ -n "${DOCKER_VERSION}" ] && valid_os=true && valid_version=true - - local finalmsg - if [ "$valid_os" = true ]; then - log_write "${TICK} Distro: ${COL_GREEN}${detected_os^}${COL_NC}" - - if [ "$valid_version" = true ]; then - log_write "${TICK} Version: ${COL_GREEN}${detected_version}${COL_NC}" - finalmsg="${TICK} ${COL_GREEN}Distro and version supported${COL_NC}" - else - log_write "${CROSS} Version: ${COL_RED}${detected_version}${COL_NC}" - finalmsg="${CROSS} Error: ${COL_RED}${detected_os^} is supported but version ${detected_version} is currently unsupported ${COL_NC}(${FAQ_HARDWARE_REQUIREMENTS})${COL_NC}" - fi - else - log_write "${CROSS} Distro: ${COL_RED}${detected_os^}${COL_NC}" - finalmsg="${CROSS} Error: ${COL_RED}${detected_os^} is not a supported distro ${COL_NC}(${FAQ_HARDWARE_REQUIREMENTS})${COL_NC}" - fi - - # Print dig response and the final check result - log_write "${TICK} dig return code: ${COL_GREEN}${digReturnCode}${COL_NC}" - log_write "${INFO} dig response: ${response}" - log_write "${finalmsg}" - fi -} - diagnose_operating_system() { # error message in a variable so we can easily modify it later (or reuse it) local error_msg="Distribution unknown -- most likely you are on an unsupported platform and may run into issues." + local detected_os + local detected_version + # Display the current test that is running echo_current_diagnostic "Operating system" @@ -389,8 +310,13 @@ diagnose_operating_system() { # If there is a /etc/*release file, it's probably a supported operating system, so we can if ls /etc/*release 1> /dev/null 2>&1; then - # display the attributes to the user from the function made earlier - os_check + # display the attributes to the user + + detected_os=$(grep "\bID\b" /etc/os-release | cut -d '=' -f2 | tr -d '"') + detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') + + log_write "${INFO} Distro: ${detected_os^}" + log_write "${INFO} Version: ${detected_version}" else # If it doesn't exist, it's not a system we currently support and link to FAQ log_write "${CROSS} ${COL_RED}${error_msg}${COL_NC} (${FAQ_HARDWARE_REQUIREMENTS})" @@ -488,7 +414,9 @@ run_and_print_command() { local output output=$(${cmd} 2>&1) # If the command was successful, - if [[ $? -eq 0 ]]; then + local return_code + return_code=$? + if [[ "${return_code}" -eq 0 ]]; then # show the output log_write "${output}" else @@ -933,7 +861,6 @@ parse_file() { # Get the lines that are in the file(s) and store them in an array for parsing later local file_info if [[ -f "$filename" ]]; then - #shellcheck disable=SC2016 IFS=$'\r\n' command eval 'file_info=( $(cat "${filename}") )' else read -r -a file_info <<< "$filename" diff --git a/advanced/Scripts/piholeLogFlush.sh b/advanced/Scripts/piholeLogFlush.sh index 34d96318..ac0c196f 100755 --- a/advanced/Scripts/piholeLogFlush.sh +++ b/advanced/Scripts/piholeLogFlush.sh @@ -9,12 +9,20 @@ # Please see LICENSE file for your rights under this license. colfile="/opt/pihole/COL_TABLE" +# shellcheck source="./advanced/Scripts/COL_TABLE" source ${colfile} readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +# shellcheck source="./advanced/Scripts/utils.sh" source "${utilsfile}" +SKIP_INSTALL="true" +# shellcheck source="./automated install/basic-install.sh" +source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" +# stop_service() is defined in basic-install.sh +# restart_service() is defined in basic-install.sh + # In case we're running at the same time as a system logrotate, use a # separate logrotate state file to prevent stepping on each other's # toes. @@ -35,6 +43,46 @@ FTLFILE=$(getFTLConfigValue "files.log.ftl") if [ -z "$FTLFILE" ]; then FTLFILE="/var/log/pihole/FTL.log" fi +WEBFILE=$(getFTLConfigValue "files.log.webserver") +if [ -z "$WEBFILE" ]; then + WEBFILE="/var/log/pihole/webserver.log" +fi + +# Helper function to handle log rotation for a single file +rotate_log() { + # This function copies x.log over to x.log.1 + # and then empties x.log + # Note that moving the file is not an option, as + # dnsmasq would happily continue writing into the + # moved file (it will have the same file handler) + local logfile="$1" + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Rotating ${logfile} ..." + fi + cp -p "${logfile}" "${logfile}.1" + echo " " > "${logfile}" + chmod 640 "${logfile}" + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Rotated ${logfile} ..." + fi +} + +# Helper function to handle log flushing for a single file +flush_log() { + local logfile="$1" + if [[ "$*" != *"quiet"* ]]; then + echo -ne " ${INFO} Flushing ${logfile} ..." + fi + echo " " > "${logfile}" + chmod 640 "${logfile}" + if [ -f "${logfile}.1" ]; then + echo " " > "${logfile}.1" + chmod 640 "${logfile}.1" + fi + if [[ "$*" != *"quiet"* ]]; then + echo -e "${OVER} ${TICK} Flushed ${logfile} ..." + fi +} if [[ "$*" == *"once"* ]]; then # Nightly logrotation @@ -46,75 +94,30 @@ if [[ "$*" == *"once"* ]]; then fi /usr/sbin/logrotate --force --state "${STATEFILE}" /etc/pihole/logrotate else - # Copy pihole.log over to pihole.log.1 - # and empty out pihole.log - # Note that moving the file is not an option, as - # dnsmasq would happily continue writing into the - # moved file (it will have the same file handler) - if [[ "$*" != *"quiet"* ]]; then - echo -ne " ${INFO} Rotating ${LOGFILE} ..." - fi - cp -p "${LOGFILE}" "${LOGFILE}.1" - echo " " > "${LOGFILE}" - chmod 640 "${LOGFILE}" - if [[ "$*" != *"quiet"* ]]; then - echo -e "${OVER} ${TICK} Rotated ${LOGFILE} ..." - fi - # Copy FTL.log over to FTL.log.1 - # and empty out FTL.log - if [[ "$*" != *"quiet"* ]]; then - echo -ne " ${INFO} Rotating ${FTLFILE} ..." - fi - cp -p "${FTLFILE}" "${FTLFILE}.1" - echo " " > "${FTLFILE}" - chmod 640 "${FTLFILE}" - if [[ "$*" != *"quiet"* ]]; then - echo -e "${OVER} ${TICK} Rotated ${FTLFILE} ..." - fi + # Handle rotation for each log file + rotate_log "${LOGFILE}" + rotate_log "${FTLFILE}" + rotate_log "${WEBFILE}" fi else # Manual flushing - - # Flush both pihole.log and pihole.log.1 (if existing) - if [[ "$*" != *"quiet"* ]]; then - echo -ne " ${INFO} Flushing ${LOGFILE} ..." - fi - echo " " > "${LOGFILE}" - chmod 640 "${LOGFILE}" - if [ -f "${LOGFILE}.1" ]; then - echo " " > "${LOGFILE}.1" - chmod 640 "${LOGFILE}.1" - fi - if [[ "$*" != *"quiet"* ]]; then - echo -e "${OVER} ${TICK} Flushed ${LOGFILE} ..." - fi - - # Flush both FTL.log and FTL.log.1 (if existing) - if [[ "$*" != *"quiet"* ]]; then - echo -ne " ${INFO} Flushing ${FTLFILE} ..." - fi - echo " " > "${FTLFILE}" - chmod 640 "${FTLFILE}" - if [ -f "${FTLFILE}.1" ]; then - echo " " > "${FTLFILE}.1" - chmod 640 "${FTLFILE}.1" - fi - if [[ "$*" != *"quiet"* ]]; then - echo -e "${OVER} ${TICK} Flushed ${FTLFILE} ..." - fi + flush_log "${LOGFILE}" + flush_log "${FTLFILE}" + flush_log "${WEBFILE}" if [[ "$*" != *"quiet"* ]]; then echo -ne " ${INFO} Flushing database, DNS resolution temporarily unavailable ..." fi # Stop FTL to make sure it doesn't write to the database while we're deleting data - service pihole-FTL stop + stop_service pihole-FTL >/dev/null + # Delete most recent 24 hours from FTL's database, leave even older data intact (don't wipe out all history) deleted=$(pihole-FTL sqlite3 -ni "${DBFILE}" "DELETE FROM query_storage WHERE timestamp >= strftime('%s','now')-86400; select changes() from query_storage limit 1") # Restart FTL - service pihole-FTL restart + restart_service pihole-FTL >/dev/null if [[ "$*" != *"quiet"* ]]; then echo -e "${OVER} ${TICK} Deleted ${deleted} queries from long-term query database" fi diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh index 3340bdd2..18c018dc 100755 --- a/advanced/Scripts/query.sh +++ b/advanced/Scripts/query.sh @@ -1,9 +1,4 @@ #!/usr/bin/env sh -# shellcheck disable=SC1090 - -# Ignore warning about `local` being undefinded in POSIX -# shellcheck disable=SC3043 -# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions # Pi-hole: A black hole for Internet advertisements # (c) 2023 Pi-hole, LLC (https://pi-hole.net) @@ -22,9 +17,11 @@ domain="" # Source color table colfile="/opt/pihole/COL_TABLE" +# shellcheck source="./advanced/Scripts/COL_TABLE" . "${colfile}" # Source api functions +# shellcheck source="./advanced/Scripts/api.sh" . "${PI_HOLE_INSTALL_DIR}/api.sh" Help() { diff --git a/advanced/Scripts/update.sh b/advanced/Scripts/update.sh index 6c6167c0..51c1b1a1 100755 --- a/advanced/Scripts/update.sh +++ b/advanced/Scripts/update.sh @@ -12,26 +12,31 @@ # Variables readonly ADMIN_INTERFACE_GIT_URL="https://github.com/pi-hole/web.git" -readonly ADMIN_INTERFACE_DIR="/var/www/html/admin" readonly PI_HOLE_GIT_URL="https://github.com/pi-hole/pi-hole.git" readonly PI_HOLE_FILES_DIR="/etc/.pihole" -# shellcheck disable=SC2034 SKIP_INSTALL=true # when --check-only is passed to this script, it will not perform the actual update CHECK_ONLY=false -# shellcheck disable=SC1090 +# shellcheck source="./automated install/basic-install.sh" source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" -# shellcheck disable=SC1091 +# shellcheck source=./advanced/Scripts/COL_TABLE source "/opt/pihole/COL_TABLE" +# shellcheck source="./advanced/Scripts/utils.sh" +source "${PI_HOLE_INSTALL_DIR}/utils.sh" # is_repo() sourced from basic-install.sh # make_repo() sourced from basic-install.sh # update_repo() source from basic-install.sh # getGitFiles() sourced from basic-install.sh # FTLcheckUpdate() sourced from basic-install.sh +# getFTLConfigValue() sourced from utils.sh + +# Honour configured paths for the web application. +ADMIN_INTERFACE_DIR=$(getFTLConfigValue "webserver.paths.webroot")$(getFTLConfigValue "webserver.paths.webhome") +readonly ADMIN_INTERFACE_DIR GitCheckUpdateAvail() { local directory @@ -107,8 +112,6 @@ main() { web_update=false FTL_update=false - # Perform an OS check to ensure we're on an appropriate operating system - os_check # Install packages used by this installation script (necessary if users have removed e.g. git from their systems) package_manager_detect @@ -209,7 +212,7 @@ main() { echo "" echo -e " ${INFO} Pi-hole Web Admin files out of date, updating local repo." getGitFiles "${ADMIN_INTERFACE_DIR}" "${ADMIN_INTERFACE_GIT_URL}" - echo -e " ${INFO} If you had made any changes in '/var/www/html/admin/', they have been stashed using 'git stash'" + echo -e " ${INFO} If you had made any changes in '${ADMIN_INTERFACE_DIR}', they have been stashed using 'git stash'" fi if [[ "${FTL_update}" == true ]]; then diff --git a/advanced/Scripts/updatecheck.sh b/advanced/Scripts/updatecheck.sh index b325ee9c..44f21419 100755 --- a/advanced/Scripts/updatecheck.sh +++ b/advanced/Scripts/updatecheck.sh @@ -39,9 +39,12 @@ function get_remote_hash() { } # Source the utils file for addOrEditKeyValPair() -# shellcheck disable=SC1091 +# shellcheck source="./advanced/Scripts/utils.sh" . /opt/pihole/utils.sh +ADMIN_INTERFACE_DIR=$(getFTLConfigValue "webserver.paths.webroot")$(getFTLConfigValue "webserver.paths.webhome") +readonly ADMIN_INTERFACE_DIR + # Remove the below three legacy files if they exist rm -f "/etc/pihole/GitHubVersions" rm -f "/etc/pihole/localbranches" @@ -85,13 +88,13 @@ addOrEditKeyValPair "${VERSION_FILE}" "GITHUB_CORE_HASH" "${GITHUB_CORE_HASH}" # get Web versions -WEB_VERSION="$(get_local_version /var/www/html/admin)" +WEB_VERSION="$(get_local_version "${ADMIN_INTERFACE_DIR}")" addOrEditKeyValPair "${VERSION_FILE}" "WEB_VERSION" "${WEB_VERSION}" -WEB_BRANCH="$(get_local_branch /var/www/html/admin)" +WEB_BRANCH="$(get_local_branch "${ADMIN_INTERFACE_DIR}")" addOrEditKeyValPair "${VERSION_FILE}" "WEB_BRANCH" "${WEB_BRANCH}" -WEB_HASH="$(get_local_hash /var/www/html/admin)" +WEB_HASH="$(get_local_hash "${ADMIN_INTERFACE_DIR}")" addOrEditKeyValPair "${VERSION_FILE}" "WEB_HASH" "${WEB_HASH}" GITHUB_WEB_VERSION="$(get_remote_version web "${WEB_BRANCH}")" diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh index 63d51f87..d4a6957c 100755 --- a/advanced/Scripts/utils.sh +++ b/advanced/Scripts/utils.sh @@ -1,5 +1,4 @@ #!/usr/bin/env sh -# shellcheck disable=SC3043 #https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions # Pi-hole: A black hole for Internet advertisements # (c) 2017 Pi-hole, LLC (https://pi-hole.net) @@ -88,8 +87,8 @@ getFTLConfigValue(){ ####################### setFTLConfigValue(){ pihole-FTL --config "${1}" "${2}" >/dev/null - if [[ $? -eq 5 ]]; then - echo -e " ${CROSS} ${1} set by environment variable. Please unset it to use this function" + if [ $? -eq 5 ]; then + printf " %s %s set by environment variable. Please unset it to use this function\n" "${CROSS}" "${1}" exit 5 fi } diff --git a/advanced/Scripts/version.sh b/advanced/Scripts/version.sh index 540924c2..e932fe63 100755 --- a/advanced/Scripts/version.sh +++ b/advanced/Scripts/version.sh @@ -8,20 +8,16 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. -# Ignore warning about `local` being undefinded in POSIX -# shellcheck disable=SC3043 -# https://github.com/koalaman/shellcheck/wiki/SC3043#exceptions - # Source the versions file populated by updatechecker.sh cachedVersions="/etc/pihole/versions" if [ -f ${cachedVersions} ]; then - # shellcheck disable=SC1090 + # shellcheck source=/dev/null . "$cachedVersions" else echo "Could not find /etc/pihole/versions. Running update now." pihole updatechecker - # shellcheck disable=SC1090 + # shellcheck source=/dev/null . "$cachedVersions" fi diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql index 021f6f67..0187e4e6 100644 --- a/advanced/Templates/gravity.db.sql +++ b/advanced/Templates/gravity.db.sql @@ -43,8 +43,8 @@ CREATE TABLE adlist CREATE TABLE adlist_by_group ( - adlist_id INTEGER NOT NULL REFERENCES adlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), + adlist_id INTEGER NOT NULL REFERENCES adlist (id) ON DELETE CASCADE, + group_id INTEGER NOT NULL REFERENCES "group" (id) ON DELETE CASCADE, PRIMARY KEY (adlist_id, group_id) ); @@ -75,8 +75,8 @@ INSERT INTO "info" VALUES('gravity_restored','false'); CREATE TABLE domainlist_by_group ( - domainlist_id INTEGER NOT NULL REFERENCES domainlist (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), + domainlist_id INTEGER NOT NULL REFERENCES domainlist (id) ON DELETE CASCADE, + group_id INTEGER NOT NULL REFERENCES "group" (id) ON DELETE CASCADE, PRIMARY KEY (domainlist_id, group_id) ); @@ -91,8 +91,8 @@ CREATE TABLE client CREATE TABLE client_by_group ( - client_id INTEGER NOT NULL REFERENCES client (id), - group_id INTEGER NOT NULL REFERENCES "group" (id), + client_id INTEGER NOT NULL REFERENCES client (id) ON DELETE CASCADE, + group_id INTEGER NOT NULL REFERENCES "group" (id) ON DELETE CASCADE, PRIMARY KEY (client_id, group_id) ); diff --git a/advanced/Templates/pihole-FTL-poststop.sh b/advanced/Templates/pihole-FTL-poststop.sh index b5ddbc97..504e2382 100755 --- a/advanced/Templates/pihole-FTL-poststop.sh +++ b/advanced/Templates/pihole-FTL-poststop.sh @@ -3,7 +3,7 @@ # Source utils.sh for getFTLConfigValue() PI_HOLE_SCRIPT_DIR='/opt/pihole' utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" -# shellcheck disable=SC1090 +# shellcheck source="./advanced/Scripts/utils.sh" . "${utilsfile}" # Get file paths diff --git a/advanced/Templates/pihole-FTL-prestart.sh b/advanced/Templates/pihole-FTL-prestart.sh index 37d750a2..579309d3 100755 --- a/advanced/Templates/pihole-FTL-prestart.sh +++ b/advanced/Templates/pihole-FTL-prestart.sh @@ -3,7 +3,7 @@ # Source utils.sh for getFTLConfigValue() PI_HOLE_SCRIPT_DIR='/opt/pihole' utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" -# shellcheck disable=SC1090 +# shellcheck source="./advanced/Scripts/utils.sh" . "${utilsfile}" # Get file paths @@ -12,6 +12,10 @@ FTL_PID_FILE="$(getFTLConfigValue files.pid)" # Ensure that permissions are set so that pihole-FTL can edit all necessary files mkdir -p /var/log/pihole chown -R pihole:pihole /etc/pihole/ /var/log/pihole/ + +# allow all users read version file (and use pihole -v) +chmod 0644 /etc/pihole/versions + # allow pihole to access subdirs in /etc/pihole (sets execution bit on dirs) find /etc/pihole/ /var/log/pihole/ -type d -exec chmod 0755 {} + # Set all files (except TLS-related ones) to u+rw g+r diff --git a/advanced/Templates/pihole-FTL.service b/advanced/Templates/pihole-FTL.service index 151d4f90..7c7e9962 100644 --- a/advanced/Templates/pihole-FTL.service +++ b/advanced/Templates/pihole-FTL.service @@ -12,7 +12,7 @@ # Source utils.sh for getFTLConfigValue(), getFTLPID() PI_HOLE_SCRIPT_DIR="/opt/pihole" utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" -# shellcheck disable=SC1090 +# shellcheck source="./advanced/Scripts/utils.sh" . "${utilsfile}" @@ -57,13 +57,16 @@ start() { stop() { if is_running; then kill "${FTL_PID}" - for i in 1 2 3 4 5; do + # Give FTL 60 seconds to gracefully stop + i=1 + while [ "${i}" -le 60 ]; do if ! is_running; then break fi printf "." sleep 1 + i=$((i + 1)) done echo diff --git a/advanced/Templates/pihole-FTL.systemd b/advanced/Templates/pihole-FTL.systemd index 0a3d270e..fcbb8d8d 100644 --- a/advanced/Templates/pihole-FTL.systemd +++ b/advanced/Templates/pihole-FTL.systemd @@ -28,7 +28,7 @@ ExecReload=/bin/kill -HUP $MAINPID ExecStopPost=/opt/pihole/pihole-FTL-poststop.sh # Use graceful shutdown with a reasonable timeout -TimeoutStopSec=10s +TimeoutStopSec=60s # Make /usr, /boot, /etc and possibly some more folders read-only... ProtectSystem=full diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh index e69256ff..6cc69008 100755 --- a/automated install/basic-install.sh +++ b/automated install/basic-install.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# shellcheck disable=SC1090 # Pi-hole: A black hole for Internet advertisements # (c) Pi-hole (https://pi-hole.net) @@ -49,7 +48,6 @@ Google (ECS, DNSSEC);8.8.8.8;8.8.4.4;2001:4860:4860:0:0:0:0:8888;2001:4860:4860: OpenDNS (ECS, DNSSEC);208.67.222.222;208.67.220.220;2620:119:35::35;2620:119:53::53 Level3;4.2.2.1;4.2.2.2;; Comodo;8.26.56.26;8.20.247.20;; -DNS.WATCH (DNSSEC);84.200.69.80;84.200.70.40;2001:1608:10:25:0:0:1c04:b12f;2001:1608:10:25:0:0:9249:d69b Quad9 (filtered, DNSSEC);9.9.9.9;149.112.112.112;2620:fe::fe;2620:fe::9 Quad9 (unfiltered, no DNSSEC);9.9.9.10;149.112.112.10;2620:fe::10;2620:fe::fe:10 Quad9 (filtered, ECS, DNSSEC);9.9.9.11;149.112.112.11;2620:fe::11;2620:fe::fe:11 @@ -57,6 +55,17 @@ Cloudflare (DNSSEC);1.1.1.1;1.0.0.1;2606:4700:4700::1111;2606:4700:4700::1001 EOM ) +DNS_SERVERS_IPV6_ONLY=$( + cat < Architecture: all Description: Pi-hole dependency meta package -Depends: grep,dnsutils,binutils,git,iproute2,dialog,ca-certificates,cron,curl,iputils-ping,psmisc,sudo,unzip,libcap2-bin,dns-root-data,libcap2,netcat-openbsd,procps,jq,lshw,bash-completion +Depends: awk,bash-completion,binutils,ca-certificates,cron|cron-daemon,curl,dialog,dnsutils,dns-root-data,git,grep,iproute2,iputils-ping,jq,libcap2,libcap2-bin,lshw,netcat-openbsd,procps,psmisc,sudo,unzip +Section: contrib/metapackages +Priority: optional EOM ) @@ -118,12 +130,12 @@ EOM PIHOLE_META_PACKAGE_CONTROL_RPM=$( cat </dev/null 2>&1 } -os_check_dig(){ - local protocol="$1" - local domain="$2" - local nameserver="$3" - local response +is_pid1() { + # Checks to see if the given command runs as PID 1 + local is_pid1="$1" - response="$(dig -"${protocol}" +short -t txt "${domain}" "${nameserver}" 2>&1 - echo $? - )" - echo "${response}" -} - -os_check_dig_response(){ - # Checks the reply from the dig command to determine if it's a valid response - local digReply="$1" - local response - - # Dig returned 0 (success), so get the actual response, and loop through it to determine if the detected variables above are valid - response="${digReply%%$'\n'*}" - # If the value of ${response} is a single 0, then this is the return code, not an actual response. - if [ "${response}" == 0 ]; then - echo false - else - echo true - fi -} - -os_check() { - if [ "$PIHOLE_SKIP_OS_CHECK" != true ]; then - # This function gets a list of supported OS versions from a TXT record at versions.pi-hole.net - # and determines whether or not the script is running on one of those systems - local remote_os_domain valid_os valid_version valid_response detected_os detected_version display_warning cmdResult digReturnCode response - local piholeNameserver="@ns1.pi-hole.net" - remote_os_domain=${OS_CHECK_DOMAIN_NAME:-"versions.pi-hole.net"} - - detected_os=$(grep '^ID=' /etc/os-release | cut -d '=' -f2 | tr -d '"') - detected_version=$(grep VERSION_ID /etc/os-release | cut -d '=' -f2 | tr -d '"') - - # Test via IPv4 and hardcoded nameserver ns1.pi-hole.net - cmdResult=$(os_check_dig 4 "${remote_os_domain}" "${piholeNameserver}") - - # Gets the return code of the previous command (last line) - digReturnCode="${cmdResult##*$'\n'}" - - if [ ! "${digReturnCode}" == "0" ]; then - valid_response=false - else - valid_response=$(os_check_dig_response cmdResult) - fi - - # Try again via IPv6 and hardcoded nameserver ns1.pi-hole.net - if [ "$valid_response" = false ]; then - unset valid_response - unset cmdResult - unset digReturnCode - - cmdResult=$(os_check_dig 6 "${remote_os_domain}" "${piholeNameserver}") - # Gets the return code of the previous command (last line) - digReturnCode="${cmdResult##*$'\n'}" - - if [ ! "${digReturnCode}" == "0" ]; then - valid_response=false - else - valid_response=$(os_check_dig_response cmdResult) - fi - fi - - # Try again without hardcoded nameserver - if [ "$valid_response" = false ]; then - unset valid_response - unset cmdResult - unset digReturnCode - - cmdResult=$(os_check_dig 4 "${remote_os_domain}") - # Gets the return code of the previous command (last line) - digReturnCode="${cmdResult##*$'\n'}" - - if [ ! "${digReturnCode}" == "0" ]; then - valid_response=false - else - valid_response=$(os_check_dig_response cmdResult) - fi - fi - - if [ "$valid_response" = false ]; then - unset valid_response - unset cmdResult - unset digReturnCode - - cmdResult=$(os_check_dig 6 "${remote_os_domain}") - # Gets the return code of the previous command (last line) - digReturnCode="${cmdResult##*$'\n'}" - - if [ ! "${digReturnCode}" == "0" ]; then - valid_response=false - else - valid_response=$(os_check_dig_response cmdResult) - fi - fi - - if [ "$valid_response" = true ]; then - response="${cmdResult%%$'\n'*}" - IFS=" " read -r -a supportedOS < <(echo "${response}" | tr -d '"') - for distro_and_versions in "${supportedOS[@]}"; do - distro_part="${distro_and_versions%%=*}" - versions_part="${distro_and_versions##*=}" - - # If the distro part is a (case-insensitive) substring of the computer OS - if [[ "${detected_os^^}" =~ ${distro_part^^} ]]; then - valid_os=true - IFS="," read -r -a supportedVer <<<"${versions_part}" - for version in "${supportedVer[@]}"; do - if [[ "${detected_version}" =~ $version ]]; then - valid_version=true - break - fi - done - break - fi - done - fi - - if [ "$valid_os" = true ] && [ "$valid_version" = true ] && [ "$valid_response" = true ]; then - display_warning=false - fi - - if [ "$display_warning" != false ]; then - if [ "$valid_response" = false ]; then - - if [ "${digReturnCode}" -eq 0 ]; then - errStr="dig succeeded, but response was blank. Please contact support" - else - errStr="dig failed with return code ${digReturnCode}" - fi - printf " %b %bRetrieval of supported OS list failed. %s. %b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${errStr}" "${COL_NC}" - printf " %bUnable to determine if the detected OS (%s %s) is supported%b\\n" "${COL_LIGHT_RED}" "${detected_os^}" "${detected_version}" "${COL_NC}" - printf " Possible causes for this include:\\n" - printf " - Firewall blocking DNS lookups from Pi-hole device to ns1.pi-hole.net\\n" - printf " - DNS resolution issues of the host system\\n" - printf " - Other internet connectivity issues\\n" - else - printf " %b %bUnsupported OS detected: %s %s%b\\n" "${CROSS}" "${COL_LIGHT_RED}" "${detected_os^}" "${detected_version}" "${COL_NC}" - printf " If you are seeing this message and you do have a supported OS, please contact support.\\n" - fi - printf "\\n" - printf " %bhttps://docs.pi-hole.net/main/prerequisites/#supported-operating-systems%b\\n" "${COL_LIGHT_GREEN}" "${COL_NC}" - printf "\\n" - printf " If you wish to attempt to continue anyway, you can try one of the following commands to skip this check:\\n" - printf "\\n" - printf " e.g: If you are seeing this message on a fresh install, you can run:\\n" - printf " %bcurl -sSL https://install.pi-hole.net | sudo PIHOLE_SKIP_OS_CHECK=true bash%b\\n" "${COL_LIGHT_GREEN}" "${COL_NC}" - printf "\\n" - printf " If you are seeing this message after having run pihole -up:\\n" - printf " %bsudo PIHOLE_SKIP_OS_CHECK=true pihole -r%b\\n" "${COL_LIGHT_GREEN}" "${COL_NC}" - printf " (In this case, your previous run of pihole -up will have already updated the local repository)\\n" - printf "\\n" - printf " It is possible that the installation will still fail at this stage due to an unsupported configuration.\\n" - printf " If that is the case, you can feel free to ask the community on Discourse with the %bCommunity Help%b category:\\n" "${COL_LIGHT_RED}" "${COL_NC}" - printf " %bhttps://discourse.pi-hole.net/c/bugs-problems-issues/community-help/%b\\n" "${COL_LIGHT_GREEN}" "${COL_NC}" - printf "\\n" - exit 1 - - else - printf " %b %bSupported OS detected%b\\n" "${TICK}" "${COL_LIGHT_GREEN}" "${COL_NC}" - fi - else - printf " %b %bPIHOLE_SKIP_OS_CHECK env variable set to true - installer will continue%b\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" - fi + # select PID 1, format output to show only CMD column without header + # quietly grep for a match on the function passed parameter + ps --pid 1 --format comm= | grep -q "${is_pid1}" } # Compatibility @@ -686,7 +541,11 @@ find_IPv4_information() { local IPv4bare # Find IP used to route to outside world by checking the route to Google's public DNS server - route=$(ip route get 8.8.8.8) + if ! route="$(ip route get 8.8.8.8 2> /dev/null)"; then + printf " %b No IPv4 route was detected.\n" "${INFO}" + IPV4_ADDRESS="" + return + fi # Get just the interface IPv4 address # shellcheck disable=SC2059,SC2086 @@ -702,10 +561,32 @@ find_IPv4_information() { IPV4_ADDRESS=$(ip -oneline -family inet address show | grep "${IPv4bare}/" | awk '{print $4}' | awk 'END {print}') } +confirm_ipv6_only() { + # Confirm from user before IPv6 only install + + dialog --no-shadow --output-fd 1 \ +--no-button "Exit" --yes-button "Install IPv6 ONLY" \ +--yesno "\\n\\nWARNING - no valid IPv4 route detected.\\n\\n\ +This may be due to a temporary connectivity issue,\\n\ +or you may be installing on an IPv6 only system.\\n\\n\ +Do you wish to continue with an IPv6-only installation?\\n\\n" \ + "${r}" "${c}" && result=0 || result="$?" + + case "${result}" in + "${DIALOG_CANCEL}" | "${DIALOG_ESC}") + printf " %b Installer exited at IPv6 only message.\\n" "${INFO}" + exit 1 + ;; + esac + + DNS_SERVERS="$DNS_SERVERS_IPV6_ONLY" + printf " %b Proceeding with IPv6 only installation.\\n" "${INFO}" +} + # Get available interfaces that are UP get_available_interfaces() { # There may be more than one so it's all stored in a variable - availableInterfaces=$(ip --oneline link show up | grep -v "lo" | awk '{print $2}' | cut -d':' -f1 | cut -d'@' -f1) + availableInterfaces=$(ip --oneline link show up | awk '{print $2}' | grep -v "^lo" | cut -d':' -f1 | cut -d'@' -f1) } # A function for displaying the dialogs the user sees when first running the installer @@ -767,7 +648,6 @@ chooseInterface() { # All further interfaces are deselected status="OFF" done - # shellcheck disable=SC2086 # Disable check for double quote here as we are passing a string with spaces PIHOLE_INTERFACE=$(dialog --no-shadow --keep-tite --output-fd 1 \ --cancel-label "Exit" --ok-label "Select" \ @@ -856,6 +736,9 @@ collect_v4andv6_information() { printf " %b IPv4 address: %s\\n" "${INFO}" "${IPV4_ADDRESS}" find_IPv6_information printf " %b IPv6 address: %s\\n" "${INFO}" "${IPV6_ADDRESS}" + if [ "$IPV4_ADDRESS" == "" ] && [ "$IPV6_ADDRESS" != "" ]; then + confirm_ipv6_only + fi } # Check an IP address to see if it is a valid one @@ -1281,8 +1164,7 @@ installConfigs() { fi # Install pihole-FTL systemd or init.d service, based on whether systemd is the init system or not - # Follow debhelper logic, which checks for /run/systemd/system to derive whether systemd is the init system - if [[ -d '/run/systemd/system' ]]; then + if is_pid1 systemd; then install -T -m 0644 "${PI_HOLE_LOCAL_REPO}/advanced/Templates/pihole-FTL.systemd" '/etc/systemd/system/pihole-FTL.service' # Remove init.d service if present @@ -1350,9 +1232,12 @@ stop_service() { # Can softfail, as process may not be installed when this is called local str="Stopping ${1} service" printf " %b %s..." "${INFO}" "${str}" - if is_command systemctl; then + # If systemd is PID 1, + if is_pid1 systemd; then + # use that to restart the service systemctl -q stop "${1}" || true else + # Otherwise, fall back to the service command service "${1}" stop >/dev/null || true fi printf "%b %b %s...\\n" "${OVER}" "${TICK}" "${str}" @@ -1363,8 +1248,8 @@ restart_service() { # Local, named variables local str="Restarting ${1} service" printf " %b %s..." "${INFO}" "${str}" - # If systemctl exists, - if is_command systemctl; then + # If systemd is PID 1, + if is_pid1 systemd; then # use that to restart the service systemctl -q restart "${1}" else @@ -1379,8 +1264,8 @@ enable_service() { # Local, named variables local str="Enabling ${1} service to start on reboot" printf " %b %s..." "${INFO}" "${str}" - # If systemctl exists, - if is_command systemctl; then + # If systemd is PID1, + if is_pid1 systemd; then # use that to enable the service systemctl -q enable "${1}" else @@ -1395,8 +1280,8 @@ disable_service() { # Local, named variables local str="Disabling ${1} service" printf " %b %s..." "${INFO}" "${str}" - # If systemctl exists, - if is_command systemctl; then + # If systemd is PID1, + if is_pid1 systemd; then # use that to disable the service systemctl -q disable "${1}" else @@ -1407,8 +1292,8 @@ disable_service() { } check_service_active() { - # If systemctl exists, - if is_command systemctl; then + # If systemd is PID1, + if is_pid1 systemd; then # use that to check the status of the service systemctl -q is-enabled "${1}" 2>/dev/null else @@ -1750,7 +1635,8 @@ checkSelinux() { check_download_exists() { # Check if the download exists and we can reach the server - local status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) + local status + status=$(curl --head --silent "https://ftl.pi-hole.net/${1}" | head -n 1) # Check the status code if grep -q "200" <<<"$status"; then @@ -1869,7 +1755,6 @@ clone_or_reset_repos() { # Download FTL binary to random temp directory and install FTL binary # Disable directive for SC2120 a value _can_ be passed to this function, but it is passed from an external script that sources this one -# shellcheck disable=SC2120 FTLinstall() { # Local, named variables local str="Downloading and Installing FTL" @@ -2067,13 +1952,13 @@ FTLcheckUpdate() { path="${ftlBranch}/${binary}" # Check whether or not the binary for this FTL branch actually exists. If not, then there is no update! - # shellcheck disable=SC1090 if ! check_download_exists "$path"; then - if [ $? -eq 1 ]; then + local status + status=$? + if [ "${status}" -eq 1 ]; then printf " %b Branch \"%s\" is not available.\\n" "${INFO}" "${ftlBranch}" printf " %b Use %bpihole checkout ftl [branchname]%b to switch to a valid branch.\\n" "${INFO}" "${COL_LIGHT_GREEN}" "${COL_NC}" - return 2 - elif [ $? -eq 2 ]; then + elif [ "${status}" -eq 2 ]; then printf " %b Unable to download from ftl.pi-hole.net. Please check your Internet connection and try again later.\\n" "${CROSS}" return 3 else @@ -2101,12 +1986,14 @@ FTLcheckUpdate() { # same as the remote one local FTLversion FTLversion=$(/usr/bin/pihole-FTL tag) - local FTLlatesttag # Get the latest version from the GitHub API - if ! FTLlatesttag=$(curl -sI https://github.com/pi-hole/FTL/releases/latest | grep --color=never -i Location: | awk -F / '{print $NF}' | tr -d '[:cntrl:]'); then + local FTLlatesttag + FTLlatesttag=$(curl -s https://api.github.com/repos/pi-hole/FTL/releases/latest | jq -sRr 'fromjson? | .tag_name | values') + + if [ -z "${FTLlatesttag}" ]; then # There was an issue while retrieving the latest version - printf " %b Failed to retrieve latest FTL release metadata" "${CROSS}" + printf " %b Failed to retrieve latest FTL release metadata\\n" "${CROSS}" return 3 fi @@ -2124,6 +2011,7 @@ FTLcheckUpdate() { # Continue further down... fi else + # FTL not installed, then download return 0 fi fi @@ -2329,8 +2217,6 @@ main() { # Install Pi-hole dependencies install_dependent_packages - # Check that the installed OS is officially supported - display warning if not - os_check # Check if there is a usable FTL binary available on this architecture - do # this early on as FTL is a hard dependency for Pi-hole @@ -2398,7 +2284,7 @@ main() { # /opt/pihole/utils.sh should be installed by installScripts now, so we can use it if [ -f "${PI_HOLE_INSTALL_DIR}/utils.sh" ]; then - # shellcheck disable=SC1091 + # shellcheck source="./advanced/Scripts/utils.sh" source "${PI_HOLE_INSTALL_DIR}/utils.sh" else printf " %b Failure: /opt/pihole/utils.sh does not exist .\\n" "${CROSS}" @@ -2411,16 +2297,28 @@ main() { # Migrate existing install to v6.0 migrate_dnsmasq_configs + # Cleanup old v5 sudoers file if it exists + sudoers_file="/etc/sudoers.d/pihole" + if [[ -f "${sudoers_file}" ]]; then + # only remove the file if it contains the Pi-hole header + if grep -q "Pi-hole: A black hole for Internet advertisements" "${sudoers_file}"; then + rm -f "${sudoers_file}" + fi + fi + # Check for and disable systemd-resolved-DNSStubListener before reloading resolved # DNSStubListener needs to remain in place for installer to download needed files, # so this change needs to be made after installation is complete, # but before starting or resttarting the ftl service disable_resolved_stublistener - # Check if gravity database needs to be upgraded. If so, do it without rebuilding - # gravity altogether. This may be a very long running task needlessly blocking - # the update process. - /opt/pihole/gravity.sh --upgrade + if [[ "${fresh_install}" == false ]]; then + # Check if gravity database needs to be upgraded. If so, do it without rebuilding + # gravity altogether. This may be a very long running task needlessly blocking + # the update process. + # Only do this on updates, not on fresh installs as the database does not exit yet + /opt/pihole/gravity.sh --upgrade + fi printf " %b Restarting services...\\n" "${INFO}" # Start services @@ -2452,6 +2350,10 @@ main() { if [ -n "${PRIVACY_LEVEL}" ]; then setFTLConfigValue "misc.privacylevel" "${PRIVACY_LEVEL}" fi + + if [ -n "${PIHOLE_INTERFACE}" ]; then + setFTLConfigValue "dns.interface" "${PIHOLE_INTERFACE}" + fi fi # Download and compile the aggregated block list diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh index 39c13037..eb1e9e29 100755 --- a/automated install/uninstall.sh +++ b/automated install/uninstall.sh @@ -8,7 +8,18 @@ # This file is copyright under the latest version of the EUPL. # Please see LICENSE file for your rights under this license. +# shellcheck source="./advanced/Scripts/COL_TABLE" source "/opt/pihole/COL_TABLE" +# shellcheck source="./advanced/Scripts/utils.sh" +source "/opt/pihole/utils.sh" + +SKIP_INSTALL="true" +# shellcheck source="./automated install/basic-install.sh" +source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" +# stop_service() is defined in basic-install.sh + +ADMIN_INTERFACE_DIR=$(getFTLConfigValue "webserver.paths.webroot")$(getFTLConfigValue "webserver.paths.webhome") +readonly ADMIN_INTERFACE_DIR while true; do read -rp " ${QST} Are you sure you would like to remove ${COL_WHITE}Pi-hole${COL_NC}? [y/N] " answer @@ -37,6 +48,7 @@ fi readonly PI_HOLE_FILES_DIR="/etc/.pihole" SKIP_INSTALL="true" +# shellcheck source="./automated install/basic-install.sh" source "${PI_HOLE_FILES_DIR}/automated install/basic-install.sh" # package_manager_detect() sourced from basic-install.sh @@ -53,17 +65,9 @@ removeMetaPackage() { } removePiholeFiles() { - # Only web directories/files that are created by Pi-hole should be removed + # Remove the web interface of Pi-hole echo -ne " ${INFO} Removing Web Interface..." - ${SUDO} rm -rf /var/www/html/admin &> /dev/null - - - # If the web directory is empty after removing these files, then the parent html directory can be removed. - if [ -d "/var/www/html" ]; then - if [[ ! "$(ls -A /var/www/html)" ]]; then - ${SUDO} rm -rf /var/www/html &> /dev/null - fi - fi + ${SUDO} rm -rf "${ADMIN_INTERFACE_DIR}" &> /dev/null echo -e "${OVER} ${TICK} Removed Web Interface" # Attempt to preserve backwards compatibility with older versions @@ -103,11 +107,7 @@ removePiholeFiles() { # Remove FTL if command -v pihole-FTL &> /dev/null; then echo -ne " ${INFO} Removing pihole-FTL..." - if [[ -x "$(command -v systemctl)" ]]; then - systemctl stop pihole-FTL - else - service pihole-FTL stop - fi + stop_service pihole-FTL ${SUDO} rm -f /etc/systemd/system/pihole-FTL.service if [[ -d '/etc/systemd/system/pihole-FTL.service.d' ]]; then read -rp " ${QST} FTL service override directory /etc/systemd/system/pihole-FTL.service.d detected. Do you wish to remove this from your system? [y/N] " answer diff --git a/gravity.sh b/gravity.sh index 493f2b15..16e459c6 100755 --- a/gravity.sh +++ b/gravity.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash -# shellcheck disable=SC1090 # Pi-hole: A black hole for Internet advertisements # (c) 2017 Pi-hole, LLC (https://pi-hole.net) @@ -16,13 +15,13 @@ export LC_ALL=C PI_HOLE_SCRIPT_DIR="/opt/pihole" # Source utils.sh for GetFTLConfigValue utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" -# shellcheck disable=SC1090 +# shellcheck source=./advanced/Scripts/utils.sh . "${utilsfile}" coltable="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" -# shellcheck disable=SC1090 +# shellcheck source=./advanced/Scripts/COL_TABLE . "${coltable}" -# shellcheck disable=SC1091 +# shellcheck source=./advanced/Scripts/database_migration/gravity-db.sh . "/etc/.pihole/advanced/Scripts/database_migration/gravity-db.sh" basename="pihole" @@ -58,7 +57,7 @@ fi # Set this only after sourcing pihole-FTL.conf as the gravity database path may # have changed gravityDBfile="${GRAVITYDB}" -gravityDBfile_default="/etc/pihole/gravity.db" +gravityDBfile_default="${piholeDir}/gravity.db" gravityTEMPfile="${GRAVITYDB}_temp" gravityDIR="$(dirname -- "${gravityDBfile}")" gravityOLDfile="${gravityDIR}/gravity_old.db" @@ -127,7 +126,7 @@ gravity_swap_databases() { oldAvail=false if [ "${availableBlocks}" -gt "$((gravityBlocks * 2))" ] && [ -f "${gravityDBfile}" ]; then oldAvail=true - cp "${gravityDBfile}" "${gravityOLDfile}" + cp -p "${gravityDBfile}" "${gravityOLDfile}" fi # Drop the gravity and antigravity tables + subsequent VACUUM the current @@ -140,7 +139,7 @@ gravity_swap_databases() { else # Check if the backup directory exists if [ ! -d "${gravityBCKdir}" ]; then - mkdir -p "${gravityBCKdir}" + mkdir -p "${gravityBCKdir}" && chown pihole:pihole "${gravityBCKdir}" fi # If multiple gravityBCKfile's are present (appended with a number), rotate them @@ -306,7 +305,7 @@ migrate_to_database() { fi # Check if gravity database needs to be updated - upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" + upgrade_gravityDB "${gravityDBfile}" # Migrate list files to new database if [ -e "${adListFile}" ]; then @@ -334,7 +333,7 @@ migrate_to_database() { fi # Check if gravity database needs to be updated - upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" + upgrade_gravityDB "${gravityDBfile}" } # Determine if DNS resolution is available before proceeding @@ -349,17 +348,24 @@ gravity_CheckDNSResolutionAvailable() { echo -e " ${CROSS} DNS resolution is currently unavailable" fi - str="Waiting until DNS resolution is available..." + str="Waiting up to 120 seconds for DNS resolution..." echo -ne " ${INFO} ${str}" - until getent hosts github.com &> /dev/null; do - # Append one dot for each second waiting - str="${str}." - echo -ne " ${OVER} ${INFO} ${str}" - sleep 1 + + # Default DNS timeout is two seconds, plus 1 second for each dot > 120 seconds + for ((i = 0; i < 40; i++)); do + if getent hosts github.com &> /dev/null; then + # If we reach this point, DNS resolution is available + echo -e "${OVER} ${TICK} DNS resolution is available" + return 0 + fi + # Append one dot for each second waiting + echo -ne "." + sleep 1 done - # If we reach this point, DNS resolution is available - echo -e "${OVER} ${TICK} DNS resolution is available" + # DNS resolution is still unavailable after 120 seconds + return 1 + } # Function: try_restore_backup @@ -418,7 +424,7 @@ gravity_DownloadBlocklists() { echo -e " ${INFO} Storing gravity database in ${COL_BOLD}${gravityDBfile}${COL_NC}" fi - local url domain str target compression adlist_type directory success + local url domain str compression adlist_type directory success echo "" # Prepare new gravity database @@ -567,12 +573,12 @@ gravity_DownloadBlocklists() { if [[ "${check_url}" =~ ${regex} ]]; then echo -e " ${CROSS} Invalid Target" else - timeit gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${target}" "${compression}" "${adlist_type}" "${domain}" + timeit gravity_DownloadBlocklistFromUrl "${url}" "${sourceIDs[$i]}" "${saveLocation}" "${compression}" "${adlist_type}" "${domain}" fi echo "" done - gravity_Blackbody=true + DownloadBlocklists_done=true } compareLists() { @@ -601,7 +607,7 @@ compareLists() { # Download specified URL and perform checks on HTTP status and file content gravity_DownloadBlocklistFromUrl() { - local url="${1}" adlistID="${2}" saveLocation="${3}" target="${4}" compression="${5}" gravity_type="${6}" domain="${7}" + local url="${1}" adlistID="${2}" saveLocation="${3}" compression="${4}" gravity_type="${5}" domain="${6}" local modifiedOptions="" listCurlBuffer str httpCode success="" ip cmd_ext local file_path permissions ip_addr port blocked=false download=true @@ -647,32 +653,6 @@ gravity_DownloadBlocklistFromUrl() { str="Status:" echo -ne " ${INFO} ${str} Pending..." blocked=false - case $(getFTLConfigValue dns.blocking.mode) in - "IP-NODATA-AAAA" | "IP") - # Get IP address of this domain - ip="$(dig "${domain}" +short)" - # Check if this IP matches any IP of the system - if [[ -n "${ip}" && $(grep -Ec "inet(|6) ${ip}" <<<"$(ip a)") -gt 0 ]]; then - blocked=true - fi - ;; - "NXDOMAIN") - if [[ $(dig "${domain}" | grep "NXDOMAIN" -c) -ge 1 ]]; then - blocked=true - fi - ;; - "NODATA") - if [[ $(dig "${domain}" | grep "NOERROR" -c) -ge 1 ]] && [[ -z $(dig +short "${domain}") ]]; then - blocked=true - fi - ;; - "NULL" | *) - if [[ $(dig "${domain}" +short | grep "0.0.0.0" -c) -ge 1 ]]; then - blocked=true - fi - ;; - esac - # Check if this domain is blocked by Pi-hole but only if the domain is not a # local file or empty if [[ $url != "file"* ]] && [[ -n "${domain}" ]]; then @@ -770,6 +750,7 @@ gravity_DownloadBlocklistFromUrl() { fi if [[ "${download}" == true ]]; then + # See https://github.com/pi-hole/pi-hole/issues/6159 for justification of the below disable directive # shellcheck disable=SC2086 httpCode=$(curl --connect-timeout ${curl_connect_timeout} -s -L ${compression} ${cmd_ext} ${modifiedOptions} -w "%{http_code}" "${url}" -o "${listCurlBuffer}" 2>/dev/null) fi @@ -821,11 +802,11 @@ gravity_DownloadBlocklistFromUrl() { done="true" # Check if $listCurlBuffer is a non-zero length file elif [[ -s "${listCurlBuffer}" ]]; then - # Determine if blocklist is non-standard and parse as appropriate - gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}" - # Remove curl buffer file after its use - rm "${listCurlBuffer}" - # Compare lists if are they identical + # Move the downloaded list to the final location + mv "${listCurlBuffer}" "${saveLocation}" + # Ensure the file has the correct permissions + fix_owner_permissions "${saveLocation}" + # Compare lists if they are identical compareLists "${adlistID}" "${saveLocation}" # Add domains to database table file pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}" @@ -854,37 +835,6 @@ gravity_DownloadBlocklistFromUrl() { fi } -# Parse source files into domains format -gravity_ParseFileIntoDomains() { - local src="${1}" destination="${2}" - - # Remove comments and print only the domain name - # Most of the lists downloaded are already in hosts file format but the spacing/formatting is not contiguous - # This helps with that and makes it easier to read - # It also helps with debugging so each stage of the script can be researched more in depth - # 1) Convert all characters to lowercase - tr '[:upper:]' '[:lower:]' <"${src}" >"${destination}" - - # 2) Remove carriage returns - # 3) Remove lines starting with ! (ABP Comments) - # 4) Remove lines starting with [ (ABP Header) - # 5) Remove lines containing ABP extended CSS selectors ("##", "#$#", "#@#", "#?#") and Adguard JavaScript (#%#) preceded by a letter - # 6) Remove comments (text starting with "#", include possible spaces before the hash sign) - # 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses) - # 8) Remove empty lines - - sed -i -r \ - -e 's/\r$//' \ - -e 's/\s*!.*//g' \ - -e 's/\s*\[.*//g' \ - -e '/[a-z]\#[$?@%]{0,3}\#/d' \ - -e 's/\s*#.*//g' \ - -e 's/^.*\s+//g' \ - -e '/^$/d' "${destination}" - - fix_owner_permissions "${destination}" -} - # Report number of entries in a table gravity_Table_Count() { local table="${1}" @@ -932,13 +882,13 @@ gravity_Cleanup() { # invalid_domains location rm "${GRAVITY_TMPDIR}"/*.ph-non-domains 2>/dev/null - # Ensure this function only runs when gravity_SetDownloadOptions() has completed - if [[ "${gravity_Blackbody:-}" == true ]]; then - # Remove any unused .domains files - for file in "${piholeDir}"/*."${domainsExtension}"; do - # If list is not in active array, then remove it + # Ensure this function only runs when gravity_DownloadBlocklists() has completed + if [[ "${DownloadBlocklists_done:-}" == true ]]; then + # Remove any unused .domains/.etag/.sha files + for file in "${listsCacheDir}"/*."${domainsExtension}"; do + # If list is not in active array, then remove it and all associated files if [[ ! "${activeDomains[*]}" == *"${file}"* ]]; then - rm -f "${file}" 2>/dev/null || + rm -f "${file}"* 2>/dev/null || echo -e " ${CROSS} Failed to remove ${file##*/}" fi done @@ -1072,7 +1022,7 @@ migrate_to_listsCache_dir() { # If not, we need to migrate the old files to the new directory local str="Migrating the list's cache directory to new location" echo -ne " ${INFO} ${str}..." - mkdir -p "${listsCacheDir}" + mkdir -p "${listsCacheDir}" && chown pihole:pihole "${listsCacheDir}" # Move the old files to the new directory if mv "${piholeDir}"/list.* "${listsCacheDir}/" 2>/dev/null; then @@ -1131,13 +1081,19 @@ for var in "$@"; do "-t" | "--timeit") timed=true ;; "-r" | "--repair") repairSelector "$3" ;; "-u" | "--upgrade") - upgrade_gravityDB "${gravityDBfile}" "${piholeDir}" + upgrade_gravityDB "${gravityDBfile}" exit 0 ;; "-h" | "--help") helpFunc ;; esac done +# Check if DNS is available, no need to do any database manipulation if we're not able to download adlists +if ! timeit gravity_CheckDNSResolutionAvailable; then + echo -e " ${CROSS} No DNS resolution available. Please contact support." + exit 1 +fi + # Remove OLD (backup) gravity file, if it exists if [[ -f "${gravityOLDfile}" ]]; then rm "${gravityOLDfile}" @@ -1178,11 +1134,6 @@ if [[ "${forceDelete:-}" == true ]]; then fi # Gravity downloads blocklists next -if ! timeit gravity_CheckDNSResolutionAvailable; then - echo -e " ${CROSS} Can not complete gravity update, no DNS is available. Please contact support." - exit 1 -fi - if ! gravity_DownloadBlocklists; then echo -e " ${CROSS} Unable to create gravity database. Please try again later. If the problem persists, please contact support." exit 1 diff --git a/manpages/pihole.8 b/manpages/pihole.8 index 97a6ec68..e0c38828 100644 --- a/manpages/pihole.8 +++ b/manpages/pihole.8 @@ -23,7 +23,7 @@ pihole -r .br \fBpihole -g\fR .br -\fBpihole\fR -\fBq\fR [options] +\fBpihole\fR \fB-q\fR [options] .br \fBpihole\fR \fB-l\fR (\fBon|off|off noflush\fR) .br @@ -43,7 +43,7 @@ pihole -r .br \fBpihole\fR \fBcheckout\fR repo [branch] .br -\fBpihole\fR \api\fR endpoint +\fBpihole\fR \fBapi\fR [verbose] endpoint .br \fBpihole\fR \fBhelp\fR .br @@ -234,10 +234,14 @@ Available commands and options: branchname Update subsystems to the specified branchname .br -\fBapi\fR endpoint +\fBapi\fR [verbose] endpoint .br Query the Pi-hole API at .br + + verbose Show authentication and status messages +.br + .SH "EXAMPLE" Some usage examples @@ -323,6 +327,11 @@ Switching Pi-hole subsystem branches Queries FTL for the stats/summary endpoint .br +\fBpihole api verbose stats/summary\fR +.br + Same as above, but shows authentication and status messages +.br + .SH "COLOPHON" Get sucked into the latest news and community activity by entering Pi-hole's orbit. Information about Pi-hole, and the latest version of the software can be found at https://pi-hole.net. diff --git a/pihole b/pihole index bf662a82..1d5093c6 100755 --- a/pihole +++ b/pihole @@ -17,13 +17,16 @@ readonly PI_HOLE_SCRIPT_DIR="/opt/pihole" PI_HOLE_BIN_DIR="/usr/local/bin" readonly colfile="${PI_HOLE_SCRIPT_DIR}/COL_TABLE" +# shellcheck source=./advanced/Scripts/COL_TABLE source "${colfile}" readonly utilsfile="${PI_HOLE_SCRIPT_DIR}/utils.sh" +# shellcheck source=./advanced/Scripts/utils.sh source "${utilsfile}" # Source api functions readonly apifile="${PI_HOLE_SCRIPT_DIR}/api.sh" +# shellcheck source=./advanced/Scripts/api.sh source "${apifile}" versionsfile="/etc/pihole/versions" @@ -31,6 +34,7 @@ if [ -f "${versionsfile}" ]; then # Only source versionsfile if the file exits # fixes a warning during installation where versionsfile does not exist yet # but gravity calls `pihole -status` and thereby sourcing the file + # shellcheck source=/dev/null source "${versionsfile}" fi @@ -247,12 +251,14 @@ Time: data=$(PostFTLData "dns/blocking" "{ \"blocking\": ${1}, \"timer\": ${tt} }") # Check the response - local extra=" forever" - local timer="$(echo "${data}"| jq --raw-output '.timer' )" + local extra timer + extra=" forever" + timer="$(echo "${data}"| jq --raw-output '.timer' )" if [[ "${timer}" != "null" ]]; then extra=" for ${timer}s" fi - local str="Pi-hole $(echo "${data}" | jq --raw-output '.blocking')${extra}" + local str + str="Pi-hole $(echo "${data}" | jq --raw-output '.blocking')${extra}" # Logout from the API LogoutAPI @@ -375,14 +381,16 @@ statusFunc() { tailFunc() { # Warn user if Pi-hole's logging is disabled - local logging_enabled=$(getFTLConfigValue dns.queryLogging) + local logging_enabled + logging_enabled=$(getFTLConfigValue dns.queryLogging) if [[ "${logging_enabled}" != "true" ]]; then echo " ${CROSS} Warning: Query logging is disabled" fi echo -e " ${INFO} Press Ctrl-C to exit" # Get logfile path - readonly LOGFILE=$(getFTLConfigValue files.log.dnsmasq) + readonly LOGFILE + LOGFILE=$(getFTLConfigValue files.log.dnsmasq) # Strip date from each line # Color blocklist/denylist/wildcard entries as red @@ -423,6 +431,7 @@ piholeCheckoutFunc() { exit 0 fi + #shellcheck source=./advanced/Scripts/piholeCheckout.sh source "${PI_HOLE_SCRIPT_DIR}"/piholeCheckout.sh shift checkout "$@" @@ -484,6 +493,7 @@ Debugging Options: Add an optional argument to filter the log (regular expressions are supported) api Query the Pi-hole API at + Precede with 'verbose' option to show authentication and status messages Options: @@ -592,6 +602,6 @@ case "${1}" in "updatechecker" ) shift; updateCheckFunc "$@";; "arpflush" ) arpFunc "$@";; "-t" | "tail" ) tailFunc "$2";; - "api" ) apiFunc "$2";; + "api" ) shift; apiFunc "$@";; * ) helpFunc;; esac diff --git a/test/_centos_10.Dockerfile b/test/_centos_10.Dockerfile new file mode 100644 index 00000000..c6b2ca75 --- /dev/null +++ b/test/_centos_10.Dockerfile @@ -0,0 +1,19 @@ +FROM quay.io/centos/centos:stream10 +# Disable SELinux +RUN echo "SELINUX=disabled" > /etc/selinux/config +RUN yum install -y --allowerasing curl git + +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL=true + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_centos_9.Dockerfile b/test/_centos_9.Dockerfile index a5e7cf0b..0e12edab 100644 --- a/test/_centos_9.Dockerfile +++ b/test/_centos_9.Dockerfile @@ -1,7 +1,7 @@ FROM quay.io/centos/centos:stream9 # Disable SELinux RUN echo "SELINUX=disabled" > /etc/selinux/config -RUN yum install -y --allowerasing curl git initscripts +RUN yum install -y --allowerasing curl git ENV GITDIR=/etc/.pihole ENV SCRIPTDIR=/opt/pihole @@ -15,6 +15,5 @@ RUN true && \ chmod +x $SCRIPTDIR/* ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_debian_11.Dockerfile b/test/_debian_11.Dockerfile index b8107244..2389063c 100644 --- a/test/_debian_11.Dockerfile +++ b/test/_debian_11.Dockerfile @@ -12,6 +12,5 @@ RUN true && \ chmod +x $SCRIPTDIR/* ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_debian_12.Dockerfile b/test/_debian_12.Dockerfile index 7446711a..a6c5f1ed 100644 --- a/test/_debian_12.Dockerfile +++ b/test/_debian_12.Dockerfile @@ -12,6 +12,5 @@ RUN true && \ chmod +x $SCRIPTDIR/* ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_fedora_40.Dockerfile b/test/_fedora_40.Dockerfile index 20102a10..56be9d84 100644 --- a/test/_fedora_40.Dockerfile +++ b/test/_fedora_40.Dockerfile @@ -1,5 +1,5 @@ FROM fedora:40 -RUN dnf install -y git initscripts +RUN dnf install -y git ENV GITDIR=/etc/.pihole ENV SCRIPTDIR=/opt/pihole @@ -13,6 +13,5 @@ RUN true && \ chmod +x $SCRIPTDIR/* ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_fedora_41.Dockerfile b/test/_fedora_41.Dockerfile index bf5fe5d5..2a9ecf70 100644 --- a/test/_fedora_41.Dockerfile +++ b/test/_fedora_41.Dockerfile @@ -1,5 +1,5 @@ FROM fedora:41 -RUN dnf install -y git initscripts +RUN dnf install -y git ENV GITDIR=/etc/.pihole ENV SCRIPTDIR=/opt/pihole @@ -13,6 +13,5 @@ RUN true && \ chmod +x $SCRIPTDIR/* ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_fedora_42.Dockerfile b/test/_fedora_42.Dockerfile new file mode 100644 index 00000000..0d235e2d --- /dev/null +++ b/test/_fedora_42.Dockerfile @@ -0,0 +1,17 @@ +FROM fedora:42 +RUN dnf install -y git gawk + +ENV GITDIR=/etc/.pihole +ENV SCRIPTDIR=/opt/pihole + +RUN mkdir -p $GITDIR $SCRIPTDIR /etc/pihole +ADD . $GITDIR +RUN cp $GITDIR/advanced/Scripts/*.sh $GITDIR/gravity.sh $GITDIR/pihole $GITDIR/automated\ install/*.sh $GITDIR/advanced/Scripts/COL_TABLE $SCRIPTDIR/ +ENV PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$SCRIPTDIR + +RUN true && \ + chmod +x $SCRIPTDIR/* + +ENV SKIP_INSTALL=true + +#sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_20.Dockerfile b/test/_ubuntu_20.Dockerfile index 75c12673..5b8deb5d 100644 --- a/test/_ubuntu_20.Dockerfile +++ b/test/_ubuntu_20.Dockerfile @@ -12,6 +12,5 @@ RUN true && \ chmod +x $SCRIPTDIR/* ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_22.Dockerfile b/test/_ubuntu_22.Dockerfile index 9206a46a..c3be89e1 100644 --- a/test/_ubuntu_22.Dockerfile +++ b/test/_ubuntu_22.Dockerfile @@ -13,6 +13,5 @@ RUN true && \ chmod +x $SCRIPTDIR/* ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/_ubuntu_24.Dockerfile b/test/_ubuntu_24.Dockerfile index 4cab43de..cf57c2aa 100644 --- a/test/_ubuntu_24.Dockerfile +++ b/test/_ubuntu_24.Dockerfile @@ -13,6 +13,5 @@ RUN true && \ chmod +x $SCRIPTDIR/* ENV SKIP_INSTALL=true -ENV OS_CHECK_DOMAIN_NAME=dev-supportedos.pi-hole.net #sed '/# Start the installer/Q' /opt/pihole/basic-install.sh > /opt/pihole/stub_basic-install.sh && \ diff --git a/test/requirements.txt b/test/requirements.txt index fa536e25..b273c351 100644 --- a/test/requirements.txt +++ b/test/requirements.txt @@ -1,6 +1,6 @@ pyyaml == 6.0.2 pytest == 8.3.5 pytest-xdist == 3.6.1 -pytest-testinfra == 10.1.1 -tox == 4.25.0 +pytest-testinfra == 10.2.2 +tox == 4.26.0 pytest-clarity == 1.0.1 diff --git a/test/test_any_automated_install.py b/test/test_any_automated_install.py index 5fa0f065..64d8c28a 100644 --- a/test/test_any_automated_install.py +++ b/test/test_any_automated_install.py @@ -66,6 +66,14 @@ def test_installPihole_fresh_install_readableFiles(host): mock_command("dialog", {"*": ("", "0")}, host) # mock git pull mock_command_passthrough("git", {"pull": ("", "0")}, host) + # mock PID 1 to pretend to be systemd + mock_command_2( + "ps", + { + "--pid 1": ("systemd", "0"), + }, + host, + ) # mock systemctl to not start FTL mock_command_2( "systemctl", @@ -73,6 +81,7 @@ def test_installPihole_fresh_install_readableFiles(host): "enable pihole-FTL": ("", "0"), "restart pihole-FTL": ("", "0"), "start pihole-FTL": ("", "0"), + "stop pihole-FTL": ("", "0"), "*": ('echo "systemctl call with $@"', "0"), }, host, @@ -131,13 +140,6 @@ def test_installPihole_fresh_install_readableFiles(host): check_macvendor = test_cmd.format("r", "/etc/pihole/macvendor.db", piholeuser) actual_rc = host.run(check_macvendor).rc assert exit_status_success == actual_rc - # check readable and executable /etc/init.d/pihole-FTL - check_init = test_cmd.format("x", "/etc/init.d/pihole-FTL", piholeuser) - actual_rc = host.run(check_init).rc - assert exit_status_success == actual_rc - check_init = test_cmd.format("r", "/etc/init.d/pihole-FTL", piholeuser) - actual_rc = host.run(check_init).rc - assert exit_status_success == actual_rc # check readable and executable manpages if maninstalled is True: check_man = test_cmd.format("x", "/usr/local/share/man", piholeuser) @@ -245,6 +247,7 @@ def test_FTL_detect_no_errors(host, arch, detected_string, supported): { "-A /bin/sh": ("Tag_CPU_arch: " + arch, "0"), "-A /usr/bin/sh": ("Tag_CPU_arch: " + arch, "0"), + "-A /usr/sbin/sh": ("Tag_CPU_arch: " + arch, "0"), }, host, ) @@ -465,50 +468,6 @@ def test_validate_ip(host): test_address("0.0.0.0#00001", False) -def test_os_check_fails(host): - """Confirms install fails on unsupported OS""" - host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - build_dependency_package - install_dependent_packages - cat < /etc/os-release -ID=UnsupportedOS -VERSION_ID="2" -EOT - """ - ) - detectOS = host.run( - """t - source /opt/pihole/basic-install.sh - os_check - """ - ) - expected_stdout = "Unsupported OS detected: UnsupportedOS" - assert expected_stdout in detectOS.stdout - - -def test_os_check_passes(host): - """Confirms OS meets the requirements""" - host.run( - """ - source /opt/pihole/basic-install.sh - package_manager_detect - build_dependency_package - install_dependent_packages - """ - ) - detectOS = host.run( - """ - source /opt/pihole/basic-install.sh - os_check - """ - ) - expected_stdout = "Supported OS detected" - assert expected_stdout in detectOS.stdout - - def test_package_manager_has_pihole_deps(host): """Confirms OS is able to install the required packages for Pi-hole""" mock_command("dialog", {"*": ("", "0")}, host) diff --git a/test/tox.centos_10.ini b/test/tox.centos_10.ini new file mode 100644 index 00000000..1a15c766 --- /dev/null +++ b/test/tox.centos_10.ini @@ -0,0 +1,10 @@ +[tox] +envlist = py3 + +[testenv:py3] +allowlist_externals = docker +deps = -rrequirements.txt +setenv = + COLUMNS=120 +commands = docker buildx build --load --progress plain -f _centos_10.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py diff --git a/test/tox.fedora_42.ini b/test/tox.fedora_42.ini new file mode 100644 index 00000000..67eb77e4 --- /dev/null +++ b/test/tox.fedora_42.ini @@ -0,0 +1,10 @@ +[tox] +envlist = py3 + +[testenv] +allowlist_externals = docker +deps = -rrequirements.txt +setenv = + COLUMNS=120 +commands = docker buildx build --load --progress plain -f _fedora_42.Dockerfile -t pytest_pihole:test_container ../ + pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py