From 5ecdfb53c23712a03f22a2d0bbbc78843cf4f68e Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 11 Feb 2023 10:57:20 +0000
Subject: [PATCH 01/32] Bump tox from 4.4.4 to 4.4.5 in /test

Bumps [tox](https://github.com/tox-dev/tox) from 4.4.4 to 4.4.5.
- [Release notes](https://github.com/tox-dev/tox/releases)
- [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst)
- [Commits](https://github.com/tox-dev/tox/compare/4.4.4...4.4.5)

---
updated-dependencies:
- dependency-name: tox
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 test/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/requirements.txt b/test/requirements.txt
index 08e7027a..1670e765 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -2,5 +2,5 @@ docker-compose == 1.29.2
 pytest == 7.2.1
 pytest-xdist == 3.1.0
 pytest-testinfra == 7.0.0
-tox == 4.4.4
+tox == 4.4.5
 

From d6275cdd7c7822545d022b0393d308f37b11fcb7 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Feb 2023 09:04:19 +0000
Subject: [PATCH 02/32] Bump pytest-xdist from 3.1.0 to 3.2.0 in /test

Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.1.0 to 3.2.0.
- [Release notes](https://github.com/pytest-dev/pytest-xdist/releases)
- [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.1.0...v3.2.0)

---
updated-dependencies:
- dependency-name: pytest-xdist
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 test/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/requirements.txt b/test/requirements.txt
index 1670e765..229d0135 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -1,6 +1,6 @@
 docker-compose == 1.29.2
 pytest == 7.2.1
-pytest-xdist == 3.1.0
+pytest-xdist == 3.2.0
 pytest-testinfra == 7.0.0
 tox == 4.4.5
 

From 46fe37b4dafdd860fe26decee88e1454d21feb1b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Tue, 14 Feb 2023 21:11:54 +0100
Subject: [PATCH 03/32] Use new worksteal distribution algorithm
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 test/tox.centos_8.ini  | 2 +-
 test/tox.centos_9.ini  | 2 +-
 test/tox.debian_10.ini | 2 +-
 test/tox.debian_11.ini | 2 +-
 test/tox.fedora_36.ini | 2 +-
 test/tox.fedora_37.ini | 2 +-
 test/tox.ubuntu_20.ini | 2 +-
 test/tox.ubuntu_22.ini | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini
index dac10e97..5e94baa6 100644
--- a/test/tox.centos_8.ini
+++ b/test/tox.centos_8.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands =  docker build -f _centos_8.Dockerfile -t pytest_pihole:test_container ../
-            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
+            pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini
index aa7009e1..d1de2c54 100644
--- a/test/tox.centos_9.ini
+++ b/test/tox.centos_9.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _centos_9.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
+           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini
index a012bda4..4913b173 100644
--- a/test/tox.debian_10.ini
+++ b/test/tox.debian_10.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _debian_10.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
+           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini
index 48dc9df1..fa8412a8 100644
--- a/test/tox.debian_11.ini
+++ b/test/tox.debian_11.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _debian_11.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
+           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_36.ini
index 0cc6f29c..47a4e08e 100644
--- a/test/tox.fedora_36.ini
+++ b/test/tox.fedora_36.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
+           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini
index d6f44533..4a1d4f25 100644
--- a/test/tox.fedora_37.ini
+++ b/test/tox.fedora_37.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
+           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini
index 88ee0b54..51715e91 100644
--- a/test/tox.ubuntu_20.ini
+++ b/test/tox.ubuntu_20.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
+           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.ubuntu_22.ini b/test/tox.ubuntu_22.ini
index cb5527ab..70a0c820 100644
--- a/test/tox.ubuntu_22.ini
+++ b/test/tox.ubuntu_22.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
+           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py

From 4b503a080b0ed21455c1e9215cb92360e1a0329e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Tue, 14 Feb 2023 22:13:24 +0100
Subject: [PATCH 04/32] Revert "Use new worksteal distribution algorithm"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 46fe37b4dafdd860fe26decee88e1454d21feb1b.

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 test/tox.centos_8.ini  | 2 +-
 test/tox.centos_9.ini  | 2 +-
 test/tox.debian_10.ini | 2 +-
 test/tox.debian_11.ini | 2 +-
 test/tox.fedora_36.ini | 2 +-
 test/tox.fedora_37.ini | 2 +-
 test/tox.ubuntu_20.ini | 2 +-
 test/tox.ubuntu_22.ini | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini
index 5e94baa6..dac10e97 100644
--- a/test/tox.centos_8.ini
+++ b/test/tox.centos_8.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands =  docker build -f _centos_8.Dockerfile -t pytest_pihole:test_container ../
-            pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
+            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini
index d1de2c54..aa7009e1 100644
--- a/test/tox.centos_9.ini
+++ b/test/tox.centos_9.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _centos_9.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini
index 4913b173..a012bda4 100644
--- a/test/tox.debian_10.ini
+++ b/test/tox.debian_10.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _debian_10.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini
index fa8412a8..48dc9df1 100644
--- a/test/tox.debian_11.ini
+++ b/test/tox.debian_11.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _debian_11.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_36.ini
index 47a4e08e..0cc6f29c 100644
--- a/test/tox.fedora_36.ini
+++ b/test/tox.fedora_36.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini
index 4a1d4f25..d6f44533 100644
--- a/test/tox.fedora_37.ini
+++ b/test/tox.fedora_37.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini
index 51715e91..88ee0b54 100644
--- a/test/tox.ubuntu_20.ini
+++ b/test/tox.ubuntu_20.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.ubuntu_22.ini b/test/tox.ubuntu_22.ini
index 70a0c820..cb5527ab 100644
--- a/test/tox.ubuntu_22.ini
+++ b/test/tox.ubuntu_22.ini
@@ -5,4 +5,4 @@ envlist = py3
 allowlist_externals = docker
 deps = -rrequirements.txt
 commands = docker build -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../
-           pytest {posargs:-vv -n auto --dist worksteal} ./test_any_automated_install.py ./test_any_utils.py
+           pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

From a4bdf2454bf6b8676fc199db7d81a699536c9b8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Thu, 16 Feb 2023 10:34:19 +0100
Subject: [PATCH 05/32] Don't use intermediate strings to filter false
 positives in gravity
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 gravity.sh | 31 +++++++------------------------
 1 file changed, 7 insertions(+), 24 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index c2795442..28c28a9e 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -524,7 +524,7 @@ num_total_imported_domains=0
 num_domains=0
 num_non_domains=0
 parseList() {
-  local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains tmp_non_domains_str false_positive
+  local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains
   # This sed does the following things:
   # 1. Remove all lines containing no domains
   # 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_)
@@ -542,30 +542,13 @@ parseList() {
   # A list of items of common local hostnames not to report as unusable
   # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files
   # but flagging them as unusable causes more confusion than it's worth - so we suppress them from the output
-  false_positives=(
-    "localhost"
-    "localhost.localdomain"
-    "local"
-    "broadcasthost"
-    "localhost"
-    "ip6-localhost"
-    "ip6-loopback"
-    "lo0 localhost"
-    "ip6-localnet"
-    "ip6-mcastprefix"
-    "ip6-allnodes"
-    "ip6-allrouters"
-    "ip6-allhosts"
-    )
+  false_positives="localhost|localhost.localdomain|local|broadcasthost|localhost|ip6-localhost|ip6-loopback|lo0 localhost|ip6-localnet|ip6-mcastprefix|ip6-allnodes|ip6-allrouters|ip6-allhosts"
 
-  # Read the unusable lines into a string
-  tmp_non_domains_str=" ${non_domains[*]} "
-  for false_positive in "${false_positives[@]}"; do
-    # Remove false positives from tmp_non_domains_str
-    tmp_non_domains_str="${tmp_non_domains_str/ ${false_positive} / }"
-  done
-  # Read the string back into an array
-  IFS=" " read -r -a non_domains <<< "${tmp_non_domains_str}"
+  # if there are any non-domains, filter the array for false-positives
+  # Credit: https://stackoverflow.com/a/40264051
+  if [[ "${#non_domains[@]}" -gt 0 ]]; then
+    mapfile -d $'\0' -t non_domains < <(printf '%s\0' "${non_domains[@]}" | grep -Ezv "^${false_positives}")
+  fi
 
   # Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated)
   IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')"

From 90da155053dcebb205f33f438af8fdd4f9bcb3a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Thu, 16 Feb 2023 21:02:49 +0100
Subject: [PATCH 06/32] Use buildx to create docker test images
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 test/tox.centos_8.ini  | 2 +-
 test/tox.centos_9.ini  | 2 +-
 test/tox.debian_10.ini | 2 +-
 test/tox.debian_11.ini | 2 +-
 test/tox.fedora_36.ini | 2 +-
 test/tox.fedora_37.ini | 2 +-
 test/tox.ubuntu_20.ini | 2 +-
 test/tox.ubuntu_22.ini | 2 +-
 8 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/test/tox.centos_8.ini b/test/tox.centos_8.ini
index dac10e97..dca77c93 100644
--- a/test/tox.centos_8.ini
+++ b/test/tox.centos_8.ini
@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands =  docker build -f _centos_8.Dockerfile -t pytest_pihole:test_container ../
+commands =  docker buildx build --load --progress plain -f _centos_8.Dockerfile -t pytest_pihole:test_container ../
             pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
diff --git a/test/tox.centos_9.ini b/test/tox.centos_9.ini
index aa7009e1..a69c336a 100644
--- a/test/tox.centos_9.ini
+++ b/test/tox.centos_9.ini
@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _centos_9.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _centos_9.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_centos_common_support.py
diff --git a/test/tox.debian_10.ini b/test/tox.debian_10.ini
index a012bda4..f107300f 100644
--- a/test/tox.debian_10.ini
+++ b/test/tox.debian_10.ini
@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _debian_10.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _debian_10.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.debian_11.ini b/test/tox.debian_11.ini
index 48dc9df1..c38a15fb 100644
--- a/test/tox.debian_11.ini
+++ b/test/tox.debian_11.ini
@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _debian_11.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _debian_11.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.fedora_36.ini b/test/tox.fedora_36.ini
index 0cc6f29c..515487ed 100644
--- a/test/tox.fedora_36.ini
+++ b/test/tox.fedora_36.ini
@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _fedora_36.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
diff --git a/test/tox.fedora_37.ini b/test/tox.fedora_37.ini
index d6f44533..2a8ef398 100644
--- a/test/tox.fedora_37.ini
+++ b/test/tox.fedora_37.ini
@@ -4,5 +4,5 @@ envlist = py3
 [testenv]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _fedora_37.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py ./test_centos_fedora_common_support.py ./test_fedora_support.py
diff --git a/test/tox.ubuntu_20.ini b/test/tox.ubuntu_20.ini
index 88ee0b54..49a6153e 100644
--- a/test/tox.ubuntu_20.ini
+++ b/test/tox.ubuntu_20.ini
@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _ubuntu_20.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py
diff --git a/test/tox.ubuntu_22.ini b/test/tox.ubuntu_22.ini
index cb5527ab..8014d6d6 100644
--- a/test/tox.ubuntu_22.ini
+++ b/test/tox.ubuntu_22.ini
@@ -4,5 +4,5 @@ envlist = py3
 [testenv:py3]
 allowlist_externals = docker
 deps = -rrequirements.txt
-commands = docker build -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../
+commands = docker buildx build --load --progress plain -f _ubuntu_22.Dockerfile -t pytest_pihole:test_container ../
            pytest {posargs:-vv -n auto} ./test_any_automated_install.py ./test_any_utils.py

From 4fd0f15d903472e2e56ee3604a16d48d982ab01b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Thu, 16 Feb 2023 15:21:18 +0100
Subject: [PATCH 07/32] Ignore commented lines when reding PRIVACYLEVEL from
 config file Create dedicated getVal function in utils.sh as it might be
 useful somewhere else Account for tailing comments and $key not being on the
 first line
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 advanced/Scripts/utils.sh          | 22 ++++++++++++++++++++--
 automated install/basic-install.sh |  3 ++-
 test/test_any_utils.py             | 16 ++++++++++++++++
 3 files changed, 38 insertions(+), 3 deletions(-)

diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh
index 37516472..a1178265 100755
--- a/advanced/Scripts/utils.sh
+++ b/advanced/Scripts/utils.sh
@@ -44,7 +44,7 @@ addOrEditKeyValPair() {
 }
 
 #######################
-# Takes two arguments: file, and key.
+# Takes two arguments: file and key.
 # Adds a key to target file
 #
 # Example usage:
@@ -64,7 +64,7 @@ addKey(){
 }
 
 #######################
-# Takes two arguments: file, and key.
+# Takes two arguments: file and key.
 # Deletes a key or key/value pair from target file
 #
 # Example usage:
@@ -76,6 +76,24 @@ removeKey() {
   sed -i "/^${key}/d" "${file}"
 }
 
+#######################
+# Takes two arguments: file and key.
+# Returns the value of a given key from target file
+# - ignores all commented lines
+# - only returns the first value if multiple identical keys exist
+#
+#
+# Example usage:
+# getVal "/etc/pihole/setupVars.conf" "PIHOLE_DNS_1"
+#######################
+getVal() {
+  local file="${1}"
+  local key="${2}"
+  local value
+  value=$(sed -e '/^[[:blank:]]*#/d' "${file}" | grep "${key}" | awk -F "=" 'NR==1{printf$2}')
+  printf "%s" "$value"
+}
+
 
 #######################
 # returns FTL's current telnet API port based on the setting in /etc/pihole-FTL.conf
diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh
index 05bc0e4e..a9398d90 100755
--- a/automated install/basic-install.sh	
+++ b/automated install/basic-install.sh	
@@ -2612,7 +2612,8 @@ main() {
 
         # Get the privacy level if it exists (default is 0)
         if [[ -f "${FTL_CONFIG_FILE}" ]]; then
-            PRIVACY_LEVEL=$(sed -ne 's/PRIVACYLEVEL=\(.*\)/\1/p' "${FTL_CONFIG_FILE}")
+            # use getVal from utils.sh to get PRIVACYLEVEL
+            PRIVACY_LEVEL=$(getVal "${FTL_CONFIG_FILE}" "PRIVACYLEVEL")
 
             # If no setting was found, default to 0
             PRIVACY_LEVEL="${PRIVACY_LEVEL:-0}"
diff --git a/test/test_any_utils.py b/test/test_any_utils.py
index 5b4075d9..6c920161 100644
--- a/test/test_any_utils.py
+++ b/test/test_any_utils.py
@@ -62,6 +62,22 @@ def test_key_removal_works(host):
     assert expected_stdout == output.stdout
 
 
+def test_get_value_works(host):
+    """Confirms getVal returns the correct value for a given key"""
+    output = host.run(
+        """
+    source /opt/pihole/utils.sh
+    echo "Somekey=xxx" >> /tmp/testfile
+    echo "#Testkey=1234" >> /tmp/testfile
+    echo "Testkey=5678" >> /tmp/testfile
+    echo "Testkey=abcd" >> /tmp/testfile
+    getVal "/tmp/testfile" "Testkey"
+    """
+    )
+    expected_stdout = "5678"
+    assert expected_stdout == output.stdout
+
+
 def test_getFTLAPIPort_default(host):
     """Confirms getFTLAPIPort returns the default API port"""
     output = host.run(

From ca74152d1d5a5fc179314458a0a96a1843156f0d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Thu, 16 Feb 2023 23:11:00 +0100
Subject: [PATCH 08/32] Allow adding ABP style blocklists
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Co-authored-by: Adam Warner <me@adamwarner.co.uk>
Signed-off-by: Christian König <ckoenig@posteo.de>
---
 gravity.sh | 47 ++++++++++++++++++++++++-----------------------
 1 file changed, 24 insertions(+), 23 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 28c28a9e..d3bb4a07 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -519,25 +519,31 @@ gravity_DownloadBlocklists() {
   gravity_Blackbody=true
 }
 
-# num_total_imported_domains increases for each list processed
-num_total_imported_domains=0
-num_domains=0
-num_non_domains=0
 parseList() {
-  local adlistID="${1}" src="${2}" target="${3}" non_domains sample_non_domains
-  # This sed does the following things:
-  # 1. Remove all lines containing no domains
-  # 2. Remove all domains containing invalid characters. Valid are: a-z, A-Z, 0-9, dot (.), minus (-), underscore (_)
-  # 3. Append ,adlistID to every line
-  # 4. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701)
-  # 5. Ensures there is a newline on the last line
-  sed -r  "/([^\.]+\.)+[^\.]{2,}/!d;/[^a-zA-Z0-9.\_-]/d;s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${src}" >> "${target}"
+  local adlistID="${1}" src="${2}" target="${3}" temp_file non_domains sample_non_domains
 
-  # Find lines containing no domains or with invalid characters (see above)
+  # Create a temporary file for the sed magic instead of using "${target}" directly
+  # this allows to split the sed commands to improve readability
+  temp_file="$(mktemp -p "/tmp" --suffix=".gravity")"
+
+  # 1. Add all valid domains (adapted from https://stackoverflow.com/a/30007882)
+  # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already
+  sed -r "/^([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/!d" "${src}" > "${temp_file}"
+  # 2. Add all supported ABP style lines (||subdomain.domain.tlp^)
+  sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}"
+
+  # Find lines containing no domains or with invalid characters (not matching regex above)
+  # This is simply everything that is not in $temp_file compared to $src
   # Remove duplicates from the list
-  mapfile -t non_domains <<< "$(sed -r "/([^\.]+\.)+[^\.]{2,}/d" < "${src}")"
-  mapfile -t -O "${#non_domains[@]}" non_domains <<< "$(sed -r "/[^a-zA-Z0-9.\_-]/!d" < "${src}")"
-  IFS=" " read -r -a non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | sort -u | tr '\n' ' ')"
+  mapfile -t non_domains < <(grep -Fvf "${temp_file}" "${src}" | sort -u )
+
+  # 3. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701)
+  # 4. Append ,adlistID to every line
+  # 5. Ensures there is a newline on the last line
+  sed -i "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}"
+
+  # concatenate the temporary file to the target file
+  cat "${temp_file}" >> "${target}"
 
   # A list of items of common local hostnames not to report as unusable
   # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files
@@ -553,13 +559,8 @@ parseList() {
   # Get a sample of non-domain entries, limited to 5 (the list should already have been de-duplicated)
   IFS=" " read -r -a sample_non_domains <<< "$(tr ' ' '\n' <<< "${non_domains[@]}" | head -n 5 | tr '\n' ' ')"
 
-  local tmp_new_imported_total
-  # Get the new number of domains in destination file
-  tmp_new_imported_total="$(grep -c "^" "${target}")"
-  # Number of imported lines for this file is the difference between the new total and the old total. (Or, the number of domains we just added.)
-  num_domains="$(( tmp_new_imported_total-num_total_imported_domains ))"
-  # Replace the running total with the new total.
-  num_total_imported_domains="$tmp_new_imported_total"
+  # Get the number of domains added
+  num_domains="$(grep -c "^" "${temp_file}")"
   # Get the number of non_domains (this is the number of entries left after stripping the source of comments/duplicates/false positives/domains)
   num_non_domains="${#non_domains[@]}"
 

From 1f3f8491065949f38faac75879764f7edf9ce94e Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sun, 19 Feb 2023 17:47:10 +0000
Subject: [PATCH 09/32] Remove lines starting with ! or [ to account for ABP
 style comments and header

Also splits the piped "one-liner" in ParseFileIntoDomains into individually commented commands (makes for easier reading and debugging)

Signed-off-by: Adam Warner <me@adamwarner.co.uk>
---
 gravity.sh | 32 ++++++++++++++++++++++----------
 1 file changed, 22 insertions(+), 10 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index d3bb4a07..fb7f42f1 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -745,18 +745,30 @@ gravity_ParseFileIntoDomains() {
   # Most of the lists downloaded are already in hosts file format but the spacing/formatting is not contiguous
   # This helps with that and makes it easier to read
   # It also helps with debugging so each stage of the script can be researched more in depth
-  # 1) Remove carriage returns
-  # 2) Convert all characters to lowercase
-  # 3) Remove comments (text starting with "#", include possible spaces before the hash sign)
+  # 1) Convert all characters to lowercase
+  tr '[:upper:]' '[:lower:]' < "${src}" > "${destination}"
+
+  # 2) Remove carriage returns
+  sed -i 's/\r$//' "${destination}"
+
+  # 3a) Remove comments (text starting with "#", include possible spaces before the hash sign)
+  sed -i 's/\s*#.*//g' "${destination}"
+
+  # 3b) Remove lines starting with ! (ABP Comments)
+  sed -i 's/\s*!.*//g' "${destination}"
+
+  # 3c) Remove lines starting with [ (ABP Header)
+  sed -i 's/\s*\[.*//g' "${destination}"
+
   # 4) Remove lines containing "/"
-  # 5) Remove leading tabs, spaces, etc.
+  sed -i -r '/(\/).*$/d' "${destination}"
+
+  # 5) Remove leading tabs, spaces, etc. (Also removes leading IP addresses)
+  sed -i -r 's/^.*\s+//g' "${destination}"
+
   # 6) Remove empty lines
-  < "${src}" tr -d '\r' | \
-  tr '[:upper:]' '[:lower:]' | \
-  sed 's/\s*#.*//g' | \
-  sed -r '/(\/).*$/d' | \
-  sed -r 's/^.*\s+//g' | \
-  sed '/^$/d'>  "${destination}"
+  sed -i '/^$/d' "${destination}"
+
   chmod 644 "${destination}"
 }
 

From c5faf3d1744f100bbbfc800702c70ffc79eda02e Mon Sep 17 00:00:00 2001
From: Adam Warner <me@adamwarner.co.uk>
Date: Sun, 19 Feb 2023 18:12:03 +0000
Subject: [PATCH 10/32] Use ',' as the separator char in query rather than the
 default '|' as we now expect some valid results to contain '|'

Signed-off-by: Adam Warner <me@adamwarner.co.uk>
---
 advanced/Scripts/query.sh | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index d48e9363..8717d328 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -116,7 +116,7 @@ scanDatabaseTable() {
     fi
 
     # Send prepared query to gravity database
-    result="$(pihole-FTL sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
+    result="$(pihole-FTL sqlite3 -separator ',' "${gravityDBfile}" "${querystr}")" 2> /dev/null
     if [[ -z "${result}" ]]; then
         # Return early when there are no matches in this table
         return
@@ -136,8 +136,8 @@ scanDatabaseTable() {
     # Loop over results and print them
     mapfile -t results <<< "${result}"
     for result in "${results[@]}"; do
-        domain="${result/|*}"
-        if [[ "${result#*|}" == "0" ]]; then
+        domain="${result/,*}"
+        if [[ "${result#*,}" == "0" ]]; then
             extra=" (disabled)"
         else
             extra=""
@@ -212,10 +212,10 @@ if [[ -n "${exact}" ]]; then
 fi
 
 for result in "${results[@]}"; do
-    match="${result/|*/}"
-    extra="${result#*|}"
-    adlistAddress="${extra/|*/}"
-    extra="${extra#*|}"
+    match="${result/,*/}"
+    extra="${result#*,}"
+    adlistAddress="${extra/,*/}"
+    extra="${extra#*,}"
     if [[ "${extra}" == "0" ]]; then
         extra=" (disabled)"
     else

From eedd93d7823ace35e2915e83a6dcc86b13abfd68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Wed, 22 Feb 2023 21:14:27 +0100
Subject: [PATCH 11/32] Remove temporary files
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 gravity.sh | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index fb7f42f1..57959b25 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -524,7 +524,13 @@ parseList() {
 
   # Create a temporary file for the sed magic instead of using "${target}" directly
   # this allows to split the sed commands to improve readability
+  # we use a file handle here and remove the temporary file immediately so the content will be deleted in any case
+  # when the script stops
   temp_file="$(mktemp -p "/tmp" --suffix=".gravity")"
+  exec 3>"$temp_file"
+  rm "${temp_file}"
+  temp_file="/proc/$$/fd/3"
+
 
   # 1. Add all valid domains (adapted from https://stackoverflow.com/a/30007882)
   # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already
@@ -540,10 +546,8 @@ parseList() {
   # 3. Remove trailing period (see https://github.com/pi-hole/pi-hole/issues/4701)
   # 4. Append ,adlistID to every line
   # 5. Ensures there is a newline on the last line
-  sed -i "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}"
-
-  # concatenate the temporary file to the target file
-  cat "${temp_file}" >> "${target}"
+  # and write everything to the target file
+  sed "s/\.$//;s/$/,${adlistID}/;/.$/a\\" "${temp_file}" >> "${target}"
 
   # A list of items of common local hostnames not to report as unusable
   # Some lists (i.e StevenBlack's) contain these as they are supposed to be used as HOST files
@@ -575,6 +579,9 @@ parseList() {
   else
     echo "  ${INFO} Imported ${num_domains} domains"
   fi
+
+  # close file handle
+  exec 3<&-
 }
 
 compareLists() {

From 821c7dc190cdabc28f024e4688bff3670f028731 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Fri, 24 Feb 2023 22:18:41 +0100
Subject: [PATCH 12/32] Add info when list cotains ABP style domains
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 gravity.sh | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 57959b25..451a40fc 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -520,23 +520,29 @@ gravity_DownloadBlocklists() {
 }
 
 parseList() {
-  local adlistID="${1}" src="${2}" target="${3}" temp_file non_domains sample_non_domains
+  local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains
 
   # Create a temporary file for the sed magic instead of using "${target}" directly
   # this allows to split the sed commands to improve readability
   # we use a file handle here and remove the temporary file immediately so the content will be deleted in any case
   # when the script stops
-  temp_file="$(mktemp -p "/tmp" --suffix=".gravity")"
-  exec 3>"$temp_file"
-  rm "${temp_file}"
+  temp_file_base="$(mktemp -p "/tmp" --suffix=".gravity")"
+  exec 3>"$temp_file_base"
+  rm "${temp_file_base}"
   temp_file="/proc/$$/fd/3"
 
 
   # 1. Add all valid domains (adapted from https://stackoverflow.com/a/30007882)
   # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already
   sed -r "/^([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/!d" "${src}" > "${temp_file}"
-  # 2. Add all supported ABP style lines (||subdomain.domain.tlp^)
-  sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}"
+
+  # if there is at least one ABP style domains
+  if  grep -E "^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" -m 1 -q "${src}"; then
+    echo "  ${INFO} List contained AdBlock Plus style domains"
+    # 2. Add all supported ABP style lines (||subdomain.domain.tlp^)
+    sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}"
+  fi
+
 
   # Find lines containing no domains or with invalid characters (not matching regex above)
   # This is simply everything that is not in $temp_file compared to $src

From 31a9e189972c24c42ac959d12a09b1da689d4d1a Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 25 Feb 2023 10:57:18 +0000
Subject: [PATCH 13/32] Bump tox from 4.4.5 to 4.4.6 in /test

Bumps [tox](https://github.com/tox-dev/tox) from 4.4.5 to 4.4.6.
- [Release notes](https://github.com/tox-dev/tox/releases)
- [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst)
- [Commits](https://github.com/tox-dev/tox/compare/4.4.5...4.4.6)

---
updated-dependencies:
- dependency-name: tox
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 test/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/requirements.txt b/test/requirements.txt
index 229d0135..c7848e8d 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -2,5 +2,5 @@ docker-compose == 1.29.2
 pytest == 7.2.1
 pytest-xdist == 3.2.0
 pytest-testinfra == 7.0.0
-tox == 4.4.5
+tox == 4.4.6
 

From 6cb0be82caca4ca7b642a292f3d5a16ab1b47bcd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Sun, 26 Feb 2023 10:34:17 +0100
Subject: [PATCH 14/32] Add flag abp_domains into info table to signal if abp
 domains have been found
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 gravity.sh | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index 451a40fc..39ac2ded 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -137,6 +137,18 @@ update_gravity_timestamp() {
   return 0
 }
 
+# Update timestamp when the gravity table was last updated successfully
+set_abp_info() {
+  pihole-FTL sqlite3 "${gravityDBfile}" "INSERT OR REPLACE INTO info (property,value) VALUES ('abp_domains',${abp_domains});"
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to update ABP domain status in database ${gravityDBfile}\\n  ${output}"
+    return 1
+  fi
+  return 0
+}
+
 # Import domains from file and store them in the specified database table
 database_table_from_file() {
   # Define locals
@@ -519,6 +531,10 @@ gravity_DownloadBlocklists() {
   gravity_Blackbody=true
 }
 
+
+# global variable to indicate if we found ABP style domains during the gravity run
+# is saved in gravtiy's info table to signal FTL if such domains are available
+abp_domains=0
 parseList() {
   local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains
 
@@ -536,9 +552,10 @@ parseList() {
   # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already
   sed -r "/^([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/!d" "${src}" > "${temp_file}"
 
-  # if there is at least one ABP style domains
+  # if there is at least one ABP style domain
   if  grep -E "^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" -m 1 -q "${src}"; then
     echo "  ${INFO} List contained AdBlock Plus style domains"
+    abp_domains=1
     # 2. Add all supported ABP style lines (||subdomain.domain.tlp^)
     sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}"
   fi
@@ -1014,6 +1031,9 @@ fi
 # Update gravity timestamp
 update_gravity_timestamp
 
+# Set abp_domain info field
+set_abp_info
+
 # Ensure proper permissions are set for the database
 chown pihole:pihole "${gravityDBfile}"
 chmod g+w "${piholeDir}" "${gravityDBfile}"

From 16385af3ef473e4f148224bfc735e2582e666a3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Sun, 26 Feb 2023 21:16:45 +0100
Subject: [PATCH 15/32] Use dedicated pattern variable to make RegEx reusable
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 gravity.sh | 22 ++++++++++++++--------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 39ac2ded..8db728a4 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -536,7 +536,7 @@ gravity_DownloadBlocklists() {
 # is saved in gravtiy's info table to signal FTL if such domains are available
 abp_domains=0
 parseList() {
-  local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains
+  local adlistID="${1}" src="${2}" target="${3}" temp_file temp_file_base non_domains sample_non_domains valid_domain_pattern abp_domain_pattern
 
   # Create a temporary file for the sed magic instead of using "${target}" directly
   # this allows to split the sed commands to improve readability
@@ -547,17 +547,23 @@ parseList() {
   rm "${temp_file_base}"
   temp_file="/proc/$$/fd/3"
 
-
-  # 1. Add all valid domains (adapted from https://stackoverflow.com/a/30007882)
+  # define valid domain patterns
   # no need to include uppercase letters, as we convert to lowercase in gravity_ParseFileIntoDomains() already
-  sed -r "/^([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/!d" "${src}" > "${temp_file}"
+  # adapted from https://stackoverflow.com/a/30007882
+  # supported ABP style: ||subdomain.domain.tlp^
 
-  # if there is at least one ABP style domain
-  if  grep -E "^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]" -m 1 -q "${src}"; then
+  valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]"
+  abp_domain_pattern="\|\|${valid_domain_pattern}\^"
+
+
+  # 1. Add all valid domains
+    sed -r "/^${valid_domain_pattern}$/!d" "${src}" > "${temp_file}"
+
+  # 2. Add valid ABP style domains if there is at least one such domain
+  if  grep -E "^${abp_domain_pattern}$" -m 1 -q "${src}"; then
     echo "  ${INFO} List contained AdBlock Plus style domains"
     abp_domains=1
-    # 2. Add all supported ABP style lines (||subdomain.domain.tlp^)
-    sed -r "/^\|\|([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\^$/!d" "${src}" >> "${temp_file}"
+    sed -r "/^${abp_domain_pattern}$/!d" "${src}" >> "${temp_file}"
   fi
 
 

From 73de49323c13f1e87f68e272e9a71e12d03e3f65 Mon Sep 17 00:00:00 2001
From: MichaIng <micha@dietpi.com>
Date: Tue, 21 Feb 2023 21:49:03 +0100
Subject: [PATCH 16/32] Remove systemd service and optionally override configs
 on uninstall

This has been forgotten when adding the new native systemd service.

Signed-off-by: MichaIng <micha@dietpi.com>
---
 automated install/uninstall.sh | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/automated install/uninstall.sh b/automated install/uninstall.sh
index c36027fc..7a1a290d 100755
--- a/automated install/uninstall.sh	
+++ b/automated install/uninstall.sh	
@@ -193,6 +193,18 @@ removeNoPurge() {
         else
             service pihole-FTL stop
         fi
+        ${SUDO} rm -f /etc/systemd/system/pihole-FTL.service
+        if [[ -d '/etc/systemd/system/pihole-FTL.service.d' ]]; then
+            read -rp "  ${QST} FTL service override directory /etc/systemd/system/pihole-FTL.service.d detected. Do you wish to remove this from your system? [y/N] " answer
+            case $answer in
+                [yY]*)
+                    echo -ne "  ${INFO} Removing /etc/systemd/system/pihole-FTL.service.d..."
+                    ${SUDO} rm -R /etc/systemd/system/pihole-FTL.service.d
+                    echo -e "${OVER}  ${INFO} Removed /etc/systemd/system/pihole-FTL.service.d"
+                ;;
+                *) echo -e "  ${INFO} Leaving /etc/systemd/system/pihole-FTL.service.d in place.";;
+            esac
+        fi
         ${SUDO} rm -f /etc/init.d/pihole-FTL
         ${SUDO} rm -f /usr/bin/pihole-FTL
         echo -e "${OVER}  ${TICK} Removed pihole-FTL"

From ddf972cede32c3b80fc25cfdab356c101d08f36e Mon Sep 17 00:00:00 2001
From: Alex <aleksandrosansan@gmail.com>
Date: Thu, 8 Dec 2022 20:00:33 +0200
Subject: [PATCH 17/32] build: harden workflow permissions

Signed-off-by: Alex <aleksandrosansan@gmail.com>
---
 .github/workflows/sync-back-to-dev.yml | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml
index f689ae36..8572ffde 100644
--- a/.github/workflows/sync-back-to-dev.yml
+++ b/.github/workflows/sync-back-to-dev.yml
@@ -5,8 +5,30 @@ on:
     branches:
       - master
 
+# The section is needed to drop the default write-all permissions for all jobs
+# that are granted on `push` event. By specifying any permission explicitly
+# all others are set to none. By using the principle of least privilege the damage a compromised
+# workflow can do (because of an injection or compromised third party tool or
+# action) is restricted. Adding labels to issues, commenting
+# on pull-requests, etc. may need additional permissions:
+#
+# Syntax for this section:
+# https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#permissions
+#
+# Reference for how to assign permissions on a job-by-job basis:
+# https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+#
+# Reference for available permissions that we can enable if needed:
+# https://docs.github.com/en/actions/security-guides/automatic-token-authentication#permissions-for-the-github_token
+permissions: {}
+
 jobs:
   sync-branches:
+    # The job needs to be able to pull the code and create a pull request.
+    permissions:
+      contents: read #  for actions/checkout
+      pull-requests: write #  to create pull request
+
     runs-on: ubuntu-latest
     name: Syncing branches
     steps:

From 0b5da9f0dabccaf902aada60527c3a4b5855b925 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Sat, 4 Mar 2023 10:36:07 +0100
Subject: [PATCH 18/32] Allow final dot (root zone)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 gravity.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index 8db728a4..281d5d54 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -552,7 +552,7 @@ parseList() {
   # adapted from https://stackoverflow.com/a/30007882
   # supported ABP style: ||subdomain.domain.tlp^
 
-  valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]"
+  valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\.?"
   abp_domain_pattern="\|\|${valid_domain_pattern}\^"
 
 

From 0b60601f863fff2453b5a1e9cb9f678d33e72b13 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 4 Mar 2023 10:57:42 +0000
Subject: [PATCH 19/32] Bump pytest from 7.2.1 to 7.2.2 in /test

Bumps [pytest](https://github.com/pytest-dev/pytest) from 7.2.1 to 7.2.2.
- [Release notes](https://github.com/pytest-dev/pytest/releases)
- [Changelog](https://github.com/pytest-dev/pytest/blob/main/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest/compare/7.2.1...7.2.2)

---
updated-dependencies:
- dependency-name: pytest
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 test/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/requirements.txt b/test/requirements.txt
index c7848e8d..e64e8c66 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -1,5 +1,5 @@
 docker-compose == 1.29.2
-pytest == 7.2.1
+pytest == 7.2.2
 pytest-xdist == 3.2.0
 pytest-testinfra == 7.0.0
 tox == 4.4.6

From d6f5552ccf4ee96c3ca137184b99307cf81b1496 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Mon, 6 Mar 2023 21:16:51 +0100
Subject: [PATCH 20/32] Convert domain to lowercase in pihole -q
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 advanced/Scripts/query.sh | 57 ++++++++++++++++++++-------------------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index d48e9363..4061e17c 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -30,33 +30,6 @@ gravityDBfile="${GRAVITYDB}"
 colfile="/opt/pihole/COL_TABLE"
 source "${colfile}"
 
-# Scan an array of files for matching strings
-scanList(){
-    # Escape full stops
-    local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}"
-
-    # Prevent grep from printing file path
-    cd "$piholeDir" || exit 1
-
-    # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX
-    export LC_CTYPE=C
-
-    # /dev/null forces filename to be printed when only one list has been generated
-    case "${list_type}" in
-        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
-        # Iterate through each regexp and check whether it matches the domainQuery
-        # If it does, print the matching regexp and continue looping
-        # Input 1 - regexps | Input 2 - domainQuery
-        "regex" )
-            for list in ${lists}; do
-                if [[ "${domain}" =~ ${list} ]]; then
-                    printf "%b\n" "${list}";
-                fi
-            done;;
-        *       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
-    esac
-}
-
 if [[ "${options}" == "-h" ]] || [[ "${options}" == "--help" ]]; then
     echo "Usage: pihole -q [option] <domain>
 Example: 'pihole -q -exact domain.com'
@@ -88,11 +61,41 @@ case "${options}" in
     *              ) domainQuery="${options}";;
 esac
 
+# convert the domain to lowercase
+domainQuery=$(echo "${domainQuery}" | tr '[:upper:]' '[:lower:]')
+
 if [[ -n "${str:-}" ]]; then
     echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information."
     exit 1
 fi
 
+# Scan an array of files for matching strings
+scanList(){
+    # Escape full stops
+    local domain="${1}" esc_domain="${1//./\\.}" lists="${2}" list_type="${3:-}"
+
+    # Prevent grep from printing file path
+    cd "$piholeDir" || exit 1
+
+    # Prevent grep -i matching slowly: https://bit.ly/2xFXtUX
+    export LC_CTYPE=C
+
+    # /dev/null forces filename to be printed when only one list has been generated
+    case "${list_type}" in
+        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+        # Iterate through each regexp and check whether it matches the domainQuery
+        # If it does, print the matching regexp and continue looping
+        # Input 1 - regexps | Input 2 - domainQuery
+        "regex" )
+            for list in ${lists}; do
+                if [[ "${domain}" =~ ${list} ]]; then
+                    printf "%b\n" "${list}";
+                fi
+            done;;
+        *       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
+    esac
+}
+
 scanDatabaseTable() {
     local domain table list_type querystr result extra
     domain="$(printf "%q" "${1}")"

From 309ee789036708b77453e2c71f1a2a22180d85ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Tue, 7 Mar 2023 20:40:16 +0100
Subject: [PATCH 21/32] Use distinct variabled
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 advanced/Scripts/query.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 4061e17c..99c1cf0f 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -57,12 +57,12 @@ options=$(sed -E 's/ ?-(all|exact) ?//g' <<< "${options}")
 case "${options}" in
     ""             ) str="No domain specified";;
     *" "*          ) str="Unknown query option specified";;
-    *[![:ascii:]]* ) domainQuery=$(idn2 "${options}");;
-    *              ) domainQuery="${options}";;
+    *[![:ascii:]]* ) rawDomainQuery=$(idn2 "${options}");;
+    *              ) rawDomainQuery="${options}";;
 esac
 
 # convert the domain to lowercase
-domainQuery=$(echo "${domainQuery}" | tr '[:upper:]' '[:lower:]')
+domainQuery=$(echo "${rawDomainQuery}" | tr '[:upper:]' '[:lower:]')
 
 if [[ -n "${str:-}" ]]; then
     echo -e "${str}${COL_NC}\\nTry 'pihole -q --help' for more information."
@@ -82,7 +82,7 @@ scanList(){
 
     # /dev/null forces filename to be printed when only one list has been generated
     case "${list_type}" in
-        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" ${lists} /dev/null 2>/dev/null;;
+        "exact" ) grep -i -E -l "(^|(?<!#)\\s)${esc_domain}($|\\s|#)" "${lists}" /dev/null 2>/dev/null;;
         # Iterate through each regexp and check whether it matches the domainQuery
         # If it does, print the matching regexp and continue looping
         # Input 1 - regexps | Input 2 - domainQuery
@@ -92,7 +92,7 @@ scanList(){
                     printf "%b\n" "${list}";
                 fi
             done;;
-        *       ) grep -i "${esc_domain}" ${lists} /dev/null 2>/dev/null;;
+        *       ) grep -i "${esc_domain}" "${lists}" /dev/null 2>/dev/null;;
     esac
 }
 

From 71e262c37f5582d54e5c7d9f05b4f61533ca81a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <ckoenig@posteo.de>
Date: Tue, 14 Mar 2023 19:42:05 +0100
Subject: [PATCH 22/32] Revert "Allow final dot (root zone)"
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

This reverts commit 0b5da9f0dabccaf902aada60527c3a4b5855b925.

Signed-off-by: Christian König <ckoenig@posteo.de>
---
 advanced/Scripts/piholeDebug.sh | 2 +-
 gravity.sh                      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/piholeDebug.sh b/advanced/Scripts/piholeDebug.sh
index ad25e866..fa1cebbb 100755
--- a/advanced/Scripts/piholeDebug.sh
+++ b/advanced/Scripts/piholeDebug.sh
@@ -230,7 +230,7 @@ initialize_debug() {
 
 # This is a function for visually displaying the current test that is being run.
 # Accepts one variable: the name of what is being diagnosed
-# Colors do not show in the dasboard, but the icons do: [i], [✓], and [✗]
+# Colors do not show in the dashboard, but the icons do: [i], [✓], and [✗]
 echo_current_diagnostic() {
     # Colors are used for visually distinguishing each test in the output
     # These colors do not show in the GUI, but the formatting will
diff --git a/gravity.sh b/gravity.sh
index 281d5d54..8db728a4 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -552,7 +552,7 @@ parseList() {
   # adapted from https://stackoverflow.com/a/30007882
   # supported ABP style: ||subdomain.domain.tlp^
 
-  valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]\.?"
+  valid_domain_pattern="([a-z0-9]([a-z0-9_-]{0,61}[a-z0-9]){0,1}\.)+[a-z0-9][a-z0-9-]{0,61}[a-z0-9]"
   abp_domain_pattern="\|\|${valid_domain_pattern}\^"
 
 

From c35ed6805159e3df319d0bb663cc145d5987a8af Mon Sep 17 00:00:00 2001
From: RD WebDesign <github@rdwebdesign.com.br>
Date: Mon, 27 Feb 2023 01:46:12 -0300
Subject: [PATCH 23/32] Allow `pihole -q` matching ABP subdomains

Signed-off-by: RD WebDesign <github@rdwebdesign.com.br>
---
 advanced/Scripts/query.sh | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 463b0901..71309b56 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -102,6 +102,16 @@ scanDatabaseTable() {
     table="${2}"
     list_type="${3:-}"
 
+    # Create search string for ABP entries
+    local abpentry="${domain}" searchstr
+
+    searchstr="'||${abpentry}^'"
+    while [ "${abpentry}" != "${abpentry/./}" ]
+    do
+        abpentry=$(echo "${abpentry}" | cut -f 2- -d '.')
+        searchstr=$(echo "$searchstr, '||${abpentry}^'")
+    done
+
     # As underscores are legitimate parts of domains, we escape them when using the LIKE operator.
     # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
@@ -109,12 +119,12 @@ scanDatabaseTable() {
     if [[ "${table}" == "gravity" ]]; then
         case "${exact}" in
             "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";;
-            *       ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
+            *       ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE (domain IN (${searchstr}) OR domain LIKE '%${domain//_/\\_}%' ESCAPE '\\')";;
         esac
     else
         case "${exact}" in
             "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";;
-            *       ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
+            *       ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND (domain IN (${searchstr}) OR domain LIKE '%${domain//_/\\_}%' ESCAPE '\\')";;
         esac
     fi
 

From 20f8c6af3c3b10fa4be6bd23365ecd7b89c93cea Mon Sep 17 00:00:00 2001
From: RD WebDesign <github@rdwebdesign.com.br>
Date: Tue, 28 Feb 2023 15:55:02 -0300
Subject: [PATCH 24/32] Search for ABP entries only if they exist in gravity.db

and use `abp_domains` property.

Signed-off-by: RD WebDesign <github@rdwebdesign.com.br>
---
 advanced/Scripts/query.sh | 40 +++++++++++++++++++++++++++------------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 71309b56..c6a932b7 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -102,29 +102,45 @@ scanDatabaseTable() {
     table="${2}"
     list_type="${3:-}"
 
-    # Create search string for ABP entries
-    local abpentry="${domain}" searchstr
-
-    searchstr="'||${abpentry}^'"
-    while [ "${abpentry}" != "${abpentry/./}" ]
-    do
-        abpentry=$(echo "${abpentry}" | cut -f 2- -d '.')
-        searchstr=$(echo "$searchstr, '||${abpentry}^'")
-    done
-
     # As underscores are legitimate parts of domains, we escape them when using the LIKE operator.
     # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     if [[ "${table}" == "gravity" ]]; then
+        local abpquerystr, abpfound, abpentry, searchstr
+
+        # Are there ABP entries on gravity?
+        # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set
+        abpquerystr="SELECT EXISTS (SELECT 1 FROM info WHERE property='abp_domains' and value='1')"
+        abpfound="$(pihole-FTL sqlite3 "${gravityDBfile}" "${abpquerystr}")" 2> /dev/null
+
+        # Create search string for ABP entries only if needed
+        if [ "${abpfound}" -eq 1 ]; then
+            abpentry="${domain}"
+
+            searchstr="'||${abpentry}^'"
+
+            # While a dot is found ...
+            while [ "${abpentry}" != "${abpentry/./}" ]
+            do
+                # ... remove text before the dot (including the dot) and append the result to $searchstr
+                abpentry=$(echo "${abpentry}" | cut -f 2- -d '.')
+                searchstr="$searchstr, '||${abpentry}^'"
+            done
+
+            # The final search string will look like:
+            # "domain IN ('||sub2.sub1.domain.com^', '||sub1.domain.com^', '||domain.com^', '||com^') OR"
+            searchstr="domain IN (${searchstr}) OR "
+        fi
+
         case "${exact}" in
             "exact" ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE domain = '${domain}'";;
-            *       ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE (domain IN (${searchstr}) OR domain LIKE '%${domain//_/\\_}%' ESCAPE '\\')";;
+            *       ) querystr="SELECT gravity.domain,adlist.address,adlist.enabled FROM gravity LEFT JOIN adlist ON adlist.id = gravity.adlist_id WHERE ${searchstr} domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
         esac
     else
         case "${exact}" in
             "exact" ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain = '${domain}'";;
-            *       ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND (domain IN (${searchstr}) OR domain LIKE '%${domain//_/\\_}%' ESCAPE '\\')";;
+            *       ) querystr="SELECT domain,enabled FROM domainlist WHERE type = '${list_type}' AND domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
         esac
     fi
 

From b9a6970bfd04b8fbe9f78bc849b5bddc776202eb Mon Sep 17 00:00:00 2001
From: William Blew <william@kulian.org>
Date: Thu, 16 Mar 2023 19:36:22 -0700
Subject: [PATCH 25/32] Fix addKey to handle substrings of existing keys

Fix addKey to handle the case where a key is being added, and that key
is the leading substring of an already existing key within that file.

For example: add "server=192.168.1.1", when "server=192.168.1.178"
already exists within the /etc/dnsmasq.d/01-pihole.conf file.

Check pihole docker with PIHOLE_DNS="192.168.1.178;192.168.1.1". Its
/etc/dnsmasq/01-pihole.conf will be missing its second server= entry.

Add the test_key_addition_substr, to test addKey when
its adding a substring key of an existing key in the file.

Signed-off-by: William Blew <william@kulian.org>
---
 advanced/Scripts/utils.sh |  2 +-
 test/test_any_utils.py    | 20 ++++++++++++++++++++
 2 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh
index 37516472..9e714606 100755
--- a/advanced/Scripts/utils.sh
+++ b/advanced/Scripts/utils.sh
@@ -57,7 +57,7 @@ addKey(){
   # touch file to prevent grep error if file does not exist yet
   touch "${file}"
 
-  if ! grep -q "^${key}" "${file}"; then
+  if ! grep -q "^${key}$" "${file}"; then
     # Key does not exist, add it.
     echo "${key}" >> "${file}"
   fi
diff --git a/test/test_any_utils.py b/test/test_any_utils.py
index 5b4075d9..b3fabe6c 100644
--- a/test/test_any_utils.py
+++ b/test/test_any_utils.py
@@ -40,6 +40,26 @@ def test_key_addition_works(host):
     assert expected_stdout == output.stdout
 
 
+def test_key_addition_substr(host):
+    """Confirms addKey adds substring keys (no value) to a file"""
+    host.run(
+        """
+    source /opt/pihole/utils.sh
+    addKey "./testoutput" "KEY_ONE"
+    addKey "./testoutput" "KEY_O"
+    addKey "./testoutput" "KEY_TWO"
+    addKey "./testoutput" "Y_TWO"
+    """
+    )
+    output = host.run(
+        """
+    cat ./testoutput
+    """
+    )
+    expected_stdout = "KEY_ONE\nKEY_O\nKEY_TWO\nY_TWO\n"
+    assert expected_stdout == output.stdout
+
+
 def test_key_removal_works(host):
     """Confirms removeKey removes a key or key/value pair"""
     host.run(

From 3c91b6558dcc947736e1df1631a9a7dfc7d32f9b Mon Sep 17 00:00:00 2001
From: William Blew <william@kulian.org>
Date: Fri, 17 Mar 2023 11:47:26 -0700
Subject: [PATCH 26/32] restore the addKey comment, reworded for anchors

Per @dschaper, restore the addKey clarifying comment. It has
been reworded to describe the use of anchors where before it
referenced using grep's  'match only an entire line' argument.

Signed-off-by: William Blew <william@kulian.org>
---
 advanced/Scripts/utils.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/advanced/Scripts/utils.sh b/advanced/Scripts/utils.sh
index 9e714606..f655e56c 100755
--- a/advanced/Scripts/utils.sh
+++ b/advanced/Scripts/utils.sh
@@ -57,6 +57,10 @@ addKey(){
   # touch file to prevent grep error if file does not exist yet
   touch "${file}"
 
+  # Match key against entire line, using both anchors. We assume
+  # that the file's keys never have bounding whitespace. Anchors
+  # are necessary to ensure the key is considered absent when it
+  # is a substring of another key present in the file.
   if ! grep -q "^${key}$" "${file}"; then
     # Key does not exist, add it.
     echo "${key}" >> "${file}"

From 3a592e56ba6b6e41d11f8fd3769a42892e4a3059 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 18 Mar 2023 10:56:52 +0000
Subject: [PATCH 27/32] Bump actions/checkout from 3.3.0 to 3.4.0

Bumps [actions/checkout](https://github.com/actions/checkout) from 3.3.0 to 3.4.0.
- [Release notes](https://github.com/actions/checkout/releases)
- [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md)
- [Commits](https://github.com/actions/checkout/compare/v3.3.0...v3.4.0)

---
updated-dependencies:
- dependency-name: actions/checkout
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 .github/workflows/codeql-analysis.yml  | 2 +-
 .github/workflows/sync-back-to-dev.yml | 2 +-
 .github/workflows/test.yml             | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
index 5539cec9..01be8b25 100644
--- a/.github/workflows/codeql-analysis.yml
+++ b/.github/workflows/codeql-analysis.yml
@@ -25,7 +25,7 @@ jobs:
     steps:
     -
       name: Checkout repository
-      uses: actions/checkout@v3.3.0
+      uses: actions/checkout@v3.4.0
     # Initializes the CodeQL tools for scanning.
     -
       name: Initialize CodeQL
diff --git a/.github/workflows/sync-back-to-dev.yml b/.github/workflows/sync-back-to-dev.yml
index 25c3a7f7..0fe850d1 100644
--- a/.github/workflows/sync-back-to-dev.yml
+++ b/.github/workflows/sync-back-to-dev.yml
@@ -33,7 +33,7 @@ jobs:
     name: Syncing branches
     steps:
       - name: Checkout
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3.4.0
       - name: Opening pull request
         run: gh pr create -B development -H master --title 'Sync master back into development' --body 'Created by Github action' --label 'internal'
         env:
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index ce948e09..27867ef3 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3.4.0
 
       - name: Check scripts in repository are executable
         run: |
@@ -62,7 +62,7 @@ jobs:
       DISTRO: ${{matrix.distro}}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3.3.0
+        uses: actions/checkout@v3.4.0
 
       - name: Set up Python 3.10
         uses: actions/setup-python@v4.5.0

From 8a2829de874aa1c90fba1ee38bfe44f5b480d3fe Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 18 Mar 2023 10:58:49 +0000
Subject: [PATCH 28/32] Bump pytest-xdist from 3.2.0 to 3.2.1 in /test

Bumps [pytest-xdist](https://github.com/pytest-dev/pytest-xdist) from 3.2.0 to 3.2.1.
- [Release notes](https://github.com/pytest-dev/pytest-xdist/releases)
- [Changelog](https://github.com/pytest-dev/pytest-xdist/blob/master/CHANGELOG.rst)
- [Commits](https://github.com/pytest-dev/pytest-xdist/compare/v3.2.0...v3.2.1)

---
updated-dependencies:
- dependency-name: pytest-xdist
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 test/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/requirements.txt b/test/requirements.txt
index e64e8c66..b2f0e8c4 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -1,6 +1,6 @@
 docker-compose == 1.29.2
 pytest == 7.2.2
-pytest-xdist == 3.2.0
+pytest-xdist == 3.2.1
 pytest-testinfra == 7.0.0
 tox == 4.4.6
 

From 686da5a9480fc1791c0623f7a035c84dceb23a06 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Sat, 18 Mar 2023 10:58:57 +0000
Subject: [PATCH 29/32] Bump tox from 4.4.6 to 4.4.7 in /test

Bumps [tox](https://github.com/tox-dev/tox) from 4.4.6 to 4.4.7.
- [Release notes](https://github.com/tox-dev/tox/releases)
- [Changelog](https://github.com/tox-dev/tox/blob/main/docs/changelog.rst)
- [Commits](https://github.com/tox-dev/tox/compare/4.4.6...4.4.7)

---
updated-dependencies:
- dependency-name: tox
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <support@github.com>
---
 test/requirements.txt | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/requirements.txt b/test/requirements.txt
index e64e8c66..ff2f49d0 100644
--- a/test/requirements.txt
+++ b/test/requirements.txt
@@ -2,5 +2,5 @@ docker-compose == 1.29.2
 pytest == 7.2.2
 pytest-xdist == 3.2.0
 pytest-testinfra == 7.0.0
-tox == 4.4.6
+tox == 4.4.7
 

From 6b919f3a2e36f724e8dd2a8f3b4e518d28de20a1 Mon Sep 17 00:00:00 2001
From: RD WebDesign <github@rdwebdesign.com.br>
Date: Tue, 21 Mar 2023 12:27:20 -0300
Subject: [PATCH 30/32] Removing unnecessary commas

Signed-off-by: RD WebDesign <github@rdwebdesign.com.br>
---
 advanced/Scripts/query.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index c6a932b7..604ac2ed 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -107,7 +107,7 @@ scanDatabaseTable() {
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     if [[ "${table}" == "gravity" ]]; then
-        local abpquerystr, abpfound, abpentry, searchstr
+        local abpquerystr abpfound abpentry searchstr
 
         # Are there ABP entries on gravity?
         # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set

From 66ed7c9ea374cc7c322807b6a0fc5cb33ac45d80 Mon Sep 17 00:00:00 2001
From: RD WebDesign <github@rdwebdesign.com.br>
Date: Tue, 21 Mar 2023 16:34:50 -0300
Subject: [PATCH 31/32] Declaring all local variables under the function
 declaration line

Signed-off-by: RD WebDesign <github@rdwebdesign.com.br>
---
 advanced/Scripts/query.sh | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 604ac2ed..12295fbc 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -97,7 +97,7 @@ scanList(){
 }
 
 scanDatabaseTable() {
-    local domain table list_type querystr result extra
+    local domain table list_type querystr result extra abpquerystr abpfound abpentry searchstr
     domain="$(printf "%q" "${1}")"
     table="${2}"
     list_type="${3:-}"
@@ -107,7 +107,6 @@ scanDatabaseTable() {
     # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
     # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
     if [[ "${table}" == "gravity" ]]; then
-        local abpquerystr abpfound abpentry searchstr
 
         # Are there ABP entries on gravity?
         # Return 1 if abp_domain=1 or Zero if abp_domain=0 or not set

From 76b7453f902c50364a0ca688ef06eb106116f0f4 Mon Sep 17 00:00:00 2001
From: kot0dama <89980752+kot0dama@users.noreply.github.com>
Date: Sun, 19 Mar 2023 05:32:46 +0100
Subject: [PATCH 32/32] Add configurable GRAVITY_TMPDIR variable into setupVars
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Loïc Gomez <89980752+kot0dama@users.noreply.github.com>
---
 gravity.sh | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 8db728a4..ca859e38 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -52,6 +52,14 @@ else
   exit 1
 fi
 
+# Set up tmp dir variable in case it's not configured
+: "${GRAVITY_TMPDIR:=/tmp}"
+
+if [ ! -d "${GRAVITY_TMPDIR}" ] || [ ! -w "${GRAVITY_TMPDIR}" ]; then
+  echo -e "  ${COL_LIGHT_RED}Gravity temporary directory does not exist or is not a writeable directory, falling back to /tmp. ${COL_NC}"
+  GRAVITY_TMPDIR="/tmp"
+fi
+
 # Source pihole-FTL from install script
 pihole_FTL="${piholeDir}/pihole-FTL.conf"
 if [[ -f "${pihole_FTL}" ]]; then
@@ -157,7 +165,7 @@ database_table_from_file() {
   src="${2}"
   backup_path="${piholeDir}/migration_backup"
   backup_file="${backup_path}/$(basename "${2}")"
-  tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")"
+  tmpFile="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")"
 
   local timestamp
   timestamp="$(date --utc +'%s')"
@@ -430,7 +438,7 @@ gravity_DownloadBlocklists() {
     echo -e "${OVER}  ${TICK} ${str}"
   fi
 
-  target="$(mktemp -p "/tmp" --suffix=".gravity")"
+  target="$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".gravity")"
 
   # Use compression to reduce the amount of data that is transferred
   # between the Pi-hole and the ad list provider. Use this feature
@@ -643,7 +651,7 @@ gravity_DownloadBlocklistFromUrl() {
   local heisenbergCompensator="" patternBuffer str httpCode success="" ip
 
   # Create temp file to store content on disk instead of RAM
-  patternBuffer=$(mktemp -p "/tmp" --suffix=".phgpb")
+  patternBuffer=$(mktemp -p "${GRAVITY_TMPDIR}" --suffix=".phgpb")
 
   # Determine if $saveLocation has read permission
   if [[ -r "${saveLocation}" && $url != "file"* ]]; then
@@ -860,7 +868,7 @@ gravity_Cleanup() {
   # Delete tmp content generated by Gravity
   rm ${piholeDir}/pihole.*.txt 2> /dev/null
   rm ${piholeDir}/*.tmp 2> /dev/null
-  rm /tmp/*.phgpb 2> /dev/null
+  rm "${GRAVITY_TMPDIR}"/*.phgpb 2> /dev/null
 
   # Ensure this function only runs when gravity_SetDownloadOptions() has completed
   if [[ "${gravity_Blackbody:-}" == true ]]; then