From 3aa838bbe47aee2f4ec804873fc2b267422d8f3c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 12:47:25 +0200
Subject: [PATCH 1/7] Implement black- and whitelist searching with SQL
statements. We use the ESCAPE clause in the LIKE query as the underscore "_"
wildcard matches any single character but we want to suppress this behavior
(underscores can be legitimate part of domains)
Signed-off-by: DL6ER <dl6er@dl6er.de>
---
advanced/Scripts/query.sh | 70 +++++++++++++++++++++++++++------------
1 file changed, 49 insertions(+), 21 deletions(-)
diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index a4ac895b..9b7a0fab 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -102,29 +102,57 @@ if [[ -n "${str:-}" ]]; then
exit 1
fi
-# Scan Whitelist and Blacklist
-lists="whitelist.txt blacklist.txt"
-mapfile -t results <<< "$(scanList "${domainQuery}" "${lists}" "${exact}")"
-if [[ -n "${results[*]}" ]]; then
- wbMatch=true
- # Loop through each result in order to print unique file title once
- for result in "${results[@]}"; do
- fileName="${result%%.*}"
- if [[ -n "${blockpage}" ]]; then
- echo "π ${result}"
- exit 0
- elif [[ -n "${exact}" ]]; then
- echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
- else
- # Only print filename title once per file
- if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+scanDatabaseTable() {
+ local domain table type querystr result
+ domain="${1}"
+ table="${2}"
+ type="${3:-}"
+
+ # As underscores are legitimate parts of domains, we escape possible them when using the LIKE operator.
+ # Underscores are a SQLite wildcard matching exactly one character. We obviously want to suppress this
+ # behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
+ # as a literal underscore character.
+ case "${type}" in
+ "exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
+ * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";;
+ esac
+
+ # Send prepared query to gravity database
+ result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
+ if [[ -n "${result}" ]]; then
+ # Prepend listname (separated by a colon) if we found at least one result
+ # and output result
+ results="$(sed "s/^/${table}:/g;" <<< "${result}")"
+ else
+ # Output empty string as the database query didn't return any result
+ return
+ fi
+ mapfile -t results <<< "${results}"
+ if [[ -n "${results[*]}" ]]; then
+ wbMatch=true
+ # Loop through each result in order to print unique file title once
+ for result in "${results[@]}"; do
+ fileName="${result%%:*}"
+ if [[ -n "${blockpage}" ]]; then
+ echo "π ${result}"
+ exit 0
+ elif [[ -n "${exact}" ]]; then
echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
- fileName_prev="${fileName}"
+ else
+ # Only print filename title once per file
+ if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
+ echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
+ fileName_prev="${fileName}"
+ fi
+ echo " ${result#*:}"
fi
- echo " ${result#*:}"
- fi
- done
-fi
+ done
+ fi
+}
+
+# Scan Whitelist and Blacklist
+scanDatabaseTable "${domainQuery}" "whitelist" "${exact}"
+scanDatabaseTable "${domainQuery}" "blacklist" "${exact}"
# Scan Wildcards
if [[ -e "${wildcardlist}" ]]; then
From 6ba58896d23c5ac7bf497413192a1fdd225e03b1 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 13:15:30 +0200
Subject: [PATCH 2/7] Simplify code
Signed-off-by: DL6ER <dl6er@dl6er.de>
---
advanced/Scripts/query.sh | 48 +++++++++++++++++----------------------
1 file changed, 21 insertions(+), 27 deletions(-)
diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 9b7a0fab..66b55e87 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -103,7 +103,7 @@ if [[ -n "${str:-}" ]]; then
fi
scanDatabaseTable() {
- local domain table type querystr result
+ local domain table type querystr result table_prev
domain="${1}"
table="${2}"
type="${3:-}"
@@ -119,35 +119,29 @@ scanDatabaseTable() {
# Send prepared query to gravity database
result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
- if [[ -n "${result}" ]]; then
- # Prepend listname (separated by a colon) if we found at least one result
- # and output result
- results="$(sed "s/^/${table}:/g;" <<< "${result}")"
- else
- # Output empty string as the database query didn't return any result
+ if [[ -z "${result}" ]]; then
+ # Return early when we have no results
return
fi
- mapfile -t results <<< "${results}"
- if [[ -n "${results[*]}" ]]; then
- wbMatch=true
- # Loop through each result in order to print unique file title once
- for result in "${results[@]}"; do
- fileName="${result%%:*}"
- if [[ -n "${blockpage}" ]]; then
- echo "π ${result}"
- exit 0
- elif [[ -n "${exact}" ]]; then
- echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
- else
- # Only print filename title once per file
- if [[ ! "${fileName}" == "${fileName_prev:-}" ]]; then
- echo " ${matchType^} found in ${COL_BOLD}${fileName^}${COL_NC}"
- fileName_prev="${fileName}"
- fi
- echo " ${result#*:}"
+
+ wbMatch=true
+ mapfile -t results <<< "${result}"
+ # Loop through each result
+ for result in "${results[@]}"; do
+ if [[ -n "${blockpage}" ]]; then
+ echo "π ${result}"
+ exit 0
+ elif [[ -n "${exact}" ]]; then
+ echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
+ else
+ # Only print table name once
+ if [[ ! "${table}" == "${table_prev:-}" ]]; then
+ echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
+ table_prev="${table}"
fi
- done
- fi
+ echo " ${result}"
+ fi
+ done
}
# Scan Whitelist and Blacklist
From f80fdd7e83b4bb23edb1671316f986b9ed791b68 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 13:19:50 +0200
Subject: [PATCH 3/7] Improve comments
Signed-off-by: DL6ER <dl6er@dl6er.de>
---
advanced/Scripts/query.sh | 11 ++++++-----
1 file changed, 6 insertions(+), 5 deletions(-)
diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 66b55e87..a7cc9dfb 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -108,10 +108,10 @@ scanDatabaseTable() {
table="${2}"
type="${3:-}"
- # As underscores are legitimate parts of domains, we escape possible them when using the LIKE operator.
- # Underscores are a SQLite wildcard matching exactly one character. We obviously want to suppress this
+ # As underscores are legitimate parts of domains, we escape them when using the LIKE operator.
+ # Underscores are SQLite wildcards matching exactly one character. We obviously want to suppress this
# behavior. The "ESCAPE '\'" clause specifies that an underscore preceded by an '\' should be matched
- # as a literal underscore character.
+ # as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
case "${type}" in
"exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
* ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";;
@@ -120,13 +120,14 @@ scanDatabaseTable() {
# Send prepared query to gravity database
result="$(sqlite3 "${gravityDBfile}" "${querystr}")" 2> /dev/null
if [[ -z "${result}" ]]; then
- # Return early when we have no results
+ # Return early when there are no matches in this table
return
fi
+ # Mark domain as having been white-/blacklist matched (global variable)
wbMatch=true
- mapfile -t results <<< "${result}"
# Loop through each result
+ mapfile -t results <<< "${result}"
for result in "${results[@]}"; do
if [[ -n "${blockpage}" ]]; then
echo "π ${result}"
From 5246b3e49672edb7e397418160ca2c327cef0ccc Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 13:24:36 +0200
Subject: [PATCH 4/7] Explicitly escape backslash in ESCAPE clause. This has
been suggested by Stickler bot.
Signed-off-by: DL6ER <dl6er@dl6er.de>
---
advanced/Scripts/query.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index a7cc9dfb..d4de380c 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -114,7 +114,7 @@ scanDatabaseTable() {
# as a literal underscore character. We pretreat the $domain variable accordingly to escape underscores.
case "${type}" in
"exact" ) querystr="SELECT domain FROM vw_${table} WHERE domain = '${domain}'";;
- * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\'";;
+ * ) querystr="SELECT domain FROM vw_${table} WHERE domain LIKE '%${domain//_/\\_}%' ESCAPE '\\'";;
esac
# Send prepared query to gravity database
From a904c183dfc2de7a503466f5ff39d1af3a4d1aca Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sat, 4 May 2019 18:25:11 +0200
Subject: [PATCH 5/7] Use printf to escape domain content. This prevents
possible SQL injection issues
Signed-off-by: DL6ER <dl6er@dl6er.de>
---
advanced/Scripts/query.sh | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index d4de380c..d2e16e79 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -104,7 +104,7 @@ fi
scanDatabaseTable() {
local domain table type querystr result table_prev
- domain="${1}"
+ domain="$(printf "%q" "${1}")"
table="${2}"
type="${3:-}"
From 5060605626950007aa0fc35153356ea49c96a187 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 30 May 2019 21:44:47 +0200
Subject: [PATCH 6/7] Print table name before entering the loop for the sake of
simplicity
Signed-off-by: DL6ER <dl6er@dl6er.de>
---
advanced/Scripts/query.sh | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index d2e16e79..88ec2173 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -103,7 +103,7 @@ if [[ -n "${str:-}" ]]; then
fi
scanDatabaseTable() {
- local domain table type querystr result table_prev
+ local domain table type querystr result
domain="$(printf "%q" "${1}")"
table="${2}"
type="${3:-}"
@@ -126,22 +126,20 @@ scanDatabaseTable() {
# Mark domain as having been white-/blacklist matched (global variable)
wbMatch=true
- # Loop through each result
+
+ # Print table name
+ if [[ ! -z "${result}" ]]; then
+ echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
+ fi
+
+ # Loop over results and print them
mapfile -t results <<< "${result}"
for result in "${results[@]}"; do
if [[ -n "${blockpage}" ]]; then
echo "π ${result}"
exit 0
- elif [[ -n "${exact}" ]]; then
- echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
- else
- # Only print table name once
- if [[ ! "${table}" == "${table_prev:-}" ]]; then
- echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
- table_prev="${table}"
- fi
- echo " ${result}"
fi
+ echo " ${result}"
done
}
From 44f8fcb600cc78f7b4dd8c6c9c8bba73c64c718c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 31 May 2019 08:39:18 +0200
Subject: [PATCH 7/7] We can print the table name without any extra check as we
already returned early in case there are no results.
Signed-off-by: DL6ER <dl6er@dl6er.de>
---
advanced/Scripts/query.sh | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 88ec2173..9cb298df 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -128,9 +128,7 @@ scanDatabaseTable() {
wbMatch=true
# Print table name
- if [[ ! -z "${result}" ]]; then
- echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
- fi
+ echo " ${matchType^} found in ${COL_BOLD}${table^}${COL_NC}"
# Loop over results and print them
mapfile -t results <<< "${result}"