From 09c4c88a6d1794a3ddb3ee8acb96e2cfb03ad7c2 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 13:04:31 +0100
Subject: [PATCH 01/66] Create and fill gravity.db during pihole -g

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.schema | 13 +++++++
 gravity.sh                           | 56 ++++++++++++++++++++++++++--
 2 files changed, 65 insertions(+), 4 deletions(-)
 create mode 100644 advanced/Templates/gravity.db.schema

diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema
new file mode 100644
index 00000000..1eee8491
--- /dev/null
+++ b/advanced/Templates/gravity.db.schema
@@ -0,0 +1,13 @@
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME);
+
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME);
+
+CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
+
+CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
+FROM gravity a
+WHERE a.domain NOT IN (SELECT domain from whitelist);
+
+CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
+FROM blacklist a
+WHERE a.domain NOT IN (SELECT domain from whitelist);
diff --git a/gravity.sh b/gravity.sh
index 75a51608..ac045a93 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -35,6 +35,10 @@ blackList="${piholeDir}/black.list"
 localList="${piholeDir}/local.list"
 VPNList="/etc/openvpn/ipp.txt"
 
+piholeGitDir="/etc/.pihole"
+gravityDBfile="${piholeDir}/gravity.db"
+gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema"
+
 domainsExtension="domains"
 matterAndLight="${basename}.0.matterandlight.txt"
 parsedMatter="${basename}.1.parsedmatter.txt"
@@ -83,6 +87,11 @@ if [[ -r "${piholeDir}/pihole.conf" ]]; then
   echo -e "  ${COL_LIGHT_RED}Ignoring overrides specified within pihole.conf! ${COL_NC}"
 fi
 
+# Generate new sqlite3 file from schema template
+generate_gravity_database() {
+  sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
+}
+
 # Determine if Pi-hole blocking is disabled
 # If this is the case, we want to update
 #  gravity.list.bck and black.list.bck instead of
@@ -582,12 +591,38 @@ gravity_ParseBlacklistDomains() {
     cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}"
   fi
 
-  # Move the file over as /etc/pihole/gravity.list so dnsmasq can use it
-  output=$( { mv "${piholeDir}/${accretionDisc}" "${adList}"; } 2>&1 )
+  # Create database file if not present
+  if [ ! -e "${gravityDBfile}" ]; then
+    generate_gravity_database
+  fi
+
+  # Backup gravity database
+  cp "${gravityDBfile}" "${gravityDBfile}.bck"
+
+  # Empty domains
+  output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM gravity;"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to move ${accretionDisc} from ${piholeDir}\\n  ${output}"
+    echo -e "\\n  ${CROSS} Unable to truncate gravity database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+
+  # Store domains in gravity database
+  output=$( { sqlite3 "${gravityDBfile}" <<< ".import ${piholeDir}/${accretionDisc} gravity"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to create gravity database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+
+  # Empty $adList if it already exists, otherwise, create it
+  output=$( { : > "${adList}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to create empty ${adList}\\n  ${output}"
     gravity_Cleanup "error"
   fi
 }
@@ -633,6 +668,19 @@ gravity_Cleanup() {
 
   echo -e "${OVER}  ${TICK} ${str}"
 
+  str="Optimizing domains database"
+  echo -ne "  ${INFO} ${str}..."
+  # Store
+  output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  else
+    echo -e "${OVER}  ${TICK} ${str}"
+  fi
+
   # Only restart DNS service if offline
   if ! pidof ${resolver} &> /dev/null; then
     "${PIHOLE_COMMAND}" restartdns
@@ -707,7 +755,7 @@ gravity_ShowBlockCount
 
 # Perform when downloading blocklists, or modifying the white/blacklist (not wildcards)
 if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then
-  str="Parsing domains into hosts format"
+  str="Parsing domains"
   echo -ne "  ${INFO} ${str}..."
 
   gravity_ParseUserDomains

From 8a91fe699276cdb1b1671d61f719653c197794b9 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 13:21:26 +0100
Subject: [PATCH 02/66] Store domains in gravit tables: gravity, whitelist,
 blacklist

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 104 ++++++++++++++++++++++++++---------------------------
 1 file changed, 52 insertions(+), 52 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index ac045a93..a8c92e8a 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -33,6 +33,7 @@ regexFile="${piholeDir}/regex.list"
 adList="${piholeDir}/gravity.list"
 blackList="${piholeDir}/black.list"
 localList="${piholeDir}/local.list"
+whiteList="${piholeDir}/white.list"
 VPNList="/etc/openvpn/ipp.txt"
 
 piholeGitDir="/etc/.pihole"
@@ -42,8 +43,6 @@ gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema"
 domainsExtension="domains"
 matterAndLight="${basename}.0.matterandlight.txt"
 parsedMatter="${basename}.1.parsedmatter.txt"
-whitelistMatter="${basename}.2.whitelistmatter.txt"
-accretionDisc="${basename}.3.accretionDisc.txt"
 preEventHorizon="list.preEventHorizon"
 
 skipDownload="false"
@@ -92,6 +91,49 @@ generate_gravity_database() {
   sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
 }
 
+# Import domains from file and store them in the specified database table
+gravity_store_in_database() {
+  # Define locals
+  local table="${1}"
+  local source="${2}"
+  local template="${3}"
+
+  # Create database file if not present
+  if [ ! -e "${gravityDBfile}" ]; then
+    generate_gravity_database
+  fi
+
+  # Backup gravity database
+  cp "${gravityDBfile}" "${gravityDBfile}.bck"
+
+  # Empty domains
+  output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to truncate ${table} database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+
+  # Store domains in gravity database
+  output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to create ${table} database ${gravityDBfile}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+
+  # Empty $adList if it already exists, otherwise, create it
+  output=$( { : > "${template}"; } 2>&1 )
+  status="$?"
+
+  if [[ "${status}" -ne 0 ]]; then
+    echo -e "\\n  ${CROSS} Unable to create empty ${template}\\n  ${output}"
+    gravity_Cleanup "error"
+  fi
+}
+
 # Determine if Pi-hole blocking is disabled
 # If this is the case, we want to update
 #  gravity.list.bck and black.list.bck instead of
@@ -476,7 +518,7 @@ gravity_SortAndFilterConsolidatedList() {
     echo -ne "  ${INFO} ${str}..."
   fi
 
-  # Parse into hosts file
+  # Parse into file
   gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
 
   # Format $parsedMatter line total as currency
@@ -514,8 +556,8 @@ gravity_Whitelist() {
   str="Number of whitelisted domains: ${num}"
   echo -ne "  ${INFO} ${str}..."
 
-  # Print everything from preEventHorizon into whitelistMatter EXCEPT domains in $whitelistFile
-  comm -23 "${piholeDir}/${preEventHorizon}" <(sort "${whitelistFile}") > "${piholeDir}/${whitelistMatter}"
+  # Store whitelisted files in gravity database
+  gravity_store_in_database "whitelist" "${whitelistFile}" "${whiteList}"
 
   echo -e "${OVER}  ${INFO} ${str}"
 }
@@ -581,50 +623,8 @@ gravity_ParseLocalDomains() {
 gravity_ParseBlacklistDomains() {
   local output status
 
-  # Empty $accretionDisc if it already exists, otherwise, create it
-  : > "${piholeDir}/${accretionDisc}"
-
-  if [[ -f "${piholeDir}/${whitelistMatter}" ]]; then
-    mv "${piholeDir}/${whitelistMatter}" "${piholeDir}/${accretionDisc}"
-  else
-    # There was no whitelist file, so use preEventHorizon instead of whitelistMatter.
-    cp "${piholeDir}/${preEventHorizon}" "${piholeDir}/${accretionDisc}"
-  fi
-
-  # Create database file if not present
-  if [ ! -e "${gravityDBfile}" ]; then
-    generate_gravity_database
-  fi
-
-  # Backup gravity database
-  cp "${gravityDBfile}" "${gravityDBfile}.bck"
-
-  # Empty domains
-  output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM gravity;"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to truncate gravity database ${gravityDBfile}\\n  ${output}"
-    gravity_Cleanup "error"
-  fi
-
-  # Store domains in gravity database
-  output=$( { sqlite3 "${gravityDBfile}" <<< ".import ${piholeDir}/${accretionDisc} gravity"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to create gravity database ${gravityDBfile}\\n  ${output}"
-    gravity_Cleanup "error"
-  fi
-
-  # Empty $adList if it already exists, otherwise, create it
-  output=$( { : > "${adList}"; } 2>&1 )
-  status="$?"
-
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to create empty ${adList}\\n  ${output}"
-    gravity_Cleanup "error"
-  fi
+  # Store gravity domains in gravity database
+  gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" "${adList}"
 }
 
 # Create user-added blacklist entries
@@ -632,9 +632,9 @@ gravity_ParseUserDomains() {
   if [[ ! -f "${blacklistFile}" ]]; then
     return 0
   fi
-  # Copy the file over as /etc/pihole/black.list so dnsmasq can use it
-  cp "${blacklistFile}" "${blackList}" 2> /dev/null || \
-    echo -e "\\n  ${CROSS} Unable to move ${blacklistFile##*/} to ${piholeDir}"
+
+  # Fill database table
+  gravity_store_in_database "blacklist" "${blacklistFile}" "${blackList}"
 }
 
 # Trap Ctrl-C

From 710036adae672486abf35be122a5d571d681d428 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 13:34:08 +0100
Subject: [PATCH 03/66] Add DISABLED field to white- and blacklist defaulting
 to false. The two views respect the DISABLED fields for both lists.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.schema | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema
index 1eee8491..2fd9a9d1 100644
--- a/advanced/Templates/gravity.db.schema
+++ b/advanced/Templates/gravity.db.schema
@@ -1,13 +1,14 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME);
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME);
 
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, DateAdded DATETIME);
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME);
 
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 
 CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
 FROM gravity a
-WHERE a.domain NOT IN (SELECT domain from whitelist);
+WHERE a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1);
 
 CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
 FROM blacklist a
-WHERE a.domain NOT IN (SELECT domain from whitelist);
+WHERE a.disabled != 1 AND
+      a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1);

From dcf0a605cf1b6e101b34206a3d5bd359f3c331a5 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 14:14:14 +0100
Subject: [PATCH 04/66] Use views for all tables and set disabled column to
 false for those tables that support it

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.schema | 10 ++++++----
 gravity.sh                           | 17 ++++++++++++++---
 2 files changed, 20 insertions(+), 7 deletions(-)

diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema
index 2fd9a9d1..83141f94 100644
--- a/advanced/Templates/gravity.db.schema
+++ b/advanced/Templates/gravity.db.schema
@@ -1,7 +1,5 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME);
-
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, comment TEXT, disabled BOOLEAN DEFAULT 0, DateAdded DATETIME);
-
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME);
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 
 CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
@@ -12,3 +10,7 @@ CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
 FROM blacklist a
 WHERE a.disabled != 1 AND
       a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1);
+
+CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
+FROM whitelist a
+WHERE a.disabled != 1;
diff --git a/gravity.sh b/gravity.sh
index a8c92e8a..21338119 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -115,16 +115,27 @@ gravity_store_in_database() {
     gravity_Cleanup "error"
   fi
 
-  # Store domains in gravity database
+  # Store domains in gravity database table ${table}
   output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to create ${table} database ${gravityDBfile}\\n  ${output}"
+    echo -e "\\n  ${CROSS} Unable to create ${table} in database ${gravityDBfile}\\n  ${output}"
     gravity_Cleanup "error"
   fi
 
-  # Empty $adList if it already exists, otherwise, create it
+  if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ]; then
+    # Set disabled to false
+    output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 )
+    status="$?"
+
+    if [[ "${status}" -ne 0 ]]; then
+      echo -e "\\n  ${CROSS} Unable to set disabled states (${table}) in database ${gravityDBfile}\\n  ${output}"
+      gravity_Cleanup "error"
+    fi
+  fi
+
+  # Empty $template if it already exists, otherwise, create it
   output=$( { : > "${template}"; } 2>&1 )
   status="$?"
 

From 93f1859babd7a28de799edb84fdeadf39300daba Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 15:13:18 +0100
Subject: [PATCH 05/66] Store regex in gravity.db as well

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.schema |  5 +++++
 gravity.sh                           | 20 +++++++++++++-------
 2 files changed, 18 insertions(+), 7 deletions(-)

diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema
index 83141f94..b7b3808a 100644
--- a/advanced/Templates/gravity.db.schema
+++ b/advanced/Templates/gravity.db.schema
@@ -1,5 +1,6 @@
 CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME);
 CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME);
+CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 
 CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
@@ -14,3 +15,7 @@ WHERE a.disabled != 1 AND
 CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
 FROM whitelist a
 WHERE a.disabled != 1;
+
+CREATE VIEW vw_regex AS SELECT DISTINCT a.domain
+FROM regex a
+WHERE a.disabled != 1;
diff --git a/gravity.sh b/gravity.sh
index 21338119..e3e78528 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -124,7 +124,7 @@ gravity_store_in_database() {
     gravity_Cleanup "error"
   fi
 
-  if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ]; then
+  if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then
     # Set disabled to false
     output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 )
     status="$?"
@@ -135,13 +135,16 @@ gravity_store_in_database() {
     fi
   fi
 
-  # Empty $template if it already exists, otherwise, create it
-  output=$( { : > "${template}"; } 2>&1 )
-  status="$?"
+  # Only create template file if asked to
+  if [ "${template}" != "-" ]; then
+    # Empty $template if it already exists, otherwise, create it
+    output=$( { : > "${template}"; } 2>&1 )
+    status="$?"
 
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to create empty ${template}\\n  ${output}"
-    gravity_Cleanup "error"
+    if [[ "${status}" -ne 0 ]]; then
+      echo -e "\\n  ${CROSS} Unable to create empty ${template}\\n  ${output}"
+      gravity_Cleanup "error"
+    fi
   fi
 }
 
@@ -586,6 +589,9 @@ gravity_ShowBlockCount() {
     num=$(grep -cv "^#" "${regexFile}")
     echo -e "  ${INFO} Number of regex filters: ${num}"
   fi
+
+  # Store regex files in gravity database
+  gravity_store_in_database "regex" "${regexFile}" "-"
 }
 
 # Parse list of domains into hosts format

From fc62cf7e2f043cc28d163386aaf7cd0134dce306 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 16:44:05 +0100
Subject: [PATCH 06/66] Only run VACUUM on "full" gravity runs

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 29 ++++++++++++++++-------------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index e3e78528..0a1d4e8d 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -39,6 +39,7 @@ VPNList="/etc/openvpn/ipp.txt"
 piholeGitDir="/etc/.pihole"
 gravityDBfile="${piholeDir}/gravity.db"
 gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema"
+optimize_database=true
 
 domainsExtension="domains"
 matterAndLight="${basename}.0.matterandlight.txt"
@@ -685,17 +686,19 @@ gravity_Cleanup() {
 
   echo -e "${OVER}  ${TICK} ${str}"
 
-  str="Optimizing domains database"
-  echo -ne "  ${INFO} ${str}..."
-  # Store
-  output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 )
-  status="$?"
+  if ${optimize_database} ; then
+    str="Optimizing domains database"
+    echo -ne "  ${INFO} ${str}..."
+    # Store
+    output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 )
+    status="$?"
 
-  if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n  ${output}"
-    gravity_Cleanup "error"
-  else
-    echo -e "${OVER}  ${TICK} ${str}"
+    if [[ "${status}" -ne 0 ]]; then
+      echo -e "\\n  ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n  ${output}"
+      gravity_Cleanup "error"
+    else
+      echo -e "${OVER}  ${TICK} ${str}"
+    fi
   fi
 
   # Only restart DNS service if offline
@@ -726,9 +729,9 @@ for var in "$@"; do
     "-f" | "--force" ) forceDelete=true;;
     "-h" | "--help" ) helpFunc;;
     "-sd" | "--skip-download" ) skipDownload=true;;
-    "-b" | "--blacklist-only" ) listType="blacklist";;
-    "-w" | "--whitelist-only" ) listType="whitelist";;
-    "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
+    "-b" | "--blacklist-only" ) listType="blacklist"; optimize_database=false;;
+    "-w" | "--whitelist-only" ) listType="whitelist"; optimize_database=false;;
+    "-wild" | "--wildcard-only" ) listType="wildcard"; optimize_database=false; dnsRestartType="restart";;
   esac
 done
 

From d5feffa117c2093e8dbd17f814bf95a7167ab2ba Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 16:45:19 +0100
Subject: [PATCH 07/66] Do not backup the database

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 0a1d4e8d..0d312b3f 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -104,9 +104,6 @@ gravity_store_in_database() {
     generate_gravity_database
   fi
 
-  # Backup gravity database
-  cp "${gravityDBfile}" "${gravityDBfile}.bck"
-
   # Empty domains
   output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
   status="$?"

From 1f9a7d0b0cdba219cc2e3ad7b5eb91e744ab915b Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 17:01:38 +0100
Subject: [PATCH 08/66] Only run VACUUM on "pihole -g -o"

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 0d312b3f..d0b44c24 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -39,7 +39,7 @@ VPNList="/etc/openvpn/ipp.txt"
 piholeGitDir="/etc/.pihole"
 gravityDBfile="${piholeDir}/gravity.db"
 gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema"
-optimize_database=true
+optimize_database=false
 
 domainsExtension="domains"
 matterAndLight="${basename}.0.matterandlight.txt"
@@ -724,11 +724,12 @@ Options:
 for var in "$@"; do
   case "${var}" in
     "-f" | "--force" ) forceDelete=true;;
+    "-o" | "--optimize" ) optimize_database=true;;
     "-h" | "--help" ) helpFunc;;
     "-sd" | "--skip-download" ) skipDownload=true;;
-    "-b" | "--blacklist-only" ) listType="blacklist"; optimize_database=false;;
-    "-w" | "--whitelist-only" ) listType="whitelist"; optimize_database=false;;
-    "-wild" | "--wildcard-only" ) listType="wildcard"; optimize_database=false; dnsRestartType="restart";;
+    "-b" | "--blacklist-only" ) listType="blacklist";;
+    "-w" | "--whitelist-only" ) listType="whitelist";;
+    "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
   esac
 done
 

From b2f4385232f7293d16e2a24eee6b60e3e7011308 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 3 Feb 2019 17:05:00 +0100
Subject: [PATCH 09/66] Avoid infinity loop if gravity optimization fails

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index d0b44c24..06e75a90 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -692,7 +692,7 @@ gravity_Cleanup() {
 
     if [[ "${status}" -ne 0 ]]; then
       echo -e "\\n  ${CROSS} Unable to optimize gravity database ${gravityDBfile}\\n  ${output}"
-      gravity_Cleanup "error"
+      error="error"
     else
       echo -e "${OVER}  ${TICK} ${str}"
     fi

From dbbf21071b4d3a13522d63bdb873ead883f2e13f Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 5 Feb 2019 19:05:11 +0100
Subject: [PATCH 10/66] Invert logic to use ENABLED instead of DISABLED. We
 make sure to set "enabled = 1 WHERE enabled IS NULL" in
 gravity_store_in_database()

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.schema | 21 ---------------------
 advanced/Templates/gravity.db.sql    | 21 +++++++++++++++++++++
 gravity.sh                           |  8 ++++----
 3 files changed, 25 insertions(+), 25 deletions(-)
 delete mode 100644 advanced/Templates/gravity.db.schema
 create mode 100644 advanced/Templates/gravity.db.sql

diff --git a/advanced/Templates/gravity.db.schema b/advanced/Templates/gravity.db.schema
deleted file mode 100644
index b7b3808a..00000000
--- a/advanced/Templates/gravity.db.schema
+++ /dev/null
@@ -1,21 +0,0 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME);
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME);
-CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, disabled BOOLEAN DEFAULT 0, comment TEXT, DateAdded DATETIME);
-CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
-
-CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
-FROM gravity a
-WHERE a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1);
-
-CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
-FROM blacklist a
-WHERE a.disabled != 1 AND
-      a.domain NOT IN (SELECT domain from whitelist WHERE disabled != 1);
-
-CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
-FROM whitelist a
-WHERE a.disabled != 1;
-
-CREATE VIEW vw_regex AS SELECT DISTINCT a.domain
-FROM regex a
-WHERE a.disabled != 1;
diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
new file mode 100644
index 00000000..72d64bc1
--- /dev/null
+++ b/advanced/Templates/gravity.db.sql
@@ -0,0 +1,21 @@
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME);
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME);
+CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME);
+CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
+
+CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
+FROM gravity a
+WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
+
+CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
+FROM blacklist a
+WHERE a.enabled == 1 AND
+      a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
+
+CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
+FROM whitelist a
+WHERE a.enabled == 1;
+
+CREATE VIEW vw_regex AS SELECT DISTINCT a.domain
+FROM regex a
+WHERE a.enabled == 1;
diff --git a/gravity.sh b/gravity.sh
index 06e75a90..c47f7487 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -38,7 +38,7 @@ VPNList="/etc/openvpn/ipp.txt"
 
 piholeGitDir="/etc/.pihole"
 gravityDBfile="${piholeDir}/gravity.db"
-gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.schema"
+gravityDBschema="${piholeGitDir}/advanced/Templates/gravity.db.sql"
 optimize_database=false
 
 domainsExtension="domains"
@@ -123,12 +123,12 @@ gravity_store_in_database() {
   fi
 
   if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then
-    # Set disabled to false
-    output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET disabled = 0 WHERE disabled IS NULL;"; } 2>&1 )
+    # Set enabled to true where it is unspecified
+    output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET enabled = 1 WHERE enabled IS NULL;"; } 2>&1 )
     status="$?"
 
     if [[ "${status}" -ne 0 ]]; then
-      echo -e "\\n  ${CROSS} Unable to set disabled states (${table}) in database ${gravityDBfile}\\n  ${output}"
+      echo -e "\\n  ${CROSS} Unable to set enabled states (${table}) in database ${gravityDBfile}\\n  ${output}"
       gravity_Cleanup "error"
     fi
   fi

From 8a2363621d084bff58ab160cc01633ae32aa4f19 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 5 Feb 2019 19:06:46 +0100
Subject: [PATCH 11/66] Rename fields from DateAdded to dateadded

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 72d64bc1..4470c21c 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,6 +1,6 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME);
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME);
-CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, DateAdded DATETIME);
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
+CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 
 CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain

From c3c60e10f18db58e6a3cb09beb573a4eef26f8c8 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 5 Feb 2019 19:07:12 +0100
Subject: [PATCH 12/66] Rename regex field from domain to filter because this
 described better what it is

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 4470c21c..5290e83b 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,6 +1,6 @@
 CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
 CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
-CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
+CREATE TABLE regex     (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 
 CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain

From ec5b16ef851e25167997fe02fd23944be65c7b4d Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 5 Feb 2019 19:08:08 +0100
Subject: [PATCH 13/66] Fix incomplete comment

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index c47f7487..c45e124e 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -686,7 +686,7 @@ gravity_Cleanup() {
   if ${optimize_database} ; then
     str="Optimizing domains database"
     echo -ne "  ${INFO} ${str}..."
-    # Store
+    # Run VACUUM command on database to optimize it
     output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 )
     status="$?"
 

From 9984647ebbc053f21881c8e4473a4ac5220050cf Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Tue, 5 Feb 2019 19:41:11 +0100
Subject: [PATCH 14/66] After renaming column in regex, it also needs to be
 renamed in vw_regex

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 5290e83b..23efa64a 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -16,6 +16,6 @@ CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
 FROM whitelist a
 WHERE a.enabled == 1;
 
-CREATE VIEW vw_regex AS SELECT DISTINCT a.domain
+CREATE VIEW vw_regex AS SELECT DISTINCT a.filter
 FROM regex a
 WHERE a.enabled == 1;

From 644ec36e6462a9b46f2e63fc5185040e3a803ebd Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 6 Feb 2019 18:57:48 +0100
Subject: [PATCH 15/66] Review comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql |  9 +++++---
 gravity.sh                        | 34 ++++++++++++++++++++-----------
 2 files changed, 28 insertions(+), 15 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 23efa64a..3a2230c8 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,7 +1,10 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
-CREATE TABLE regex     (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN DEFAULT 1, comment TEXT, dateadded DATETIME);
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
+CREATE TABLE regex     (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
+CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
+
+INSERT INTO info VALUES("version","1");
 
 CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
 FROM gravity a
diff --git a/gravity.sh b/gravity.sh
index c45e124e..0c3c8dee 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -20,6 +20,8 @@ source "${regexconverter}"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
+PIHOLE_USER="pihole"
+PIHOLE_GROUP="pihole"
 
 piholeDir="/etc/${basename}"
 
@@ -90,6 +92,7 @@ fi
 # Generate new sqlite3 file from schema template
 generate_gravity_database() {
   sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
+  chown $PIHOLE_USER:$PIHOLE_GROUP "${gravityDBfile}"
 }
 
 # Import domains from file and store them in the specified database table
@@ -113,25 +116,32 @@ gravity_store_in_database() {
     gravity_Cleanup "error"
   fi
 
+  local tmpFile=$(mktemp -p "/tmp" --suffix=".gravity")
+  if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then
+    # Apply format for white-, blacklist, and regex tables, prevent globbing
+    set -f
+    for domain in $(cat < "${source}"); do
+      echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}"
+    done
+    set +f
+    inputfile="${tmpFile}"
+  else
+    # No need to modify the input data for the gravity table
+    inputfile="${source}"
+  fi
   # Store domains in gravity database table ${table}
-  output=$( { sqlite3 "${gravityDBfile}" <<< ".import \"${source}\" ${table}"; } 2>&1 )
+  # Use printf as .mode and .import need to be on separate lines
+  # see https://unix.stackexchange.com/a/445615/83260
+  output=$( { printf ".mode csv\n.import \"${inputfile}\" ${table}\n" | sqlite3 "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
-    echo -e "\\n  ${CROSS} Unable to create ${table} in database ${gravityDBfile}\\n  ${output}"
+    echo -e "\\n  ${CROSS} Unable to fill table ${table} in database ${gravityDBfile}\\n  ${output}"
     gravity_Cleanup "error"
   fi
 
-  if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then
-    # Set enabled to true where it is unspecified
-    output=$( { sqlite3 "${gravityDBfile}" <<< "UPDATE ${table} SET enabled = 1 WHERE enabled IS NULL;"; } 2>&1 )
-    status="$?"
-
-    if [[ "${status}" -ne 0 ]]; then
-      echo -e "\\n  ${CROSS} Unable to set enabled states (${table}) in database ${gravityDBfile}\\n  ${output}"
-      gravity_Cleanup "error"
-    fi
-  fi
+  # Delete tmpfile
+  #rm "$tmpFile" > /dev/null 2>&1 || true
 
   # Only create template file if asked to
   if [ "${template}" != "-" ]; then

From 491c828661ba203e679acc74df48a22970f2aee5 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 6 Feb 2019 19:09:09 +0100
Subject: [PATCH 16/66] Improved code, ensure that gravity.sh passes shellcheck
 as a whole

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 0c3c8dee..129d0ed4 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -116,14 +116,15 @@ gravity_store_in_database() {
     gravity_Cleanup "error"
   fi
 
-  local tmpFile=$(mktemp -p "/tmp" --suffix=".gravity")
+  local tmpFile
+  tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")"
   if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then
-    # Apply format for white-, blacklist, and regex tables, prevent globbing
-    set -f
-    for domain in $(cat < "${source}"); do
+    # Apply format for white-, blacklist, and regex tables
+    # Read file line by line
+    grep -v '^ *#' < "${source}" | while IFS= read -r domain
+    do
       echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}"
     done
-    set +f
     inputfile="${tmpFile}"
   else
     # No need to modify the input data for the gravity table
@@ -132,7 +133,7 @@ gravity_store_in_database() {
   # Store domains in gravity database table ${table}
   # Use printf as .mode and .import need to be on separate lines
   # see https://unix.stackexchange.com/a/445615/83260
-  output=$( { printf ".mode csv\n.import \"${inputfile}\" ${table}\n" | sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".mode csv\n.import \"%s\" ${table}\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -412,10 +413,10 @@ gravity_ParseFileIntoDomains() {
     # Last awk command takes non-commented lines and if they have 2 fields, take the right field (the domain) and leave
     # the left (IP address), otherwise grab the single field.
 
-    < ${source} awk -F '#' '{print $1}' | \
+    < "${source}" awk -F '#' '{print $1}' | \
     awk -F '/' '{print $1}' | \
     awk '($1 !~ /^#/) { if (NF>1) {print $2} else {print $1}}' | \
-    sed -nr -e 's/\.{2,}/./g' -e '/\./p' >  ${destination}
+    sed -nr -e 's/\.{2,}/./g' -e '/\./p' > "${destination}"
     return 0
   fi
 

From f9250d91a52a0bf2c62223ed51b96acf4a80b692 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 6 Feb 2019 19:13:31 +0100
Subject: [PATCH 17/66] Explictly escape the escape.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index 129d0ed4..c742a699 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -133,7 +133,7 @@ gravity_store_in_database() {
   # Store domains in gravity database table ${table}
   # Use printf as .mode and .import need to be on separate lines
   # see https://unix.stackexchange.com/a/445615/83260
-  output=$( { printf ".mode csv\n.import \"%s\" ${table}\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".mode csv\\n.import \"%s\" ${table}\\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then

From 61840edb301005178101ed821da01b05a22e1c0d Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 22 Feb 2019 22:46:19 +0100
Subject: [PATCH 18/66] Fill comment field with NULL (default) instead of empty
 string and use a variable for the timestamp instead of periodically calling
 date

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index c742a699..6ace8108 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -118,12 +118,14 @@ gravity_store_in_database() {
 
   local tmpFile
   tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")"
+  local timestamp
+  timestamp="$(date --utc +'%s')"
   if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then
     # Apply format for white-, blacklist, and regex tables
     # Read file line by line
     grep -v '^ *#' < "${source}" | while IFS= read -r domain
     do
-      echo "\"${domain}\",1,$(date --utc +'%s')," >> "${tmpFile}"
+      echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}"
     done
     inputfile="${tmpFile}"
   else

From bbc392caeb631e46f840d1e87d6bebe07442545a Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 22 Feb 2019 22:49:02 +0100
Subject: [PATCH 19/66] More review comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 6ace8108..b09ddc0c 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -120,6 +120,7 @@ gravity_store_in_database() {
   tmpFile="$(mktemp -p "/tmp" --suffix=".gravity")"
   local timestamp
   timestamp="$(date --utc +'%s')"
+  local inputfile
   if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then
     # Apply format for white-, blacklist, and regex tables
     # Read file line by line
@@ -135,7 +136,7 @@ gravity_store_in_database() {
   # Store domains in gravity database table ${table}
   # Use printf as .mode and .import need to be on separate lines
   # see https://unix.stackexchange.com/a/445615/83260
-  output=$( { printf ".mode csv\\n.import \"%s\" ${table}\\n" "${inputfile}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
+  output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
   status="$?"
 
   if [[ "${status}" -ne 0 ]]; then
@@ -144,7 +145,7 @@ gravity_store_in_database() {
   fi
 
   # Delete tmpfile
-  #rm "$tmpFile" > /dev/null 2>&1 || true
+  rm "$tmpFile" > /dev/null 2>&1 || true
 
   # Only create template file if asked to
   if [ "${template}" != "-" ]; then

From de4456330169a3328c6e22ab4860a4f42a916c22 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 24 Apr 2019 18:51:38 +0200
Subject: [PATCH 20/66] Remove addn-hosts from 01-pihole.conf

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/01-pihole.conf | 2 --
 1 file changed, 2 deletions(-)

diff --git a/advanced/01-pihole.conf b/advanced/01-pihole.conf
index 502293bf..40a117fe 100644
--- a/advanced/01-pihole.conf
+++ b/advanced/01-pihole.conf
@@ -18,8 +18,6 @@
 #                    WITHIN /etc/dnsmasq.d/yourname.conf                      #
 ###############################################################################
 
-addn-hosts=/etc/pihole/gravity.list
-addn-hosts=/etc/pihole/black.list
 addn-hosts=/etc/pihole/local.list
 
 domain-needed

From 6594a0a6e75ed60c5b15f422a9b319f14e312330 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 24 Apr 2019 18:53:20 +0200
Subject: [PATCH 21/66] Fix detection of blocked query in pihole -t

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 pihole | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pihole b/pihole
index 84a5623f..6b572fa9 100755
--- a/pihole
+++ b/pihole
@@ -313,7 +313,7 @@ tailFunc() {
   # Colour everything else as gray
   tail -f /var/log/pihole.log | sed -E \
     -e "s,($(date +'%b %d ')| dnsmasq[.*[0-9]]),,g" \
-    -e "s,(.*(gravity.list|black.list|regex.list| config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
+    -e "s,(.*(gravity |black |regex | config ).* is (0.0.0.0|::|NXDOMAIN|${IPV4_ADDRESS%/*}|${IPV6_ADDRESS:-NULL}).*),${COL_RED}&${COL_NC}," \
     -e "s,.*(query\\[A|DHCP).*,${COL_NC}&${COL_NC}," \
     -e "s,.*,${COL_GRAY}&${COL_NC},"
   exit 0

From c13ebd3a9d9cab33e0d6e8d389b5e69e8115d14d Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 24 Apr 2019 18:54:55 +0200
Subject: [PATCH 22/66] Replace Pi-hole ID string with another line we intend
 to keep

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 automated install/basic-install.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh
index 110399f8..ffc1bdf6 100755
--- a/automated install/basic-install.sh	
+++ b/automated install/basic-install.sh	
@@ -1224,7 +1224,7 @@ version_check_dnsmasq() {
     # Local, named variables
     local dnsmasq_conf="/etc/dnsmasq.conf"
     local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig"
-    local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list"
+    local dnsmasq_pihole_id_string="# Dnsmasq config for Pi-hole's FTLDNS"
     local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original"
     local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf"
     local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf"
@@ -1232,7 +1232,7 @@ version_check_dnsmasq() {
     # If the dnsmasq config file exists
     if [[ -f "${dnsmasq_conf}" ]]; then
         printf "  %b Existing dnsmasq.conf found..." "${INFO}"
-        # If gravity.list is found within this file, we presume it's from older versions on Pi-hole,
+        # If a specific string is found within this file, we presume it's from older versions on Pi-hole,
         if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then
             printf " it is from a previous Pi-hole install.\\n"
             printf "  %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}"

From d7ee44960e25b3c3bb89c2e81578c1a8cff14087 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 24 Apr 2019 18:56:44 +0200
Subject: [PATCH 23/66] Remove moving around of the gravity and blacklist files
 as we do not use them any longer

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 pihole | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/pihole b/pihole
index 6b572fa9..71c286c6 100755
--- a/pihole
+++ b/pihole
@@ -10,10 +10,8 @@
 # Please see LICENSE file for your rights under this license.
 
 readonly PI_HOLE_SCRIPT_DIR="/opt/pihole"
-readonly gravitylist="/etc/pihole/gravity.list"
-readonly blacklist="/etc/pihole/black.list"
 
-# setupVars is not readonly here because in some funcitons (checkout),
+# setupVars is not readonly here because in some functions (checkout),
 # it might get set again when the installer is sourced. This causes an
 # error due to modifying a readonly variable.
 setupVars="/etc/pihole/setupVars.conf"
@@ -148,14 +146,6 @@ Time:
       echo -e "  ${INFO} Blocking already disabled, nothing to do"
       exit 0
     fi
-    if [[ -e "${gravitylist}" ]]; then
-      mv "${gravitylist}" "${gravitylist}.bck"
-      echo "" > "${gravitylist}"
-    fi
-    if [[ -e "${blacklist}" ]]; then
-      mv "${blacklist}" "${blacklist}.bck"
-      echo "" > "${blacklist}"
-    fi
     if [[ $# > 1 ]]; then
       local error=false
       if [[ "${2}" == *"s" ]]; then
@@ -204,12 +194,6 @@ Time:
     echo -e "  ${INFO} Enabling blocking"
     local str="Pi-hole Enabled"
 
-    if [[ -e "${gravitylist}.bck" ]]; then
-      mv "${gravitylist}.bck" "${gravitylist}"
-    fi
-    if [[ -e "${blacklist}.bck" ]]; then
-      mv "${blacklist}.bck" "${blacklist}"
-    fi
     sed -i "/BLOCKING_ENABLED=/d" "${setupVars}"
     echo "BLOCKING_ENABLED=true" >> "${setupVars}"
   fi

From ee20164207508ca1e2dbb08e05b0858e074dd370 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 24 Apr 2019 19:01:31 +0200
Subject: [PATCH 24/66] Remove detect_pihole_blocking_status() as we can modify
 the database at any time

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 18 +-----------------
 1 file changed, 1 insertion(+), 17 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index b09ddc0c..b287653d 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -160,20 +160,6 @@ gravity_store_in_database() {
   fi
 }
 
-# Determine if Pi-hole blocking is disabled
-# If this is the case, we want to update
-#  gravity.list.bck and black.list.bck instead of
-#  gravity.list and black.list
-detect_pihole_blocking_status() {
-  if [[ "${BLOCKING_ENABLED}" == false ]]; then
-    echo -e "  ${INFO} Pi-hole blocking is disabled"
-    adList="${adList}.bck"
-    blackList="${blackList}.bck"
-  else
-    echo -e "  ${INFO} Pi-hole blocking is enabled"
-  fi
-}
-
 # Determine if DNS resolution is available before proceeding
 gravity_CheckDNSResolutionAvailable() {
   local lookupDomain="pi.hole"
@@ -235,7 +221,7 @@ gravity_GetBlocklistUrls() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
 
   if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
-    # Remove superceded $adListDefault file
+    # Remove superseded $adListDefault file
     rm "${adListDefault}" 2> /dev/null || \
       echo -e "  ${CROSS} Unable to remove ${adListDefault}"
   fi
@@ -758,8 +744,6 @@ if [[ "${forceDelete:-}" == true ]]; then
   echo -e "${OVER}  ${TICK} ${str}"
 fi
 
-detect_pihole_blocking_status
-
 # Determine which functions to run
 if [[ "${skipDownload}" == false ]]; then
   # Gravity needs to download blocklists

From e57d0fb93c24e6d7f336f449dc9bd1fc7627062d Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 24 Apr 2019 19:05:08 +0200
Subject: [PATCH 25/66] Remove the empty templates gravity.list, white.list,
 black.list. The old source files whitelist.txt, blacklist.txt, and regex.list
 still exist and need to be removed in a follow-up commit

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 23 ++++-------------------
 1 file changed, 4 insertions(+), 19 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index b287653d..6313b12c 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -32,10 +32,7 @@ whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
 regexFile="${piholeDir}/regex.list"
 
-adList="${piholeDir}/gravity.list"
-blackList="${piholeDir}/black.list"
 localList="${piholeDir}/local.list"
-whiteList="${piholeDir}/white.list"
 VPNList="/etc/openvpn/ipp.txt"
 
 piholeGitDir="/etc/.pihole"
@@ -146,18 +143,6 @@ gravity_store_in_database() {
 
   # Delete tmpfile
   rm "$tmpFile" > /dev/null 2>&1 || true
-
-  # Only create template file if asked to
-  if [ "${template}" != "-" ]; then
-    # Empty $template if it already exists, otherwise, create it
-    output=$( { : > "${template}"; } 2>&1 )
-    status="$?"
-
-    if [[ "${status}" -ne 0 ]]; then
-      echo -e "\\n  ${CROSS} Unable to create empty ${template}\\n  ${output}"
-      gravity_Cleanup "error"
-    fi
-  fi
 }
 
 # Determine if DNS resolution is available before proceeding
@@ -569,7 +554,7 @@ gravity_Whitelist() {
   echo -ne "  ${INFO} ${str}..."
 
   # Store whitelisted files in gravity database
-  gravity_store_in_database "whitelist" "${whitelistFile}" "${whiteList}"
+  gravity_store_in_database "whitelist" "${whitelistFile}"
 
   echo -e "${OVER}  ${INFO} ${str}"
 }
@@ -589,7 +574,7 @@ gravity_ShowBlockCount() {
   fi
 
   # Store regex files in gravity database
-  gravity_store_in_database "regex" "${regexFile}" "-"
+  gravity_store_in_database "regex" "${regexFile}"
 }
 
 # Parse list of domains into hosts format
@@ -639,7 +624,7 @@ gravity_ParseBlacklistDomains() {
   local output status
 
   # Store gravity domains in gravity database
-  gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}" "${adList}"
+  gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}"
 }
 
 # Create user-added blacklist entries
@@ -649,7 +634,7 @@ gravity_ParseUserDomains() {
   fi
 
   # Fill database table
-  gravity_store_in_database "blacklist" "${blacklistFile}" "${blackList}"
+  gravity_store_in_database "blacklist" "${blacklistFile}"
 }
 
 # Trap Ctrl-C

From 61cfd2f9f9ec4294e041937748f08368b80e88d5 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 24 Apr 2019 19:48:06 +0200
Subject: [PATCH 26/66] Add adlists table

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 1 +
 1 file changed, 1 insertion(+)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 3a2230c8..5442987f 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,6 +1,7 @@
 CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
 CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
 CREATE TABLE regex     (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
+CREATE TABLE adlists   (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
 

From c1277705d897ce5ca15aa4d76b1ba253b42f7e6a Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 24 Apr 2019 19:55:05 +0200
Subject: [PATCH 27/66] Copy existing whitelist.txt, blacklist.txt, regex.list,
 and adlists.list to the database. We remove the files afterwards as the
 content lives in the database now

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 62 ++++++++++++++++++++++++++++++++++++++----------------
 1 file changed, 44 insertions(+), 18 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 6313b12c..4fb601b4 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -25,12 +25,11 @@ PIHOLE_GROUP="pihole"
 
 piholeDir="/etc/${basename}"
 
-adListFile="${piholeDir}/adlists.list"
-adListDefault="${piholeDir}/adlists.default"
-
+# Legacy (pre v5.0) list file locations
 whitelistFile="${piholeDir}/whitelist.txt"
 blacklistFile="${piholeDir}/blacklist.txt"
 regexFile="${piholeDir}/regex.list"
+adListFile="${piholeDir}/adlists.list"
 
 localList="${piholeDir}/local.list"
 VPNList="/etc/openvpn/ipp.txt"
@@ -93,18 +92,20 @@ generate_gravity_database() {
 }
 
 # Import domains from file and store them in the specified database table
-gravity_store_in_database() {
+database_table_from_file() {
   # Define locals
   local table="${1}"
   local source="${2}"
-  local template="${3}"
 
   # Create database file if not present
   if [ ! -e "${gravityDBfile}" ]; then
+    echo -e "  ${INFO} Creating new gravity database"
     generate_gravity_database
   fi
 
-  # Empty domains
+  echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${source} into database"
+
+  # Truncate table
   output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
   status="$?"
 
@@ -118,19 +119,19 @@ gravity_store_in_database() {
   local timestamp
   timestamp="$(date --utc +'%s')"
   local inputfile
-  if [ "$table" == "whitelist" ] || [ "$table" == "blacklist" ] || [ "$table" == "regex" ]; then
-    # Apply format for white-, blacklist, and regex tables
+  if [[ "${table}" == "gravity" ]]; then
+    # No need to modify the input data for the gravity table
+    inputfile="${source}"
+  else
+    # Apply format for white-, blacklist, regex, and adlists tables
     # Read file line by line
     grep -v '^ *#' < "${source}" | while IFS= read -r domain
     do
       echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}"
     done
     inputfile="${tmpFile}"
-  else
-    # No need to modify the input data for the gravity table
-    inputfile="${source}"
   fi
-  # Store domains in gravity database table ${table}
+  # Store domains in database table specified by ${table}
   # Use printf as .mode and .import need to be on separate lines
   # see https://unix.stackexchange.com/a/445615/83260
   output=$( { printf ".mode csv\\n.import \"%s\" %s\\n" "${inputfile}" "${table}" | sqlite3 "${gravityDBfile}"; } 2>&1 )
@@ -145,6 +146,31 @@ gravity_store_in_database() {
   rm "$tmpFile" > /dev/null 2>&1 || true
 }
 
+migrate_to_database() {
+ls -lh
+  # Migrate pre-v5.0 list files to database-based Pi-hole versions
+  if [[ -e "${whitelistFile}" ]]; then
+    # Store whitelisted domains in database
+    database_table_from_file "whitelist" "${whitelistFile}"
+    rm "${whitelistFile}"
+  fi
+  if [[ -e "${blacklistFile}" ]]; then
+    # Store blacklisted domains in database
+    database_table_from_file "blacklist" "${blacklistFile}"
+    rm "${blacklistFile}"
+  fi
+  if [[ -e "${regexFile}" ]]; then
+    # Store regex domains in database
+    database_table_from_file "regex" "${regexFile}"
+    rm "${regexFile}"
+  fi
+  if [[ -e "${adListFile}" ]]; then
+    # Store adlists domains in database
+    database_table_from_file "adlists" "${adListFile}"
+    rm "${adListFile}"
+  fi
+}
+
 # Determine if DNS resolution is available before proceeding
 gravity_CheckDNSResolutionAvailable() {
   local lookupDomain="pi.hole"
@@ -553,9 +579,6 @@ gravity_Whitelist() {
   str="Number of whitelisted domains: ${num}"
   echo -ne "  ${INFO} ${str}..."
 
-  # Store whitelisted files in gravity database
-  gravity_store_in_database "whitelist" "${whitelistFile}"
-
   echo -e "${OVER}  ${INFO} ${str}"
 }
 
@@ -574,7 +597,7 @@ gravity_ShowBlockCount() {
   fi
 
   # Store regex files in gravity database
-  gravity_store_in_database "regex" "${regexFile}"
+  database_table_from_file "regex" "${regexFile}"
 }
 
 # Parse list of domains into hosts format
@@ -624,7 +647,7 @@ gravity_ParseBlacklistDomains() {
   local output status
 
   # Store gravity domains in gravity database
-  gravity_store_in_database "gravity" "${piholeDir}/${preEventHorizon}"
+  database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
 }
 
 # Create user-added blacklist entries
@@ -634,7 +657,7 @@ gravity_ParseUserDomains() {
   fi
 
   # Fill database table
-  gravity_store_in_database "blacklist" "${blacklistFile}"
+  database_table_from_file "blacklist" "${blacklistFile}"
 }
 
 # Trap Ctrl-C
@@ -721,6 +744,9 @@ done
 # Trap Ctrl-C
 gravity_Trap
 
+# Move possibly existing legacy files to the gravity database
+migrate_to_database
+
 if [[ "${forceDelete:-}" == true ]]; then
   str="Deleting existing list cache"
   echo -ne "${INFO} ${str}..."

From 2664ac0efc6dab8720170d55b09c2ea50d857f18 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 25 Apr 2019 10:18:37 +0200
Subject: [PATCH 28/66] Improve error reporting

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 4fb601b4..870027a1 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -143,7 +143,12 @@ database_table_from_file() {
   fi
 
   # Delete tmpfile
-  rm "$tmpFile" > /dev/null 2>&1 || true
+  rm "${tmpFile}" > /dev/null 2>&1 || \
+      echo -e "  ${CROSS} Unable to remove ${tmpFile}"
+
+  # Delete source file
+  rm "${source}" 2> /dev/null || \
+      echo -e "  ${CROSS} Unable to remove ${source}"
 }
 
 migrate_to_database() {
@@ -152,22 +157,18 @@ ls -lh
   if [[ -e "${whitelistFile}" ]]; then
     # Store whitelisted domains in database
     database_table_from_file "whitelist" "${whitelistFile}"
-    rm "${whitelistFile}"
   fi
   if [[ -e "${blacklistFile}" ]]; then
     # Store blacklisted domains in database
     database_table_from_file "blacklist" "${blacklistFile}"
-    rm "${blacklistFile}"
   fi
   if [[ -e "${regexFile}" ]]; then
     # Store regex domains in database
     database_table_from_file "regex" "${regexFile}"
-    rm "${regexFile}"
   fi
   if [[ -e "${adListFile}" ]]; then
     # Store adlists domains in database
     database_table_from_file "adlists" "${adListFile}"
-    rm "${adListFile}"
   fi
 }
 

From 465a39427b1a0f34b83ede17a9dbfb000a19a3bb Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 25 Apr 2019 10:46:18 +0200
Subject: [PATCH 29/66] Source adlists from gravity database

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 15 ++++-----------
 1 file changed, 4 insertions(+), 11 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 870027a1..0333a25d 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -151,9 +151,8 @@ database_table_from_file() {
       echo -e "  ${CROSS} Unable to remove ${source}"
 }
 
+# Migrate pre-v5.0 list files to database-based Pi-hole versions
 migrate_to_database() {
-ls -lh
-  # Migrate pre-v5.0 list files to database-based Pi-hole versions
   if [[ -e "${whitelistFile}" ]]; then
     # Store whitelisted domains in database
     database_table_from_file "whitelist" "${whitelistFile}"
@@ -232,15 +231,9 @@ gravity_CheckDNSResolutionAvailable() {
 gravity_GetBlocklistUrls() {
   echo -e "  ${INFO} ${COL_BOLD}Neutrino emissions detected${COL_NC}..."
 
-  if [[ -f "${adListDefault}" ]] && [[ -f "${adListFile}" ]]; then
-    # Remove superseded $adListDefault file
-    rm "${adListDefault}" 2> /dev/null || \
-      echo -e "  ${CROSS} Unable to remove ${adListDefault}"
-  fi
-
-  # Retrieve source URLs from $adListFile
-  # Logic: Remove comments and empty lines
-  mapfile -t sources <<< "$(grep -v -E "^(#|$)" "${adListFile}" 2> /dev/null)"
+  # Retrieve source URLs from gravity database
+  # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true)
+  mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM adlists WHERE enabled = 1;" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(

From d94bbfca7f728258ed3f0574b9eb9bf783114652 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 25 Apr 2019 11:18:54 +0200
Subject: [PATCH 30/66] Finish migrating gravity.db to using our new gravity
 database. This removes the --skip-download, --blacklist-only,
 --whitelist-only, and --wildcard-only options as it is not necessary to run
 gravity when modifying them from now on (gravity really only builds the
 gravity database and the local.list file).

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 171 +++++++++++++++--------------------------------------
 1 file changed, 49 insertions(+), 122 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 0333a25d..09e7b19c 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -44,8 +44,6 @@ matterAndLight="${basename}.0.matterandlight.txt"
 parsedMatter="${basename}.1.parsedmatter.txt"
 preEventHorizon="list.preEventHorizon"
 
-skipDownload="false"
-
 resolver="pihole-FTL"
 
 haveSourceUrls=true
@@ -103,8 +101,6 @@ database_table_from_file() {
     generate_gravity_database
   fi
 
-  echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${source} into database"
-
   # Truncate table
   output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
   status="$?"
@@ -153,22 +149,26 @@ database_table_from_file() {
 
 # Migrate pre-v5.0 list files to database-based Pi-hole versions
 migrate_to_database() {
-  if [[ -e "${whitelistFile}" ]]; then
-    # Store whitelisted domains in database
-    database_table_from_file "whitelist" "${whitelistFile}"
+  if [[ -e "${adListFile}" ]]; then
+    # Store adlists domains in database
+    echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${adListFile} into database"
+    database_table_from_file "adlists" "${adListFile}"
   fi
   if [[ -e "${blacklistFile}" ]]; then
     # Store blacklisted domains in database
+    echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${blacklistFile} into database"
     database_table_from_file "blacklist" "${blacklistFile}"
   fi
+  if [[ -e "${whitelistFile}" ]]; then
+    # Store whitelisted domains in database
+    echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${whitelistFile} into database"
+    database_table_from_file "whitelist" "${whitelistFile}"
+  fi
   if [[ -e "${regexFile}" ]]; then
     # Store regex domains in database
+    echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${regexFile} into database"
     database_table_from_file "regex" "${regexFile}"
   fi
-  if [[ -e "${adListFile}" ]]; then
-    # Store adlists domains in database
-    database_table_from_file "adlists" "${adListFile}"
-  fi
 }
 
 # Determine if DNS resolution is available before proceeding
@@ -282,11 +282,9 @@ gravity_SetDownloadOptions() {
       *) cmd_ext="";;
     esac
 
-    if [[ "${skipDownload}" == false ]]; then
-      echo -e "  ${INFO} Target: ${domain} (${url##*/})"
-      gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
-      echo ""
-    fi
+    echo -e "  ${INFO} Target: ${domain} (${url##*/})"
+    gravity_DownloadBlocklistFromUrl "${url}" "${cmd_ext}" "${agent}"
+    echo ""
   done
   gravity_Blackbody=true
 }
@@ -543,55 +541,35 @@ gravity_SortAndFilterConsolidatedList() {
   if [[ "${haveSourceUrls}" == true ]]; then
     echo -e "${OVER}  ${TICK} ${str}"
   fi
-  echo -e "  ${INFO} Number of domains being pulled in by gravity: ${COL_BLUE}${num}${COL_NC}"
+  echo -e "  ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains"
 
   str="Removing duplicate domains"
-  if [[ "${haveSourceUrls}" == true ]]; then
-    echo -ne "  ${INFO} ${str}..."
-  fi
-
+  echo -ne "  ${INFO} ${str}..."
   sort -u "${piholeDir}/${parsedMatter}" > "${piholeDir}/${preEventHorizon}"
+  echo -e "${OVER}  ${TICK} ${str}"
 
-  if [[ "${haveSourceUrls}" == true ]]; then
-    echo -e "${OVER}  ${TICK} ${str}"
-    # Format $preEventHorizon line total as currency
-    num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-    echo -e "  ${INFO} Number of unique domains trapped in the Event Horizon: ${COL_BLUE}${num}${COL_NC}"
-  fi
+  # Format $preEventHorizon line total as currency
+  num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
+  str="Storing ${COL_BLUE}${num}${COL_NC} unique blocking domains in database"
+  echo -ne "  ${INFO} ${str}..."
+  database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
+  echo -e "${OVER}  ${TICK} ${str}"
 }
 
-# Whitelist user-defined domains
-gravity_Whitelist() {
-  local num str
-
-  if [[ ! -f "${whitelistFile}" ]]; then
-    echo -e "  ${INFO} Nothing to whitelist!"
-    return 0
-  fi
-
-  num=$(wc -l < "${whitelistFile}")
-  str="Number of whitelisted domains: ${num}"
-  echo -ne "  ${INFO} ${str}..."
-
-  echo -e "${OVER}  ${INFO} ${str}"
+# Report number of entries in a table
+gravity_Table_Count() {
+  local table="${1}"
+  local str="${2}"
+  local num
+  num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${table} WHERE enabled = 1;")"
+  echo -e "  ${INFO} Number of ${str}: ${num}"
 }
 
 # Output count of blacklisted domains and regex filters
-gravity_ShowBlockCount() {
-  local num
-
-  if [[ -f "${blacklistFile}" ]]; then
-    num=$(printf "%'.0f" "$(wc -l < "${blacklistFile}")")
-    echo -e "  ${INFO} Number of blacklisted domains: ${num}"
-  fi
-
-  if [[ -f "${regexFile}" ]]; then
-    num=$(grep -cv "^#" "${regexFile}")
-    echo -e "  ${INFO} Number of regex filters: ${num}"
-  fi
-
-  # Store regex files in gravity database
-  database_table_from_file "regex" "${regexFile}"
+gravity_ShowCount() {
+  gravity_Table_Count "blacklist" "blacklisted domains"
+  gravity_Table_Count "whitelist" "whitelisted domains"
+  gravity_Table_Count "regex" "regex filters"
 }
 
 # Parse list of domains into hosts format
@@ -611,7 +589,7 @@ gravity_ParseDomainsIntoHosts() {
 }
 
 # Create "localhost" entries into hosts format
-gravity_ParseLocalDomains() {
+gravity_generateLocalList() {
   local hostname
 
   if [[ -s "/etc/hostname" ]]; then
@@ -636,24 +614,6 @@ gravity_ParseLocalDomains() {
   fi
 }
 
-# Create primary blacklist entries
-gravity_ParseBlacklistDomains() {
-  local output status
-
-  # Store gravity domains in gravity database
-  database_table_from_file "gravity" "${piholeDir}/${preEventHorizon}"
-}
-
-# Create user-added blacklist entries
-gravity_ParseUserDomains() {
-  if [[ ! -f "${blacklistFile}" ]]; then
-    return 0
-  fi
-
-  # Fill database table
-  database_table_from_file "blacklist" "${blacklistFile}"
-}
-
 # Trap Ctrl-C
 gravity_Trap() {
   trap '{ echo -e "\\n\\n  ${INFO} ${COL_LIGHT_RED}User-abort detected${COL_NC}"; gravity_Cleanup "error"; }' INT
@@ -689,7 +649,7 @@ gravity_Cleanup() {
     str="Optimizing domains database"
     echo -ne "  ${INFO} ${str}..."
     # Run VACUUM command on database to optimize it
-    output=$( { sqlite3 "${gravityDBfile}" <<< "VACUUM;"; } 2>&1 )
+    output=$( { sqlite3 "${gravityDBfile}" "VACUUM;"; } 2>&1 )
     status="$?"
 
     if [[ "${status}" -ne 0 ]]; then
@@ -728,10 +688,6 @@ for var in "$@"; do
     "-f" | "--force" ) forceDelete=true;;
     "-o" | "--optimize" ) optimize_database=true;;
     "-h" | "--help" ) helpFunc;;
-    "-sd" | "--skip-download" ) skipDownload=true;;
-    "-b" | "--blacklist-only" ) listType="blacklist";;
-    "-w" | "--whitelist-only" ) listType="whitelist";;
-    "-wild" | "--wildcard-only" ) listType="wildcard"; dnsRestartType="restart";;
   esac
 done
 
@@ -749,54 +705,25 @@ if [[ "${forceDelete:-}" == true ]]; then
   echo -e "${OVER}  ${TICK} ${str}"
 fi
 
-# Determine which functions to run
-if [[ "${skipDownload}" == false ]]; then
-  # Gravity needs to download blocklists
-  gravity_CheckDNSResolutionAvailable
-  gravity_GetBlocklistUrls
-  if [[ "${haveSourceUrls}" == true ]]; then
-    gravity_SetDownloadOptions
-  fi
-  gravity_ConsolidateDownloadedBlocklists
-  gravity_SortAndFilterConsolidatedList
-else
-  # Gravity needs to modify Blacklist/Whitelist/Wildcards
-  echo -e "  ${INFO} Using cached Event Horizon list..."
-  numberOf=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${preEventHorizon}")")
-  echo -e "  ${INFO} ${COL_BLUE}${numberOf}${COL_NC} unique domains trapped in the Event Horizon"
+# Gravity downloads blocklists next
+gravity_CheckDNSResolutionAvailable
+gravity_GetBlocklistUrls
+if [[ "${haveSourceUrls}" == true ]]; then
+gravity_SetDownloadOptions
 fi
+# Build preEventHorizon
+gravity_ConsolidateDownloadedBlocklists
+gravity_SortAndFilterConsolidatedList
 
-# Perform when downloading blocklists, or modifying the whitelist
-if [[ "${skipDownload}" == false ]] || [[ "${listType}" == "whitelist" ]]; then
-  gravity_Whitelist
-fi
-
-convert_wildcard_to_regex
-gravity_ShowBlockCount
-
-# Perform when downloading blocklists, or modifying the white/blacklist (not wildcards)
-if [[ "${skipDownload}" == false ]] || [[ "${listType}" == *"list" ]]; then
-  str="Parsing domains"
-  echo -ne "  ${INFO} ${str}..."
-
-  gravity_ParseUserDomains
-
-  # Perform when downloading blocklists
-  if [[ ! "${listType:-}" == "blacklist" ]]; then
-    gravity_ParseLocalDomains
-    gravity_ParseBlacklistDomains
-  fi
-
-  echo -e "${OVER}  ${TICK} ${str}"
-
-  gravity_Cleanup
-fi
+# Create local.list
+gravity_generateLocalList
+gravity_ShowCount
 
+gravity_Cleanup
 echo ""
 
 # Determine if DNS has been restarted by this instance of gravity
 if [[ -z "${dnsWasOffline:-}" ]]; then
-  # Use "force-reload" when restarting dnsmasq for everything but Wildcards
-  "${PIHOLE_COMMAND}" restartdns "${dnsRestartType:-force-reload}"
+  "${PIHOLE_COMMAND}" restartdns reload
 fi
 "${PIHOLE_COMMAND}" status

From e5d1cb5a2ef11ab4642bd3b8f80cb2edcd9cb2b5 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 25 Apr 2019 12:10:42 +0200
Subject: [PATCH 31/66] Rewrite list.sh to operate on database-based lists

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 247 +++++++++++++++++----------------------
 1 file changed, 104 insertions(+), 143 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index c1d95aae..1a2094de 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -11,10 +11,8 @@
 # Globals
 basename=pihole
 piholeDir=/etc/"${basename}"
-whitelist="${piholeDir}"/whitelist.txt
-blacklist="${piholeDir}"/blacklist.txt
+gravityDBfile="${piholeDir}/gravity.db"
 
-readonly regexlist="/etc/pihole/regex.list"
 reload=false
 addmode=true
 verbose=true
@@ -22,35 +20,34 @@ wildcard=false
 
 domList=()
 
-listMain=""
-listAlt=""
+listType=""
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
 
 
 helpFunc() {
-    if [[ "${listMain}" == "${whitelist}" ]]; then
+    if [[ "${listType}" == "whitelist" ]]; then
         param="w"
-        type="white"
-    elif [[ "${listMain}" == "${regexlist}" && "${wildcard}" == true ]]; then
+        type="whitelist"
+    elif [[ "${listType}" == "regex" && "${wildcard}" == true ]]; then
         param="-wild"
-        type="wildcard black"
-    elif [[ "${listMain}" == "${regexlist}" ]]; then
+        type="wildcard blacklist"
+    elif [[ "${listType}" == "regex" ]]; then
         param="-regex"
-        type="regex black"
+        type="regex filter"
     else
         param="b"
-        type="black"
+        type="blacklist"
     fi
 
     echo "Usage: pihole -${param} [options] <domain> <domain2 ...>
 Example: 'pihole -${param} site.com', or 'pihole -${param} site1.com site2.com'
-${type^}list one or more domains
+${type^} one or more domains
 
 Options:
-  -d, --delmode       Remove domain(s) from the ${type}list
-  -nr, --noreload     Update ${type}list without refreshing dnsmasq
+  -d, --delmode       Remove domain(s) from the ${type}
+  -nr, --noreload     Update ${type} without reloading the DNS server
   -q, --quiet         Make output less verbose
   -h, --help          Show this help dialog
   -l, --list          Display all your ${type}listed domains
@@ -73,7 +70,7 @@ HandleOther() {
 
     # Check validity of domain (don't check for regex entries)
     if [[ "${#domain}" -le 253 ]]; then
-        if [[ "${listMain}" == "${regexlist}" && "${wildcard}" == false ]]; then
+        if [[ "${listType}" == "regex" && "${wildcard}" == false ]]; then
             validDomain="${domain}"
         else
             validDomain=$(grep -P "^((-|_)*[a-z\\d]((-|_)*[a-z\\d])*(-|_)*)(\\.(-|_)*([a-z\\d]((-|_)*[a-z\\d])*))*$" <<< "${domain}") # Valid chars check
@@ -88,169 +85,134 @@ HandleOther() {
     fi
 }
 
-PoplistFile() {
-    # Check whitelist file exists, and if not, create it
-    if [[ ! -f "${whitelist}" ]]; then
-        touch "${whitelist}"
-    fi
-
-    # Check blacklist file exists, and if not, create it
-    if [[ ! -f "${blacklist}" ]]; then
-        touch "${blacklist}"
-    fi
-
+ProcessDomainList() {
     for dom in "${domList[@]}"; do
         # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
-            AddDomain "${dom}" "${listMain}"
+            AddDomain "${dom}" "${listType}"
             RemoveDomain "${dom}" "${listAlt}"
         else
-            RemoveDomain "${dom}" "${listMain}"
+            RemoveDomain "${dom}" "${listType}"
         fi
   done
 }
 
 AddDomain() {
+    local domain list listname sqlitekey num
+    domain="$1"
     list="$2"
-    domain=$(EscapeRegexp "$1")
 
-    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
-    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
-
-    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
-        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
-        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
-        bool=true
-        # Is the domain in the list we want to add it to?
-        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
-
-        if [[ "${bool}" == false ]]; then
-            # Domain not found in the whitelist file, add it!
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} Adding ${1} to ${listname}..."
-            fi
-            reload=true
-            # Add it to the list we want to add it to
-            echo "$1" >> "${list}"
-        else
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
-            fi
+    if [[ "${list}" == "regex" ]]; then
+        listname="regex filters"
+        sqlitekey="filter"
+        if [[ "${wildcard}" == true ]]; then
+            domain="(^|\\.)${domain//\./\\.}$"
         fi
-    elif [[ "${list}" == "${regexlist}" ]]; then
-        [[ -z "${type}" ]] && type="--wildcard-only"
-        bool=true
-        domain="${1}"
+    else
+        # Whitelist / Blacklist
+        listname="${list}list"
+        sqlitekey="domain"
+    fi
 
-        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+    # Is the domain in the list we want to add it to?
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
 
-        # Is the domain in the list?
-        # Search only for exactly matching lines
-        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
-
-        if [[ "${bool}" == false ]]; then
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} Adding ${domain} to regex list..."
-            fi
-            reload="restart"
-            echo "$domain" >> "${regexlist}"
-        else
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${domain} already exists in regex list, no need to add!"
-            fi
+    if [[ "${num}" -eq 0 ]]; then
+        # Domain not found in the file, add it!
+        if [[ "${verbose}" == true ]]; then
+            echo -e "  ${INFO} Adding ${1} to ${listname}..."
+        fi
+        reload=true
+        # Add it to the list we want to add it to
+        local timestamp
+        timestamp="$(date --utc +'%s')"
+        sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},enabled,date_added) VALUES (\"${domain}\",1,${timestamp});"
+    else
+        if [[ "${verbose}" == true ]]; then
+            echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
         fi
     fi
 }
 
 RemoveDomain() {
+    local domain list listname sqlitekey num
+    domain="$1"
     list="$2"
-    domain=$(EscapeRegexp "$1")
 
-    [[ "${list}" == "${whitelist}" ]] && listname="whitelist"
-    [[ "${list}" == "${blacklist}" ]] && listname="blacklist"
-
-    if [[ "${list}" == "${whitelist}" || "${list}" == "${blacklist}" ]]; then
-        bool=true
-        [[ "${list}" == "${whitelist}" && -z "${type}" ]] && type="--whitelist-only"
-        [[ "${list}" == "${blacklist}" && -z "${type}" ]] && type="--blacklist-only"
-        # Is it in the list? Logic follows that if its whitelisted it should not be blacklisted and vice versa
-        grep -Ex -q "${domain}" "${list}" > /dev/null 2>&1 || bool=false
-        if [[ "${bool}" == true ]]; then
-            # Remove it from the other one
-            echo -e "  ${INFO} Removing $1 from ${listname}..."
-            # /I flag: search case-insensitive
-            sed -i "/${domain}/Id" "${list}"
-            reload=true
-        else
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
-            fi
+    if [[ "${list}" == "regex" ]]; then
+        listname="regex filters"
+        sqlitekey="filter"
+        if [[ "${wildcard}" == true ]]; then
+            domain="(^|\\.)${domain//\./\\.}$"
         fi
-    elif [[ "${list}" == "${regexlist}" ]]; then
-        [[ -z "${type}" ]] && type="--wildcard-only"
-        domain="${1}"
+    else
+        # Whitelist / Blacklist
+        listname="${list}list"
+        sqlitekey="domain"
+    fi
 
-        [[ "${wildcard}" == true ]] && domain="(^|\\.)${domain//\./\\.}$"
+    # Is the domain in the list we want to remove it from?
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
 
-        bool=true
-        # Is it in the list?
-        grep -Fx "${domain}" "${regexlist}" > /dev/null 2>&1 || bool=false
-        if [[ "${bool}" == true ]]; then
-            # Remove it from the other one
-            echo -e "  ${INFO} Removing $domain from regex list..."
-            local lineNumber
-            lineNumber=$(grep -Fnx "$domain" "${list}" | cut -f1 -d:)
-            sed -i "${lineNumber}d" "${list}"
-            reload=true
-        else
-            if [[ "${verbose}" == true ]]; then
-                echo -e "  ${INFO} ${domain} does not exist in regex list, no need to remove!"
-            fi
+    if [[ "${num}" -ne 0 ]]; then
+        # Domain found in the file, remove it!
+        if [[ "${verbose}" == true ]]; then
+            echo -e "  ${INFO} Removing ${1} from ${listname}..."
+        fi
+        reload=true
+        # Remove it from the current list
+        local timestamp
+        timestamp="$(date --utc +'%s')"
+        sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";"
+    else
+        if [[ "${verbose}" == true ]]; then
+            echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
         fi
     fi
 }
 
-# Update Gravity
-Reload() {
-    echo ""
-    pihole -g --skip-download "${type:-}"
-}
-
 Displaylist() {
-    if [[ -f ${listMain} ]]; then
-        if [[ "${listMain}" == "${whitelist}" ]]; then
-            string="gravity resistant domains"
-        else
-            string="domains caught in the sinkhole"
-        fi
-        verbose=false
-        echo -e "Displaying $string:\n"
-        count=1
-        while IFS= read -r RD || [ -n "${RD}" ]; do
-            echo "  ${count}: ${RD}"
-            count=$((count+1))
-        done < "${listMain}"
+    local domain list listname count status
+
+    if [[ "${listType}" == "regex" ]]; then
+        listname="regex filters list"
     else
-        echo -e "  ${COL_LIGHT_RED}${listMain} does not exist!${COL_NC}"
+        # Whitelist / Blacklist
+        listname="${listType}"
+    fi
+    data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)"
+
+    if [[ -z $data ]]; then
+        echo -e "Not showing empty ${listname}"
+    else
+        echo -e "Displaying ${listname}:"
+        count=1
+        while IFS= read -r line
+        do
+            domain="$(cut -d'|' -f1 <<< "${line}")"
+            enabled="$(cut -d'|' -f2 <<< "${line}")"
+            if [[ "${enabled}" -eq 1 ]]; then
+                status="enabled"
+            else
+                status="disabled"
+            fi
+            echo "  ${count}: ${domain} (${status})"
+            count=$((count+1))
+        done <<< "${data}"
+        exit 0;
     fi
-    exit 0;
 }
 
 NukeList() {
-    if [[ -f "${listMain}" ]]; then
-        # Back up original list
-        cp "${listMain}" "${listMain}.bck~"
-        # Empty out file
-        echo "" > "${listMain}"
-    fi
+    sqlite3 "${gravityDBfile}" "DELETE FROM ${listType};"
 }
 
 for var in "$@"; do
     case "${var}" in
-        "-w" | "whitelist"   ) listMain="${whitelist}"; listAlt="${blacklist}";;
-        "-b" | "blacklist"   ) listMain="${blacklist}"; listAlt="${whitelist}";;
-        "--wild" | "wildcard" ) listMain="${regexlist}"; wildcard=true;;
-        "--regex" | "regex"   ) listMain="${regexlist}";;
+        "-w" | "whitelist"   ) listType="whitelist"; listAlt="blacklist";;
+        "-b" | "blacklist"   ) listType="blacklist"; listAlt="whitelist";;
+        "--wild" | "wildcard" ) listType="regex"; wildcard=true;;
+        "--regex" | "regex"   ) listType="regex";;
         "-nr"| "--noreload"  ) reload=false;;
         "-d" | "--delmode"   ) addmode=false;;
         "-q" | "--quiet"     ) verbose=false;;
@@ -267,9 +229,8 @@ if [[ $# = 0 ]]; then
     helpFunc
 fi
 
-PoplistFile
+ProcessDomainList
 
 if [[ "${reload}" != false ]]; then
-    # Ensure that "restart" is used for Wildcard updates
-    Reload "${reload}"
+    pihole restartdns reload
 fi

From 788cd783212828229fb8708f38841b4a89472267 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 25 Apr 2019 12:30:38 +0200
Subject: [PATCH 32/66] Reduce code duplication and add special handling of |
 character as it might appear in regex filter string

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 70 +++++++++++++++++++++-------------------
 1 file changed, 36 insertions(+), 34 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 1a2094de..727919fe 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -21,9 +21,11 @@ wildcard=false
 domList=()
 
 listType=""
+listname=""
+sqlitekey=""
 
-colfile="/opt/pihole/COL_TABLE"
-source ${colfile}
+# shellcheck source=/opt/pihole/COL_TABLE
+source "/opt/pihole/COL_TABLE"
 
 
 helpFunc() {
@@ -61,7 +63,7 @@ EscapeRegexp() {
     # string in our regular expressions
     # This sed is intentionally executed in three steps to ease maintainability
     # The first sed removes any amount of leading dots
-    echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
+    echo "$@" | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
 }
 
 HandleOther() {
@@ -86,11 +88,28 @@ HandleOther() {
 }
 
 ProcessDomainList() {
+    if [[ "${listType}" == "regex" ]]; then
+        # Regex filter list
+        listname="regex filters"
+        sqlitekey="filter"
+    else
+        # Whitelist / Blacklist
+        listname="${listType}"
+        sqlitekey="domain"
+    fi
+
     for dom in "${domList[@]}"; do
-        # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the other
+        # Format domain into regex filter if requested
+        if [[ "${wildcard}" == true ]]; then
+            dom="(^|\\.)${dom//\./\\.}$"
+        fi
+
+        # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the othe
         if ${addmode}; then
             AddDomain "${dom}" "${listType}"
-            RemoveDomain "${dom}" "${listAlt}"
+            if [[ ! "${listType}" == "regex" ]]; then
+                RemoveDomain "${dom}" "${listAlt}"
+            fi
         else
             RemoveDomain "${dom}" "${listType}"
         fi
@@ -98,22 +117,10 @@ ProcessDomainList() {
 }
 
 AddDomain() {
-    local domain list listname sqlitekey num
+    local domain list num
     domain="$1"
     list="$2"
 
-    if [[ "${list}" == "regex" ]]; then
-        listname="regex filters"
-        sqlitekey="filter"
-        if [[ "${wildcard}" == true ]]; then
-            domain="(^|\\.)${domain//\./\\.}$"
-        fi
-    else
-        # Whitelist / Blacklist
-        listname="${list}list"
-        sqlitekey="domain"
-    fi
-
     # Is the domain in the list we want to add it to?
     num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
 
@@ -135,22 +142,10 @@ AddDomain() {
 }
 
 RemoveDomain() {
-    local domain list listname sqlitekey num
+    local domain list num
     domain="$1"
     list="$2"
 
-    if [[ "${list}" == "regex" ]]; then
-        listname="regex filters"
-        sqlitekey="filter"
-        if [[ "${wildcard}" == true ]]; then
-            domain="(^|\\.)${domain//\./\\.}$"
-        fi
-    else
-        # Whitelist / Blacklist
-        listname="${list}list"
-        sqlitekey="domain"
-    fi
-
     # Is the domain in the list we want to remove it from?
     num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
 
@@ -172,7 +167,7 @@ RemoveDomain() {
 }
 
 Displaylist() {
-    local domain list listname count status
+    local list listname count num_pipes domain enabled status
 
     if [[ "${listType}" == "regex" ]]; then
         listname="regex filters list"
@@ -189,13 +184,20 @@ Displaylist() {
         count=1
         while IFS= read -r line
         do
-            domain="$(cut -d'|' -f1 <<< "${line}")"
-            enabled="$(cut -d'|' -f2 <<< "${line}")"
+            # Count number of pipes seen in this line
+            # This is necessary because we can only detect the pipe separating the fields
+            # from the end backwards as the domain (which is the first field) may contain
+            # pipe symbols as they are perfectly valid regex filter control characters
+            num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")"
+
+            domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")"
+            enabled="$(cut -d'|' -f$((num_pipes-1)) <<< "${line}")"
             if [[ "${enabled}" -eq 1 ]]; then
                 status="enabled"
             else
                 status="disabled"
             fi
+
             echo "  ${count}: ${domain} (${status})"
             count=$((count+1))
         done <<< "${data}"

From 43bced7997772d5604d1302e1c8fbc6369ec45e1 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 25 Apr 2019 12:45:08 +0200
Subject: [PATCH 33/66] Add more comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 727919fe..5fd24f6b 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -24,8 +24,8 @@ listType=""
 listname=""
 sqlitekey=""
 
-# shellcheck source=/opt/pihole/COL_TABLE
-source "/opt/pihole/COL_TABLE"
+colfile="/opt/pihole/COL_TABLE"
+source ${colfile}
 
 
 helpFunc() {
@@ -63,7 +63,7 @@ EscapeRegexp() {
     # string in our regular expressions
     # This sed is intentionally executed in three steps to ease maintainability
     # The first sed removes any amount of leading dots
-    echo "$@" | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
+    echo $* | sed 's/^\.*//' | sed "s/[]\.|$(){}?+*^]/\\\\&/g" | sed "s/\\//\\\\\//g"
 }
 
 HandleOther() {
@@ -190,8 +190,11 @@ Displaylist() {
             # pipe symbols as they are perfectly valid regex filter control characters
             num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")"
 
+            # Extract domain and enabled status based on the obtained number of pipe characters
             domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")"
-            enabled="$(cut -d'|' -f$((num_pipes-1)) <<< "${line}")"
+            enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")"
+
+            # Translate boolean status into human readable string
             if [[ "${enabled}" -eq 1 ]]; then
                 status="enabled"
             else

From cce66a13b69ece62037b9dd2a01cf4aab18a3abe Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 25 Apr 2019 14:54:41 +0200
Subject: [PATCH 34/66] Fix output in list.sh when adding a domain that is
 already present in the target list

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 5fd24f6b..1ad99468 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -161,7 +161,7 @@ RemoveDomain() {
         sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";"
     else
         if [[ "${verbose}" == true ]]; then
-            echo -e "  ${INFO} ${1} does not exist in ${listname}, no need to remove!"
+            echo -e "  ${INFO} ${1} does not exist in ${list}, no need to remove!"
         fi
     fi
 }

From a2a7e8000795d2b58071d7393b86cf2bc3e894fc Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 25 Apr 2019 15:01:13 +0200
Subject: [PATCH 35/66] Add --web for sending a confirmation string when
 executed from the web interface. This is necessary as the "Reloading DNS
 service" message does not always propagate through to the web interface when
 reloading pihole-FTL takes a while.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 1ad99468..969e7e44 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -17,6 +17,7 @@ reload=false
 addmode=true
 verbose=true
 wildcard=false
+web=false
 
 domList=()
 
@@ -224,6 +225,7 @@ for var in "$@"; do
         "-h" | "--help"      ) helpFunc;;
         "-l" | "--list"      ) Displaylist;;
         "--nuke"             ) NukeList;;
+        "--web"              ) web=true;;
         *                    ) HandleOther "${var}";;
     esac
 done
@@ -236,6 +238,11 @@ fi
 
 ProcessDomainList
 
+# Used on web interface
+if $web; then
+echo "DONE"
+fi
+
 if [[ "${reload}" != false ]]; then
     pihole restartdns reload
 fi

From 0efc46260f0726edf0e2986df536c0f4e0c93fdc Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 16:16:30 +0200
Subject: [PATCH 36/66] Use both old and new strings for detecting whether this
 is a Pi-hole dnsmasq config file

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 automated install/basic-install.sh | 6 ++++--
 gravity.sh                         | 8 +++++---
 2 files changed, 9 insertions(+), 5 deletions(-)

diff --git a/automated install/basic-install.sh b/automated install/basic-install.sh
index 6bcfb564..06facdf7 100755
--- a/automated install/basic-install.sh	
+++ b/automated install/basic-install.sh	
@@ -1224,7 +1224,8 @@ version_check_dnsmasq() {
     # Local, named variables
     local dnsmasq_conf="/etc/dnsmasq.conf"
     local dnsmasq_conf_orig="/etc/dnsmasq.conf.orig"
-    local dnsmasq_pihole_id_string="# Dnsmasq config for Pi-hole's FTLDNS"
+    local dnsmasq_pihole_id_string="addn-hosts=/etc/pihole/gravity.list"
+    local dnsmasq_pihole_id_string2="# Dnsmasq config for Pi-hole's FTLDNS"
     local dnsmasq_original_config="${PI_HOLE_LOCAL_REPO}/advanced/dnsmasq.conf.original"
     local dnsmasq_pihole_01_snippet="${PI_HOLE_LOCAL_REPO}/advanced/01-pihole.conf"
     local dnsmasq_pihole_01_location="/etc/dnsmasq.d/01-pihole.conf"
@@ -1233,7 +1234,8 @@ version_check_dnsmasq() {
     if [[ -f "${dnsmasq_conf}" ]]; then
         printf "  %b Existing dnsmasq.conf found..." "${INFO}"
         # If a specific string is found within this file, we presume it's from older versions on Pi-hole,
-        if grep -q ${dnsmasq_pihole_id_string} ${dnsmasq_conf}; then
+        if grep -q "${dnsmasq_pihole_id_string}" "${dnsmasq_conf}" ||
+           grep -q "${dnsmasq_pihole_id_string2}" "${dnsmasq_conf}"; then
             printf " it is from a previous Pi-hole install.\\n"
             printf "  %b Backing up dnsmasq.conf to dnsmasq.conf.orig..." "${INFO}"
             # so backup the original file
diff --git a/gravity.sh b/gravity.sh
index 09e7b19c..75057eec 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -20,8 +20,9 @@ source "${regexconverter}"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
-PIHOLE_USER="pihole"
-PIHOLE_GROUP="pihole"
+DATABASE_USER="pihole"
+DATABASE_GROUP="www-data"
+DATABASE_PERMISSIONS="0660"
 
 piholeDir="/etc/${basename}"
 
@@ -86,7 +87,8 @@ fi
 # Generate new sqlite3 file from schema template
 generate_gravity_database() {
   sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
-  chown $PIHOLE_USER:$PIHOLE_GROUP "${gravityDBfile}"
+  chown ${DATABASE_USER}:${DATABASE_GROUP} "${piholeDir}" "${gravityDBfile}"
+  chmod ${DATABASE_PERMISSIONS} "${piholeDir}" "${gravityDBfile}"
 }
 
 # Import domains from file and store them in the specified database table

From 08ee9526adde9fa3bd18cffa8b58ef148c2ccc52 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:06:01 +0200
Subject: [PATCH 37/66] Review comments

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 76 +++++++++++++++++++++-------------------
 1 file changed, 39 insertions(+), 37 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 969e7e44..af44013e 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -105,7 +105,8 @@ ProcessDomainList() {
             dom="(^|\\.)${dom//\./\\.}$"
         fi
 
-        # Logic: If addmode then add to desired list and remove from the other; if delmode then remove from desired list but do not add to the othe
+        # Logic: If addmode then add to desired list and remove from the other;
+        # if delmode then remove from desired list but do not add to the other
         if ${addmode}; then
             AddDomain "${dom}" "${listType}"
             if [[ ! "${listType}" == "regex" ]]; then
@@ -119,63 +120,64 @@ ProcessDomainList() {
 
 AddDomain() {
     local domain list num
-    domain="$1"
+    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
+    domain="$(printf "%q" "$1")"
     list="$2"
 
     # Is the domain in the list we want to add it to?
     num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
 
-    if [[ "${num}" -eq 0 ]]; then
-        # Domain not found in the file, add it!
-        if [[ "${verbose}" == true ]]; then
-            echo -e "  ${INFO} Adding ${1} to ${listname}..."
-        fi
-        reload=true
-        # Add it to the list we want to add it to
-        local timestamp
-        timestamp="$(date --utc +'%s')"
-        sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},enabled,date_added) VALUES (\"${domain}\",1,${timestamp});"
-    else
-        if [[ "${verbose}" == true ]]; then
-            echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
-        fi
+    if [[ "${num}" -ne 0 ]]; then
+      if [[ "${verbose}" == true ]]; then
+          echo -e "  ${INFO} ${1} already exists in ${listname}, no need to add!"
+      fi
+      return
     fi
+
+    # Domain not found in the table, add it!
+    if [[ "${verbose}" == true ]]; then
+        echo -e "  ${INFO} Adding ${1} to ${listname}..."
+    fi
+    reload=true
+    # Add it to the list we want to add it to
+    local timestamp
+    timestamp="$(date --utc +'%s')"
+    # Insert only domain and date_added here. The enabled fields will be filled
+    # with its default value is true.
+    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},date_added) VALUES (\"${domain}\",${timestamp});"
 }
 
 RemoveDomain() {
     local domain list num
-    domain="$1"
+    # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
+    domain="$(printf "%q" "$1")"
     list="$2"
 
     # Is the domain in the list we want to remove it from?
     num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
 
-    if [[ "${num}" -ne 0 ]]; then
-        # Domain found in the file, remove it!
-        if [[ "${verbose}" == true ]]; then
-            echo -e "  ${INFO} Removing ${1} from ${listname}..."
-        fi
-        reload=true
-        # Remove it from the current list
-        local timestamp
-        timestamp="$(date --utc +'%s')"
-        sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";"
-    else
-        if [[ "${verbose}" == true ]]; then
-            echo -e "  ${INFO} ${1} does not exist in ${list}, no need to remove!"
-        fi
+    if [[ "${num}" -eq 0 ]]; then
+      if [[ "${verbose}" == true ]]; then
+          echo -e "  ${INFO} ${1} does not exist in ${list}, no need to remove!"
+      fi
+      return
     fi
+
+    # Domain found in the table, remove it!
+    if [[ "${verbose}" == true ]]; then
+        echo -e "  ${INFO} Removing ${1} from ${listname}..."
+    fi
+    reload=true
+    # Remove it from the current list
+    local timestamp
+    timestamp="$(date --utc +'%s')"
+    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";"
 }
 
 Displaylist() {
     local list listname count num_pipes domain enabled status
 
-    if [[ "${listType}" == "regex" ]]; then
-        listname="regex filters list"
-    else
-        # Whitelist / Blacklist
-        listname="${listType}"
-    fi
+    listname="${listType}"
     data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)"
 
     if [[ -z $data ]]; then

From 6b1d2523e8bea9310e5095208f0d4732cef14cb9 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:07:16 +0200
Subject: [PATCH 38/66] Also print datetime a domain was added when listing the
 domains on the CLI

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index af44013e..d11ab96c 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -175,7 +175,7 @@ RemoveDomain() {
 }
 
 Displaylist() {
-    local list listname count num_pipes domain enabled status
+    local list listname count num_pipes domain enabled status nicedate
 
     listname="${listType}"
     data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)"
@@ -196,7 +196,9 @@ Displaylist() {
             # Extract domain and enabled status based on the obtained number of pipe characters
             domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")"
             enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")"
+            dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")"
 
+            echo "${dateadded}"
             # Translate boolean status into human readable string
             if [[ "${enabled}" -eq 1 ]]; then
                 status="enabled"
@@ -204,7 +206,10 @@ Displaylist() {
                 status="disabled"
             fi
 
-            echo "  ${count}: ${domain} (${status})"
+            # Get nice representation of numerical date stored in database
+            nicedate=$(date --rfc-2822 -d "@${dateadded}")
+
+            echo "  ${count}: ${domain} (${status}, added ${nicedate})"
             count=$((count+1))
         done <<< "${data}"
         exit 0;

From a891d6439533f64a42baacdb409df4725ac7461e Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:13:36 +0200
Subject: [PATCH 39/66] We should always exit after displaying a list, even if
 it is empty

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index d11ab96c..cd9dce52 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -212,8 +212,8 @@ Displaylist() {
             echo "  ${count}: ${domain} (${status}, added ${nicedate})"
             count=$((count+1))
         done <<< "${data}"
-        exit 0;
     fi
+    exit 0;
 }
 
 NukeList() {

From 5bb816367422c5c6394543ff88ed8b22f31cf60f Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:17:21 +0200
Subject: [PATCH 40/66] Change type of whitelist, blacklist, regex, and adlists
 time column to INTEGER and insert a DEFAULT clause that returns the current
 Unix time as such an integer

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 5442987f..cd62c94e 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,7 +1,7 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
-CREATE TABLE regex     (filter TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
-CREATE TABLE adlists   (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, comment TEXT);
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
+CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
+CREATE TABLE adlists   (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
 

From 53e3ff2b2421a491badb24be578717e3bc11e485 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:32:05 +0200
Subject: [PATCH 41/66] Only add non-empty lines when moving file content into
 database tables

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index 75057eec..1d94424f 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -125,7 +125,10 @@ database_table_from_file() {
     # Read file line by line
     grep -v '^ *#' < "${source}" | while IFS= read -r domain
     do
-      echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}"
+      # Only add non-empty lines
+      if [[ ! -z "${domain}" ]]; then
+        echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}"
+      fi
     done
     inputfile="${tmpFile}"
   fi

From bd0215524ba0191b5364a596e769027a22be88fd Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:36:00 +0200
Subject: [PATCH 42/66] Add vw_adlists view which returns only enabled adlists

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 6 +++++-
 gravity.sh                        | 2 +-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index cd62c94e..ebe97955 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -20,6 +20,10 @@ CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
 FROM whitelist a
 WHERE a.enabled == 1;
 
-CREATE VIEW vw_regex AS SELECT DISTINCT a.filter
+CREATE VIEW vw_regex AS SELECT DISTINCT a.domain
 FROM regex a
 WHERE a.enabled == 1;
+
+CREATE VIEW vw_adlists AS SELECT DISTINCT a.address
+FROM adlists a
+WHERE a.enabled == 1;
diff --git a/gravity.sh b/gravity.sh
index 1d94424f..a0652ced 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -238,7 +238,7 @@ gravity_GetBlocklistUrls() {
 
   # Retrieve source URLs from gravity database
   # We source only enabled adlists, sqlite3 stores boolean values as 0 (false) or 1 (true)
-  mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM adlists WHERE enabled = 1;" 2> /dev/null)"
+  mapfile -t sources <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)"
 
   # Parse source domains from $sources
   mapfile -t sourceDomains <<< "$(

From 3dcb9722d35fb42d8a63903d5808a493e0b8507c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:39:35 +0200
Subject: [PATCH 43/66] Remove global haveSourceUrls variable and just skip
 downloading and processing adlists if there aren't any to be downloaded

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 33 +++++++++++++--------------------
 1 file changed, 13 insertions(+), 20 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index a0652ced..3543ac15 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -47,8 +47,6 @@ preEventHorizon="list.preEventHorizon"
 
 resolver="pihole-FTL"
 
-haveSourceUrls=true
-
 # Source setupVars from install script
 setupVars="${piholeDir}/setupVars.conf"
 if [[ -f "${setupVars}" ]];then
@@ -255,11 +253,12 @@ gravity_GetBlocklistUrls() {
 
   if [[ -n "${sources[*]}" ]] && [[ -n "${sourceDomains[*]}" ]]; then
     echo -e "${OVER}  ${TICK} ${str}"
+    return 0
   else
     echo -e "${OVER}  ${CROSS} ${str}"
     echo -e "  ${INFO} No source list found, or it is empty"
     echo ""
-    haveSourceUrls=false
+    return 1
   fi
 }
 
@@ -503,9 +502,7 @@ gravity_ConsolidateDownloadedBlocklists() {
   local str lastLine
 
   str="Consolidating blocklists"
-  if [[ "${haveSourceUrls}" == true ]]; then
-    echo -ne "  ${INFO} ${str}..."
-  fi
+  echo -ne "  ${INFO} ${str}..."
 
   # Empty $matterAndLight if it already exists, otherwise, create it
   : > "${piholeDir}/${matterAndLight}"
@@ -524,9 +521,8 @@ gravity_ConsolidateDownloadedBlocklists() {
       fi
     fi
   done
-  if [[ "${haveSourceUrls}" == true ]]; then
-    echo -e "${OVER}  ${TICK} ${str}"
-  fi
+  echo -e "${OVER}  ${TICK} ${str}"
+
 }
 
 # Parse consolidated list into (filtered, unique) domains-only format
@@ -534,18 +530,15 @@ gravity_SortAndFilterConsolidatedList() {
   local str num
 
   str="Extracting domains from blocklists"
-  if [[ "${haveSourceUrls}" == true ]]; then
-    echo -ne "  ${INFO} ${str}..."
-  fi
+  echo -ne "  ${INFO} ${str}..."
 
   # Parse into file
   gravity_ParseFileIntoDomains "${piholeDir}/${matterAndLight}" "${piholeDir}/${parsedMatter}"
 
   # Format $parsedMatter line total as currency
   num=$(printf "%'.0f" "$(wc -l < "${piholeDir}/${parsedMatter}")")
-  if [[ "${haveSourceUrls}" == true ]]; then
-    echo -e "${OVER}  ${TICK} ${str}"
-  fi
+
+  echo -e "${OVER}  ${TICK} ${str}"
   echo -e "  ${INFO} Gravity pulled in ${COL_BLUE}${num}${COL_NC} domains"
 
   str="Removing duplicate domains"
@@ -713,12 +706,12 @@ fi
 # Gravity downloads blocklists next
 gravity_CheckDNSResolutionAvailable
 gravity_GetBlocklistUrls
-if [[ "${haveSourceUrls}" == true ]]; then
-gravity_SetDownloadOptions
+if gravity_GetBlocklistUrls; then
+  gravity_SetDownloadOptions
+  # Build preEventHorizon
+  gravity_ConsolidateDownloadedBlocklists
+  gravity_SortAndFilterConsolidatedList
 fi
-# Build preEventHorizon
-gravity_ConsolidateDownloadedBlocklists
-gravity_SortAndFilterConsolidatedList
 
 # Create local.list
 gravity_generateLocalList

From e04dc9d2bb20400912067d1ff39c6dcc2873f76c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:44:42 +0200
Subject: [PATCH 44/66] Remove sqlitekey variable as the first column of the
 regex table is now called domain as well

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 11 ++++-------
 1 file changed, 4 insertions(+), 7 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index cd9dce52..dcd6f580 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -23,7 +23,6 @@ domList=()
 
 listType=""
 listname=""
-sqlitekey=""
 
 colfile="/opt/pihole/COL_TABLE"
 source ${colfile}
@@ -92,11 +91,9 @@ ProcessDomainList() {
     if [[ "${listType}" == "regex" ]]; then
         # Regex filter list
         listname="regex filters"
-        sqlitekey="filter"
     else
         # Whitelist / Blacklist
         listname="${listType}"
-        sqlitekey="domain"
     fi
 
     for dom in "${domList[@]}"; do
@@ -125,7 +122,7 @@ AddDomain() {
     list="$2"
 
     # Is the domain in the list we want to add it to?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")"
 
     if [[ "${num}" -ne 0 ]]; then
       if [[ "${verbose}" == true ]]; then
@@ -144,7 +141,7 @@ AddDomain() {
     timestamp="$(date --utc +'%s')"
     # Insert only domain and date_added here. The enabled fields will be filled
     # with its default value is true.
-    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (${sqlitekey},date_added) VALUES (\"${domain}\",${timestamp});"
+    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain,date_added) VALUES (\"${domain}\",${timestamp});"
 }
 
 RemoveDomain() {
@@ -154,7 +151,7 @@ RemoveDomain() {
     list="$2"
 
     # Is the domain in the list we want to remove it from?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE ${sqlitekey} = \"${domain}\";")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")"
 
     if [[ "${num}" -eq 0 ]]; then
       if [[ "${verbose}" == true ]]; then
@@ -171,7 +168,7 @@ RemoveDomain() {
     # Remove it from the current list
     local timestamp
     timestamp="$(date --utc +'%s')"
-    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE ${sqlitekey} = \"${domain}\";"
+    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";"
 }
 
 Displaylist() {

From 8524aecfed5eb4e3984fe938d2049c47976741c7 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 17:56:11 +0200
Subject: [PATCH 45/66] Don't set gravity database permissions

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 3543ac15..98db04b8 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -20,9 +20,6 @@ source "${regexconverter}"
 
 basename="pihole"
 PIHOLE_COMMAND="/usr/local/bin/${basename}"
-DATABASE_USER="pihole"
-DATABASE_GROUP="www-data"
-DATABASE_PERMISSIONS="0660"
 
 piholeDir="/etc/${basename}"
 
@@ -85,8 +82,6 @@ fi
 # Generate new sqlite3 file from schema template
 generate_gravity_database() {
   sqlite3 "${gravityDBfile}" < "${gravityDBschema}"
-  chown ${DATABASE_USER}:${DATABASE_GROUP} "${piholeDir}" "${gravityDBfile}"
-  chmod ${DATABASE_PERMISSIONS} "${piholeDir}" "${gravityDBfile}"
 }
 
 # Import domains from file and store them in the specified database table

From 3fe43ce1d923f825b73a4f9c367b3a7f1bf344c9 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 18:15:53 +0200
Subject: [PATCH 46/66] Rely on default value (database provided) for
 date_added when adding a new domain

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index dcd6f580..053def02 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -133,15 +133,12 @@ AddDomain() {
 
     # Domain not found in the table, add it!
     if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} Adding ${1} to ${listname}..."
+        echo -e "  ${INFO} Adding ${1} to the ${listname}..."
     fi
     reload=true
-    # Add it to the list we want to add it to
-    local timestamp
-    timestamp="$(date --utc +'%s')"
-    # Insert only domain and date_added here. The enabled fields will be filled
-    # with its default value is true.
-    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain,date_added) VALUES (\"${domain}\",${timestamp});"
+    # Insert only the domain here. The enabled and date_added fields will be filled
+    # with their default values (enabled = true, date_added = current timestamp)
+    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES (\"${domain}\");"
 }
 
 RemoveDomain() {
@@ -162,12 +159,10 @@ RemoveDomain() {
 
     # Domain found in the table, remove it!
     if [[ "${verbose}" == true ]]; then
-        echo -e "  ${INFO} Removing ${1} from ${listname}..."
+        echo -e "  ${INFO} Removing ${1} from the ${listname}..."
     fi
     reload=true
     # Remove it from the current list
-    local timestamp
-    timestamp="$(date --utc +'%s')"
     sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";"
 }
 
@@ -195,7 +190,6 @@ Displaylist() {
             enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")"
             dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")"
 
-            echo "${dateadded}"
             # Translate boolean status into human readable string
             if [[ "${enabled}" -eq 1 ]]; then
                 status="enabled"

From b4ae142149f60150d583ec49876dfdef7dd3d259 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Fri, 26 Apr 2019 20:54:01 +0200
Subject: [PATCH 47/66] Remove printf escaping (we will realize it differently)
 and ensure we're using single quotes for strings (although double quotes are
 possible, too)

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 053def02..374f888b 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -118,11 +118,11 @@ ProcessDomainList() {
 AddDomain() {
     local domain list num
     # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
-    domain="$(printf "%q" "$1")"
+    domain="$1"
     list="$2"
 
     # Is the domain in the list we want to add it to?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
 
     if [[ "${num}" -ne 0 ]]; then
       if [[ "${verbose}" == true ]]; then
@@ -138,17 +138,17 @@ AddDomain() {
     reload=true
     # Insert only the domain here. The enabled and date_added fields will be filled
     # with their default values (enabled = true, date_added = current timestamp)
-    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES (\"${domain}\");"
+    sqlite3 "${gravityDBfile}" "INSERT INTO ${list} (domain) VALUES ('${domain}');"
 }
 
 RemoveDomain() {
     local domain list num
     # Use printf to escape domain. %q prints the argument in a form that can be reused as shell input
-    domain="$(printf "%q" "$1")"
+    domain="$1"
     list="$2"
 
     # Is the domain in the list we want to remove it from?
-    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = \"${domain}\";")"
+    num="$(sqlite3 "${gravityDBfile}" "SELECT COUNT(*) FROM ${list} WHERE domain = '${domain}';")"
 
     if [[ "${num}" -eq 0 ]]; then
       if [[ "${verbose}" == true ]]; then
@@ -163,7 +163,7 @@ RemoveDomain() {
     fi
     reload=true
     # Remove it from the current list
-    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = \"${domain}\";"
+    sqlite3 "${gravityDBfile}" "DELETE FROM ${list} WHERE domain = '${domain}';"
 }
 
 Displaylist() {

From 126f7101ffaa56b42891d9db1bed12e270222bd0 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 28 Apr 2019 14:42:52 +0200
Subject: [PATCH 48/66] Add (optional) date_modified field and use vw_whitelist
 view in vw_blacklist

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh          |  6 +++---
 advanced/Templates/gravity.db.sql | 17 ++++++++---------
 2 files changed, 11 insertions(+), 12 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 374f888b..73cb2878 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -186,9 +186,9 @@ Displaylist() {
             num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")"
 
             # Extract domain and enabled status based on the obtained number of pipe characters
-            domain="$(cut -d'|' -f"-$((num_pipes-2))" <<< "${line}")"
-            enabled="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")"
-            dateadded="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")"
+            domain="$(cut -d'|' -f"-$((num_pipes-3))" <<< "${line}")"
+            enabled="$(cut -d'|' -f"$((num_pipes-2))" <<< "${line}")"
+            dateadded="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")"
 
             # Translate boolean status into human readable string
             if [[ "${enabled}" -eq 1 ]]; then
diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index ebe97955..31fbfd9f 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,7 +1,7 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
-CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
-CREATE TABLE adlists   (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT);
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT);
+CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT);
+CREATE TABLE adlists   (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
 
@@ -11,15 +11,14 @@ CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
 FROM gravity a
 WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
 
-CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
-FROM blacklist a
-WHERE a.enabled == 1 AND
-      a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
-
 CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
 FROM whitelist a
 WHERE a.enabled == 1;
 
+CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
+FROM blacklist a
+WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist;
+
 CREATE VIEW vw_regex AS SELECT DISTINCT a.domain
 FROM regex a
 WHERE a.enabled == 1;

From efeba594ae791000df0461b00e82e02251892038 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 28 Apr 2019 21:39:06 +0200
Subject: [PATCH 49/66] Add support for manipulating adlists from the CLI

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/webpage.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/advanced/Scripts/webpage.sh b/advanced/Scripts/webpage.sh
index c6852896..6ac351bc 100755
--- a/advanced/Scripts/webpage.sh
+++ b/advanced/Scripts/webpage.sh
@@ -17,6 +17,8 @@ readonly FTLconf="/etc/pihole/pihole-FTL.conf"
 # 03 -> wildcards
 readonly dhcpstaticconfig="/etc/dnsmasq.d/04-pihole-static-dhcp.conf"
 
+readonly gravityDBfile="/etc/pihole/gravity.db"
+
 coltable="/opt/pihole/COL_TABLE"
 if [[ -f ${coltable} ]]; then
     source ${coltable}
@@ -385,19 +387,17 @@ SetWebUILayout() {
 }
 
 CustomizeAdLists() {
-    list="/etc/pihole/adlists.list"
+    local address
+    address="${args[3]}"
 
     if [[ "${args[2]}" == "enable" ]]; then
-        sed -i "\\@${args[3]}@s/^#http/http/g" "${list}"
+        sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 1 WHERE address = '${address}'"
     elif [[ "${args[2]}" == "disable" ]]; then
-        sed -i "\\@${args[3]}@s/^http/#http/g" "${list}"
+        sqlite3 "${gravityDBfile}" "UPDATE adlists SET enabled = 0 WHERE address = '${address}'"
     elif [[ "${args[2]}" == "add" ]]; then
-        if [[ $(grep -c "^${args[3]}$" "${list}") -eq 0 ]] ; then
-            echo "${args[3]}" >> ${list}
-        fi
+        sqlite3 "${gravityDBfile}" "INSERT OR IGNORE INTO adlists (address) VALUES ('${address}')"
     elif [[ "${args[2]}" == "del" ]]; then
-        var=$(echo "${args[3]}" | sed 's/\//\\\//g')
-        sed -i "/${var}/Id" "${list}"
+        sqlite3 "${gravityDBfile}" "DELETE FROM adlists WHERE address = '${address}'"
     else
         echo "Not permitted"
         return 1

From 4946b5907bf4d45fda807558e2e0234ed70db784 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Sun, 28 Apr 2019 22:15:47 +0200
Subject: [PATCH 50/66] Move migrated files to backup directory instead of
 deleting them. This has the benefit that gravity.db can be recreated at any
 time by: 1. deleting gravity.db, 2. copying all files from the
 migration_backup directory back to /etc/pihole, 3. running pihole -g again.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index a3456fb4..6e238e68 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -89,6 +89,8 @@ database_table_from_file() {
   # Define locals
   local table="${1}"
   local source="${2}"
+  local backup_path="${piholeDir}/migration_backup"
+  local backup_file="${backup_path}/$(basename "${2}")"
 
   # Create database file if not present
   if [ ! -e "${gravityDBfile}" ]; then
@@ -140,8 +142,9 @@ database_table_from_file() {
   rm "${tmpFile}" > /dev/null 2>&1 || \
       echo -e "  ${CROSS} Unable to remove ${tmpFile}"
 
-  # Delete source file
-  rm "${source}" 2> /dev/null || \
+  # Move source file to backup directory, create directory if not existing
+  mkdir -p "${backup_path}"
+  mv "${source}" "${backup_file}" 2> /dev/null || \
       echo -e "  ${CROSS} Unable to remove ${source}"
 }
 
@@ -702,7 +705,6 @@ fi
 
 # Gravity downloads blocklists next
 gravity_CheckDNSResolutionAvailable
-gravity_GetBlocklistUrls
 if gravity_GetBlocklistUrls; then
   gravity_SetDownloadOptions
   # Build preEventHorizon

From a932209143af5eb9e3acd4991a5bb203375ba461 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 29 Apr 2019 17:19:02 +0200
Subject: [PATCH 51/66] Set default for date_modified to be the current time

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 8 ++++----
 gravity.sh                        | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 31fbfd9f..c1066f33 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,7 +1,7 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT);
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT);
-CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT);
-CREATE TABLE adlists   (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER, comment TEXT);
+CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL,  enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
+CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL,  enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
+CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL,  enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
+CREATE TABLE adlists   (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
 
diff --git a/gravity.sh b/gravity.sh
index 6e238e68..331b632d 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -122,7 +122,7 @@ database_table_from_file() {
     do
       # Only add non-empty lines
       if [[ ! -z "${domain}" ]]; then
-        echo "\"${domain}\",1,${timestamp}" >> "${tmpFile}"
+        echo "\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}"
       fi
     done
     inputfile="${tmpFile}"
@@ -145,7 +145,7 @@ database_table_from_file() {
   # Move source file to backup directory, create directory if not existing
   mkdir -p "${backup_path}"
   mv "${source}" "${backup_file}" 2> /dev/null || \
-      echo -e "  ${CROSS} Unable to remove ${source}"
+      echo -e "  ${CROSS} Unable to backup ${source} to ${backup_path}"
 }
 
 # Migrate pre-v5.0 list files to database-based Pi-hole versions

From 2180531a84516f6720985f034fc8d0cdb589e6a7 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Mon, 29 Apr 2019 19:43:35 +0200
Subject: [PATCH 52/66] Add trigger for automatic updates of the date_modified
 column and improve formating in the gravity.db template

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 37 ++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index c1066f33..dc0f23ed 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -8,21 +8,38 @@ CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
 INSERT INTO info VALUES("version","1");
 
 CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
-FROM gravity a
-WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
+    FROM gravity a
+    WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
 
 CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
-FROM whitelist a
-WHERE a.enabled == 1;
+    FROM whitelist a
+    WHERE a.enabled == 1;
+CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
+    BEGIN
+      UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+     END;
 
 CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
-FROM blacklist a
-WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist;
+    FROM blacklist a
+    WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist;
+CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
+    BEGIN
+      UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+     END;
 
 CREATE VIEW vw_regex AS SELECT DISTINCT a.domain
-FROM regex a
-WHERE a.enabled == 1;
+    FROM regex a
+    WHERE a.enabled == 1;
+CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
+    BEGIN
+      UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
+     END;
 
 CREATE VIEW vw_adlists AS SELECT DISTINCT a.address
-FROM adlists a
-WHERE a.enabled == 1;
+    FROM adlists a
+    WHERE a.enabled == 1;
+CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists
+    BEGIN
+      UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
+     END;
+

From 487718512dabdbdb4e3858eeed98c711f3de6fa7 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 16:37:23 +0200
Subject: [PATCH 53/66] Remove DISTINCT attribute from views. They are not
 needed as the corresponding data is obtained from fields with UNIQUE
 attribute so duplications are already prevented at domain insertion. The
 elimination of the duplication check on imort through the views slightly
 reduces the load during import (I observed about 8 percent time saved on my
 NanoPi NEO).

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index dc0f23ed..dac0a50c 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -7,11 +7,11 @@ CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
 
 INSERT INTO info VALUES("version","1");
 
-CREATE VIEW vw_gravity AS SELECT DISTINCT a.domain
+CREATE VIEW vw_gravity AS SELECT a.domain
     FROM gravity a
     WHERE a.domain NOT IN (SELECT domain from whitelist WHERE enabled == 1);
 
-CREATE VIEW vw_whitelist AS SELECT DISTINCT a.domain
+CREATE VIEW vw_whitelist AS SELECT a.domain
     FROM whitelist a
     WHERE a.enabled == 1;
 CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
@@ -19,7 +19,7 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
       UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
      END;
 
-CREATE VIEW vw_blacklist AS SELECT DISTINCT a.domain
+CREATE VIEW vw_blacklist AS SELECT a.domain
     FROM blacklist a
     WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist;
 CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
@@ -27,7 +27,7 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
       UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
      END;
 
-CREATE VIEW vw_regex AS SELECT DISTINCT a.domain
+CREATE VIEW vw_regex AS SELECT a.domain
     FROM regex a
     WHERE a.enabled == 1;
 CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
@@ -35,7 +35,7 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
       UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
      END;
 
-CREATE VIEW vw_adlists AS SELECT DISTINCT a.address
+CREATE VIEW vw_adlists AS SELECT a.address
     FROM adlists a
     WHERE a.enabled == 1;
 CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists

From d5781fb1105d0c0ca1eda09c8e92f19ac135ff0c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 16:38:41 +0200
Subject: [PATCH 54/66] Fix extra space in front on END statement in CREATE
 TRIGGER commands

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index dac0a50c..0960018b 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -17,7 +17,7 @@ CREATE VIEW vw_whitelist AS SELECT a.domain
 CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
     BEGIN
       UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
-     END;
+    END;
 
 CREATE VIEW vw_blacklist AS SELECT a.domain
     FROM blacklist a
@@ -25,7 +25,7 @@ CREATE VIEW vw_blacklist AS SELECT a.domain
 CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
     BEGIN
       UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
-     END;
+    END;
 
 CREATE VIEW vw_regex AS SELECT a.domain
     FROM regex a
@@ -33,7 +33,7 @@ CREATE VIEW vw_regex AS SELECT a.domain
 CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
     BEGIN
       UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
-     END;
+    END;
 
 CREATE VIEW vw_adlists AS SELECT a.address
     FROM adlists a
@@ -41,5 +41,5 @@ CREATE VIEW vw_adlists AS SELECT a.address
 CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists
     BEGIN
       UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;
-     END;
+    END;
 

From bc9b62363895b8b6483e0f518ba46607b75f9909 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 16:56:16 +0200
Subject: [PATCH 55/66] Modify query.sh to use the gravity database

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 16 ++++------------
 1 file changed, 4 insertions(+), 12 deletions(-)
 mode change 100644 => 100755 advanced/Scripts/query.sh

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
old mode 100644
new mode 100755
index 69a3c7a4..8eb7c404
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -11,7 +11,7 @@
 
 # Globals
 piholeDir="/etc/pihole"
-adListsList="$piholeDir/adlists.list"
+gravityDBfile="${piholeDir}/gravity.db"
 wildcardlist="/etc/dnsmasq.d/03-pihole-wildcard.conf"
 options="$*"
 adlist=""
@@ -73,11 +73,6 @@ Options:
   exit 0
 fi
 
-if [[ ! -e "$adListsList" ]]; then
-    echo -e "${COL_LIGHT_RED}The file $adListsList was not found${COL_NC}"
-    exit 1
-fi
-
 # Handle valid options
 if [[ "${options}" == *"-bp"* ]]; then
     exact="exact"; blockpage=true
@@ -185,12 +180,9 @@ if [[ -z "${exact}" ]]; then
 fi
 
 # Get adlist file content as array
-if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
-    for adlistUrl in $(< "${adListsList}"); do
-        if [[ "${adlistUrl:0:4}" =~ (http|www.) ]]; then
-            adlists+=("${adlistUrl}")
-        fi
-    done
+if [[ -n "${blockpage}" ]]; then
+    # Retrieve source URLs from gravity database
+    mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)"
 fi
 
 # Print "Exact matches for" title

From b372f808ddf1ff0fed03b1f314c0a85e487766cd Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 17:06:14 +0200
Subject: [PATCH 56/66] Fix lint errors

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/query.sh | 2 +-
 gravity.sh                | 9 +++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/advanced/Scripts/query.sh b/advanced/Scripts/query.sh
index 8eb7c404..a4ac895b 100755
--- a/advanced/Scripts/query.sh
+++ b/advanced/Scripts/query.sh
@@ -180,7 +180,7 @@ if [[ -z "${exact}" ]]; then
 fi
 
 # Get adlist file content as array
-if [[ -n "${blockpage}" ]]; then
+if [[ -n "${adlist}" ]] || [[ -n "${blockpage}" ]]; then
     # Retrieve source URLs from gravity database
     mapfile -t adlists <<< "$(sqlite3 "${gravityDBfile}" "SELECT address FROM vw_adlists;" 2> /dev/null)"
 fi
diff --git a/gravity.sh b/gravity.sh
index 331b632d..9f4065e7 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -87,10 +87,11 @@ generate_gravity_database() {
 # Import domains from file and store them in the specified database table
 database_table_from_file() {
   # Define locals
-  local table="${1}"
-  local source="${2}"
-  local backup_path="${piholeDir}/migration_backup"
-  local backup_file="${backup_path}/$(basename "${2}")"
+  local table source backup_path backup_file
+  table="${1}"
+  source="${2}"
+  backup_path="${piholeDir}/migration_backup"
+  backup_file="${backup_path}/$(basename "${2}")"
 
   # Create database file if not present
   if [ ! -e "${gravityDBfile}" ]; then

From 983117d788bd32ef1521a5c2da3088a2039e23bf Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 17:31:34 +0200
Subject: [PATCH 57/66] Add id (primary key, autoincrement) field to lists
 tables

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 28 ++++++++++++++++++++++++----
 gravity.sh                        |  6 +++++-
 2 files changed, 29 insertions(+), 5 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 0960018b..65bd7448 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,7 +1,27 @@
-CREATE TABLE whitelist (domain TEXT UNIQUE NOT NULL,  enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
-CREATE TABLE blacklist (domain TEXT UNIQUE NOT NULL,  enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
-CREATE TABLE regex     (domain TEXT UNIQUE NOT NULL,  enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
-CREATE TABLE adlists   (address TEXT UNIQUE NOT NULL, enabled BOOLEAN NOT NULL DEFAULT 1, date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)), comment TEXT);
+CREATE TABLE whitelist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+                        domain TEXT UNIQUE NOT NULL,
+                        enabled BOOLEAN NOT NULL DEFAULT 1,
+                        date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+                        date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+                        comment TEXT);
+CREATE TABLE blacklist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+                        domain TEXT UNIQUE NOT NULL,
+                        enabled BOOLEAN NOT NULL DEFAULT 1,
+                        date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+                        date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+                        comment TEXT);
+CREATE TABLE regex     (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+                        domain TEXT UNIQUE NOT NULL,
+                        enabled BOOLEAN NOT NULL DEFAULT 1,
+                        date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+                        date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+                        comment TEXT);
+CREATE TABLE adlists   (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+                        address TEXT UNIQUE NOT NULL,
+                        enabled BOOLEAN NOT NULL DEFAULT 1,
+                        date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+                        date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+                        comment TEXT);
 CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
 CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
 
diff --git a/gravity.sh b/gravity.sh
index 9f4065e7..c1eb0720 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -118,12 +118,16 @@ database_table_from_file() {
     inputfile="${source}"
   else
     # Apply format for white-, blacklist, regex, and adlists tables
+    local rowid
+    declare -i rowid
+    rowid=1
     # Read file line by line
     grep -v '^ *#' < "${source}" | while IFS= read -r domain
     do
       # Only add non-empty lines
       if [[ ! -z "${domain}" ]]; then
-        echo "\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}"
+        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}"
+        rowid+=1
       fi
     done
     inputfile="${tmpFile}"

From 5e26e8245ba4f849577bb1ba232389ba00e59247 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 17:50:36 +0200
Subject: [PATCH 58/66] Order results of queries to list views by id key

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 65bd7448..d1651619 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -33,7 +33,9 @@ CREATE VIEW vw_gravity AS SELECT a.domain
 
 CREATE VIEW vw_whitelist AS SELECT a.domain
     FROM whitelist a
-    WHERE a.enabled == 1;
+    WHERE a.enabled == 1
+    ORDER BY a.id;
+
 CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
     BEGIN
       UPDATE whitelist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
@@ -41,7 +43,9 @@ CREATE TRIGGER tr_whitelist_update AFTER UPDATE ON whitelist
 
 CREATE VIEW vw_blacklist AS SELECT a.domain
     FROM blacklist a
-    WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist;
+    WHERE a.enabled == 1 AND a.domain NOT IN vw_whitelist
+    ORDER BY a.id;
+
 CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
     BEGIN
       UPDATE blacklist SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
@@ -49,7 +53,9 @@ CREATE TRIGGER tr_blacklist_update AFTER UPDATE ON blacklist
 
 CREATE VIEW vw_regex AS SELECT a.domain
     FROM regex a
-    WHERE a.enabled == 1;
+    WHERE a.enabled == 1
+    ORDER BY a.id;
+
 CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
     BEGIN
       UPDATE regex SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE domain = NEW.domain;
@@ -57,7 +63,9 @@ CREATE TRIGGER tr_regex_update AFTER UPDATE ON regex
 
 CREATE VIEW vw_adlists AS SELECT a.address
     FROM adlists a
-    WHERE a.enabled == 1;
+    WHERE a.enabled == 1
+    ORDER BY a.id;
+
 CREATE TRIGGER tr_adlists_update AFTER UPDATE ON adlists
     BEGIN
       UPDATE adlists SET date_modified = (cast(strftime('%s', 'now') as int)) WHERE address = NEW.address;

From 540c57f755c0e6255c2416738478491494f692b4 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 17:51:25 +0200
Subject: [PATCH 59/66] Add comment for automigrated list entries

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/gravity.sh b/gravity.sh
index c1eb0720..afd01aad 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -126,7 +126,7 @@ database_table_from_file() {
     do
       # Only add non-empty lines
       if [[ ! -z "${domain}" ]]; then
-        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp}" >> "${tmpFile}"
+        echo "${rowid},\"${domain}\",1,${timestamp},${timestamp},\"Migrated from ${source}\"" >> "${tmpFile}"
         rowid+=1
       fi
     done

From 6c1e3a17f7534bb1ed7ea448401d11adc4b66406 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 17:54:29 +0200
Subject: [PATCH 60/66] Remove NOT NULL constraint on list ids as this is
 implied by PRIMARY KEY (even though most, if not all, examples on the web
 still use both attributes at the same time)

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index d1651619..4e78e026 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,22 +1,22 @@
-CREATE TABLE whitelist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+CREATE TABLE whitelist (id INTEGER PRIMARY KEY AUTOINCREMENT,
                         domain TEXT UNIQUE NOT NULL,
                         enabled BOOLEAN NOT NULL DEFAULT 1,
                         date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
                         date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
                         comment TEXT);
-CREATE TABLE blacklist (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+CREATE TABLE blacklist (id INTEGER PRIMARY KEY AUTOINCREMENT,
                         domain TEXT UNIQUE NOT NULL,
                         enabled BOOLEAN NOT NULL DEFAULT 1,
                         date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
                         date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
                         comment TEXT);
-CREATE TABLE regex     (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+CREATE TABLE regex     (id INTEGER PRIMARY KEY AUTOINCREMENT,
                         domain TEXT UNIQUE NOT NULL,
                         enabled BOOLEAN NOT NULL DEFAULT 1,
                         date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
                         date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
                         comment TEXT);
-CREATE TABLE adlists   (id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+CREATE TABLE adlists   (id INTEGER PRIMARY KEY AUTOINCREMENT,
                         address TEXT UNIQUE NOT NULL,
                         enabled BOOLEAN NOT NULL DEFAULT 1,
                         date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),

From 106f079afc24dd4f43699ca0a9f494f5e72dc850 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 17:57:23 +0200
Subject: [PATCH 61/66] Improve indentation (single tab) in SQLite template

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 71 ++++++++++++++++++++-----------
 1 file changed, 45 insertions(+), 26 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index 4e78e026..a8d48608 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -1,29 +1,48 @@
-CREATE TABLE whitelist (id INTEGER PRIMARY KEY AUTOINCREMENT,
-                        domain TEXT UNIQUE NOT NULL,
-                        enabled BOOLEAN NOT NULL DEFAULT 1,
-                        date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-                        date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-                        comment TEXT);
-CREATE TABLE blacklist (id INTEGER PRIMARY KEY AUTOINCREMENT,
-                        domain TEXT UNIQUE NOT NULL,
-                        enabled BOOLEAN NOT NULL DEFAULT 1,
-                        date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-                        date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-                        comment TEXT);
-CREATE TABLE regex     (id INTEGER PRIMARY KEY AUTOINCREMENT,
-                        domain TEXT UNIQUE NOT NULL,
-                        enabled BOOLEAN NOT NULL DEFAULT 1,
-                        date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-                        date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-                        comment TEXT);
-CREATE TABLE adlists   (id INTEGER PRIMARY KEY AUTOINCREMENT,
-                        address TEXT UNIQUE NOT NULL,
-                        enabled BOOLEAN NOT NULL DEFAULT 1,
-                        date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-                        date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
-                        comment TEXT);
-CREATE TABLE gravity   (domain TEXT UNIQUE NOT NULL);
-CREATE TABLE info      (property TEXT NOT NULL, value TEXT NOT NULL);
+CREATE TABLE whitelist
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+CREATE TABLE blacklist
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+CREATE TABLE regex
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	domain TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+CREATE TABLE adlists
+(
+	id INTEGER PRIMARY KEY AUTOINCREMENT,
+	address TEXT UNIQUE NOT NULL,
+	enabled BOOLEAN NOT NULL DEFAULT 1,
+	date_added INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	date_modified INTEGER NOT NULL DEFAULT (cast(strftime('%s', 'now') as int)),
+	comment TEXT
+);
+CREATE TABLE gravity
+(
+	domain TEXT UNIQUE NOT NULL
+);
+CREATE TABLE info
+(
+	property TEXT NOT NULL,
+	value TEXT NOT NULL
+);
 
 INSERT INTO info VALUES("version","1");
 

From 2f698904e406fc978cb00bf4759aebbc22b14479 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 18:03:35 +0200
Subject: [PATCH 62/66] Use PRIMARY KEY instead of NOT NULL also in gravity and
 info tables

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index a8d48608..fd6d2b28 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -36,11 +36,11 @@ CREATE TABLE adlists
 );
 CREATE TABLE gravity
 (
-	domain TEXT UNIQUE NOT NULL
+	domain TEXT UNIQUE PRIMARY KEY
 );
 CREATE TABLE info
 (
-	property TEXT NOT NULL,
+	property TEXT PRIMARY KEY,
 	value TEXT NOT NULL
 );
 

From 008e88b84be1f818fe6518d954ddc8b4a7aca63c Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 18:04:31 +0200
Subject: [PATCH 63/66] Only migrate legacy list files when we create
 gravity.db in the same gravity run

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 51 ++++++++++++++++++++++++++-------------------------
 1 file changed, 26 insertions(+), 25 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index afd01aad..02f41f86 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -93,12 +93,6 @@ database_table_from_file() {
   backup_path="${piholeDir}/migration_backup"
   backup_file="${backup_path}/$(basename "${2}")"
 
-  # Create database file if not present
-  if [ ! -e "${gravityDBfile}" ]; then
-    echo -e "  ${INFO} Creating new gravity database"
-    generate_gravity_database
-  fi
-
   # Truncate table
   output=$( { sqlite3 "${gravityDBfile}" <<< "DELETE FROM ${table};"; } 2>&1 )
   status="$?"
@@ -155,25 +149,32 @@ database_table_from_file() {
 
 # Migrate pre-v5.0 list files to database-based Pi-hole versions
 migrate_to_database() {
-  if [[ -e "${adListFile}" ]]; then
-    # Store adlists domains in database
-    echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${adListFile} into database"
-    database_table_from_file "adlists" "${adListFile}"
-  fi
-  if [[ -e "${blacklistFile}" ]]; then
-    # Store blacklisted domains in database
-    echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${blacklistFile} into database"
-    database_table_from_file "blacklist" "${blacklistFile}"
-  fi
-  if [[ -e "${whitelistFile}" ]]; then
-    # Store whitelisted domains in database
-    echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${whitelistFile} into database"
-    database_table_from_file "whitelist" "${whitelistFile}"
-  fi
-  if [[ -e "${regexFile}" ]]; then
-    # Store regex domains in database
-    echo -e "  ${INFO} Pi-hole upgrade: Moving content of ${regexFile} into database"
-    database_table_from_file "regex" "${regexFile}"
+  # Create database file if not present
+  if [ ! -e "${gravityDBfile}" ]; then
+    echo -e "  ${INFO} Creating new gravity database"
+    generate_gravity_database
+
+    # Migrate list files to new database
+    if [[ -e "${adListFile}" ]]; then
+      # Store adlists domains in database
+      echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
+      database_table_from_file "adlists" "${adListFile}"
+    fi
+    if [[ -e "${blacklistFile}" ]]; then
+      # Store blacklisted domains in database
+      echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
+      database_table_from_file "blacklist" "${blacklistFile}"
+    fi
+    if [[ -e "${whitelistFile}" ]]; then
+      # Store whitelisted domains in database
+      echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
+      database_table_from_file "whitelist" "${whitelistFile}"
+    fi
+    if [[ -e "${regexFile}" ]]; then
+      # Store regex domains in database
+      echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
+      database_table_from_file "regex" "${regexFile}"
+    fi
   fi
 }
 

From 0cfecd31d3501c5a760207bd1780c855a90be2f9 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 21:12:22 +0200
Subject: [PATCH 64/66] Invert logic to reduce nesting

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 gravity.sh | 52 +++++++++++++++++++++++++++-------------------------
 1 file changed, 27 insertions(+), 25 deletions(-)

diff --git a/gravity.sh b/gravity.sh
index 02f41f86..76d42f32 100755
--- a/gravity.sh
+++ b/gravity.sh
@@ -149,32 +149,34 @@ database_table_from_file() {
 
 # Migrate pre-v5.0 list files to database-based Pi-hole versions
 migrate_to_database() {
-  # Create database file if not present
-  if [ ! -e "${gravityDBfile}" ]; then
-    echo -e "  ${INFO} Creating new gravity database"
-    generate_gravity_database
+  # Create database file only if not present
+  if [ -e "${gravityDBfile}" ]; then
+    return 0
+  fi
 
-    # Migrate list files to new database
-    if [[ -e "${adListFile}" ]]; then
-      # Store adlists domains in database
-      echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
-      database_table_from_file "adlists" "${adListFile}"
-    fi
-    if [[ -e "${blacklistFile}" ]]; then
-      # Store blacklisted domains in database
-      echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
-      database_table_from_file "blacklist" "${blacklistFile}"
-    fi
-    if [[ -e "${whitelistFile}" ]]; then
-      # Store whitelisted domains in database
-      echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
-      database_table_from_file "whitelist" "${whitelistFile}"
-    fi
-    if [[ -e "${regexFile}" ]]; then
-      # Store regex domains in database
-      echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
-      database_table_from_file "regex" "${regexFile}"
-    fi
+  echo -e "  ${INFO} Creating new gravity database"
+  generate_gravity_database
+
+  # Migrate list files to new database
+  if [[ -e "${adListFile}" ]]; then
+    # Store adlists domains in database
+    echo -e "  ${INFO} Migrating content of ${adListFile} into new database"
+    database_table_from_file "adlists" "${adListFile}"
+  fi
+  if [[ -e "${blacklistFile}" ]]; then
+    # Store blacklisted domains in database
+    echo -e "  ${INFO} Migrating content of ${blacklistFile} into new database"
+    database_table_from_file "blacklist" "${blacklistFile}"
+  fi
+  if [[ -e "${whitelistFile}" ]]; then
+    # Store whitelisted domains in database
+    echo -e "  ${INFO} Migrating content of ${whitelistFile} into new database"
+    database_table_from_file "whitelist" "${whitelistFile}"
+  fi
+  if [[ -e "${regexFile}" ]]; then
+    # Store regex domains in database
+    echo -e "  ${INFO} Migrating content of ${regexFile} into new database"
+    database_table_from_file "regex" "${regexFile}"
   fi
 }
 

From d46ff1d2d79dbd2cb6f2ee2f919e793a6371757b Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Wed, 1 May 2019 22:08:30 +0200
Subject: [PATCH 65/66] Remove UNIQUE on gravity table as the only column is
 already PRIMARY KEY. Primary keys must contain UNIQUE values, and cannot
 contain NULL values

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Templates/gravity.db.sql | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/advanced/Templates/gravity.db.sql b/advanced/Templates/gravity.db.sql
index fd6d2b28..372a4a29 100644
--- a/advanced/Templates/gravity.db.sql
+++ b/advanced/Templates/gravity.db.sql
@@ -36,7 +36,7 @@ CREATE TABLE adlists
 );
 CREATE TABLE gravity
 (
-	domain TEXT UNIQUE PRIMARY KEY
+	domain TEXT PRIMARY KEY
 );
 CREATE TABLE info
 (

From 0bc112ce52e717467835f052353c6c8283e8db20 Mon Sep 17 00:00:00 2001
From: DL6ER <dl6er@dl6er.de>
Date: Thu, 2 May 2019 17:11:39 +0200
Subject: [PATCH 66/66] Query only those columns we actually use when showing
 domains on the CLI. Show date of last modification instead of addition date
 for domains.

Signed-off-by: DL6ER <dl6er@dl6er.de>
---
 advanced/Scripts/list.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/advanced/Scripts/list.sh b/advanced/Scripts/list.sh
index 73cb2878..fa81348b 100755
--- a/advanced/Scripts/list.sh
+++ b/advanced/Scripts/list.sh
@@ -170,7 +170,7 @@ Displaylist() {
     local list listname count num_pipes domain enabled status nicedate
 
     listname="${listType}"
-    data="$(sqlite3 "${gravityDBfile}" "SELECT * FROM ${listType};" 2> /dev/null)"
+    data="$(sqlite3 "${gravityDBfile}" "SELECT domain,enabled,date_modified FROM ${listType};" 2> /dev/null)"
 
     if [[ -z $data ]]; then
         echo -e "Not showing empty ${listname}"
@@ -186,9 +186,9 @@ Displaylist() {
             num_pipes="$(grep -c "^" <<< "$(grep -o "|" <<< "${line}")")"
 
             # Extract domain and enabled status based on the obtained number of pipe characters
-            domain="$(cut -d'|' -f"-$((num_pipes-3))" <<< "${line}")"
-            enabled="$(cut -d'|' -f"$((num_pipes-2))" <<< "${line}")"
-            dateadded="$(cut -d'|' -f"$((num_pipes-1))" <<< "${line}")"
+            domain="$(cut -d'|' -f"-$((num_pipes-1))" <<< "${line}")"
+            enabled="$(cut -d'|' -f"$((num_pipes))" <<< "${line}")"
+            datemod="$(cut -d'|' -f"$((num_pipes+1))" <<< "${line}")"
 
             # Translate boolean status into human readable string
             if [[ "${enabled}" -eq 1 ]]; then
@@ -198,9 +198,9 @@ Displaylist() {
             fi
 
             # Get nice representation of numerical date stored in database
-            nicedate=$(date --rfc-2822 -d "@${dateadded}")
+            nicedate=$(date --rfc-2822 -d "@${datemod}")
 
-            echo "  ${count}: ${domain} (${status}, added ${nicedate})"
+            echo "  ${count}: ${domain} (${status}, last modified ${nicedate})"
             count=$((count+1))
         done <<< "${data}"
     fi