Move list parsing entirely into FTL

Signed-off-by: DL6ER <dl6er@dl6er.de>
This commit is contained in:
DL6ER 2025-03-22 09:18:02 +01:00
parent 3f5c00919b
commit dec670a6d6
No known key found for this signature in database
GPG Key ID: 00135ACBD90B28DD

View File

@ -823,11 +823,11 @@ gravity_DownloadBlocklistFromUrl() {
done="true"
# Check if $listCurlBuffer is a non-zero length file
elif [[ -s "${listCurlBuffer}" ]]; then
# Determine if blocklist is non-standard and parse as appropriate
gravity_ParseFileIntoDomains "${listCurlBuffer}" "${saveLocation}"
# Remove curl buffer file after its use
rm "${listCurlBuffer}"
# Compare lists if are they identical
# Move the downloaded list to the final location
mv "${listCurlBuffer}" "${saveLocation}"
# Ensure the file has the correct permissions
fix_owner_permissions "${saveLocation}"
# Compare lists if they are identical
compareLists "${adlistID}" "${saveLocation}"
# Add domains to database table file
pihole-FTL "${gravity_type}" parseList "${saveLocation}" "${gravityTEMPfile}" "${adlistID}"
@ -856,37 +856,6 @@ gravity_DownloadBlocklistFromUrl() {
fi
}
# Parse source files into domains format
gravity_ParseFileIntoDomains() {
local src="${1}" destination="${2}"
# Remove comments and print only the domain name
# Most of the lists downloaded are already in hosts file format but the spacing/formatting is not contiguous
# This helps with that and makes it easier to read
# It also helps with debugging so each stage of the script can be researched more in depth
# 1) Convert all characters to lowercase
tr '[:upper:]' '[:lower:]' <"${src}" >"${destination}"
# 2) Remove carriage returns
# 3) Remove lines starting with ! (ABP Comments)
# 4) Remove lines starting with [ (ABP Header)
# 5) Remove lines containing ABP extended CSS selectors ("##", "#$#", "#@#", "#?#") and Adguard JavaScript (#%#) preceded by a letter
# 6) Remove comments (text starting with "#", include possible spaces before the hash sign)
# 7) Remove leading tabs, spaces, etc. (Also removes leading IP addresses)
# 8) Remove empty lines
sed -i -r \
-e 's/\r$//' \
-e 's/\s*!.*//g' \
-e 's/\s*\[.*//g' \
-e '/[a-z]\#[$?@%]{0,3}\#/d' \
-e 's/\s*#.*//g' \
-e 's/^.*\s+//g' \
-e '/^$/d' "${destination}"
fix_owner_permissions "${destination}"
}
# Report number of entries in a table
gravity_Table_Count() {
local table="${1}"