diff --git a/blocklist-file-preparer.sh b/blocklist-file-preparer.sh index 9c4fe48b..c492d49e 100755 --- a/blocklist-file-preparer.sh +++ b/blocklist-file-preparer.sh @@ -13,19 +13,26 @@ EOF exit 43 fi -curl --silent --header 'accept: text/plain' https://big.oisd.nl > list.oisd.nl.abp -echo "Hosts from oisd big: $(wc -l list.oisd.nl.abp)" - +curl --silent --header 'accept: text/plain' https://raw.githubusercontent.com/icyflame/hosts/master/hosts_abp.txt > list.icyflame.abp +curl --silent --header 'accept: text/plain' https://big.oisd.nl > list.oisd.nl.abp & # This list is not offered in the ABP format. It blocks some domains which are already blocked by # the Big OISD list. However, I am not sure how many of the domains in the StevenBlack list are # blocked by OISD. I have not calculated this yet. So, for now, I will continue to merge this list # into the OISD list. -curl --silent --header 'accept: test/plain' https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts > list.stevenblack.hosts -rm -f list.stevenblack.hosts.cleaned +curl --silent --header 'accept: test/plain' https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts > list.stevenblack.hosts & + +wait + cat list.stevenblack.hosts | rg '^0.0.0.0' | sed 's#0.0.0.0 ##g' | sed 's#^#||#g' | sed 's#$#^#g' > list.stevenblack.abp + +echo "Hosts from icyflame: $(wc -l list.icyflame.abp)" +echo "Hosts from oisd big: $(wc -l list.oisd.nl.abp)" echo "Hosts from stevenblack: $(wc -l list.stevenblack.abp)" +cat list.icyflame.abp list.oisd.nl.abp list.stevenblack.abp > list.blocklist-in-full + +echo "Final hosts (with duplicates): $(wc -l list.blocklist-in-full)" +echo "Final hosts (without duplicates): $(sort -u list.blocklist-in-full | wc -l)" -cat list.oisd.nl.abp list.stevenblack.abp > list.blocklist-in-full mv list.blocklist-in-full $OUTFILE -rm -f list.oisd.nl.abp list.stevenblack.abp list.stevenblack.hosts +rm -f list.icyflame.abp list.oisd.nl.abp list.stevenblack.abp list.stevenblack.hosts