first commit

.gitignore

@@ -0,0 +1,10 @@
+zabov
+killfile
+killfile/*
+db/zabov.db
+binaries
+binaries/*
+build.sh
+.vscode
+.vscode/*
+

00.database.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/syndtr/goleveldb/leveldb"
+)
+
+//MyZabovKDB is the storage where we'll put domains to block
+var MyZabovKDB *leveldb.DB
+
+//MyZabovCDB is the storage where we'll put domains to cache
+var MyZabovCDB *leveldb.DB
+
+func init() {
+
+	var err error
+
+	os.RemoveAll("./db")
+
+	os.MkdirAll("./db", 0755)
+
+	MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil)
+	if err != nil {
+		fmt.Println("Cannot create Killfile db: ", err.Error())
+	} else {
+		fmt.Println("Killfile DB created")
+	}
+
+	MyZabovCDB, err = leveldb.OpenFile("./db/cache", nil)
+	if err != nil {
+		fmt.Println("Cannot create Cache db: ", err.Error())
+	} else {
+		fmt.Println("Cache DB created")
+	}
+
+}

00.memory.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"fmt"
+	"runtime"
+	"time"
+)
+
+func init() {
+
+	fmt.Println("Garbage Collector Thread Starting")
+
+	go memoryCleanerThread()
+
+}
+
+func memoryCleanerThread() {
+
+	for {
+		time.Sleep(10 * time.Minute)
+		fmt.Println("Time to clean memory...")
+		runtime.GC()
+		fmt.Println("Garbage Collection done.")
+	}
+
+}

01.conf.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"os"
+
+	"github.com/miekg/dns"
+)
+
+func init() {
+
+	//ZabovConf describes the Json we use for configuration
+	type ZabovConf struct {
+		Zabov struct {
+			Port          string `json:"port"`
+			Proto         string `json:"proto"`
+			Ipaddr        string `json:"ipaddr"`
+			Upstream      string `json:"upstream"`
+			Cachettl      int    `json:"cachettl"`
+			Killfilettl   int    `json:"killfilettl"`
+			Singlefilters string `json:"singlefilters"`
+			Doublefilters string `json:"doublefilters"`
+			Blackholeip   string `json:"blackholeip"`
+			Hostsfile     string `json:"hostsfile"`
+		} `json:"zabov"`
+	}
+
+	var MyConf ZabovConf
+
+	file, err := ioutil.ReadFile("config.json")
+
+	if err != nil {
+		log.Println("Cannot open config file", err.Error())
+		os.Exit(1)
+	}
+
+	err = json.Unmarshal([]byte(file), &MyConf)
+
+	if err != nil {
+		log.Println("Cannot marshal json: ", err.Error())
+		os.Exit(1)
+	}
+
+	// now we read configuration file
+	fmt.Println("Reading configuration file...")
+
+	ZabovPort := MyConf.Zabov.Port
+	ZabovType := MyConf.Zabov.Proto
+	ZabovAddr := MyConf.Zabov.Ipaddr
+	ZabovUpDNS = MyConf.Zabov.Upstream
+	ZabovSingleBL = MyConf.Zabov.Singlefilters
+	ZabovDoubleBL = MyConf.Zabov.Doublefilters
+	ZabovAddBL = MyConf.Zabov.Blackholeip
+	ZabovCacheTTL = MyConf.Zabov.Cachettl
+	ZabovKillTTL = MyConf.Zabov.Killfilettl
+	ZabovHostsFile = MyConf.Zabov.Hostsfile
+
+	zabovString := ZabovAddr + ":" + ZabovPort
+
+	MyDNS = new(dns.Server)
+	MyDNS.Addr = zabovString
+	MyDNS.Net = ZabovType
+
+	ZabovDNSArray = fileByLines(ZabovUpDNS)
+
+}

01.dnscheck.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+	"time"
+)
+
+//NetworkUp tells the system if the network is up or not
+var NetworkUp bool
+
+func checkNetworkUp() bool {
+	// RFC2606 test domain, should always work, unless internet is down.
+	_, err := http.Get("http://example.com")
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func checkNetworkUpThread() {
+
+	ticker := time.NewTicker(2 * time.Minute)
+
+	for range ticker.C {
+		NetworkUp = checkNetworkUp()
+	}
+
+}
+
+func init() {
+
+	fmt.Println("Network Checker starting....")
+
+	go checkNetworkUpThread()
+
+}

01.killfile.go

@@ -0,0 +1,79 @@
+package main
+
+import (
+	"fmt"
+	"strings"
+)
+
+var zabovKbucket = []byte("killfile")
+
+type killfileItem struct {
+	Kdomain string
+	Ksource string
+}
+
+var bChannel chan killfileItem
+
+func init() {
+
+	bChannel = make(chan killfileItem, 1024)
+	fmt.Println("Initializing kill channel engine.")
+
+	go bWriteThread()
+
+}
+
+func bWriteThread() {
+
+	for item := range bChannel {
+
+		writeInKillfile(item.Kdomain, item.Ksource)
+		incrementStats("BL domains from "+item.Ksource, 1)
+		incrementStats("TOTAL", 1)
+
+	}
+
+}
+
+//DomainKill stores a domain name inside the killfile
+func DomainKill(s, durl string) {
+
+	if len(s) > 2 {
+
+		s = strings.ToLower(s)
+
+		var k killfileItem
+
+		k.Kdomain = s
+		k.Ksource = durl
+
+		bChannel <- k
+
+	}
+
+}
+
+func writeInKillfile(key, value string) {
+
+	stK := []byte(key)
+	stV := []byte(value)
+
+	err := MyZabovKDB.Put(stK, stV, nil)
+	if err != nil {
+		fmt.Println("Cannot write to Killfile DB: ", err.Error())
+	}
+
+}
+
+func domainInKillfile(domain string) bool {
+
+	s := strings.ToLower(domain)
+
+	has, err := MyZabovKDB.Has([]byte(s), nil)
+	if err != nil {
+		fmt.Println("Cannot read from Killfile DB: ", err.Error())
+	}
+
+	return has
+
+}

01.stats.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"time"
+)
+
+type send struct {
+	Payload   string
+	Number    int64
+	Operation string
+}
+
+//ZabovStats is used to keep statistics to print
+var ZabovStats map[string]int64
+
+var stats chan send
+
+func init() {
+
+	stats = make(chan send, 1024)
+
+	ZabovStats = make(map[string]int64)
+
+	fmt.Println("Initializing stats engine.")
+	go reportPrintThread()
+	go statsThread()
+}
+
+func statsPrint() {
+	fmt.Println()
+	stat, _ := json.Marshal(ZabovStats)
+	fmt.Println(jsonPrettyPrint(string(stat)))
+	fmt.Println()
+}
+
+func incrementStats(key string, value int64) {
+
+	var s send
+
+	s.Payload = key
+	s.Number = value
+	s.Operation = "INC"
+
+	stats <- s
+
+}
+
+func setstatsvalue(key string, value int64) {
+
+	var s send
+
+	s.Payload = key
+	s.Number = value
+	s.Operation = "SET"
+
+	stats <- s
+
+}
+
+func reportPrintThread() {
+	for {
+		var s send
+		s.Operation = "PRI"
+		s.Payload = "-"
+		s.Number = 0
+		stats <- s
+		time.Sleep(2 * time.Minute)
+	}
+}
+
+func statsThread() {
+
+	fmt.Println("Starting Statistical Collection Thread")
+
+	for item := range stats {
+
+		switch item.Operation {
+		case "INC":
+			ZabovStats[item.Payload] += item.Number
+		case "SET":
+			ZabovStats[item.Payload] = item.Number
+		case "PRI":
+			statsPrint()
+		}
+
+	}
+
+}
+
+func jsonPrettyPrint(in string) string {
+	var out bytes.Buffer
+	err := json.Indent(&out, []byte(in), "", "\t")
+	if err != nil {
+		return in
+	}
+	return out.String()
+}

02.cache.go

@@ -0,0 +1,104 @@
+package main
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"time"
+
+	"github.com/miekg/dns"
+)
+
+
+
+type cacheItem struct {
+	Query []byte
+	Date  time.Time
+}
+
+//DomainCache stores a domain name inside the cache
+func DomainCache(s string, resp *dns.Msg) {
+
+	var domain2cache cacheItem
+	var err error
+	var dom2 bytes.Buffer
+	enc := gob.NewEncoder(&dom2)
+
+	domain2cache.Query, err = resp.Pack()
+	if err != nil {
+		fmt.Println("Problems packing the response: ", err.Error())
+	}
+	domain2cache.Date = time.Now()
+
+	err = enc.Encode(domain2cache)
+
+	if err != nil {
+		fmt.Println("Cannot GOB the domain to cache: ", err.Error())
+	}
+
+	cacheDomain(s, dom2.Bytes())
+
+}
+
+func cacheDomain(key string, domain []byte) {
+
+	err := MyZabovCDB.Put([]byte(key), domain, nil)
+	if err != nil {
+		fmt.Println("Cannot write to Cache DB: ", err.Error())
+	}
+
+}
+
+//GetDomainFromCache stores a domain name inside the cache
+func GetDomainFromCache(s string) *dns.Msg {
+
+	ret := new(dns.Msg)
+	var cache bytes.Buffer
+	dec := gob.NewDecoder(&cache)
+	var record cacheItem
+	var conf []byte
+	var errDB error
+
+	if domainInCache(s) == false {
+		return nil
+	}
+
+	conf, errDB = MyZabovCDB.Get([]byte(s), nil)
+	if errDB != nil {
+		fmt.Println("Cant READ DB :" , errDB.Error() )
+		return nil
+	}
+
+	cache.Write(conf)
+
+	err := dec.Decode(&record)
+	if err != nil {
+		fmt.Println("Decode error :", err.Error())
+		return nil
+	}
+
+	if time.Since(record.Date) > (time.Duration(ZabovCacheTTL) * time.Hour) {
+		return nil
+	}
+
+	err = ret.Unpack(record.Query)
+	if err != nil {
+		fmt.Println("Problem unpacking response: ", err.Error())
+		return nil
+	}
+
+	return ret
+
+}
+
+func domainInCache(domain string) bool {
+
+	has, err := MyZabovCDB.Has([]byte(domain), nil)
+	if err != nil {
+		fmt.Println("Cannot search Cache DB: ", err.Error())
+		return false
+	}
+
+	return has
+
+}

Dockerfile.amd64

@@ -0,0 +1,18 @@
+FROM golang:1.14.1 AS builder
+RUN apt install git -y
+RUN mkdir -p /go/src/zabov
+RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov 
+WORKDIR /go/src/zabov
+ENV GO111MODULE=auto 
+RUN go get ; go build -mod=vendor 
+
+FROM debian:latest 
+RUN apt update
+RUN apt upgrade -y
+RUN apt install ca-certificates -y
+RUN mkdir -p /opt/zabov
+WORKDIR  /opt/zabov
+COPY --from=builder /go/src/zabov /opt/zabov 
+EXPOSE 53/udp
+ENTRYPOINT ["/opt/zabov/zabov"]
+

Dockerfile.arm32v7

@@ -0,0 +1,17 @@
+FROM arm32v7/golang:1.14.1 AS builder
+RUN apt install git -y
+RUN mkdir -p /go/src/zabov
+RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov 
+WORKDIR /go/src/zabov
+ENV GO111MODULE=auto 
+RUN go get ; go build -mod=vendor 
+
+FROM arm32v7/debian:latest 
+RUN apt update
+RUN apt upgrade -y
+RUN apt install ca-certificates -y
+RUN mkdir -p /opt/zabov
+WORKDIR  /opt/zabov
+COPY --from=builder /go/src/zabov /opt/zabov 
+EXPOSE 53/udp
+ENTRYPOINT ["/opt/zabov/zabov"]

Dockerfile.arm64v8

@@ -0,0 +1,17 @@
+FROM arm64v8/golang:1.14.1 AS builder
+RUN apt install git -y
+RUN mkdir -p /go/src/zabov
+RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov 
+WORKDIR /go/src/zabov
+ENV GO111MODULE=auto 
+RUN go get ; go build -mod=vendor 
+
+FROM arm64v8/debian:latest 
+RUN apt update
+RUN apt upgrade -y
+RUN apt install ca-certificates -y
+RUN mkdir -p /opt/zabov
+WORKDIR  /opt/zabov
+COPY --from=builder /go/src/zabov /opt/zabov 
+EXPOSE 53/udp
+ENTRYPOINT ["/opt/zabov/zabov"]

LICENSE

@@ -0,0 +1,14 @@
+    Copyright (C) 2020  loweel@keinpfusch.net
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.

README.md

@@ -0,0 +1,91 @@
+# zabov
+
+Tiny replacement for piHole DNS filter
+
+Still Work in progress, usable.
+
+Idea is to produce a very simple, no-web-interface , IP DNS blocker.
+
+# INSTALL
+
+Zabov requires golang 1.13 or later.
+
+<pre>
+git clone https://git.keinpfusch.net/Loweel/zabov.git
+cd zabov
+go get
+go build -mod=vendor
+</pre>
+
+Then, edit config.json: please notice config.json must be in the same folder of the executable you run.
+
+
+Just a few words about "singlefilters" and "doublefilters":
+
+Data must be downloaded from URLs of blacklist mantainers.They may come in different formats.
+
+There are two kinds of blacklists:
+
+One is the format zabov calls "singlefilter", where we find a single column , full of domains:
+
+<pre>
+domain1.com
+domain2.com
+domain3.com
+</pre>
+
+The second is the format zabov calls "doublefilter" (a file in "/etc/hosts" format, to be precise), where there is an IP, usually localhost or 0.0.0.0 and then the domain:
+
+<pre>
+127.0.0.1 domain1.com
+127.0.0.1 domain2.com
+127.0.0.1 domain3.com
+</pre>
+
+This is why configuration file has two separated items.
+
+The config file should look like:
+
+<pre>
+{
+    "zabov": {  
+        "port":"53", 
+        "proto":"udp", 
+        "ipaddr":"127.0.0.1",
+        "upstream":"./dns-upstream.txt",
+        "cachettl": "4",
+        "killfilettl": "12",
+        "singlefilters":"./urls-hosts.txt" ,
+        "doublefilters":"./urls-domains.txt", 
+        "blackholeip":"127.0.0.1",
+        "hostsfile":"./urls-local.txt"
+    }
+
+}
+
+
+
+</pre>
+
+Where:
+
+- port is the port number. Usually is 53, you can change for docker, if you like
+- proto is the protocol. Choices are "udp", "tcp", "tcp/udp"
+- ipaddr is the port to listen to. Maybe empty, (which will result in listening to 0.0.0.0) to avoid issues with docker.
+- upstream: file containing all DNS we want to query :  each line in format IP:PORT
+- cachettl: amount of time the cache is kept (in hours)
+- killfilettl: refresh time for _killfiles_
+- singlefilters: name of the file  for blacklists following the "singlefilter" schema.(one URL per line)
+- doublefilters: name of the file, for blacklists following the "doublefilter" schema.(one URL per line)
+- blackholeip: IP address to return when the IP is banned. This is because you may want to avoid MX issues, mail loops on localhost, or you have a web server running on localhost
+- hostsfile: path where you keep your local blacklistfile : this is in the format "singlefilter", meaning one domain per line, unlike hosts file.
+
+# DOCKER
+Multistage Dockerfiles are provided for AMD64, ARMv7, ARM64V8
+
+# TODO:
+
+- ~~caching~~
+- monitoring port
+
+

adlist_hosts.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"net"
+	"net/http"
+
+	"strings"
+	"time"
+)
+
+func init() {
+	go downloadDoubleThread()
+}
+
+//DoubleIndexFilter puts the domains inside file
+func DoubleIndexFilter(durl string) error {
+
+	fmt.Println("Retrieving HostFile from: ", durl)
+
+	var err error
+
+	// Get the data
+	resp, err := http.Get(durl)
+	if err != nil {
+		fmt.Println("HTTP problem: ", err)
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 200 { // OK
+		fmt.Println(durl + " Response: OK")
+	} else {
+		fmt.Println("Server <"+durl+"> returned status code: ", resp.StatusCode)
+		return errors.New("Server <" + durl + "> returned status code: " + resp.Status)
+	}
+
+	scanner := bufio.NewScanner(resp.Body)
+	splitter := func(c rune) bool {
+		return c == ' ' || c == '\t'
+	}
+
+	var numLines int64
+
+	for scanner.Scan() {
+
+		line := scanner.Text()
+
+		h := strings.FieldsFunc(line, splitter)
+
+		if h == nil {
+			continue
+		}
+
+		if len(h) < 2 {
+			continue
+		}
+
+		if net.ParseIP(h[0]) != nil {
+			DomainKill(h[1], durl)
+
+			// fmt.Println("MATCH: ", h[1])
+			numLines++
+		} else {
+			incrementStats("Malformed HostLines "+durl, 1)
+			// fmt.Println("Malformed line: <" + line + ">")
+		}
+
+	}
+
+	fmt.Println("Finished to parse: "+durl+" ,number of lines", numLines)
+
+	return err
+
+}
+
+func getDoubleFilters() {
+
+	s := fileByLines(ZabovDoubleBL)
+
+	for _, a := range s {
+		DoubleIndexFilter(a)
+	}
+
+}
+
+func downloadDoubleThread() {
+	fmt.Println("Starting updater of DOUBLE lists, each (hours):", ZabovKillTTL)
+	for {
+		getDoubleFilters()
+		time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
+	}
+
+}

adlist_single.go

@@ -0,0 +1,93 @@
+package main
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"net/http"
+	"strings"
+	"time"
+)
+
+func init() {
+	go downloadThread()
+}
+
+//SingleIndexFilter puts the domains inside file
+func SingleIndexFilter(durl string) error {
+
+	fmt.Println("Retrieving DomainFile from: ", durl)
+
+	var err error
+
+	// Get the data
+	resp, err := http.Get(durl)
+	if err != nil {
+		fmt.Println("HTTP Problem: ", err)
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode == 200 { // OK
+		fmt.Println(durl + " Response: OK")
+	} else {
+		fmt.Println("Server <"+durl+"> returned status code: ", resp.StatusCode)
+		return errors.New("Server <" + durl + "> returned status code: " + resp.Status)
+	}
+
+	scanner := bufio.NewScanner(resp.Body)
+	splitter := func(c rune) bool {
+		return c == ' ' || c == '\t'
+	}
+
+	var numLines int64
+
+	for scanner.Scan() {
+
+		line := scanner.Text()
+
+		h := strings.FieldsFunc(line, splitter)
+
+		if h == nil {
+			continue
+		}
+
+		if len(h) < 1 {
+			continue
+		}
+
+		if !strings.Contains(h[0], "#") {
+			DomainKill(h[0], durl)
+			// fmt.Println("MATCH: ", h[1])
+			numLines++
+		} else {
+			incrementStats("Malformed DomainLines "+durl, 1)
+			// fmt.Println("Malformed line: <" + line + ">")
+		}
+
+	}
+
+	fmt.Println("Finished to parse: "+durl+" ,number of lines", numLines)
+
+	return err
+
+}
+
+func getSingleFilters() {
+
+	s := fileByLines(ZabovSingleBL)
+
+	for _, a := range s {
+		SingleIndexFilter(a)
+	}
+
+}
+
+func downloadThread() {
+	fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL)
+	for {
+		getSingleFilters()
+		time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
+	}
+
+}

config.json

@@ -0,0 +1,15 @@
+{
+    "zabov": {  
+        "port":"53", 
+        "proto":"udp", 
+        "ipaddr":"0.0.0.0",
+        "upstream":"./dns-upstream.txt"  ,
+        "cachettl": 1,
+        "killfilettl": 12,
+        "singlefilters":"./urls-domains.txt" ,
+        "doublefilters":"./urls-hosts.txt", 
+        "blackholeip":"127.0.0.1",
+        "hostsfile":"./urls-local.txt"
+    }
+
+}

dns-upstream.txt

@@ -0,0 +1,387 @@
+194.150.168.168:53
+194.25.0.68:53
+141.1.1.1:53
+213.239.204.35:53
+194.25.0.52:53
+212.211.132.4:53
+213.68.194.51:53
+195.243.214.4:53
+141.1.27.249:53
+80.237.197.14:53
+217.28.98.62:53
+82.96.64.2:53
+82.96.65.2:53
+194.25.0.60:53
+193.101.111.10:53
+193.101.111.20:53
+192.76.144.66:53
+217.69.169.25:53
+85.88.19.10:53
+85.88.19.11:53
+85.214.20.141:53
+194.169.239.10:53
+194.172.160.4:53
+212.102.225.2:53
+212.51.16.1:53
+212.51.17.1:53
+212.66.129.98:53
+212.89.130.180:53
+213.209.122.11:53
+213.23.108.129:53
+91.204.4.133:53
+84.200.69.80:53
+62.146.63.211:53
+212.77.178.83:53
+78.46.58.246:53
+85.214.102.25:53
+87.106.62.128:53
+81.20.87.84:53
+81.20.87.181:53
+217.5.159.227:53
+185.38.9.99:53
+62.146.25.130:53
+37.59.218.50:53
+212.184.191.193:53
+213.23.143.154:53
+178.32.187.10:53
+62.154.214.86:53
+85.214.151.164:53
+5.175.225.2:53
+193.105.38.142:53
+145.253.183.21:53
+178.15.146.43:53
+62.245.233.22:53
+212.224.71.71:53
+213.136.78.213:53
+80.156.196.196:53
+193.158.99.67:53
+194.95.75.230:53
+212.184.191.2:53
+213.138.38.22:53
+195.145.80.150:53
+139.18.25.34:53
+79.143.180.116:53
+213.240.172.200:53
+217.160.238.238:53
+213.61.185.238:53
+84.201.0.34:53
+82.194.105.219:53
+62.157.89.178:53
+46.189.26.123:53
+85.214.208.8:53
+87.239.128.130:53
+78.111.65.40:53
+85.214.69.126:53
+109.75.29.1:53
+80.81.19.226:53
+81.169.162.74:53
+217.14.164.35:53
+5.9.172.92:53
+62.225.102.180:53
+217.7.71.203:53
+217.6.71.61:53
+62.154.138.43:53
+146.0.38.140:53
+78.111.67.10:53
+217.6.110.20:53
+87.245.18.221:53
+62.225.66.19:53
+81.169.212.52:53
+178.162.205.123:53
+212.227.83.183:53
+139.18.25.33:53
+193.29.2.4:53
+212.91.246.11:53
+62.153.141.15:53
+148.251.120.228:53
+62.154.253.226:53
+194.25.218.2:53
+194.174.73.36:53
+62.245.226.182:53
+87.234.222.68:53
+194.25.169.130:53
+62.225.15.253:53
+176.94.20.4:53
+188.40.115.29:53
+188.40.115.22:53
+194.187.240.10:53
+80.150.109.197:53
+217.86.149.109:53
+91.208.193.1:53
+195.243.99.35:53
+62.225.102.177:53
+178.210.102.9:53
+80.228.113.125:53
+178.210.102.12:53
+130.255.121.9:53
+212.204.56.218:53
+37.59.218.151:53
+80.148.52.109:53
+194.30.174.222:53
+5.199.141.5:53
+94.135.173.22:53
+88.79.208.11:53
+141.16.180.9:53
+82.193.241.125:53
+212.8.216.37:53
+109.75.29.2:53
+78.46.17.82:53
+81.169.185.49:53
+217.244.13.14:53
+93.104.209.27:53
+79.143.182.174:53
+81.20.82.131:53
+213.136.68.181:53
+213.136.68.189:53
+193.107.145.233:53
+195.145.241.3:53
+80.242.182.182:53
+193.159.181.250:53
+195.243.124.75:53
+62.159.104.102:53
+92.222.202.244:53
+85.214.254.13:53
+85.114.128.115:53
+145.253.176.50:53
+217.7.63.1:53
+78.35.40.149:53
+81.169.187.253:53
+94.249.192.20:53
+85.214.43.157:53
+80.149.83.60:53
+178.210.102.225:53
+178.210.102.193:53
+62.154.236.126:53
+213.183.185.50:53
+212.60.229.242:53
+80.146.192.66:53
+79.133.62.62:53
+178.33.33.219:53
+62.245.225.225:53
+46.38.235.212:53
+213.136.88.31:53
+212.66.135.250:53
+194.231.138.26:53
+62.225.1.33:53
+80.148.34.131:53
+94.23.163.114:53
+80.64.189.94:53
+81.169.241.28:53
+212.38.26.132:53
+62.91.19.67:53
+87.239.128.25:53
+212.185.196.10:53
+89.221.2.171:53
+217.243.239.11:53
+213.136.69.214:53
+213.138.56.75:53
+212.122.52.11:53
+46.4.166.113:53
+77.37.30.12:53
+194.187.242.10:53
+188.40.132.212:53
+194.150.168.169:53
+85.25.105.193:53
+185.93.180.131:53
+109.234.249.10:53
+109.234.248.10:53
+138.201.120.250:53
+81.3.27.54:53
+78.46.231.161:53
+78.46.231.162:53
+212.51.16.197:53
+212.28.34.65:53
+148.251.24.48:53
+212.75.32.4:53
+91.103.112.150:53
+217.69.169.26:53
+195.63.103.144:53
+213.209.121.30:53
+88.79.149.4:53
+185.194.143.243:53
+46.182.19.48:53
+217.111.24.246:53
+62.96.37.74:53
+213.61.64.174:53
+213.61.65.226:53
+62.96.190.134:53
+217.111.123.166:53
+213.61.176.118:53
+185.216.33.82:53
+185.220.70.50:53
+188.138.57.95:53
+195.145.137.164:53
+195.167.223.164:53
+195.226.69.82:53
+195.37.174.194:53
+195.4.138.12:53
+195.63.61.189:53
+212.124.35.25:53
+212.184.191.100:53
+212.38.2.130:53
+212.51.16.193:53
+212.66.129.107:53
+212.8.216.41:53
+212.89.128.28:53
+213.133.116.14:53
+213.166.247.100:53
+217.243.173.82:53
+217.5.182.118:53
+217.7.80.40:53
+217.7.81.136:53
+217.9.50.199:53
+46.237.220.2:53
+5.189.179.105:53
+52.28.79.14:53
+52.29.2.17:53
+62.146.202.2:53
+62.146.2.48:53
+62.153.122.2:53
+62.153.237.200:53
+62.153.237.201:53
+62.154.139.99:53
+62.154.159.12:53
+62.154.159.5:53
+62.154.160.3:53
+62.209.40.75:53
+62.217.61.162:53
+62.245.225.55:53
+78.111.224.224:53
+78.111.226.226:53
+78.138.80.42:53
+80.149.112.139:53
+80.156.6.209:53
+80.190.209.218:53
+80.228.231.122:53
+80.228.231.48:53
+80.245.65.100:53
+81.14.182.169:53
+81.27.162.100:53
+83.97.23.178:53
+83.97.23.226:53
+84.16.240.43:53
+85.214.98.185:53
+89.19.228.52:53
+89.200.168.203:53
+91.217.86.4:53
+93.104.195.2:53
+94.247.43.254:53
+109.234.248.8:53
+131.220.20.199:53
+131.220.23.123:53
+144.76.173.169:53
+144.76.83.104:53
+148.251.92.241:53
+176.9.136.236:53
+195.10.195.195:53
+173.212.249.41:53
+85.214.41.155:53
+54.37.75.2:53
+194.55.13.75:53
+5.189.138.153:53
+159.69.51.18:53
+51.75.77.179:53
+138.201.169.84:53
+138.201.239.66:53
+138.68.106.109:53
+145.253.109.162:53
+159.69.68.181:53
+167.86.78.56:53
+173.212.208.116:53
+173.212.218.206:53
+173.212.219.129:53
+173.212.239.87:53
+173.212.242.89:53
+173.212.244.78:53
+173.249.41.233:53
+173.249.48.6:53
+176.9.233.171:53
+176.9.58.218:53
+178.162.199.27:53
+178.162.208.135:53
+178.238.230.127:53
+178.238.235.218:53
+18.195.121.224:53
+185.139.98.100:53
+185.40.135.11:53
+185.53.169.22:53
+185.90.131.194:53
+188.40.239.99:53
+188.68.35.145:53
+192.162.85.48:53
+193.159.232.5:53
+194.77.237.31:53
+194.77.253.32:53
+195.201.192.29:53
+195.202.52.30:53
+195.243.101.5:53
+207.180.203.42:53
+207.180.243.200:53
+207.180.247.212:53
+213.136.71.68:53
+213.136.77.39:53
+213.144.24.234:53
+217.147.96.210:53
+217.182.198.203:53
+217.6.131.248:53
+217.6.247.237:53
+217.6.64.5:53
+217.79.177.220:53
+46.163.119.155:53
+46.228.199.116:53
+5.175.26.208:53
+51.77.65.15:53
+5.189.133.151:53
+5.189.141.216:53
+5.189.186.154:53
+5.189.186.93:53
+5.189.187.34:53
+5.199.141.30:53
+5.45.96.220:53
+62.138.20.211:53
+62.144.82.252:53
+62.153.165.107:53
+62.153.201.91:53
+62.154.214.84:53
+62.157.242.85:53
+79.143.177.243:53
+79.143.183.45:53
+80.156.198.146:53
+80.156.6.206:53
+80.237.207.100:53
+80.82.223.94:53
+81.169.215.29:53
+81.169.223.126:53
+81.169.230.157:53
+81.20.80.79:53
+83.236.183.211:53
+84.16.240.224:53
+85.214.224.76:53
+85.214.238.190:53
+85.214.246.133:53
+85.214.62.160:53
+85.93.91.101:53
+87.106.63.208:53
+87.118.126.225:53
+88.198.37.146:53
+88.99.66.18:53
+89.163.150.209:53
+89.163.220.114:53
+89.19.236.152:53
+93.104.213.74:53
+93.186.196.137:53
+93.190.71.172:53
+94.177.246.221:53
+80.241.218.68:53
+172.105.81.90:53
+172.105.81.92:53
+84.200.70.40:53
+94.16.114.254:53
+93.90.207.192:53
+93.90.201.211:53
+144.91.68.146:53
+176.9.37.132:53
+176.9.93.198:53
+176.9.1.117:53
+144.91.115.47:53
+91.237.100.4:53

dns_client.go

@@ -0,0 +1,98 @@
+package main
+
+import (
+	"fmt"
+	"time"
+
+	"math/rand"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+//ForwardQuery forwards the query to the upstream server
+//first server to answer wins
+func ForwardQuery(query *dns.Msg) *dns.Msg {
+
+	go incrementStats("ForwardQueries", 1)
+
+	r := new(dns.Msg)
+	r.SetReply(query)
+	r.Authoritative = true
+
+	fqdn := strings.TrimRight(query.Question[0].Name, ".")
+
+	lfqdn := fmt.Sprintf("%d", query.Question[0].Qtype) + "." + fqdn
+	if cached := GetDomainFromCache(lfqdn); cached != nil {
+		go incrementStats("CacheHit", 1)
+		cached.SetReply(query)
+		cached.Authoritative = true
+		return cached
+
+	}
+
+	c := new(dns.Client)
+
+	c.ReadTimeout = 500 * time.Millisecond
+	c.WriteTimeout = 500 * time.Millisecond
+
+	for {
+		// round robin with retry
+
+		if !NetworkUp {
+			time.Sleep(10 * time.Second)
+			go incrementStats("Network Problems ", 1)
+			continue
+		}
+
+		d := oneTimeDNS()
+
+		in, _, err := c.Exchange(query, d)
+		if err != nil {
+			fmt.Printf("Problem with DNS %s : %s\n", d, err.Error())
+			go incrementStats("DNS Problems "+d, 1)
+			continue
+		} else {
+			go incrementStats(d, 1)
+			in.SetReply(query)
+			in.Authoritative = true
+			go DomainCache(lfqdn, in)
+			return in
+
+		}
+
+	}
+
+}
+
+func init() {
+
+	fmt.Println("DNS client engine starting")
+	NetworkUp = checkNetworkUp()
+
+	if NetworkUp {
+		fmt.Println("[OK]: Network is UP")
+	} else {
+		fmt.Println("[KO] Network is DOWN: system will check again in 2 minutes")
+	}
+
+}
+
+func oneTimeDNS() (dns string) {
+
+	rand.Seed(time.Now().Unix())
+
+	upl := ZabovDNSArray
+
+	if len(upl) < 1 {
+		fmt.Println("No DNS defined, using default 127.0.0.53:53. Hope it works!")
+		return "127.0.0.53:53"
+	}
+
+	n := rand.Intn(128*len(upl)) % len(upl)
+
+	dns = upl[n]
+
+	return
+
+}

dns_handler.go

@@ -0,0 +1,44 @@
+package main
+
+import (
+	"net"
+	"strings"
+
+	"github.com/miekg/dns"
+)
+
+func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
+	go incrementStats("TotalQueries", 1)
+
+	remIP, _, e := net.SplitHostPort(w.RemoteAddr().String())
+	if e != nil {
+		go incrementStats("CLIENT: "+remIP, 1)
+	}
+
+	msg := dns.Msg{}
+	msg.SetReply(r)
+
+	switch r.Question[0].Qtype {
+	case dns.TypeA:
+		msg.Authoritative = true
+		domain := msg.Question[0].Name
+		fqdn := strings.TrimRight(domain, ".")
+
+		if domainInKillfile(fqdn) {
+			go incrementStats("Killed", 1)
+
+			msg.Answer = append(msg.Answer, &dns.A{
+				Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
+				A:   net.ParseIP(ZabovAddBL),
+			})
+		} else {
+			ret := ForwardQuery(r)
+			w.WriteMsg(ret)
+		}
+	default:
+		ret := ForwardQuery(r)
+		w.WriteMsg(ret)
+	}
+	w.WriteMsg(&msg)
+
+}

go.mod

@@ -0,0 +1,8 @@
+module zabov
+
+go 1.13
+
+require (
+	github.com/miekg/dns v1.1.27
+	github.com/syndtr/goleveldb v1.0.0
+)

go.sum

@@ -0,0 +1,36 @@
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
+github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
+github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
+github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

hostfile.go

@@ -0,0 +1,59 @@
+package main
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+)
+
+func init() {
+
+	fmt.Println("Ingesting local hosts file")
+	ingestLocalBlacklist()
+
+}
+
+func ingestLocalBlacklist() {
+
+	file, err := os.Open(ZabovHostsFile)
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		d := scanner.Text()
+		DomainKill(d, ZabovHostsFile)
+		incrementStats("Blacklist", 1)
+
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Println(err.Error())
+	}
+
+}
+
+func fileByLines(filename string) (blurls []string) {
+
+	file, err := os.Open(filename)
+	if err != nil {
+		fmt.Println(err.Error())
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+	for scanner.Scan() {
+		d := scanner.Text()
+		blurls = append(blurls, d)
+
+	}
+
+	if err := scanner.Err(); err != nil {
+		fmt.Println(err.Error())
+	}
+
+	return
+
+}

main.go

@@ -0,0 +1,46 @@
+package main
+
+import (
+	"log"
+
+	"github.com/miekg/dns"
+)
+
+//MyDNS is my dns server
+var MyDNS *dns.Server
+
+//ZabovUpDNS keeps the name of upstream DNSs
+var ZabovUpDNS string
+
+//ZabovSingleBL list of urls returning a file with just names of domains
+var ZabovSingleBL string
+
+//ZabovDoubleBL list of urls returning a file with  IP<space>domain
+var ZabovDoubleBL string
+
+//ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1
+var ZabovAddBL string
+
+//ZabovCacheTTL is the amount of hours we cache records of DNS
+var ZabovCacheTTL int
+
+//ZabovKillTTL is the amount of hours we cache the killfile
+var ZabovKillTTL int
+
+//ZabovHostsFile is the file we use to keep our hosts
+var ZabovHostsFile string
+
+//ZabovDNSArray is the array containing all the DNS we mention
+var ZabovDNSArray []string
+
+type handler struct{}
+
+func main() {
+
+	MyDNS.Handler = &handler{}
+	if err := MyDNS.ListenAndServe(); err != nil {
+		log.Printf("Failed to set udp listener %s\n", err.Error())
+	} else {
+		log.Printf("Listener running \n")
+	}
+}

urls-domains.txt

@@ -0,0 +1,37 @@
+https://mirror1.malwaredomains.com/files/justdomains
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/adaway.org/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/adblock-nocoin-list/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/adguard-simplified/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/anudeepnd-adservers/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-ad/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-malvertising/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-malware/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-tracking/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/easyprivacy/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/eth-phishing-detect/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.2o7net/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.dead/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.risk/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.spam/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/kadhosts/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomainlist.com/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomains.com-immortaldomains/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomains.com-justdomains/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/matomo.org-spammers/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/mitchellkrogza-badd-boyz-hosts/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/pgl.yoyo.org/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/ransomwaretracker.abuse.ch/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/someonewhocares.org/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/spam404.com/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/stevenblack/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/winhelp2002.mvps.org/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/zerodot1-coinblockerlists-browser/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/zeustracker.abuse.ch/list.txt
+https://raw.githubusercontent.com/CHEF-KOCH/Audio-fingerprint-pages/master/AudioFp.txt
+https://raw.githubusercontent.com/CHEF-KOCH/Canvas-fingerprinting-pages/master/Canvas.txt
+https://raw.githubusercontent.com/CHEF-KOCH/WebRTC-tracking/master/WebRTC.txt
+https://raw.githubusercontent.com/CHEF-KOCH/CKs-FilterList/master/Anti-Corp/hosts/NSABlocklist.txt
+https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt
+https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt
+https://www.stopforumspam.com/downloads/toxic_domains_whole.txt

urls-hosts.txt

@@ -0,0 +1,12 @@
+http://sysctl.org/cameleon/hosts
+https://www.malwaredomainlist.com/hostslist/hosts.txt
+https://adaway.org/hosts.txt
+https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
+https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews/hosts
+https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling/hosts
+https://someonewhocares.org/hosts/hosts
+https://getadhell.com/standard-package.txt
+https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt
+https://raw.githubusercontent.com/notracking/hosts-blocklists/master/hostnames.txt
+https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt
+https://raw.githubusercontent.com/anudeepND/blacklist/master/facebook.txt

urls-local.txt

@@ -0,0 +1,8 @@
+blc.vodafone.com
+gab.com
+gab.ai
+freespeechextremist.com
+neckbeard.xyz
+funkwhale.it
+social.byoblu.com
+

vendor/github.com/golang/snappy/.gitignore

@@ -0,0 +1,16 @@
+cmd/snappytool/snappytool
+testdata/bench
+
+# These explicitly listed benchmark data files are for an obsolete version of
+# snappy_test.go.
+testdata/alice29.txt
+testdata/asyoulik.txt
+testdata/fireworks.jpeg
+testdata/geo.protodata
+testdata/html
+testdata/html_x_4
+testdata/kppkn.gtb
+testdata/lcet10.txt
+testdata/paper-100k.pdf
+testdata/plrabn12.txt
+testdata/urls.10K

vendor/github.com/golang/snappy/AUTHORS

@@ -0,0 +1,15 @@
+# This is the official list of Snappy-Go authors for copyright purposes.
+# This file is distinct from the CONTRIBUTORS files.
+# See the latter for an explanation.
+
+# Names should be added to this file as
+#	Name or Organization <email address>
+# The email address is not required for organizations.
+
+# Please keep the list sorted.
+
+Damian Gryski <dgryski@gmail.com>
+Google Inc.
+Jan Mercl <0xjnml@gmail.com>
+Rodolfo Carvalho <rhcarvalho@gmail.com>
+Sebastien Binet <seb.binet@gmail.com>

vendor/github.com/golang/snappy/CONTRIBUTORS

@@ -0,0 +1,37 @@
+# This is the official list of people who can contribute
+# (and typically have contributed) code to the Snappy-Go repository.
+# The AUTHORS file lists the copyright holders; this file
+# lists people.  For example, Google employees are listed here
+# but not in AUTHORS, because Google holds the copyright.
+#
+# The submission process automatically checks to make sure
+# that people submitting code are listed in this file (by email address).
+#
+# Names should be added to this file only after verifying that
+# the individual or the individual's organization has agreed to
+# the appropriate Contributor License Agreement, found here:
+#
+#     http://code.google.com/legal/individual-cla-v1.0.html
+#     http://code.google.com/legal/corporate-cla-v1.0.html
+#
+# The agreement for individuals can be filled out on the web.
+#
+# When adding J Random Contributor's name to this file,
+# either J's name or J's organization's name should be
+# added to the AUTHORS file, depending on whether the
+# individual or corporate CLA was used.
+
+# Names should be added to this file like so:
+#     Name <email address>
+
+# Please keep the list sorted.
+
+Damian Gryski <dgryski@gmail.com>
+Jan Mercl <0xjnml@gmail.com>
+Kai Backman <kaib@golang.org>
+Marc-Antoine Ruel <maruel@chromium.org>
+Nigel Tao <nigeltao@golang.org>
+Rob Pike <r@golang.org>
+Rodolfo Carvalho <rhcarvalho@gmail.com>
+Russ Cox <rsc@golang.org>
+Sebastien Binet <seb.binet@gmail.com>

vendor/github.com/golang/snappy/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+   * Redistributions of source code must retain the above copyright
+notice, this list of conditions and the following disclaimer.
+   * Redistributions in binary form must reproduce the above
+copyright notice, this list of conditions and the following disclaimer
+in the documentation and/or other materials provided with the
+distribution.
+   * Neither the name of Google Inc. nor the names of its
+contributors may be used to endorse or promote products derived from
+this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

vendor/github.com/golang/snappy/README

@@ -0,0 +1,107 @@
+The Snappy compression format in the Go programming language.
+
+To download and install from source:
+$ go get github.com/golang/snappy
+
+Unless otherwise noted, the Snappy-Go source files are distributed
+under the BSD-style license found in the LICENSE file.
+
+
+
+Benchmarks.
+
+The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten
+or so files, the same set used by the C++ Snappy code (github.com/google/snappy
+and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @
+3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29:
+
+"go test -test.bench=."
+
+_UFlat0-8         2.19GB/s ± 0%  html
+_UFlat1-8         1.41GB/s ± 0%  urls
+_UFlat2-8         23.5GB/s ± 2%  jpg
+_UFlat3-8         1.91GB/s ± 0%  jpg_200
+_UFlat4-8         14.0GB/s ± 1%  pdf
+_UFlat5-8         1.97GB/s ± 0%  html4
+_UFlat6-8          814MB/s ± 0%  txt1
+_UFlat7-8          785MB/s ± 0%  txt2
+_UFlat8-8          857MB/s ± 0%  txt3
+_UFlat9-8          719MB/s ± 1%  txt4
+_UFlat10-8        2.84GB/s ± 0%  pb
+_UFlat11-8        1.05GB/s ± 0%  gaviota
+
+_ZFlat0-8         1.04GB/s ± 0%  html
+_ZFlat1-8          534MB/s ± 0%  urls
+_ZFlat2-8         15.7GB/s ± 1%  jpg
+_ZFlat3-8          740MB/s ± 3%  jpg_200
+_ZFlat4-8         9.20GB/s ± 1%  pdf
+_ZFlat5-8          991MB/s ± 0%  html4
+_ZFlat6-8          379MB/s ± 0%  txt1
+_ZFlat7-8          352MB/s ± 0%  txt2
+_ZFlat8-8          396MB/s ± 1%  txt3
+_ZFlat9-8          327MB/s ± 1%  txt4
+_ZFlat10-8        1.33GB/s ± 1%  pb
+_ZFlat11-8         605MB/s ± 1%  gaviota
+
+
+
+"go test -test.bench=. -tags=noasm"
+
+_UFlat0-8          621MB/s ± 2%  html
+_UFlat1-8          494MB/s ± 1%  urls
+_UFlat2-8         23.2GB/s ± 1%  jpg
+_UFlat3-8         1.12GB/s ± 1%  jpg_200
+_UFlat4-8         4.35GB/s ± 1%  pdf
+_UFlat5-8          609MB/s ± 0%  html4
+_UFlat6-8          296MB/s ± 0%  txt1
+_UFlat7-8          288MB/s ± 0%  txt2
+_UFlat8-8          309MB/s ± 1%  txt3
+_UFlat9-8          280MB/s ± 1%  txt4
+_UFlat10-8         753MB/s ± 0%  pb
+_UFlat11-8         400MB/s ± 0%  gaviota
+
+_ZFlat0-8          409MB/s ± 1%  html
+_ZFlat1-8          250MB/s ± 1%  urls
+_ZFlat2-8         12.3GB/s ± 1%  jpg
+_ZFlat3-8          132MB/s ± 0%  jpg_200
+_ZFlat4-8         2.92GB/s ± 0%  pdf
+_ZFlat5-8          405MB/s ± 1%  html4
+_ZFlat6-8          179MB/s ± 1%  txt1
+_ZFlat7-8          170MB/s ± 1%  txt2
+_ZFlat8-8          189MB/s ± 1%  txt3
+_ZFlat9-8          164MB/s ± 1%  txt4
+_ZFlat10-8         479MB/s ± 1%  pb
+_ZFlat11-8         270MB/s ± 1%  gaviota
+
+
+
+For comparison (Go's encoded output is byte-for-byte identical to C++'s), here
+are the numbers from C++ Snappy's
+
+make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log
+
+BM_UFlat/0     2.4GB/s  html
+BM_UFlat/1     1.4GB/s  urls
+BM_UFlat/2    21.8GB/s  jpg
+BM_UFlat/3     1.5GB/s  jpg_200
+BM_UFlat/4    13.3GB/s  pdf
+BM_UFlat/5     2.1GB/s  html4
+BM_UFlat/6     1.0GB/s  txt1
+BM_UFlat/7   959.4MB/s  txt2
+BM_UFlat/8     1.0GB/s  txt3
+BM_UFlat/9   864.5MB/s  txt4
+BM_UFlat/10    2.9GB/s  pb
+BM_UFlat/11    1.2GB/s  gaviota
+
+BM_ZFlat/0   944.3MB/s  html (22.31 %)
+BM_ZFlat/1   501.6MB/s  urls (47.78 %)
+BM_ZFlat/2    14.3GB/s  jpg (99.95 %)
+BM_ZFlat/3   538.3MB/s  jpg_200 (73.00 %)
+BM_ZFlat/4     8.3GB/s  pdf (83.30 %)
+BM_ZFlat/5   903.5MB/s  html4 (22.52 %)
+BM_ZFlat/6   336.0MB/s  txt1 (57.88 %)
+BM_ZFlat/7   312.3MB/s  txt2 (61.91 %)
+BM_ZFlat/8   353.1MB/s  txt3 (54.99 %)
+BM_ZFlat/9   289.9MB/s  txt4 (66.26 %)