@ -0,0 +1,10 @@ | |||
zabov | |||
killfile | |||
killfile/* | |||
db/zabov.db | |||
binaries | |||
binaries/* | |||
build.sh | |||
.vscode | |||
.vscode/* | |||
@ -0,0 +1,38 @@ | |||
package main | |||
import ( | |||
"fmt" | |||
"os" | |||
"github.com/syndtr/goleveldb/leveldb" | |||
) | |||
//MyZabovKDB is the storage where we'll put domains to block | |||
var MyZabovKDB *leveldb.DB | |||
//MyZabovCDB is the storage where we'll put domains to cache | |||
var MyZabovCDB *leveldb.DB | |||
func init() { | |||
var err error | |||
os.RemoveAll("./db") | |||
os.MkdirAll("./db", 0755) | |||
MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil) | |||
if err != nil { | |||
fmt.Println("Cannot create Killfile db: ", err.Error()) | |||
} else { | |||
fmt.Println("Killfile DB created") | |||
} | |||
MyZabovCDB, err = leveldb.OpenFile("./db/cache", nil) | |||
if err != nil { | |||
fmt.Println("Cannot create Cache db: ", err.Error()) | |||
} else { | |||
fmt.Println("Cache DB created") | |||
} | |||
} |
@ -0,0 +1,26 @@ | |||
package main | |||
import ( | |||
"fmt" | |||
"runtime" | |||
"time" | |||
) | |||
func init() { | |||
fmt.Println("Garbage Collector Thread Starting") | |||
go memoryCleanerThread() | |||
} | |||
func memoryCleanerThread() { | |||
for { | |||
time.Sleep(10 * time.Minute) | |||
fmt.Println("Time to clean memory...") | |||
runtime.GC() | |||
fmt.Println("Garbage Collection done.") | |||
} | |||
} |
@ -0,0 +1,69 @@ | |||
package main | |||
import ( | |||
"encoding/json" | |||
"fmt" | |||
"io/ioutil" | |||
"log" | |||
"os" | |||
"github.com/miekg/dns" | |||
) | |||
func init() { | |||
//ZabovConf describes the Json we use for configuration | |||
type ZabovConf struct { | |||
Zabov struct { | |||
Port string `json:"port"` | |||
Proto string `json:"proto"` | |||
Ipaddr string `json:"ipaddr"` | |||
Upstream string `json:"upstream"` | |||
Cachettl int `json:"cachettl"` | |||
Killfilettl int `json:"killfilettl"` | |||
Singlefilters string `json:"singlefilters"` | |||
Doublefilters string `json:"doublefilters"` | |||
Blackholeip string `json:"blackholeip"` | |||
Hostsfile string `json:"hostsfile"` | |||
} `json:"zabov"` | |||
} | |||
var MyConf ZabovConf | |||
file, err := ioutil.ReadFile("config.json") | |||
if err != nil { | |||
log.Println("Cannot open config file", err.Error()) | |||
os.Exit(1) | |||
} | |||
err = json.Unmarshal([]byte(file), &MyConf) | |||
if err != nil { | |||
log.Println("Cannot marshal json: ", err.Error()) | |||
os.Exit(1) | |||
} | |||
// now we read configuration file | |||
fmt.Println("Reading configuration file...") | |||
ZabovPort := MyConf.Zabov.Port | |||
ZabovType := MyConf.Zabov.Proto | |||
ZabovAddr := MyConf.Zabov.Ipaddr | |||
ZabovUpDNS = MyConf.Zabov.Upstream | |||
ZabovSingleBL = MyConf.Zabov.Singlefilters | |||
ZabovDoubleBL = MyConf.Zabov.Doublefilters | |||
ZabovAddBL = MyConf.Zabov.Blackholeip | |||
ZabovCacheTTL = MyConf.Zabov.Cachettl | |||
ZabovKillTTL = MyConf.Zabov.Killfilettl | |||
ZabovHostsFile = MyConf.Zabov.Hostsfile | |||
zabovString := ZabovAddr + ":" + ZabovPort | |||
MyDNS = new(dns.Server) | |||
MyDNS.Addr = zabovString | |||
MyDNS.Net = ZabovType | |||
ZabovDNSArray = fileByLines(ZabovUpDNS) | |||
} |
@ -0,0 +1,37 @@ | |||
package main | |||
import ( | |||
"fmt" | |||
"net/http" | |||
"time" | |||
) | |||
//NetworkUp tells the system if the network is up or not | |||
var NetworkUp bool | |||
func checkNetworkUp() bool { | |||
// RFC2606 test domain, should always work, unless internet is down. | |||
_, err := http.Get("http://example.com") | |||
if err != nil { | |||
return false | |||
} | |||
return true | |||
} | |||
func checkNetworkUpThread() { | |||
ticker := time.NewTicker(2 * time.Minute) | |||
for range ticker.C { | |||
NetworkUp = checkNetworkUp() | |||
} | |||
} | |||
func init() { | |||
fmt.Println("Network Checker starting....") | |||
go checkNetworkUpThread() | |||
} |
@ -0,0 +1,79 @@ | |||
package main | |||
import ( | |||
"fmt" | |||
"strings" | |||
) | |||
var zabovKbucket = []byte("killfile") | |||
type killfileItem struct { | |||
Kdomain string | |||
Ksource string | |||
} | |||
var bChannel chan killfileItem | |||
func init() { | |||
bChannel = make(chan killfileItem, 1024) | |||
fmt.Println("Initializing kill channel engine.") | |||
go bWriteThread() | |||
} | |||
func bWriteThread() { | |||
for item := range bChannel { | |||
writeInKillfile(item.Kdomain, item.Ksource) | |||
incrementStats("BL domains from "+item.Ksource, 1) | |||
incrementStats("TOTAL", 1) | |||
} | |||
} | |||
//DomainKill stores a domain name inside the killfile | |||
func DomainKill(s, durl string) { | |||
if len(s) > 2 { | |||
s = strings.ToLower(s) | |||
var k killfileItem | |||
k.Kdomain = s | |||
k.Ksource = durl | |||
bChannel <- k | |||
} | |||
} | |||
func writeInKillfile(key, value string) { | |||
stK := []byte(key) | |||
stV := []byte(value) | |||
err := MyZabovKDB.Put(stK, stV, nil) | |||
if err != nil { | |||
fmt.Println("Cannot write to Killfile DB: ", err.Error()) | |||
} | |||
} | |||
func domainInKillfile(domain string) bool { | |||
s := strings.ToLower(domain) | |||
has, err := MyZabovKDB.Has([]byte(s), nil) | |||
if err != nil { | |||
fmt.Println("Cannot read from Killfile DB: ", err.Error()) | |||
} | |||
return has | |||
} |
@ -0,0 +1,100 @@ | |||
package main | |||
import ( | |||
"bytes" | |||
"encoding/json" | |||
"fmt" | |||
"time" | |||
) | |||
type send struct { | |||
Payload string | |||
Number int64 | |||
Operation string | |||
} | |||
//ZabovStats is used to keep statistics to print | |||
var ZabovStats map[string]int64 | |||
var stats chan send | |||
func init() { | |||
stats = make(chan send, 1024) | |||
ZabovStats = make(map[string]int64) | |||
fmt.Println("Initializing stats engine.") | |||
go reportPrintThread() | |||
go statsThread() | |||
} | |||
func statsPrint() { | |||
fmt.Println() | |||
stat, _ := json.Marshal(ZabovStats) | |||
fmt.Println(jsonPrettyPrint(string(stat))) | |||
fmt.Println() | |||
} | |||
func incrementStats(key string, value int64) { | |||
var s send | |||
s.Payload = key | |||
s.Number = value | |||
s.Operation = "INC" | |||
stats <- s | |||
} | |||
func setstatsvalue(key string, value int64) { | |||
var s send | |||
s.Payload = key | |||
s.Number = value | |||
s.Operation = "SET" | |||
stats <- s | |||
} | |||
func reportPrintThread() { | |||
for { | |||
var s send | |||
s.Operation = "PRI" | |||
s.Payload = "-" | |||
s.Number = 0 | |||
stats <- s | |||
time.Sleep(2 * time.Minute) | |||
} | |||
} | |||
func statsThread() { | |||
fmt.Println("Starting Statistical Collection Thread") | |||
for item := range stats { | |||
switch item.Operation { | |||
case "INC": | |||
ZabovStats[item.Payload] += item.Number | |||
case "SET": | |||
ZabovStats[item.Payload] = item.Number | |||
case "PRI": | |||
statsPrint() | |||
} | |||
} | |||
} | |||
func jsonPrettyPrint(in string) string { | |||
var out bytes.Buffer | |||
err := json.Indent(&out, []byte(in), "", "\t") | |||
if err != nil { | |||
return in | |||
} | |||
return out.String() | |||
} |
@ -0,0 +1,104 @@ | |||
package main | |||
import ( | |||
"bytes" | |||
"encoding/gob" | |||
"fmt" | |||
"time" | |||
"github.com/miekg/dns" | |||
) | |||
type cacheItem struct { | |||
Query []byte | |||
Date time.Time | |||
} | |||
//DomainCache stores a domain name inside the cache | |||
func DomainCache(s string, resp *dns.Msg) { | |||
var domain2cache cacheItem | |||
var err error | |||
var dom2 bytes.Buffer | |||
enc := gob.NewEncoder(&dom2) | |||
domain2cache.Query, err = resp.Pack() | |||
if err != nil { | |||
fmt.Println("Problems packing the response: ", err.Error()) | |||
} | |||
domain2cache.Date = time.Now() | |||
err = enc.Encode(domain2cache) | |||
if err != nil { | |||
fmt.Println("Cannot GOB the domain to cache: ", err.Error()) | |||
} | |||
cacheDomain(s, dom2.Bytes()) | |||
} | |||
func cacheDomain(key string, domain []byte) { | |||
err := MyZabovCDB.Put([]byte(key), domain, nil) | |||
if err != nil { | |||
fmt.Println("Cannot write to Cache DB: ", err.Error()) | |||
} | |||
} | |||
//GetDomainFromCache stores a domain name inside the cache | |||
func GetDomainFromCache(s string) *dns.Msg { | |||
ret := new(dns.Msg) | |||
var cache bytes.Buffer | |||
dec := gob.NewDecoder(&cache) | |||
var record cacheItem | |||
var conf []byte | |||
var errDB error | |||
if domainInCache(s) == false { | |||
return nil | |||
} | |||
conf, errDB = MyZabovCDB.Get([]byte(s), nil) | |||
if errDB != nil { | |||
fmt.Println("Cant READ DB :" , errDB.Error() ) | |||
return nil | |||
} | |||
cache.Write(conf) | |||
err := dec.Decode(&record) | |||
if err != nil { | |||
fmt.Println("Decode error :", err.Error()) | |||
return nil | |||
} | |||
if time.Since(record.Date) > (time.Duration(ZabovCacheTTL) * time.Hour) { | |||
return nil | |||
} | |||
err = ret.Unpack(record.Query) | |||
if err != nil { | |||
fmt.Println("Problem unpacking response: ", err.Error()) | |||
return nil | |||
} | |||
return ret | |||
} | |||
func domainInCache(domain string) bool { | |||
has, err := MyZabovCDB.Has([]byte(domain), nil) | |||
if err != nil { | |||
fmt.Println("Cannot search Cache DB: ", err.Error()) | |||
return false | |||
} | |||
return has | |||
} |
@ -0,0 +1,18 @@ | |||
FROM golang:1.14.1 AS builder | |||
RUN apt install git -y | |||
RUN mkdir -p /go/src/zabov | |||
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov | |||
WORKDIR /go/src/zabov | |||
ENV GO111MODULE=auto | |||
RUN go get ; go build -mod=vendor | |||
FROM debian:latest | |||
RUN apt update | |||
RUN apt upgrade -y | |||
RUN apt install ca-certificates -y | |||
RUN mkdir -p /opt/zabov | |||
WORKDIR /opt/zabov | |||
COPY --from=builder /go/src/zabov /opt/zabov | |||
EXPOSE 53/udp | |||
ENTRYPOINT ["/opt/zabov/zabov"] | |||
@ -0,0 +1,17 @@ | |||
FROM arm32v7/golang:1.14.1 AS builder | |||
RUN apt install git -y | |||
RUN mkdir -p /go/src/zabov | |||
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov | |||
WORKDIR /go/src/zabov | |||
ENV GO111MODULE=auto | |||
RUN go get ; go build -mod=vendor | |||
FROM arm32v7/debian:latest | |||
RUN apt update | |||
RUN apt upgrade -y | |||
RUN apt install ca-certificates -y | |||
RUN mkdir -p /opt/zabov | |||
WORKDIR /opt/zabov | |||
COPY --from=builder /go/src/zabov /opt/zabov | |||
EXPOSE 53/udp | |||
ENTRYPOINT ["/opt/zabov/zabov"] |
@ -0,0 +1,17 @@ | |||
FROM arm64v8/golang:1.14.1 AS builder | |||
RUN apt install git -y | |||
RUN mkdir -p /go/src/zabov | |||
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov | |||
WORKDIR /go/src/zabov | |||
ENV GO111MODULE=auto | |||
RUN go get ; go build -mod=vendor | |||
FROM arm64v8/debian:latest | |||
RUN apt update | |||
RUN apt upgrade -y | |||
RUN apt install ca-certificates -y | |||
RUN mkdir -p /opt/zabov | |||
WORKDIR /opt/zabov | |||
COPY --from=builder /go/src/zabov /opt/zabov | |||
EXPOSE 53/udp | |||
ENTRYPOINT ["/opt/zabov/zabov"] |
@ -0,0 +1,14 @@ | |||
Copyright (C) 2020 loweel@keinpfusch.net | |||
This program is free software: you can redistribute it and/or modify | |||
it under the terms of the GNU General Public License as published by | |||
the Free Software Foundation, either version 3 of the License, or | |||
(at your option) any later version. | |||
This program is distributed in the hope that it will be useful, | |||
but WITHOUT ANY WARRANTY; without even the implied warranty of | |||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |||
GNU General Public License for more details. | |||
You should have received a copy of the GNU General Public License | |||
along with this program. If not, see <https://www.gnu.org/licenses/>. |
@ -0,0 +1,91 @@ | |||
# zabov | |||
Tiny replacement for piHole DNS filter | |||
Still Work in progress, usable. | |||
Idea is to produce a very simple, no-web-interface , IP DNS blocker. | |||
# INSTALL | |||
Zabov requires golang 1.13 or later. | |||
<pre> | |||
git clone https://git.keinpfusch.net/Loweel/zabov.git | |||
cd zabov | |||
go get | |||
go build -mod=vendor | |||
</pre> | |||
Then, edit config.json: please notice config.json must be in the same folder of the executable you run. | |||
Just a few words about "singlefilters" and "doublefilters": | |||
Data must be downloaded from URLs of blacklist mantainers.They may come in different formats. | |||
There are two kinds of blacklists: | |||
One is the format zabov calls "singlefilter", where we find a single column , full of domains: | |||
<pre> | |||
domain1.com | |||
domain2.com | |||
domain3.com | |||
</pre> | |||
The second is the format zabov calls "doublefilter" (a file in "/etc/hosts" format, to be precise), where there is an IP, usually localhost or 0.0.0.0 and then the domain: | |||
<pre> | |||
127.0.0.1 domain1.com | |||
127.0.0.1 domain2.com | |||
127.0.0.1 domain3.com | |||
</pre> | |||
This is why configuration file has two separated items. | |||
The config file should look like: | |||
<pre> | |||
{ | |||
"zabov": { | |||
"port":"53", | |||
"proto":"udp", | |||
"ipaddr":"127.0.0.1", | |||
"upstream":"./dns-upstream.txt", | |||
"cachettl": "4", | |||
"killfilettl": "12", | |||
"singlefilters":"./urls-hosts.txt" , | |||
"doublefilters":"./urls-domains.txt", | |||
"blackholeip":"127.0.0.1", | |||
"hostsfile":"./urls-local.txt" | |||
} | |||
} | |||
</pre> | |||
Where: | |||
- port is the port number. Usually is 53, you can change for docker, if you like | |||
- proto is the protocol. Choices are "udp", "tcp", "tcp/udp" | |||
- ipaddr is the port to listen to. Maybe empty, (which will result in listening to 0.0.0.0) to avoid issues with docker. | |||
- upstream: file containing all DNS we want to query : each line in format IP:PORT | |||
- cachettl: amount of time the cache is kept (in hours) | |||
- killfilettl: refresh time for _killfiles_ | |||
- singlefilters: name of the file for blacklists following the "singlefilter" schema.(one URL per line) | |||
- doublefilters: name of the file, for blacklists following the "doublefilter" schema.(one URL per line) | |||
- blackholeip: IP address to return when the IP is banned. This is because you may want to avoid MX issues, mail loops on localhost, or you have a web server running on localhost | |||
- hostsfile: path where you keep your local blacklistfile : this is in the format "singlefilter", meaning one domain per line, unlike hosts file. | |||
# DOCKER | |||
Multistage Dockerfiles are provided for AMD64, ARMv7, ARM64V8 | |||
# TODO: | |||
- ~~caching~~ | |||
- monitoring port | |||
@ -0,0 +1,96 @@ | |||
package main | |||
import ( | |||
"bufio" | |||
"errors" | |||
"fmt" | |||
"net" | |||
"net/http" | |||
"strings" | |||
"time" | |||
) | |||
func init() { | |||
go downloadDoubleThread() | |||
} | |||
//DoubleIndexFilter puts the domains inside file | |||
func DoubleIndexFilter(durl string) error { | |||
fmt.Println("Retrieving HostFile from: ", durl) | |||
var err error | |||
// Get the data | |||
resp, err := http.Get(durl) | |||
if err != nil { | |||
fmt.Println("HTTP problem: ", err) | |||
return err | |||
} | |||
defer resp.Body.Close() | |||
if resp.StatusCode == 200 { // OK | |||
fmt.Println(durl + " Response: OK") | |||
} else { | |||
fmt.Println("Server <"+durl+"> returned status code: ", resp.StatusCode) | |||
return errors.New("Server <" + durl + "> returned status code: " + resp.Status) | |||
} | |||
scanner := bufio.NewScanner(resp.Body) | |||
splitter := func(c rune) bool { | |||
return c == ' ' || c == '\t' | |||
} | |||
var numLines int64 | |||
for scanner.Scan() { | |||
line := scanner.Text() | |||
h := strings.FieldsFunc(line, splitter) | |||
if h == nil { | |||
continue | |||
} | |||
if len(h) < 2 { | |||
continue | |||
} | |||
if net.ParseIP(h[0]) != nil { | |||
DomainKill(h[1], durl) | |||
// fmt.Println("MATCH: ", h[1]) | |||
numLines++ | |||
} else { | |||
incrementStats("Malformed HostLines "+durl, 1) | |||
// fmt.Println("Malformed line: <" + line + ">") | |||
} | |||
} | |||
fmt.Println("Finished to parse: "+durl+" ,number of lines", numLines) | |||
return err | |||
} | |||
func getDoubleFilters() { | |||
s := fileByLines(ZabovDoubleBL) | |||
for _, a := range s { | |||
DoubleIndexFilter(a) | |||
} | |||
} | |||
func downloadDoubleThread() { | |||
fmt.Println("Starting updater of DOUBLE lists, each (hours):", ZabovKillTTL) | |||
for { | |||
getDoubleFilters() | |||
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour) | |||
} | |||
} |
@ -0,0 +1,93 @@ | |||
package main | |||
import ( | |||
"bufio" | |||
"errors" | |||
"fmt" | |||
"net/http" | |||
"strings" | |||
"time" | |||
) | |||
func init() { | |||
go downloadThread() | |||
} | |||
//SingleIndexFilter puts the domains inside file | |||
func SingleIndexFilter(durl string) error { | |||
fmt.Println("Retrieving DomainFile from: ", durl) | |||
var err error | |||
// Get the data | |||
resp, err := http.Get(durl) | |||
if err != nil { | |||
fmt.Println("HTTP Problem: ", err) | |||
return err | |||
} | |||
defer resp.Body.Close() | |||
if resp.StatusCode == 200 { // OK | |||
fmt.Println(durl + " Response: OK") | |||
} else { | |||
fmt.Println("Server <"+durl+"> returned status code: ", resp.StatusCode) | |||
return errors.New("Server <" + durl + "> returned status code: " + resp.Status) | |||
} | |||
scanner := bufio.NewScanner(resp.Body) | |||
splitter := func(c rune) bool { | |||
return c == ' ' || c == '\t' | |||
} | |||
var numLines int64 | |||
for scanner.Scan() { | |||
line := scanner.Text() | |||
h := strings.FieldsFunc(line, splitter) | |||
if h == nil { | |||
continue | |||
} | |||
if len(h) < 1 { | |||
continue | |||
} | |||
if !strings.Contains(h[0], "#") { | |||
DomainKill(h[0], durl) | |||
// fmt.Println("MATCH: ", h[1]) | |||
numLines++ | |||
} else { | |||
incrementStats("Malformed DomainLines "+durl, 1) | |||
// fmt.Println("Malformed line: <" + line + ">") | |||
} | |||
} | |||
fmt.Println("Finished to parse: "+durl+" ,number of lines", numLines) | |||
return err | |||
} | |||
func getSingleFilters() { | |||
s := fileByLines(ZabovSingleBL) | |||
for _, a := range s { | |||
SingleIndexFilter(a) | |||
} | |||
} | |||
func downloadThread() { | |||
fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL) | |||
for { | |||
getSingleFilters() | |||
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour) | |||
} | |||
} |
@ -0,0 +1,15 @@ | |||
{ | |||
"zabov": { | |||
"port":"53", | |||
"proto":"udp", | |||
"ipaddr":"0.0.0.0", | |||
"upstream":"./dns-upstream.txt" , | |||
"cachettl": 1, | |||
"killfilettl": 12, | |||
"singlefilters":"./urls-domains.txt" , | |||
"doublefilters":"./urls-hosts.txt", | |||
"blackholeip":"127.0.0.1", | |||
"hostsfile":"./urls-local.txt" | |||
} | |||
} |
@ -0,0 +1,387 @@ | |||
194.150.168.168:53 | |||
194.25.0.68:53 | |||
141.1.1.1:53 | |||
213.239.204.35:53 | |||
194.25.0.52:53 | |||
212.211.132.4:53 | |||
213.68.194.51:53 | |||
195.243.214.4:53 | |||
141.1.27.249:53 | |||
80.237.197.14:53 | |||
217.28.98.62:53 | |||
82.96.64.2:53 | |||
82.96.65.2:53 | |||
194.25.0.60:53 | |||
193.101.111.10:53 | |||
193.101.111.20:53 | |||
192.76.144.66:53 | |||
217.69.169.25:53 | |||
85.88.19.10:53 | |||
85.88.19.11:53 | |||
85.214.20.141:53 | |||
194.169.239.10:53 | |||
194.172.160.4:53 | |||
212.102.225.2:53 | |||
212.51.16.1:53 | |||
212.51.17.1:53 | |||
212.66.129.98:53 | |||
212.89.130.180:53 | |||
213.209.122.11:53 | |||
213.23.108.129:53 | |||
91.204.4.133:53 | |||
84.200.69.80:53 | |||
62.146.63.211:53 | |||
212.77.178.83:53 | |||
78.46.58.246:53 | |||
85.214.102.25:53 | |||
87.106.62.128:53 | |||
81.20.87.84:53 | |||
81.20.87.181:53 | |||
217.5.159.227:53 | |||
185.38.9.99:53 | |||
62.146.25.130:53 | |||
37.59.218.50:53 | |||
212.184.191.193:53 | |||
213.23.143.154:53 | |||
178.32.187.10:53 | |||
62.154.214.86:53 | |||
85.214.151.164:53 | |||
5.175.225.2:53 | |||
193.105.38.142:53 | |||
145.253.183.21:53 | |||
178.15.146.43:53 | |||
62.245.233.22:53 | |||
212.224.71.71:53 | |||
213.136.78.213:53 | |||
80.156.196.196:53 | |||
193.158.99.67:53 | |||
194.95.75.230:53 | |||
212.184.191.2:53 | |||
213.138.38.22:53 | |||
195.145.80.150:53 | |||
139.18.25.34:53 | |||
79.143.180.116:53 | |||
213.240.172.200:53 | |||
217.160.238.238:53 | |||
213.61.185.238:53 | |||
84.201.0.34:53 | |||
82.194.105.219:53 | |||
62.157.89.178:53 | |||
46.189.26.123:53 | |||
85.214.208.8:53 | |||
87.239.128.130:53 | |||
78.111.65.40:53 | |||
85.214.69.126:53 | |||
109.75.29.1:53 | |||
80.81.19.226:53 | |||
81.169.162.74:53 | |||
217.14.164.35:53 | |||
5.9.172.92:53 | |||
62.225.102.180:53 | |||
217.7.71.203:53 | |||
217.6.71.61:53 | |||
62.154.138.43:53 | |||
146.0.38.140:53 | |||
78.111.67.10:53 | |||
217.6.110.20:53 | |||
87.245.18.221:53 | |||
62.225.66.19:53 | |||
81.169.212.52:53 | |||
178.162.205.123:53 | |||
212.227.83.183:53 | |||
139.18.25.33:53 | |||
193.29.2.4:53 | |||
212.91.246.11:53 | |||
62.153.141.15:53 | |||
148.251.120.228:53 | |||
62.154.253.226:53 | |||
194.25.218.2:53 | |||
194.174.73.36:53 | |||
62.245.226.182:53 | |||
87.234.222.68:53 | |||
194.25.169.130:53 | |||
62.225.15.253:53 | |||
176.94.20.4:53 | |||
188.40.115.29:53 | |||
188.40.115.22:53 | |||
194.187.240.10:53 | |||
80.150.109.197:53 | |||
217.86.149.109:53 | |||
91.208.193.1:53 | |||
195.243.99.35:53 | |||
62.225.102.177:53 | |||
178.210.102.9:53 | |||
80.228.113.125:53 | |||
178.210.102.12:53 | |||
130.255.121.9:53 | |||
212.204.56.218:53 | |||
37.59.218.151:53 | |||
80.148.52.109:53 | |||
194.30.174.222:53 | |||
5.199.141.5:53 | |||
94.135.173.22:53 | |||
88.79.208.11:53 | |||
141.16.180.9:53 | |||
82.193.241.125:53 | |||
212.8.216.37:53 | |||
109.75.29.2:53 | |||
78.46.17.82:53 | |||
81.169.185.49:53 | |||
217.244.13.14:53 | |||
93.104.209.27:53 | |||
79.143.182.174:53 | |||
81.20.82.131:53 | |||
213.136.68.181:53 | |||
213.136.68.189:53 | |||
193.107.145.233:53 | |||
195.145.241.3:53 | |||
80.242.182.182:53 | |||
193.159.181.250:53 | |||
195.243.124.75:53 | |||
62.159.104.102:53 | |||
92.222.202.244:53 | |||
85.214.254.13:53 | |||
85.114.128.115:53 | |||
145.253.176.50:53 | |||
217.7.63.1:53 | |||
78.35.40.149:53 | |||
81.169.187.253:53 | |||
94.249.192.20:53 | |||
85.214.43.157:53 | |||
80.149.83.60:53 | |||
178.210.102.225:53 | |||
178.210.102.193:53 | |||
62.154.236.126:53 | |||
213.183.185.50:53 | |||
212.60.229.242:53 | |||
80.146.192.66:53 | |||
79.133.62.62:53 | |||
178.33.33.219:53 | |||
62.245.225.225:53 | |||
46.38.235.212:53 | |||
213.136.88.31:53 | |||
212.66.135.250:53 | |||
194.231.138.26:53 | |||
62.225.1.33:53 | |||
80.148.34.131:53 | |||
94.23.163.114:53 | |||
80.64.189.94:53 | |||
81.169.241.28:53 | |||
212.38.26.132:53 | |||
62.91.19.67:53 | |||
87.239.128.25:53 | |||
212.185.196.10:53 | |||
89.221.2.171:53 | |||
217.243.239.11:53 | |||
213.136.69.214:53 | |||
213.138.56.75:53 | |||
212.122.52.11:53 | |||
46.4.166.113:53 | |||
77.37.30.12:53 | |||
194.187.242.10:53 | |||
188.40.132.212:53 | |||
194.150.168.169:53 | |||
85.25.105.193:53 | |||
185.93.180.131:53 | |||
109.234.249.10:53 | |||
109.234.248.10:53 | |||
138.201.120.250:53 | |||
81.3.27.54:53 | |||
78.46.231.161:53 | |||
78.46.231.162:53 | |||
212.51.16.197:53 | |||
212.28.34.65:53 | |||
148.251.24.48:53 | |||
212.75.32.4:53 | |||
91.103.112.150:53 | |||
217.69.169.26:53 | |||
195.63.103.144:53 | |||
213.209.121.30:53 | |||
88.79.149.4:53 | |||
185.194.143.243:53 | |||
46.182.19.48:53 | |||
217.111.24.246:53 | |||
62.96.37.74:53 | |||
213.61.64.174:53 | |||
213.61.65.226:53 | |||
62.96.190.134:53 | |||
217.111.123.166:53 | |||
213.61.176.118:53 | |||
185.216.33.82:53 | |||
185.220.70.50:53 | |||
188.138.57.95:53 | |||
195.145.137.164:53 | |||
195.167.223.164:53 | |||
195.226.69.82:53 | |||
195.37.174.194:53 | |||
195.4.138.12:53 | |||
195.63.61.189:53 | |||
212.124.35.25:53 | |||
212.184.191.100:53 | |||
212.38.2.130:53 | |||
212.51.16.193:53 | |||
212.66.129.107:53 | |||
212.8.216.41:53 | |||
212.89.128.28:53 | |||
213.133.116.14:53 | |||
213.166.247.100:53 | |||
217.243.173.82:53 | |||
217.5.182.118:53 | |||
217.7.80.40:53 | |||
217.7.81.136:53 | |||
217.9.50.199:53 | |||
46.237.220.2:53 | |||
5.189.179.105:53 | |||
52.28.79.14:53 | |||
52.29.2.17:53 | |||
62.146.202.2:53 | |||
62.146.2.48:53 | |||
62.153.122.2:53 | |||
62.153.237.200:53 | |||
62.153.237.201:53 | |||
62.154.139.99:53 | |||
62.154.159.12:53 | |||
62.154.159.5:53 | |||
62.154.160.3:53 | |||
62.209.40.75:53 | |||
62.217.61.162:53 | |||
62.245.225.55:53 | |||
78.111.224.224:53 | |||
78.111.226.226:53 | |||
78.138.80.42:53 | |||
80.149.112.139:53 | |||
80.156.6.209:53 | |||
80.190.209.218:53 | |||
80.228.231.122:53 | |||
80.228.231.48:53 | |||
80.245.65.100:53 | |||
81.14.182.169:53 | |||
81.27.162.100:53 | |||
83.97.23.178:53 | |||
83.97.23.226:53 | |||
84.16.240.43:53 | |||
85.214.98.185:53 | |||
89.19.228.52:53 | |||
89.200.168.203:53 | |||
91.217.86.4:53 | |||
93.104.195.2:53 | |||
94.247.43.254:53 | |||
109.234.248.8:53 | |||
131.220.20.199:53 | |||
131.220.23.123:53 | |||
144.76.173.169:53 | |||
144.76.83.104:53 | |||
148.251.92.241:53 | |||
176.9.136.236:53 | |||
195.10.195.195:53 | |||
173.212.249.41:53 | |||
85.214.41.155:53 | |||
54.37.75.2:53 | |||
194.55.13.75:53 | |||
5.189.138.153:53 | |||
159.69.51.18:53 | |||
51.75.77.179:53 | |||
138.201.169.84:53 | |||
138.201.239.66:53 | |||
138.68.106.109:53 | |||
145.253.109.162:53 | |||
159.69.68.181:53 | |||
167.86.78.56:53 | |||
173.212.208.116:53 | |||
173.212.218.206:53 | |||
173.212.219.129:53 | |||
173.212.239.87:53 | |||
173.212.242.89:53 | |||
173.212.244.78:53 | |||
173.249.41.233:53 | |||
173.249.48.6:53 | |||
176.9.233.171:53 | |||
176.9.58.218:53 | |||
178.162.199.27:53 | |||
178.162.208.135:53 | |||
178.238.230.127:53 | |||
178.238.235.218:53 | |||
18.195.121.224:53 | |||
185.139.98.100:53 | |||
185.40.135.11:53 | |||
185.53.169.22:53 | |||
185.90.131.194:53 | |||
188.40.239.99:53 | |||
188.68.35.145:53 | |||
192.162.85.48:53 | |||
193.159.232.5:53 | |||
194.77.237.31:53 | |||
194.77.253.32:53 | |||
195.201.192.29:53 | |||
195.202.52.30:53 | |||
195.243.101.5:53 | |||
207.180.203.42:53 | |||
207.180.243.200:53 | |||
207.180.247.212:53 | |||
213.136.71.68:53 | |||
213.136.77.39:53 | |||
213.144.24.234:53 | |||
217.147.96.210:53 | |||
217.182.198.203:53 | |||
217.6.131.248:53 | |||
217.6.247.237:53 | |||
217.6.64.5:53 | |||
217.79.177.220:53 | |||
46.163.119.155:53 | |||
46.228.199.116:53 | |||
5.175.26.208:53 | |||
51.77.65.15:53 | |||
5.189.133.151:53 | |||
5.189.141.216:53 | |||
5.189.186.154:53 | |||
5.189.186.93:53 | |||
5.189.187.34:53 | |||
5.199.141.30:53 | |||
5.45.96.220:53 | |||
62.138.20.211:53 | |||
62.144.82.252:53 | |||
62.153.165.107:53 | |||
62.153.201.91:53 | |||
62.154.214.84:53 | |||
62.157.242.85:53 | |||
79.143.177.243:53 | |||
79.143.183.45:53 | |||
80.156.198.146:53 | |||
80.156.6.206:53 | |||
80.237.207.100:53 | |||
80.82.223.94:53 | |||
81.169.215.29:53 | |||
81.169.223.126:53 | |||
81.169.230.157:53 | |||
81.20.80.79:53 | |||
83.236.183.211:53 | |||
84.16.240.224:53 | |||
85.214.224.76:53 | |||
85.214.238.190:53 | |||
85.214.246.133:53 | |||
85.214.62.160:53 | |||
85.93.91.101:53 | |||
87.106.63.208:53 | |||
87.118.126.225:53 | |||
88.198.37.146:53 | |||
88.99.66.18:53 | |||
89.163.150.209:53 | |||
89.163.220.114:53 | |||
89.19.236.152:53 | |||
93.104.213.74:53 | |||
93.186.196.137:53 | |||
93.190.71.172:53 | |||
94.177.246.221:53 | |||
80.241.218.68:53 | |||
172.105.81.90:53 | |||
172.105.81.92:53 | |||
84.200.70.40:53 | |||
94.16.114.254:53 | |||
93.90.207.192:53 | |||
93.90.201.211:53 | |||
144.91.68.146:53 | |||
176.9.37.132:53 | |||
176.9.93.198:53 | |||
176.9.1.117:53 | |||
144.91.115.47:53 | |||
91.237.100.4:53 |
@ -0,0 +1,98 @@ | |||
package main | |||
import ( | |||
"fmt" | |||
"time" | |||
"math/rand" | |||
"strings" | |||
"github.com/miekg/dns" | |||
) | |||
//ForwardQuery forwards the query to the upstream server | |||
//first server to answer wins | |||
func ForwardQuery(query *dns.Msg) *dns.Msg { | |||
go incrementStats("ForwardQueries", 1) | |||
r := new(dns.Msg) | |||
r.SetReply(query) | |||
r.Authoritative = true | |||
fqdn := strings.TrimRight(query.Question[0].Name, ".") | |||
lfqdn := fmt.Sprintf("%d", query.Question[0].Qtype) + "." + fqdn | |||
if cached := GetDomainFromCache(lfqdn); cached != nil { | |||
go incrementStats("CacheHit", 1) | |||
cached.SetReply(query) | |||
cached.Authoritative = true | |||
return cached | |||
} | |||
c := new(dns.Client) | |||
c.ReadTimeout = 500 * time.Millisecond | |||
c.WriteTimeout = 500 * time.Millisecond | |||
for { | |||
// round robin with retry | |||
if !NetworkUp { | |||
time.Sleep(10 * time.Second) | |||
go incrementStats("Network Problems ", 1) | |||
continue | |||
} | |||
d := oneTimeDNS() | |||
in, _, err := c.Exchange(query, d) | |||
if err != nil { | |||
fmt.Printf("Problem with DNS %s : %s\n", d, err.Error()) | |||
go incrementStats("DNS Problems "+d, 1) | |||
continue | |||
} else { | |||
go incrementStats(d, 1) | |||
in.SetReply(query) | |||
in.Authoritative = true | |||
go DomainCache(lfqdn, in) | |||
return in | |||
} | |||
} | |||
} | |||
func init() { | |||
fmt.Println("DNS client engine starting") | |||
NetworkUp = checkNetworkUp() | |||
if NetworkUp { | |||
fmt.Println("[OK]: Network is UP") | |||
} else { | |||
fmt.Println("[KO] Network is DOWN: system will check again in 2 minutes") | |||
} | |||
} | |||
func oneTimeDNS() (dns string) { | |||
rand.Seed(time.Now().Unix()) | |||
upl := ZabovDNSArray | |||
if len(upl) < 1 { | |||
fmt.Println("No DNS defined, using default 127.0.0.53:53. Hope it works!") | |||
return "127.0.0.53:53" | |||
} | |||
n := rand.Intn(128*len(upl)) % len(upl) | |||
dns = upl[n] | |||
return | |||
} |
@ -0,0 +1,44 @@ | |||
package main | |||
import ( | |||
"net" | |||
"strings" | |||
"github.com/miekg/dns" | |||
) | |||
func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { | |||
go incrementStats("TotalQueries", 1) | |||
remIP, _, e := net.SplitHostPort(w.RemoteAddr().String()) | |||
if e != nil { | |||
go incrementStats("CLIENT: "+remIP, 1) | |||
} | |||
msg := dns.Msg{} | |||
msg.SetReply(r) | |||
switch r.Question[0].Qtype { | |||
case dns.TypeA: | |||
msg.Authoritative = true | |||
domain := msg.Question[0].Name | |||
fqdn := strings.TrimRight(domain, ".") | |||
if domainInKillfile(fqdn) { | |||
go incrementStats("Killed", 1) | |||
msg.Answer = append(msg.Answer, &dns.A{ | |||
Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, | |||
A: net.ParseIP(ZabovAddBL), | |||
}) | |||
} else { | |||
ret := ForwardQuery(r) | |||
w.WriteMsg(ret) | |||
} | |||
default: | |||
ret := ForwardQuery(r) | |||
w.WriteMsg(ret) | |||
} | |||
w.WriteMsg(&msg) | |||
} |
@ -0,0 +1,8 @@ | |||
module zabov | |||
go 1.13 | |||
require ( | |||
github.com/miekg/dns v1.1.27 | |||
github.com/syndtr/goleveldb v1.0.0 | |||
) |
@ -0,0 +1,36 @@ | |||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= | |||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= | |||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= | |||
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= | |||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= | |||
github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= | |||
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= | |||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= | |||
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= | |||
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= | |||
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= | |||
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= | |||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | |||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= | |||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= | |||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= | |||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | |||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | |||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | |||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= | |||
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | |||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |||
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= | |||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | |||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | |||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= | |||
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | |||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= | |||
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= | |||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | |||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= | |||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= | |||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= | |||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= |
@ -0,0 +1,59 @@ | |||
package main | |||
import ( | |||
"bufio" | |||
"fmt" | |||
"os" | |||
) | |||
func init() { | |||
fmt.Println("Ingesting local hosts file") | |||
ingestLocalBlacklist() | |||
} | |||
func ingestLocalBlacklist() { | |||
file, err := os.Open(ZabovHostsFile) | |||
if err != nil { | |||
fmt.Println(err.Error()) | |||
} | |||
defer file.Close() | |||
scanner := bufio.NewScanner(file) | |||
for scanner.Scan() { | |||
d := scanner.Text() | |||
DomainKill(d, ZabovHostsFile) | |||
incrementStats("Blacklist", 1) | |||
} | |||
if err := scanner.Err(); err != nil { | |||
fmt.Println(err.Error()) | |||
} | |||
} | |||
func fileByLines(filename string) (blurls []string) { | |||
file, err := os.Open(filename) | |||
if err != nil { | |||
fmt.Println(err.Error()) | |||
} | |||
defer file.Close() | |||
scanner := bufio.NewScanner(file) | |||
for scanner.Scan() { | |||
d := scanner.Text() | |||
blurls = append(blurls, d) | |||
} | |||
if err := scanner.Err(); err != nil { | |||
fmt.Println(err.Error()) | |||
} | |||
return | |||
} |
@ -0,0 +1,46 @@ | |||
package main | |||
import ( | |||
"log" | |||
"github.com/miekg/dns" | |||
) | |||
//MyDNS is my dns server | |||
var MyDNS *dns.Server | |||
//ZabovUpDNS keeps the name of upstream DNSs | |||
var ZabovUpDNS string | |||
//ZabovSingleBL list of urls returning a file with just names of domains | |||
var ZabovSingleBL string | |||
//ZabovDoubleBL list of urls returning a file with IP<space>domain | |||
var ZabovDoubleBL string | |||
//ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1 | |||
var ZabovAddBL string | |||
//ZabovCacheTTL is the amount of hours we cache records of DNS | |||
var ZabovCacheTTL int | |||
//ZabovKillTTL is the amount of hours we cache the killfile | |||
var ZabovKillTTL int | |||
//ZabovHostsFile is the file we use to keep our hosts | |||
var ZabovHostsFile string | |||
//ZabovDNSArray is the array containing all the DNS we mention | |||
var ZabovDNSArray []string | |||
type handler struct{} | |||
func main() { | |||
MyDNS.Handler = &handler{} | |||
if err := MyDNS.ListenAndServe(); err != nil { | |||
log.Printf("Failed to set udp listener %s\n", err.Error()) | |||
} else { | |||
log.Printf("Listener running \n") | |||
} | |||
} |
@ -0,0 +1,37 @@ | |||
https://mirror1.malwaredomains.com/files/justdomains | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adaway.org/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adblock-nocoin-list/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adguard-simplified/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/anudeepnd-adservers/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-ad/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-malvertising/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-malware/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-tracking/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easyprivacy/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/eth-phishing-detect/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.2o7net/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.dead/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.risk/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.spam/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/kadhosts/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomainlist.com/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomains.com-immortaldomains/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomains.com-justdomains/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/matomo.org-spammers/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/mitchellkrogza-badd-boyz-hosts/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/pgl.yoyo.org/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ransomwaretracker.abuse.ch/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/someonewhocares.org/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/spam404.com/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/stevenblack/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/winhelp2002.mvps.org/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/zerodot1-coinblockerlists-browser/list.txt | |||
https://raw.githubusercontent.com/hectorm/hmirror/master/data/zeustracker.abuse.ch/list.txt | |||
https://raw.githubusercontent.com/CHEF-KOCH/Audio-fingerprint-pages/master/AudioFp.txt | |||
https://raw.githubusercontent.com/CHEF-KOCH/Canvas-fingerprinting-pages/master/Canvas.txt | |||
https://raw.githubusercontent.com/CHEF-KOCH/WebRTC-tracking/master/WebRTC.txt | |||
https://raw.githubusercontent.com/CHEF-KOCH/CKs-FilterList/master/Anti-Corp/hosts/NSABlocklist.txt | |||
https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt | |||
https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt | |||
https://www.stopforumspam.com/downloads/toxic_domains_whole.txt |
@ -0,0 +1,12 @@ | |||
http://sysctl.org/cameleon/hosts | |||
https://www.malwaredomainlist.com/hostslist/hosts.txt | |||
https://adaway.org/hosts.txt | |||
https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts | |||
https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews/hosts | |||
https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling/hosts | |||
https://someonewhocares.org/hosts/hosts | |||
https://getadhell.com/standard-package.txt | |||
https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt | |||
https://raw.githubusercontent.com/notracking/hosts-blocklists/master/hostnames.txt | |||
https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt | |||
https://raw.githubusercontent.com/anudeepND/blacklist/master/facebook.txt |
@ -0,0 +1,8 @@ | |||
blc.vodafone.com | |||
gab.com | |||
gab.ai | |||
freespeechextremist.com | |||
neckbeard.xyz | |||
funkwhale.it | |||
social.byoblu.com | |||
@ -0,0 +1,16 @@ | |||
cmd/snappytool/snappytool | |||
testdata/bench | |||
# These explicitly listed benchmark data files are for an obsolete version of | |||
# snappy_test.go. | |||
testdata/alice29.txt | |||
testdata/asyoulik.txt | |||
testdata/fireworks.jpeg | |||
testdata/geo.protodata | |||
testdata/html | |||
testdata/html_x_4 | |||
testdata/kppkn.gtb | |||
testdata/lcet10.txt | |||
testdata/paper-100k.pdf | |||
testdata/plrabn12.txt | |||
testdata/urls.10K |
@ -0,0 +1,15 @@ | |||
# This is the official list of Snappy-Go authors for copyright purposes. | |||
# This file is distinct from the CONTRIBUTORS files. | |||
# See the latter for an explanation. | |||
# Names should be added to this file as | |||
# Name or Organization <email address> | |||
# The email address is not required for organizations. | |||
# Please keep the list sorted. | |||
Damian Gryski <dgryski@gmail.com> | |||
Google Inc. | |||
Jan Mercl <0xjnml@gmail.com> | |||
Rodolfo Carvalho <rhcarvalho@gmail.com> | |||
Sebastien Binet <seb.binet@gmail.com> |
@ -0,0 +1,37 @@ | |||
# This is the official list of people who can contribute | |||
# (and typically have contributed) code to the Snappy-Go repository. | |||
# The AUTHORS file lists the copyright holders; this file | |||
# lists people. For example, Google employees are listed here | |||
# but not in AUTHORS, because Google holds the copyright. | |||
# | |||
# The submission process automatically checks to make sure | |||
# that people submitting code are listed in this file (by email address). | |||
# | |||
# Names should be added to this file only after verifying that | |||
# the individual or the individual's organization has agreed to | |||
# the appropriate Contributor License Agreement, found here: | |||
# | |||
# http://code.google.com/legal/individual-cla-v1.0.html | |||
# http://code.google.com/legal/corporate-cla-v1.0.html | |||
# | |||
# The agreement for individuals can be filled out on the web. | |||
# | |||
# When adding J Random Contributor's name to this file, | |||
# either J's name or J's organization's name should be | |||
# added to the AUTHORS file, depending on whether the | |||
# individual or corporate CLA was used. | |||
# Names should be added to this file like so: | |||
# Name <email address> | |||
# Please keep the list sorted. | |||
Damian Gryski <dgryski@gmail.com> | |||
Jan Mercl <0xjnml@gmail.com> | |||
Kai Backman <kaib@golang.org> | |||
Marc-Antoine Ruel <maruel@chromium.org> | |||
Nigel Tao <nigeltao@golang.org> | |||
Rob Pike <r@golang.org> | |||
Rodolfo Carvalho <rhcarvalho@gmail.com> | |||
Russ Cox <rsc@golang.org> | |||
Sebastien Binet <seb.binet@gmail.com> |
@ -0,0 +1,27 @@ | |||
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. | |||
Redistribution and use in source and binary forms, with or without | |||
modification, are permitted provided that the following conditions are | |||
met: | |||
* Redistributions of source code must retain the above copyright | |||
notice, this list of conditions and the following disclaimer. | |||
* Redistributions in binary form must reproduce the above | |||
copyright notice, this list of conditions and the following disclaimer | |||
in the documentation and/or other materials provided with the | |||
distribution. | |||
* Neither the name of Google Inc. nor the names of its | |||
contributors may be used to endorse or promote products derived from | |||
this software without specific prior written permission. | |||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |||
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |||
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |||
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |||
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |||
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |||
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |||
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |||
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |||
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
@ -0,0 +1,107 @@ | |||
The Snappy compression format in the Go programming language. | |||
To download and install from source: | |||
$ go get github.com/golang/snappy | |||
Unless otherwise noted, the Snappy-Go source files are distributed | |||
under the BSD-style license found in the LICENSE file. | |||
Benchmarks. | |||
The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten | |||
or so files, the same set used by the C++ Snappy code (github.com/google/snappy | |||
and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @ | |||
3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29: | |||
"go test -test.bench=." | |||
_UFlat0-8 2.19GB/s ± 0% html | |||
_UFlat1-8 1.41GB/s ± 0% urls | |||
_UFlat2-8 23.5GB/s ± 2% jpg | |||
_UFlat3-8 1.91GB/s ± 0% jpg_200 | |||
_UFlat4-8 14.0GB/s ± 1% pdf | |||
_UFlat5-8 1.97GB/s ± 0% html4 | |||
_UFlat6-8 814MB/s ± 0% txt1 | |||
_UFlat7-8 785MB/s ± 0% txt2 | |||
_UFlat8-8 857MB/s ± 0% txt3 | |||
_UFlat9-8 719MB/s ± 1% txt4 | |||
_UFlat10-8 2.84GB/s ± 0% pb | |||
_UFlat11-8 1.05GB/s ± 0% gaviota | |||
_ZFlat0-8 1.04GB/s ± 0% html | |||
_ZFlat1-8 534MB/s ± 0% urls | |||
_ZFlat2-8 15.7GB/s ± 1% jpg | |||
_ZFlat3-8 740MB/s ± 3% jpg_200 | |||
_ZFlat4-8 9.20GB/s ± 1% pdf | |||
_ZFlat5-8 991MB/s ± 0% html4 | |||
_ZFlat6-8 379MB/s ± 0% txt1 | |||
_ZFlat7-8 352MB/s ± 0% txt2 | |||
_ZFlat8-8 396MB/s ± 1% txt3 | |||
_ZFlat9-8 327MB/s ± 1% txt4 | |||
_ZFlat10-8 1.33GB/s ± 1% pb | |||
_ZFlat11-8 605MB/s ± 1% gaviota | |||
"go test -test.bench=. -tags=noasm" | |||
_UFlat0-8 621MB/s ± 2% html | |||
_UFlat1-8 494MB/s ± 1% urls | |||
_UFlat2-8 23.2GB/s ± 1% jpg | |||
_UFlat3-8 1.12GB/s ± 1% jpg_200 | |||
_UFlat4-8 4.35GB/s ± 1% pdf | |||
_UFlat5-8 609MB/s ± 0% html4 | |||
_UFlat6-8 296MB/s ± 0% txt1 | |||
_UFlat7-8 288MB/s ± 0% txt2 | |||
_UFlat8-8 309MB/s ± 1% txt3 | |||
_UFlat9-8 280MB/s ± 1% txt4 | |||
_UFlat10-8 753MB/s ± 0% pb | |||
_UFlat11-8 400MB/s ± 0% gaviota | |||
_ZFlat0-8 409MB/s ± 1% html | |||
_ZFlat1-8 250MB/s ± 1% urls | |||
_ZFlat2-8 12.3GB/s ± 1% jpg | |||
_ZFlat3-8 132MB/s ± 0% jpg_200 | |||
_ZFlat4-8 2.92GB/s ± 0% pdf | |||
_ZFlat5-8 405MB/s ± 1% html4 | |||
_ZFlat6-8 179MB/s ± 1% txt1 | |||
_ZFlat7-8 170MB/s ± 1% txt2 | |||
_ZFlat8-8 189MB/s ± 1% txt3 | |||
_ZFlat9-8 164MB/s ± 1% txt4 | |||
_ZFlat10-8 479MB/s ± 1% pb | |||
_ZFlat11-8 270MB/s ± 1% gaviota | |||
For comparison (Go's encoded output is byte-for-byte identical to C++'s), here | |||
are the numbers from C++ Snappy's | |||
make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log | |||
BM_UFlat/0 2.4GB/s html | |||
BM_UFlat/1 1.4GB/s urls | |||
BM_UFlat/2 21.8GB/s jpg | |||
BM_UFlat/3 1.5GB/s jpg_200 | |||
BM_UFlat/4 13.3GB/s pdf | |||
BM_UFlat/5 2.1GB/s html4 | |||
BM_UFlat/6 1.0GB/s txt1 | |||
BM_UFlat/7 959.4MB/s txt2 | |||
BM_UFlat/8 1.0GB/s txt3 | |||
BM_UFlat/9 864.5MB/s txt4 | |||
BM_UFlat/10 2.9GB/s pb | |||
BM_UFlat/11 1.2GB/s gaviota | |||
BM_ZFlat/0 944.3MB/s html (22.31 %) | |||
BM_ZFlat/1 501.6MB/s urls (47.78 %) | |||
BM_ZFlat/2 14.3GB/s jpg (99.95 %) | |||
BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %) | |||
BM_ZFlat/4 8.3GB/s pdf (83.30 %) | |||
BM_ZFlat/5 903.5MB/s html4 (22.52 %) | |||
BM_ZFlat/6 336.0MB/s txt1 (57.88 %) | |||
BM_ZFlat/7 312.3MB/s txt2 (61.91 %) | |||
BM_ZFlat/8 353.1MB/s txt3 (54.99 %) | |||
BM_ZFlat/9 289.9MB/s txt4 (66.26 %) | |||