Browse Source

first commit

master 1.0.0
LowEel 2 weeks ago
parent
commit
5c5e4e4417
685 changed files with 301506 additions and 0 deletions
  1. +10
    -0
      .gitignore
  2. +38
    -0
      00.database.go
  3. +26
    -0
      00.memory.go
  4. +69
    -0
      01.conf.go
  5. +37
    -0
      01.dnscheck.go
  6. +79
    -0
      01.killfile.go
  7. +100
    -0
      01.stats.go
  8. +104
    -0
      02.cache.go
  9. +18
    -0
      Dockerfile.amd64
  10. +17
    -0
      Dockerfile.arm32v7
  11. +17
    -0
      Dockerfile.arm64v8
  12. +14
    -0
      LICENSE
  13. +91
    -0
      README.md
  14. +96
    -0
      adlist_hosts.go
  15. +93
    -0
      adlist_single.go
  16. +15
    -0
      config.json
  17. +387
    -0
      dns-upstream.txt
  18. +98
    -0
      dns_client.go
  19. +44
    -0
      dns_handler.go
  20. +8
    -0
      go.mod
  21. +36
    -0
      go.sum
  22. +59
    -0
      hostfile.go
  23. +46
    -0
      main.go
  24. +37
    -0
      urls-domains.txt
  25. +12
    -0
      urls-hosts.txt
  26. +8
    -0
      urls-local.txt
  27. +16
    -0
      vendor/github.com/golang/snappy/.gitignore
  28. +15
    -0
      vendor/github.com/golang/snappy/AUTHORS
  29. +37
    -0
      vendor/github.com/golang/snappy/CONTRIBUTORS
  30. +27
    -0
      vendor/github.com/golang/snappy/LICENSE
  31. +107
    -0
      vendor/github.com/golang/snappy/README
  32. +237
    -0
      vendor/github.com/golang/snappy/decode.go
  33. +14
    -0
      vendor/github.com/golang/snappy/decode_amd64.go
  34. +490
    -0
      vendor/github.com/golang/snappy/decode_amd64.s
  35. +101
    -0
      vendor/github.com/golang/snappy/decode_other.go
  36. +285
    -0
      vendor/github.com/golang/snappy/encode.go
  37. +29
    -0
      vendor/github.com/golang/snappy/encode_amd64.go
  38. +730
    -0
      vendor/github.com/golang/snappy/encode_amd64.s
  39. +238
    -0
      vendor/github.com/golang/snappy/encode_other.go
  40. +98
    -0
      vendor/github.com/golang/snappy/snappy.go
  41. +8
    -0
      vendor/github.com/miekg/dns/.codecov.yml
  42. +4
    -0
      vendor/github.com/miekg/dns/.gitignore
  43. +17
    -0
      vendor/github.com/miekg/dns/.travis.yml
  44. +1
    -0
      vendor/github.com/miekg/dns/AUTHORS
  45. +1
    -0
      vendor/github.com/miekg/dns/CODEOWNERS
  46. +10
    -0
      vendor/github.com/miekg/dns/CONTRIBUTORS
  47. +9
    -0
      vendor/github.com/miekg/dns/COPYRIGHT
  48. +30
    -0
      vendor/github.com/miekg/dns/LICENSE
  49. +33
    -0
      vendor/github.com/miekg/dns/Makefile.fuzz
  50. +52
    -0
      vendor/github.com/miekg/dns/Makefile.release
  51. +175
    -0
      vendor/github.com/miekg/dns/README.md
  52. +61
    -0
      vendor/github.com/miekg/dns/acceptfunc.go
  53. +415
    -0
      vendor/github.com/miekg/dns/client.go
  54. +135
    -0
      vendor/github.com/miekg/dns/clientconfig.go
  55. +43
    -0
      vendor/github.com/miekg/dns/dane.go
  56. +378
    -0
      vendor/github.com/miekg/dns/defaults.go
  57. +134
    -0
      vendor/github.com/miekg/dns/dns.go
  58. +794
    -0
      vendor/github.com/miekg/dns/dnssec.go
  59. +140
    -0
      vendor/github.com/miekg/dns/dnssec_keygen.go
  60. +322
    -0
      vendor/github.com/miekg/dns/dnssec_keyscan.go
  61. +94
    -0
      vendor/github.com/miekg/dns/dnssec_privkey.go
  62. +268
    -0
      vendor/github.com/miekg/dns/doc.go
  63. +38
    -0
      vendor/github.com/miekg/dns/duplicate.go
  64. +675
    -0
      vendor/github.com/miekg/dns/edns.go
  65. +93
    -0
      vendor/github.com/miekg/dns/format.go
  66. +32
    -0
      vendor/github.com/miekg/dns/fuzz.go
  67. +247
    -0
      vendor/github.com/miekg/dns/generate.go
  68. +11
    -0
      vendor/github.com/miekg/dns/go.mod
  69. +39
    -0
      vendor/github.com/miekg/dns/go.sum
  70. +212
    -0
      vendor/github.com/miekg/dns/labels.go
  71. +44
    -0
      vendor/github.com/miekg/dns/listen_go111.go
  72. +23
    -0
      vendor/github.com/miekg/dns/listen_go_not111.go
  73. +1196
    -0
      vendor/github.com/miekg/dns/msg.go
  74. +810
    -0
      vendor/github.com/miekg/dns/msg_helpers.go
  75. +111
    -0
      vendor/github.com/miekg/dns/msg_truncate.go
  76. +95
    -0
      vendor/github.com/miekg/dns/nsecx.go
  77. +114
    -0
      vendor/github.com/miekg/dns/privaterr.go
  78. +52
    -0
      vendor/github.com/miekg/dns/reverse.go
  79. +86
    -0
      vendor/github.com/miekg/dns/sanitize.go
  80. +1408
    -0
      vendor/github.com/miekg/dns/scan.go
  81. +1764
    -0
      vendor/github.com/miekg/dns/scan_rr.go
  82. +123
    -0
      vendor/github.com/miekg/dns/serve_mux.go
  83. +764
    -0
      vendor/github.com/miekg/dns/server.go
  84. +209
    -0
      vendor/github.com/miekg/dns/sig0.go
  85. +61
    -0
      vendor/github.com/miekg/dns/singleinflight.go
  86. +44
    -0
      vendor/github.com/miekg/dns/smimea.go
  87. +44
    -0
      vendor/github.com/miekg/dns/tlsa.go
  88. +389
    -0
      vendor/github.com/miekg/dns/tsig.go
  89. +1527
    -0
      vendor/github.com/miekg/dns/types.go
  90. +102
    -0
      vendor/github.com/miekg/dns/udp.go
  91. +35
    -0
      vendor/github.com/miekg/dns/udp_windows.go
  92. +110
    -0
      vendor/github.com/miekg/dns/update.go
  93. +15
    -0
      vendor/github.com/miekg/dns/version.go
  94. +266
    -0
      vendor/github.com/miekg/dns/xfr.go
  95. +1157
    -0
      vendor/github.com/miekg/dns/zduplicate.go
  96. +2741
    -0
      vendor/github.com/miekg/dns/zmsg.go
  97. +898
    -0
      vendor/github.com/miekg/dns/ztypes.go
  98. +24
    -0
      vendor/github.com/syndtr/goleveldb/LICENSE
  99. +349
    -0
      vendor/github.com/syndtr/goleveldb/leveldb/batch.go
  100. +704
    -0
      vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go

+ 10
- 0
.gitignore View File

@ -0,0 +1,10 @@
zabov
killfile
killfile/*
db/zabov.db
binaries
binaries/*
build.sh
.vscode
.vscode/*

+ 38
- 0
00.database.go View File

@ -0,0 +1,38 @@
package main
import (
"fmt"
"os"
"github.com/syndtr/goleveldb/leveldb"
)
//MyZabovKDB is the storage where we'll put domains to block
var MyZabovKDB *leveldb.DB
//MyZabovCDB is the storage where we'll put domains to cache
var MyZabovCDB *leveldb.DB
func init() {
var err error
os.RemoveAll("./db")
os.MkdirAll("./db", 0755)
MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil)
if err != nil {
fmt.Println("Cannot create Killfile db: ", err.Error())
} else {
fmt.Println("Killfile DB created")
}
MyZabovCDB, err = leveldb.OpenFile("./db/cache", nil)
if err != nil {
fmt.Println("Cannot create Cache db: ", err.Error())
} else {
fmt.Println("Cache DB created")
}
}

+ 26
- 0
00.memory.go View File

@ -0,0 +1,26 @@
package main
import (
"fmt"
"runtime"
"time"
)
func init() {
fmt.Println("Garbage Collector Thread Starting")
go memoryCleanerThread()
}
func memoryCleanerThread() {
for {
time.Sleep(10 * time.Minute)
fmt.Println("Time to clean memory...")
runtime.GC()
fmt.Println("Garbage Collection done.")
}
}

+ 69
- 0
01.conf.go View File

@ -0,0 +1,69 @@
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"github.com/miekg/dns"
)
func init() {
//ZabovConf describes the Json we use for configuration
type ZabovConf struct {
Zabov struct {
Port string `json:"port"`
Proto string `json:"proto"`
Ipaddr string `json:"ipaddr"`
Upstream string `json:"upstream"`
Cachettl int `json:"cachettl"`
Killfilettl int `json:"killfilettl"`
Singlefilters string `json:"singlefilters"`
Doublefilters string `json:"doublefilters"`
Blackholeip string `json:"blackholeip"`
Hostsfile string `json:"hostsfile"`
} `json:"zabov"`
}
var MyConf ZabovConf
file, err := ioutil.ReadFile("config.json")
if err != nil {
log.Println("Cannot open config file", err.Error())
os.Exit(1)
}
err = json.Unmarshal([]byte(file), &MyConf)
if err != nil {
log.Println("Cannot marshal json: ", err.Error())
os.Exit(1)
}
// now we read configuration file
fmt.Println("Reading configuration file...")
ZabovPort := MyConf.Zabov.Port
ZabovType := MyConf.Zabov.Proto
ZabovAddr := MyConf.Zabov.Ipaddr
ZabovUpDNS = MyConf.Zabov.Upstream
ZabovSingleBL = MyConf.Zabov.Singlefilters
ZabovDoubleBL = MyConf.Zabov.Doublefilters
ZabovAddBL = MyConf.Zabov.Blackholeip
ZabovCacheTTL = MyConf.Zabov.Cachettl
ZabovKillTTL = MyConf.Zabov.Killfilettl
ZabovHostsFile = MyConf.Zabov.Hostsfile
zabovString := ZabovAddr + ":" + ZabovPort
MyDNS = new(dns.Server)
MyDNS.Addr = zabovString
MyDNS.Net = ZabovType
ZabovDNSArray = fileByLines(ZabovUpDNS)
}

+ 37
- 0
01.dnscheck.go View File

@ -0,0 +1,37 @@
package main
import (
"fmt"
"net/http"
"time"
)
//NetworkUp tells the system if the network is up or not
var NetworkUp bool
func checkNetworkUp() bool {
// RFC2606 test domain, should always work, unless internet is down.
_, err := http.Get("http://example.com")
if err != nil {
return false
}
return true
}
func checkNetworkUpThread() {
ticker := time.NewTicker(2 * time.Minute)
for range ticker.C {
NetworkUp = checkNetworkUp()
}
}
func init() {
fmt.Println("Network Checker starting....")
go checkNetworkUpThread()
}

+ 79
- 0
01.killfile.go View File

@ -0,0 +1,79 @@
package main
import (
"fmt"
"strings"
)
var zabovKbucket = []byte("killfile")
type killfileItem struct {
Kdomain string
Ksource string
}
var bChannel chan killfileItem
func init() {
bChannel = make(chan killfileItem, 1024)
fmt.Println("Initializing kill channel engine.")
go bWriteThread()
}
func bWriteThread() {
for item := range bChannel {
writeInKillfile(item.Kdomain, item.Ksource)
incrementStats("BL domains from "+item.Ksource, 1)
incrementStats("TOTAL", 1)
}
}
//DomainKill stores a domain name inside the killfile
func DomainKill(s, durl string) {
if len(s) > 2 {
s = strings.ToLower(s)
var k killfileItem
k.Kdomain = s
k.Ksource = durl
bChannel <- k
}
}
func writeInKillfile(key, value string) {
stK := []byte(key)
stV := []byte(value)
err := MyZabovKDB.Put(stK, stV, nil)
if err != nil {
fmt.Println("Cannot write to Killfile DB: ", err.Error())
}
}
func domainInKillfile(domain string) bool {
s := strings.ToLower(domain)
has, err := MyZabovKDB.Has([]byte(s), nil)
if err != nil {
fmt.Println("Cannot read from Killfile DB: ", err.Error())
}
return has
}

+ 100
- 0
01.stats.go View File

@ -0,0 +1,100 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"time"
)
type send struct {
Payload string
Number int64
Operation string
}
//ZabovStats is used to keep statistics to print
var ZabovStats map[string]int64
var stats chan send
func init() {
stats = make(chan send, 1024)
ZabovStats = make(map[string]int64)
fmt.Println("Initializing stats engine.")
go reportPrintThread()
go statsThread()
}
func statsPrint() {
fmt.Println()
stat, _ := json.Marshal(ZabovStats)
fmt.Println(jsonPrettyPrint(string(stat)))
fmt.Println()
}
func incrementStats(key string, value int64) {
var s send
s.Payload = key
s.Number = value
s.Operation = "INC"
stats <- s
}
func setstatsvalue(key string, value int64) {
var s send
s.Payload = key
s.Number = value
s.Operation = "SET"
stats <- s
}
func reportPrintThread() {
for {
var s send
s.Operation = "PRI"
s.Payload = "-"
s.Number = 0
stats <- s
time.Sleep(2 * time.Minute)
}
}
func statsThread() {
fmt.Println("Starting Statistical Collection Thread")
for item := range stats {
switch item.Operation {
case "INC":
ZabovStats[item.Payload] += item.Number
case "SET":
ZabovStats[item.Payload] = item.Number
case "PRI":
statsPrint()
}
}
}
func jsonPrettyPrint(in string) string {
var out bytes.Buffer
err := json.Indent(&out, []byte(in), "", "\t")
if err != nil {
return in
}
return out.String()
}

+ 104
- 0
02.cache.go View File

@ -0,0 +1,104 @@
package main
import (
"bytes"
"encoding/gob"
"fmt"
"time"
"github.com/miekg/dns"
)
type cacheItem struct {
Query []byte
Date time.Time
}
//DomainCache stores a domain name inside the cache
func DomainCache(s string, resp *dns.Msg) {
var domain2cache cacheItem
var err error
var dom2 bytes.Buffer
enc := gob.NewEncoder(&dom2)
domain2cache.Query, err = resp.Pack()
if err != nil {
fmt.Println("Problems packing the response: ", err.Error())
}
domain2cache.Date = time.Now()
err = enc.Encode(domain2cache)
if err != nil {
fmt.Println("Cannot GOB the domain to cache: ", err.Error())
}
cacheDomain(s, dom2.Bytes())
}
func cacheDomain(key string, domain []byte) {
err := MyZabovCDB.Put([]byte(key), domain, nil)
if err != nil {
fmt.Println("Cannot write to Cache DB: ", err.Error())
}
}
//GetDomainFromCache stores a domain name inside the cache
func GetDomainFromCache(s string) *dns.Msg {
ret := new(dns.Msg)
var cache bytes.Buffer
dec := gob.NewDecoder(&cache)
var record cacheItem
var conf []byte
var errDB error
if domainInCache(s) == false {
return nil
}
conf, errDB = MyZabovCDB.Get([]byte(s), nil)
if errDB != nil {
fmt.Println("Cant READ DB :" , errDB.Error() )
return nil
}
cache.Write(conf)
err := dec.Decode(&record)
if err != nil {
fmt.Println("Decode error :", err.Error())
return nil
}
if time.Since(record.Date) > (time.Duration(ZabovCacheTTL) * time.Hour) {
return nil
}
err = ret.Unpack(record.Query)
if err != nil {
fmt.Println("Problem unpacking response: ", err.Error())
return nil
}
return ret
}
func domainInCache(domain string) bool {
has, err := MyZabovCDB.Has([]byte(domain), nil)
if err != nil {
fmt.Println("Cannot search Cache DB: ", err.Error())
return false
}
return has
}

+ 18
- 0
Dockerfile.amd64 View File

@ -0,0 +1,18 @@
FROM golang:1.14.1 AS builder
RUN apt install git -y
RUN mkdir -p /go/src/zabov
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov
WORKDIR /go/src/zabov
ENV GO111MODULE=auto
RUN go get ; go build -mod=vendor
FROM debian:latest
RUN apt update
RUN apt upgrade -y
RUN apt install ca-certificates -y
RUN mkdir -p /opt/zabov
WORKDIR /opt/zabov
COPY --from=builder /go/src/zabov /opt/zabov
EXPOSE 53/udp
ENTRYPOINT ["/opt/zabov/zabov"]

+ 17
- 0
Dockerfile.arm32v7 View File

@ -0,0 +1,17 @@
FROM arm32v7/golang:1.14.1 AS builder
RUN apt install git -y
RUN mkdir -p /go/src/zabov
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov
WORKDIR /go/src/zabov
ENV GO111MODULE=auto
RUN go get ; go build -mod=vendor
FROM arm32v7/debian:latest
RUN apt update
RUN apt upgrade -y
RUN apt install ca-certificates -y
RUN mkdir -p /opt/zabov
WORKDIR /opt/zabov
COPY --from=builder /go/src/zabov /opt/zabov
EXPOSE 53/udp
ENTRYPOINT ["/opt/zabov/zabov"]

+ 17
- 0
Dockerfile.arm64v8 View File

@ -0,0 +1,17 @@
FROM arm64v8/golang:1.14.1 AS builder
RUN apt install git -y
RUN mkdir -p /go/src/zabov
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov
WORKDIR /go/src/zabov
ENV GO111MODULE=auto
RUN go get ; go build -mod=vendor
FROM arm64v8/debian:latest
RUN apt update
RUN apt upgrade -y
RUN apt install ca-certificates -y
RUN mkdir -p /opt/zabov
WORKDIR /opt/zabov
COPY --from=builder /go/src/zabov /opt/zabov
EXPOSE 53/udp
ENTRYPOINT ["/opt/zabov/zabov"]

+ 14
- 0
LICENSE View File

@ -0,0 +1,14 @@
Copyright (C) 2020 loweel@keinpfusch.net
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.

+ 91
- 0
README.md View File

@ -0,0 +1,91 @@
# zabov
Tiny replacement for piHole DNS filter
Still Work in progress, usable.
Idea is to produce a very simple, no-web-interface , IP DNS blocker.
# INSTALL
Zabov requires golang 1.13 or later.
<pre>
git clone https://git.keinpfusch.net/Loweel/zabov.git
cd zabov
go get
go build -mod=vendor
</pre>
Then, edit config.json: please notice config.json must be in the same folder of the executable you run.
Just a few words about "singlefilters" and "doublefilters":
Data must be downloaded from URLs of blacklist mantainers.They may come in different formats.
There are two kinds of blacklists:
One is the format zabov calls "singlefilter", where we find a single column , full of domains:
<pre>
domain1.com
domain2.com
domain3.com
</pre>
The second is the format zabov calls "doublefilter" (a file in "/etc/hosts" format, to be precise), where there is an IP, usually localhost or 0.0.0.0 and then the domain:
<pre>
127.0.0.1 domain1.com
127.0.0.1 domain2.com
127.0.0.1 domain3.com
</pre>
This is why configuration file has two separated items.
The config file should look like:
<pre>
{
"zabov": {
"port":"53",
"proto":"udp",
"ipaddr":"127.0.0.1",
"upstream":"./dns-upstream.txt",
"cachettl": "4",
"killfilettl": "12",
"singlefilters":"./urls-hosts.txt" ,
"doublefilters":"./urls-domains.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
}
}
</pre>
Where:
- port is the port number. Usually is 53, you can change for docker, if you like
- proto is the protocol. Choices are "udp", "tcp", "tcp/udp"
- ipaddr is the port to listen to. Maybe empty, (which will result in listening to 0.0.0.0) to avoid issues with docker.
- upstream: file containing all DNS we want to query : each line in format IP:PORT
- cachettl: amount of time the cache is kept (in hours)
- killfilettl: refresh time for _killfiles_
- singlefilters: name of the file for blacklists following the "singlefilter" schema.(one URL per line)
- doublefilters: name of the file, for blacklists following the "doublefilter" schema.(one URL per line)
- blackholeip: IP address to return when the IP is banned. This is because you may want to avoid MX issues, mail loops on localhost, or you have a web server running on localhost
- hostsfile: path where you keep your local blacklistfile : this is in the format "singlefilter", meaning one domain per line, unlike hosts file.
# DOCKER
Multistage Dockerfiles are provided for AMD64, ARMv7, ARM64V8
# TODO:
- ~~caching~~
- monitoring port

+ 96
- 0
adlist_hosts.go View File

@ -0,0 +1,96 @@
package main
import (
"bufio"
"errors"
"fmt"
"net"
"net/http"
"strings"
"time"
)
func init() {
go downloadDoubleThread()
}
//DoubleIndexFilter puts the domains inside file
func DoubleIndexFilter(durl string) error {
fmt.Println("Retrieving HostFile from: ", durl)
var err error
// Get the data
resp, err := http.Get(durl)
if err != nil {
fmt.Println("HTTP problem: ", err)
return err
}
defer resp.Body.Close()
if resp.StatusCode == 200 { // OK
fmt.Println(durl + " Response: OK")
} else {
fmt.Println("Server <"+durl+"> returned status code: ", resp.StatusCode)
return errors.New("Server <" + durl + "> returned status code: " + resp.Status)
}
scanner := bufio.NewScanner(resp.Body)
splitter := func(c rune) bool {
return c == ' ' || c == '\t'
}
var numLines int64
for scanner.Scan() {
line := scanner.Text()
h := strings.FieldsFunc(line, splitter)
if h == nil {
continue
}
if len(h) < 2 {
continue
}
if net.ParseIP(h[0]) != nil {
DomainKill(h[1], durl)
// fmt.Println("MATCH: ", h[1])
numLines++
} else {
incrementStats("Malformed HostLines "+durl, 1)
// fmt.Println("Malformed line: <" + line + ">")
}
}
fmt.Println("Finished to parse: "+durl+" ,number of lines", numLines)
return err
}
func getDoubleFilters() {
s := fileByLines(ZabovDoubleBL)
for _, a := range s {
DoubleIndexFilter(a)
}
}
func downloadDoubleThread() {
fmt.Println("Starting updater of DOUBLE lists, each (hours):", ZabovKillTTL)
for {
getDoubleFilters()
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
}
}

+ 93
- 0
adlist_single.go View File

@ -0,0 +1,93 @@
package main
import (
"bufio"
"errors"
"fmt"
"net/http"
"strings"
"time"
)
func init() {
go downloadThread()
}
//SingleIndexFilter puts the domains inside file
func SingleIndexFilter(durl string) error {
fmt.Println("Retrieving DomainFile from: ", durl)
var err error
// Get the data
resp, err := http.Get(durl)
if err != nil {
fmt.Println("HTTP Problem: ", err)
return err
}
defer resp.Body.Close()
if resp.StatusCode == 200 { // OK
fmt.Println(durl + " Response: OK")
} else {
fmt.Println("Server <"+durl+"> returned status code: ", resp.StatusCode)
return errors.New("Server <" + durl + "> returned status code: " + resp.Status)
}
scanner := bufio.NewScanner(resp.Body)
splitter := func(c rune) bool {
return c == ' ' || c == '\t'
}
var numLines int64
for scanner.Scan() {
line := scanner.Text()
h := strings.FieldsFunc(line, splitter)
if h == nil {
continue
}
if len(h) < 1 {
continue
}
if !strings.Contains(h[0], "#") {
DomainKill(h[0], durl)
// fmt.Println("MATCH: ", h[1])
numLines++
} else {
incrementStats("Malformed DomainLines "+durl, 1)
// fmt.Println("Malformed line: <" + line + ">")
}
}
fmt.Println("Finished to parse: "+durl+" ,number of lines", numLines)
return err
}
func getSingleFilters() {
s := fileByLines(ZabovSingleBL)
for _, a := range s {
SingleIndexFilter(a)
}
}
func downloadThread() {
fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL)
for {
getSingleFilters()
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
}
}

+ 15
- 0
config.json View File

@ -0,0 +1,15 @@
{
"zabov": {
"port":"53",
"proto":"udp",
"ipaddr":"0.0.0.0",
"upstream":"./dns-upstream.txt" ,
"cachettl": 1,
"killfilettl": 12,
"singlefilters":"./urls-domains.txt" ,
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
}
}

+ 387
- 0
dns-upstream.txt View File

@ -0,0 +1,387 @@
194.150.168.168:53
194.25.0.68:53
141.1.1.1:53
213.239.204.35:53
194.25.0.52:53
212.211.132.4:53
213.68.194.51:53
195.243.214.4:53
141.1.27.249:53
80.237.197.14:53
217.28.98.62:53
82.96.64.2:53
82.96.65.2:53
194.25.0.60:53
193.101.111.10:53
193.101.111.20:53
192.76.144.66:53
217.69.169.25:53
85.88.19.10:53
85.88.19.11:53
85.214.20.141:53
194.169.239.10:53
194.172.160.4:53
212.102.225.2:53
212.51.16.1:53
212.51.17.1:53
212.66.129.98:53
212.89.130.180:53
213.209.122.11:53
213.23.108.129:53
91.204.4.133:53
84.200.69.80:53
62.146.63.211:53
212.77.178.83:53
78.46.58.246:53
85.214.102.25:53
87.106.62.128:53
81.20.87.84:53
81.20.87.181:53
217.5.159.227:53
185.38.9.99:53
62.146.25.130:53
37.59.218.50:53
212.184.191.193:53
213.23.143.154:53
178.32.187.10:53
62.154.214.86:53
85.214.151.164:53
5.175.225.2:53
193.105.38.142:53
145.253.183.21:53
178.15.146.43:53
62.245.233.22:53
212.224.71.71:53
213.136.78.213:53
80.156.196.196:53
193.158.99.67:53
194.95.75.230:53
212.184.191.2:53
213.138.38.22:53
195.145.80.150:53
139.18.25.34:53
79.143.180.116:53
213.240.172.200:53
217.160.238.238:53
213.61.185.238:53
84.201.0.34:53
82.194.105.219:53
62.157.89.178:53
46.189.26.123:53
85.214.208.8:53
87.239.128.130:53
78.111.65.40:53
85.214.69.126:53
109.75.29.1:53
80.81.19.226:53
81.169.162.74:53
217.14.164.35:53
5.9.172.92:53
62.225.102.180:53
217.7.71.203:53
217.6.71.61:53
62.154.138.43:53
146.0.38.140:53
78.111.67.10:53
217.6.110.20:53
87.245.18.221:53
62.225.66.19:53
81.169.212.52:53
178.162.205.123:53
212.227.83.183:53
139.18.25.33:53
193.29.2.4:53
212.91.246.11:53
62.153.141.15:53
148.251.120.228:53
62.154.253.226:53
194.25.218.2:53
194.174.73.36:53
62.245.226.182:53
87.234.222.68:53
194.25.169.130:53
62.225.15.253:53
176.94.20.4:53
188.40.115.29:53
188.40.115.22:53
194.187.240.10:53
80.150.109.197:53
217.86.149.109:53
91.208.193.1:53
195.243.99.35:53
62.225.102.177:53
178.210.102.9:53
80.228.113.125:53
178.210.102.12:53
130.255.121.9:53
212.204.56.218:53
37.59.218.151:53
80.148.52.109:53
194.30.174.222:53
5.199.141.5:53
94.135.173.22:53
88.79.208.11:53
141.16.180.9:53
82.193.241.125:53
212.8.216.37:53
109.75.29.2:53
78.46.17.82:53
81.169.185.49:53
217.244.13.14:53
93.104.209.27:53
79.143.182.174:53
81.20.82.131:53
213.136.68.181:53
213.136.68.189:53
193.107.145.233:53
195.145.241.3:53
80.242.182.182:53
193.159.181.250:53
195.243.124.75:53
62.159.104.102:53
92.222.202.244:53
85.214.254.13:53
85.114.128.115:53
145.253.176.50:53
217.7.63.1:53
78.35.40.149:53
81.169.187.253:53
94.249.192.20:53
85.214.43.157:53
80.149.83.60:53
178.210.102.225:53
178.210.102.193:53
62.154.236.126:53
213.183.185.50:53
212.60.229.242:53
80.146.192.66:53
79.133.62.62:53
178.33.33.219:53
62.245.225.225:53
46.38.235.212:53
213.136.88.31:53
212.66.135.250:53
194.231.138.26:53
62.225.1.33:53
80.148.34.131:53
94.23.163.114:53
80.64.189.94:53
81.169.241.28:53
212.38.26.132:53
62.91.19.67:53
87.239.128.25:53
212.185.196.10:53
89.221.2.171:53
217.243.239.11:53
213.136.69.214:53
213.138.56.75:53
212.122.52.11:53
46.4.166.113:53
77.37.30.12:53
194.187.242.10:53
188.40.132.212:53
194.150.168.169:53
85.25.105.193:53
185.93.180.131:53
109.234.249.10:53
109.234.248.10:53
138.201.120.250:53
81.3.27.54:53
78.46.231.161:53
78.46.231.162:53
212.51.16.197:53
212.28.34.65:53
148.251.24.48:53
212.75.32.4:53
91.103.112.150:53
217.69.169.26:53
195.63.103.144:53
213.209.121.30:53
88.79.149.4:53
185.194.143.243:53
46.182.19.48:53
217.111.24.246:53
62.96.37.74:53
213.61.64.174:53
213.61.65.226:53
62.96.190.134:53
217.111.123.166:53
213.61.176.118:53
185.216.33.82:53
185.220.70.50:53
188.138.57.95:53
195.145.137.164:53
195.167.223.164:53
195.226.69.82:53
195.37.174.194:53
195.4.138.12:53
195.63.61.189:53
212.124.35.25:53
212.184.191.100:53
212.38.2.130:53
212.51.16.193:53
212.66.129.107:53
212.8.216.41:53
212.89.128.28:53
213.133.116.14:53
213.166.247.100:53
217.243.173.82:53
217.5.182.118:53
217.7.80.40:53
217.7.81.136:53
217.9.50.199:53
46.237.220.2:53
5.189.179.105:53
52.28.79.14:53
52.29.2.17:53
62.146.202.2:53
62.146.2.48:53
62.153.122.2:53
62.153.237.200:53
62.153.237.201:53
62.154.139.99:53
62.154.159.12:53
62.154.159.5:53
62.154.160.3:53
62.209.40.75:53
62.217.61.162:53
62.245.225.55:53
78.111.224.224:53
78.111.226.226:53
78.138.80.42:53
80.149.112.139:53
80.156.6.209:53
80.190.209.218:53
80.228.231.122:53
80.228.231.48:53
80.245.65.100:53
81.14.182.169:53
81.27.162.100:53
83.97.23.178:53
83.97.23.226:53
84.16.240.43:53
85.214.98.185:53
89.19.228.52:53
89.200.168.203:53
91.217.86.4:53
93.104.195.2:53
94.247.43.254:53
109.234.248.8:53
131.220.20.199:53
131.220.23.123:53
144.76.173.169:53
144.76.83.104:53
148.251.92.241:53
176.9.136.236:53
195.10.195.195:53
173.212.249.41:53
85.214.41.155:53
54.37.75.2:53
194.55.13.75:53
5.189.138.153:53
159.69.51.18:53
51.75.77.179:53
138.201.169.84:53
138.201.239.66:53
138.68.106.109:53
145.253.109.162:53
159.69.68.181:53
167.86.78.56:53
173.212.208.116:53
173.212.218.206:53
173.212.219.129:53
173.212.239.87:53
173.212.242.89:53
173.212.244.78:53
173.249.41.233:53
173.249.48.6:53
176.9.233.171:53
176.9.58.218:53
178.162.199.27:53
178.162.208.135:53
178.238.230.127:53
178.238.235.218:53
18.195.121.224:53
185.139.98.100:53
185.40.135.11:53
185.53.169.22:53
185.90.131.194:53
188.40.239.99:53
188.68.35.145:53
192.162.85.48:53
193.159.232.5:53
194.77.237.31:53
194.77.253.32:53
195.201.192.29:53
195.202.52.30:53
195.243.101.5:53
207.180.203.42:53
207.180.243.200:53
207.180.247.212:53
213.136.71.68:53
213.136.77.39:53
213.144.24.234:53
217.147.96.210:53
217.182.198.203:53
217.6.131.248:53
217.6.247.237:53
217.6.64.5:53
217.79.177.220:53
46.163.119.155:53
46.228.199.116:53
5.175.26.208:53
51.77.65.15:53
5.189.133.151:53
5.189.141.216:53
5.189.186.154:53
5.189.186.93:53
5.189.187.34:53
5.199.141.30:53
5.45.96.220:53
62.138.20.211:53
62.144.82.252:53
62.153.165.107:53
62.153.201.91:53
62.154.214.84:53
62.157.242.85:53
79.143.177.243:53
79.143.183.45:53
80.156.198.146:53
80.156.6.206:53
80.237.207.100:53
80.82.223.94:53
81.169.215.29:53
81.169.223.126:53
81.169.230.157:53
81.20.80.79:53
83.236.183.211:53
84.16.240.224:53
85.214.224.76:53
85.214.238.190:53
85.214.246.133:53
85.214.62.160:53
85.93.91.101:53
87.106.63.208:53
87.118.126.225:53
88.198.37.146:53
88.99.66.18:53
89.163.150.209:53
89.163.220.114:53
89.19.236.152:53
93.104.213.74:53
93.186.196.137:53
93.190.71.172:53
94.177.246.221:53
80.241.218.68:53
172.105.81.90:53
172.105.81.92:53
84.200.70.40:53
94.16.114.254:53
93.90.207.192:53
93.90.201.211:53
144.91.68.146:53
176.9.37.132:53
176.9.93.198:53
176.9.1.117:53
144.91.115.47:53
91.237.100.4:53

+ 98
- 0
dns_client.go View File

@ -0,0 +1,98 @@
package main
import (
"fmt"
"time"
"math/rand"
"strings"
"github.com/miekg/dns"
)
//ForwardQuery forwards the query to the upstream server
//first server to answer wins
func ForwardQuery(query *dns.Msg) *dns.Msg {
go incrementStats("ForwardQueries", 1)
r := new(dns.Msg)
r.SetReply(query)
r.Authoritative = true
fqdn := strings.TrimRight(query.Question[0].Name, ".")
lfqdn := fmt.Sprintf("%d", query.Question[0].Qtype) + "." + fqdn
if cached := GetDomainFromCache(lfqdn); cached != nil {
go incrementStats("CacheHit", 1)
cached.SetReply(query)
cached.Authoritative = true
return cached
}
c := new(dns.Client)
c.ReadTimeout = 500 * time.Millisecond
c.WriteTimeout = 500 * time.Millisecond
for {
// round robin with retry
if !NetworkUp {
time.Sleep(10 * time.Second)
go incrementStats("Network Problems ", 1)
continue
}
d := oneTimeDNS()
in, _, err := c.Exchange(query, d)
if err != nil {
fmt.Printf("Problem with DNS %s : %s\n", d, err.Error())
go incrementStats("DNS Problems "+d, 1)
continue
} else {
go incrementStats(d, 1)
in.SetReply(query)
in.Authoritative = true
go DomainCache(lfqdn, in)
return in
}
}
}
func init() {
fmt.Println("DNS client engine starting")
NetworkUp = checkNetworkUp()
if NetworkUp {
fmt.Println("[OK]: Network is UP")
} else {
fmt.Println("[KO] Network is DOWN: system will check again in 2 minutes")
}
}
func oneTimeDNS() (dns string) {
rand.Seed(time.Now().Unix())
upl := ZabovDNSArray
if len(upl) < 1 {
fmt.Println("No DNS defined, using default 127.0.0.53:53. Hope it works!")
return "127.0.0.53:53"
}
n := rand.Intn(128*len(upl)) % len(upl)
dns = upl[n]
return
}

+ 44
- 0
dns_handler.go View File

@ -0,0 +1,44 @@
package main
import (
"net"
"strings"
"github.com/miekg/dns"
)
func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
go incrementStats("TotalQueries", 1)
remIP, _, e := net.SplitHostPort(w.RemoteAddr().String())
if e != nil {
go incrementStats("CLIENT: "+remIP, 1)
}
msg := dns.Msg{}
msg.SetReply(r)
switch r.Question[0].Qtype {
case dns.TypeA:
msg.Authoritative = true
domain := msg.Question[0].Name
fqdn := strings.TrimRight(domain, ".")
if domainInKillfile(fqdn) {
go incrementStats("Killed", 1)
msg.Answer = append(msg.Answer, &dns.A{
Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
A: net.ParseIP(ZabovAddBL),
})
} else {
ret := ForwardQuery(r)
w.WriteMsg(ret)
}
default:
ret := ForwardQuery(r)
w.WriteMsg(ret)
}
w.WriteMsg(&msg)
}

+ 8
- 0
go.mod View File

@ -0,0 +1,8 @@
module zabov
go 1.13
require (
github.com/miekg/dns v1.1.27
github.com/syndtr/goleveldb v1.0.0
)

+ 36
- 0
go.sum View File

@ -0,0 +1,36 @@
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

+ 59
- 0
hostfile.go View File

@ -0,0 +1,59 @@
package main
import (
"bufio"
"fmt"
"os"
)
func init() {
fmt.Println("Ingesting local hosts file")
ingestLocalBlacklist()
}
func ingestLocalBlacklist() {
file, err := os.Open(ZabovHostsFile)
if err != nil {
fmt.Println(err.Error())
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
d := scanner.Text()
DomainKill(d, ZabovHostsFile)
incrementStats("Blacklist", 1)
}
if err := scanner.Err(); err != nil {
fmt.Println(err.Error())
}
}
func fileByLines(filename string) (blurls []string) {
file, err := os.Open(filename)
if err != nil {
fmt.Println(err.Error())
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
d := scanner.Text()
blurls = append(blurls, d)
}
if err := scanner.Err(); err != nil {
fmt.Println(err.Error())
}
return
}

+ 46
- 0
main.go View File

@ -0,0 +1,46 @@
package main
import (
"log"
"github.com/miekg/dns"
)
//MyDNS is my dns server
var MyDNS *dns.Server
//ZabovUpDNS keeps the name of upstream DNSs
var ZabovUpDNS string
//ZabovSingleBL list of urls returning a file with just names of domains
var ZabovSingleBL string
//ZabovDoubleBL list of urls returning a file with IP<space>domain
var ZabovDoubleBL string
//ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1
var ZabovAddBL string
//ZabovCacheTTL is the amount of hours we cache records of DNS
var ZabovCacheTTL int
//ZabovKillTTL is the amount of hours we cache the killfile
var ZabovKillTTL int
//ZabovHostsFile is the file we use to keep our hosts
var ZabovHostsFile string
//ZabovDNSArray is the array containing all the DNS we mention
var ZabovDNSArray []string
type handler struct{}
func main() {
MyDNS.Handler = &handler{}
if err := MyDNS.ListenAndServe(); err != nil {
log.Printf("Failed to set udp listener %s\n", err.Error())
} else {
log.Printf("Listener running \n")
}
}

+ 37
- 0
urls-domains.txt View File

@ -0,0 +1,37 @@
https://mirror1.malwaredomains.com/files/justdomains
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adaway.org/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adblock-nocoin-list/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/adguard-simplified/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/anudeepnd-adservers/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-ad/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-malvertising/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-malware/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-tracking/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/easyprivacy/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/eth-phishing-detect/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.2o7net/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.dead/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.risk/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.spam/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/kadhosts/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomainlist.com/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomains.com-immortaldomains/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomains.com-justdomains/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/matomo.org-spammers/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/mitchellkrogza-badd-boyz-hosts/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/pgl.yoyo.org/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/ransomwaretracker.abuse.ch/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/someonewhocares.org/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/spam404.com/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/stevenblack/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/winhelp2002.mvps.org/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/zerodot1-coinblockerlists-browser/list.txt
https://raw.githubusercontent.com/hectorm/hmirror/master/data/zeustracker.abuse.ch/list.txt
https://raw.githubusercontent.com/CHEF-KOCH/Audio-fingerprint-pages/master/AudioFp.txt
https://raw.githubusercontent.com/CHEF-KOCH/Canvas-fingerprinting-pages/master/Canvas.txt
https://raw.githubusercontent.com/CHEF-KOCH/WebRTC-tracking/master/WebRTC.txt
https://raw.githubusercontent.com/CHEF-KOCH/CKs-FilterList/master/Anti-Corp/hosts/NSABlocklist.txt
https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt
https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt
https://www.stopforumspam.com/downloads/toxic_domains_whole.txt

+ 12
- 0
urls-hosts.txt View File

@ -0,0 +1,12 @@
http://sysctl.org/cameleon/hosts
https://www.malwaredomainlist.com/hostslist/hosts.txt
https://adaway.org/hosts.txt
https://raw.githubusercontent.com/StevenBlack/hosts/master/hosts
https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/fakenews/hosts
https://raw.githubusercontent.com/StevenBlack/hosts/master/alternates/gambling/hosts
https://someonewhocares.org/hosts/hosts
https://getadhell.com/standard-package.txt
https://raw.githubusercontent.com/hoshsadiq/adblock-nocoin-list/master/hosts.txt
https://raw.githubusercontent.com/notracking/hosts-blocklists/master/hostnames.txt
https://raw.githubusercontent.com/anudeepND/blacklist/master/adservers.txt
https://raw.githubusercontent.com/anudeepND/blacklist/master/facebook.txt

+ 8
- 0
urls-local.txt View File

@ -0,0 +1,8 @@
blc.vodafone.com
gab.com
gab.ai
freespeechextremist.com
neckbeard.xyz
funkwhale.it
social.byoblu.com

+ 16
- 0
vendor/github.com/golang/snappy/.gitignore View File

@ -0,0 +1,16 @@
cmd/snappytool/snappytool
testdata/bench
# These explicitly listed benchmark data files are for an obsolete version of
# snappy_test.go.
testdata/alice29.txt
testdata/asyoulik.txt
testdata/fireworks.jpeg
testdata/geo.protodata
testdata/html
testdata/html_x_4
testdata/kppkn.gtb
testdata/lcet10.txt
testdata/paper-100k.pdf
testdata/plrabn12.txt
testdata/urls.10K

+ 15
- 0
vendor/github.com/golang/snappy/AUTHORS View File

@ -0,0 +1,15 @@
# This is the official list of Snappy-Go authors for copyright purposes.
# This file is distinct from the CONTRIBUTORS files.
# See the latter for an explanation.
# Names should be added to this file as
# Name or Organization <email address>
# The email address is not required for organizations.
# Please keep the list sorted.
Damian Gryski <dgryski@gmail.com>
Google Inc.
Jan Mercl <0xjnml@gmail.com>
Rodolfo Carvalho <rhcarvalho@gmail.com>
Sebastien Binet <seb.binet@gmail.com>

+ 37
- 0
vendor/github.com/golang/snappy/CONTRIBUTORS View File

@ -0,0 +1,37 @@
# This is the official list of people who can contribute
# (and typically have contributed) code to the Snappy-Go repository.
# The AUTHORS file lists the copyright holders; this file
# lists people. For example, Google employees are listed here
# but not in AUTHORS, because Google holds the copyright.
#
# The submission process automatically checks to make sure
# that people submitting code are listed in this file (by email address).
#
# Names should be added to this file only after verifying that
# the individual or the individual's organization has agreed to
# the appropriate Contributor License Agreement, found here:
#
# http://code.google.com/legal/individual-cla-v1.0.html
# http://code.google.com/legal/corporate-cla-v1.0.html
#
# The agreement for individuals can be filled out on the web.
#
# When adding J Random Contributor's name to this file,
# either J's name or J's organization's name should be
# added to the AUTHORS file, depending on whether the
# individual or corporate CLA was used.
# Names should be added to this file like so:
# Name <email address>
# Please keep the list sorted.
Damian Gryski <dgryski@gmail.com>
Jan Mercl <0xjnml@gmail.com>
Kai Backman <kaib@golang.org>
Marc-Antoine Ruel <maruel@chromium.org>
Nigel Tao <nigeltao@golang.org>
Rob Pike <r@golang.org>
Rodolfo Carvalho <rhcarvalho@gmail.com>
Russ Cox <rsc@golang.org>
Sebastien Binet <seb.binet@gmail.com>

+ 27
- 0
vendor/github.com/golang/snappy/LICENSE View File

@ -0,0 +1,27 @@
Copyright (c) 2011 The Snappy-Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

+ 107
- 0
vendor/github.com/golang/snappy/README View File

@ -0,0 +1,107 @@
The Snappy compression format in the Go programming language.
To download and install from source:
$ go get github.com/golang/snappy
Unless otherwise noted, the Snappy-Go source files are distributed
under the BSD-style license found in the LICENSE file.
Benchmarks.
The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten
or so files, the same set used by the C++ Snappy code (github.com/google/snappy
and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @
3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29:
"go test -test.bench=."
_UFlat0-8 2.19GB/s ± 0% html
_UFlat1-8 1.41GB/s ± 0% urls
_UFlat2-8 23.5GB/s ± 2% jpg
_UFlat3-8 1.91GB/s ± 0% jpg_200
_UFlat4-8 14.0GB/s ± 1% pdf
_UFlat5-8 1.97GB/s ± 0% html4
_UFlat6-8 814MB/s ± 0% txt1
_UFlat7-8 785MB/s ± 0% txt2
_UFlat8-8 857MB/s ± 0% txt3
_UFlat9-8 719MB/s ± 1% txt4
_UFlat10-8 2.84GB/s ± 0% pb
_UFlat11-8 1.05GB/s ± 0% gaviota
_ZFlat0-8 1.04GB/s ± 0% html
_ZFlat1-8 534MB/s ± 0% urls
_ZFlat2-8 15.7GB/s ± 1% jpg
_ZFlat3-8 740MB/s ± 3% jpg_200
_ZFlat4-8 9.20GB/s ± 1% pdf
_ZFlat5-8 991MB/s ± 0% html4
_ZFlat6-8 379MB/s ± 0% txt1
_ZFlat7-8 352MB/s ± 0% txt2
_ZFlat8-8 396MB/s ± 1% txt3
_ZFlat9-8 327MB/s ± 1% txt4
_ZFlat10-8 1.33GB/s ± 1% pb
_ZFlat11-8 605MB/s ± 1% gaviota
"go test -test.bench=. -tags=noasm"
_UFlat0-8 621MB/s ± 2% html
_UFlat1-8 494MB/s ± 1% urls
_UFlat2-8 23.2GB/s ± 1% jpg
_UFlat3-8 1.12GB/s ± 1% jpg_200
_UFlat4-8 4.35GB/s ± 1% pdf
_UFlat5-8 609MB/s ± 0% html4
_UFlat6-8 296MB/s ± 0% txt1
_UFlat7-8 288MB/s ± 0% txt2
_UFlat8-8 309MB/s ± 1% txt3
_UFlat9-8 280MB/s ± 1% txt4
_UFlat10-8 753MB/s ± 0% pb
_UFlat11-8 400MB/s ± 0% gaviota
_ZFlat0-8 409MB/s ± 1% html
_ZFlat1-8 250MB/s ± 1% urls
_ZFlat2-8 12.3GB/s ± 1% jpg
_ZFlat3-8 132MB/s ± 0% jpg_200
_ZFlat4-8 2.92GB/s ± 0% pdf
_ZFlat5-8 405MB/s ± 1% html4
_ZFlat6-8 179MB/s ± 1% txt1
_ZFlat7-8 170MB/s ± 1% txt2
_ZFlat8-8 189MB/s ± 1% txt3
_ZFlat9-8 164MB/s ± 1% txt4
_ZFlat10-8 479MB/s ± 1% pb
_ZFlat11-8 270MB/s ± 1% gaviota
For comparison (Go's encoded output is byte-for-byte identical to C++'s), here
are the numbers from C++ Snappy's
make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log
BM_UFlat/0 2.4GB/s html
BM_UFlat/1 1.4GB/s urls
BM_UFlat/2 21.8GB/s jpg
BM_UFlat/3 1.5GB/s jpg_200
BM_UFlat/4 13.3GB/s pdf
BM_UFlat/5 2.1GB/s html4
BM_UFlat/6 1.0GB/s txt1
BM_UFlat/7 959.4MB/s txt2
BM_UFlat/8 1.0GB/s txt3
BM_UFlat/9 864.5MB/s txt4
BM_UFlat/10 2.9GB/s pb
BM_UFlat/11 1.2GB/s gaviota
BM_ZFlat/0 944.3MB/s html (22.31 %)
BM_ZFlat/1 501.6MB/s urls (47.78 %)
BM_ZFlat/2 14.3GB/s jpg (99.95 %)
BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %)
BM_ZFlat/4 8.3GB/s pdf (83.30 %)
BM_ZFlat/5 903.5MB/s html4 (22.52 %)
BM_ZFlat/6 336.0MB/s txt1 (57.88 %)
BM_ZFlat/7 312.3MB/s txt2 (61.91 %)
BM_ZFlat/8 353.1MB/s txt3 (54.99 %)
BM_ZFlat/9 289.9MB/s txt4 (66.26 %)