bloved
/
zabov
forked from loweel/zabov
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Merge pull request 'master' (#1) from bloved/zabov:master into master 

Reviewed-on: loweel/zabov#1
golang-1.15-vendoring
Uriel Fanelli 2021-01-22 16:07:13 +00:00
parent 5c5e4e4417 0752a7e443
commit 13dd24b7b5
17 changed files with 993 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
00.database.go
View File

@ -7,12 +7,12 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
//MyZabovKDB is the storage where we'll put domains to block
var MyZabovKDB *leveldb.DB
//MyZabovCDB is the storage where we'll put domains to cache
//MyZabovCDB is the storage where we'll put domains to cache (global for all configs)
var MyZabovCDB *leveldb.DB
//MyZabovKDBs is the storage where we'll put domains to block (one for each config)
var MyZabovKDBs map[string]*leveldb.DB
func init() {
var err error
@ -21,13 +21,6 @@ func init() {
os.MkdirAll("./db", 0755)
MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil)
if err != nil {
fmt.Println("Cannot create Killfile db: ", err.Error())
} else {
fmt.Println("Killfile DB created")
}
MyZabovCDB, err = leveldb.OpenFile("./db/cache", nil)
if err != nil {
fmt.Println("Cannot create Cache db: ", err.Error())
@ -35,4 +28,21 @@ func init() {
fmt.Println("Cache DB created")
}
MyZabovKDBs = map[string]*leveldb.DB{}
}
// ZabovCreateKDB creates Kill DBs
func ZabovCreateKDB(conf string) {
var err error
dbname := "./db/killfile_" + conf
KDB, err := leveldb.OpenFile(dbname, nil)
if err != nil {
fmt.Println("Cannot create Killfile db: ", err.Error())
} else {
fmt.Println("Killfile DB created:", dbname)
}
MyZabovKDBs[conf] = KDB
}

275
01.conf.go
View File

@ -5,30 +5,19 @@ import (
"fmt"
"io/ioutil"
"log"
"net"
"os"
"strconv"
"strings"
"github.com/miekg/dns"
)
type stringarray []string
type urlsMap map[string]stringarray
func init() {
//ZabovConf describes the Json we use for configuration
type ZabovConf struct {
Zabov struct {
Port string `json:"port"`
Proto string `json:"proto"`
Ipaddr string `json:"ipaddr"`
Upstream string `json:"upstream"`
Cachettl int `json:"cachettl"`
Killfilettl int `json:"killfilettl"`
Singlefilters string `json:"singlefilters"`
Doublefilters string `json:"doublefilters"`
Blackholeip string `json:"blackholeip"`
Hostsfile string `json:"hostsfile"`
} `json:"zabov"`
}
var MyConf ZabovConf
var MyConfRaw interface{}
file, err := ioutil.ReadFile("config.json")
@ -37,26 +26,53 @@ func init() {
os.Exit(1)
}
err = json.Unmarshal([]byte(file), &MyConf)
err = json.Unmarshal([]byte(file), &MyConfRaw)
if err != nil {
log.Println("Cannot marshal json: ", err.Error())
log.Println("Cannot unmarshal json: ", err.Error())
os.Exit(1)
}
// now we read configuration file
fmt.Println("Reading configuration file...")
ZabovPort := MyConf.Zabov.Port
ZabovType := MyConf.Zabov.Proto
ZabovAddr := MyConf.Zabov.Ipaddr
ZabovUpDNS = MyConf.Zabov.Upstream
ZabovSingleBL = MyConf.Zabov.Singlefilters
ZabovDoubleBL = MyConf.Zabov.Doublefilters
ZabovAddBL = MyConf.Zabov.Blackholeip
ZabovCacheTTL = MyConf.Zabov.Cachettl
ZabovKillTTL = MyConf.Zabov.Killfilettl
ZabovHostsFile = MyConf.Zabov.Hostsfile
MyConf := MyConfRaw.(map[string]interface{})
//******************************
// zabov section (global config)
//******************************
zabov := MyConf["zabov"].(map[string]interface{})
ZabovPort := zabov["port"].(string)
ZabovType := zabov["proto"].(string)
ZabovAddr := zabov["ipaddr"].(string)
ZabovCacheTTL = int(zabov["cachettl"].(float64))
ZabovKillTTL = int(zabov["killfilettl"].(float64))
if zabov["debug"] != nil {
ZabovDebug = zabov["debug"].(string) == "true"
}
if zabov["debugdbpath"] != nil {
ZabovDebugDBPath = (zabov["debugdbpath"].(string))
}
if MyConf["configs"] == nil {
log.Println("configs not set: you shall set at least 'default' config")
os.Exit(1)
}
configs := MyConf["configs"].(map[string]interface{})
if len(configs) == 0 {
log.Println("you shall set at least 'default' config")
os.Exit(1)
}
if configs["default"] == nil {
log.Println("'default' config is required")
os.Exit(1)
}
zabovString := ZabovAddr + ":" + ZabovPort
@ -64,6 +80,203 @@ func init() {
MyDNS.Addr = zabovString
MyDNS.Net = ZabovType
ZabovDNSArray = fileByLines(ZabovUpDNS)
ZabovConfigs = map[string]*ZabovConfig{}
ZabovIPGroups = []ZabovIPGroup{}
ZabovTimetables = map[string]*ZabovTimetable{}
ZabovIPAliases = map[string]string{}
//*******************
// IP aliases section
//*******************
if MyConf["ipaliases"] != nil {
IPAliasesRaw := MyConf["ipaliases"].(map[string]interface{})
for alias, ip := range IPAliasesRaw {
fmt.Println("IP Alias:", alias, ip)
ZabovIPAliases[alias] = ip.(string)
}
}
//****************
// configs section
//****************
for name, v := range configs {
fmt.Println("evaluaing config name:", name)
confRaw := v.(map[string]interface{})
var conf ZabovConfig
conf.ZabovUpDNS = confRaw["upstream"].(string)
conf.ZabovSingleBL = confRaw["singlefilters"].(string)
conf.ZabovDoubleBL = confRaw["doublefilters"].(string)
conf.ZabovAddBL = confRaw["blackholeip"].(string)
conf.ZabovHostsFile = confRaw["hostsfile"].(string)
conf.ZabovDNSArray = fileByLines(conf.ZabovUpDNS)
ZabovConfigs[name] = &conf
}
// default config is mandatory
ZabovConfigs["default"].references++
//*******************
// timetables section
//*******************
if MyConf["timetables"] != nil {
timetables := MyConf["timetables"].(map[string]interface{})
for name, v := range timetables {
fmt.Println("evaluaing timetable name:", name)
timetableRaw := v.(map[string]interface{})
var timetable ZabovTimetable
timetable.cfgin = timetableRaw["cfgin"].(string)
timetable.cfgout = timetableRaw["cfgout"].(string)
if timetable.cfgin == "" {
timetable.cfgin = "default"
}
if timetable.cfgout == "" {
timetable.cfgout = "default"
}
refConfig, ok := ZabovConfigs[timetable.cfgin]
if !ok {
log.Println("timetable: inexistent cfgin:", timetable.cfgin)
os.Exit(1)
}
refConfig.references++
refConfig, ok = ZabovConfigs[timetable.cfgout]
if !ok {
log.Println("timetable: inexistent cfgout:", timetable.cfgout)
os.Exit(1)
}
refConfig.references++
tables := timetableRaw["tables"].([]interface{})
for i := range tables {
table := tables[i].(map[string]interface{})
var ttEntry ZabovTimetableEntry
ttEntry.times = []*ZabovTimeRange{}
for _, tRaw := range strings.Split(table["times"].(string), ";") {
tRawArr := strings.Split(tRaw, "-")
if len(tRawArr) > 1 {
startArr := strings.Split(tRawArr[0], ":")
stopArr := strings.Split(tRawArr[1], ":")
if len(startArr) > 1 && len(stopArr) > 1 {
hourStart, _ := strconv.Atoi(startArr[0])
minuteStart, _ := strconv.Atoi(startArr[1])
start := ZabovTime{hour: hourStart, minute: minuteStart}
hourStop, _ := strconv.Atoi(stopArr[0])
minuteStop, _ := strconv.Atoi(stopArr[1])
stop := ZabovTime{hour: hourStop, minute: minuteStop}
t := ZabovTimeRange{start: start, stop: stop}
ttEntry.times = append(ttEntry.times, &t)
}
}
}
ttEntry.days = map[string]bool{}
for _, day := range strings.Split(table["days"].(string), ";") {
ttEntry.days[day] = true
}
timetable.table = append(timetable.table, &ttEntry)
}
ZabovTimetables[name] = &timetable
}
}
//******************
// IP groups section
//******************
if MyConf["ipgroups"] != nil {
IPGroups := MyConf["ipgroups"].([]interface{})
fmt.Println("evaluating IP Groups: ", len(IPGroups))
for i := range IPGroups {
fmt.Println("evaluating IP Group n.", i)
var groupStruct ZabovIPGroup
groupMap := IPGroups[i].(map[string]interface{})
IPsRaw := groupMap["ips"].([]interface{})
groupStruct.ips = []net.IP{}
for x := range IPsRaw {
ipRaw := IPsRaw[x].(string)
ip := net.ParseIP(ipRaw)
fmt.Println("adding IP ", ipRaw)
alias, ok := ZabovIPAliases[ipRaw]
if ok {
fmt.Println("IP alias: ", ipRaw, alias)
ip = net.ParseIP(alias)
}
groupStruct.ips = append(groupStruct.ips, ip)
}
groupStruct.cfg = groupMap["cfg"].(string)
groupStruct.timetable = groupMap["timetable"].(string)
if len(groupStruct.cfg) > 0 {
refConfig, ok := ZabovConfigs[groupStruct.cfg]
if !ok {
log.Println("ipgroups: inexistent cfg:", groupStruct.cfg)
os.Exit(1)
} else {
refConfig.references++
}
}
fmt.Println("cfg:", groupStruct.cfg)
fmt.Println("timetable:", groupStruct.timetable)
_, ok := ZabovTimetables[groupStruct.timetable]
if !ok {
log.Println("inexistent timetable:", groupStruct.timetable)
os.Exit(1)
}
ZabovIPGroups = append(ZabovIPGroups, groupStruct)
}
}
if zabov["timetable"] != nil {
ZabovDefaultTimetable = zabov["timetable"].(string)
_, ok := ZabovTimetables[ZabovDefaultTimetable]
if !ok {
log.Println("inexistent timetable:", ZabovDefaultTimetable)
os.Exit(1)
}
}
//************************
// Local responser section
//************************
if MyConf["localresponder"] != nil {
localresponder := MyConf["localresponder"].(map[string]interface{})
if localresponder != nil {
if localresponder["responder"] != nil {
ZabovLocalResponder = localresponder["responder"].(string)
if len(ZabovLocalResponder) > 0 {
local := ZabovConfig{ZabovDNSArray: []string{ZabovLocalResponder}, references: 1}
ZabovConfigs["__localresponder__"] = &local
fmt.Println("ZabovLocalResponder:", ZabovLocalResponder)
}
}
if localresponder["localdomain"] != nil {
ZabovLocalDomain = localresponder["localdomain"].(string)
}
}
}
//******************************************
// clearing unused configs to save resources
//******************************************
for name, conf := range ZabovConfigs {
if conf.references == 0 {
log.Println("WARNING: disabling unused configuration:", name)
delete(ZabovConfigs, name)
} else {
ZabovCreateKDB(name)
}
}
}

31
01.killfile.go
View File

@ -5,11 +5,10 @@ import (
"strings"
)
var zabovKbucket = []byte("killfile")
type killfileItem struct {
Kdomain string
Ksource string
Kdomain string
Ksource string
Kconfigs stringarray
}
var bChannel chan killfileItem
@ -27,16 +26,25 @@ func bWriteThread() {
for item := range bChannel {
writeInKillfile(item.Kdomain, item.Ksource)
incrementStats("BL domains from "+item.Ksource, 1)
incrementStats("TOTAL", 1)
alreadyInSomeDB := false
for _, config := range item.Kconfigs {
if !alreadyInSomeDB {
alreadyInSomeDB = domainInKillfile(item.Kdomain, config)
}
writeInKillfile(item.Kdomain, item.Ksource, config)
}
if !alreadyInSomeDB {
incrementStats("BL domains from "+item.Ksource, 1)
incrementStats("TOTAL", 1)
}
}
}
//DomainKill stores a domain name inside the killfile
func DomainKill(s, durl string) {
func DomainKill(s, durl string, configs stringarray) {
if len(s) > 2 {
@ -46,6 +54,7 @@ func DomainKill(s, durl string) {
k.Kdomain = s
k.Ksource = durl
k.Kconfigs = configs
bChannel <- k
@ -53,11 +62,12 @@ func DomainKill(s, durl string) {
}
func writeInKillfile(key, value string) {
func writeInKillfile(key, value string, config string) {
stK := []byte(key)
stV := []byte(value)
MyZabovKDB := MyZabovKDBs[config]
err := MyZabovKDB.Put(stK, stV, nil)
if err != nil {
fmt.Println("Cannot write to Killfile DB: ", err.Error())
@ -65,10 +75,11 @@ func writeInKillfile(key, value string) {
}
func domainInKillfile(domain string) bool {
func domainInKillfile(domain string, config string) bool {
s := strings.ToLower(domain)
MyZabovKDB := MyZabovKDBs[config]
has, err := MyZabovKDB.Has([]byte(s), nil)
if err != nil {
fmt.Println("Cannot read from Killfile DB: ", err.Error())

7
01.stats.go
View File

@ -81,7 +81,12 @@ func statsThread() {
case "INC":
ZabovStats[item.Payload] += item.Number
case "SET":
ZabovStats[item.Payload] = item.Number
if item.Number == 0 {
delete(ZabovStats, item.Payload)
} else {
ZabovStats[item.Payload] = item.Number
}
case "PRI":
statsPrint()
}

5
Dockerfile.amd64
View File

@ -1,4 +1,4 @@
FROM golang:1.14.1 AS builder
FROM arm64v8/golang:1.15.6 AS builder
RUN apt install git -y
RUN mkdir -p /go/src/zabov
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov
@ -10,9 +10,10 @@ FROM debian:latest
RUN apt update
RUN apt upgrade -y
RUN apt install ca-certificates -y
RUN apt install tzdata -y
RUN mkdir -p /opt/zabov
WORKDIR /opt/zabov
COPY --from=builder /go/src/zabov /opt/zabov
EXPOSE 53/udp
ENV TZ Europe/Rome
ENTRYPOINT ["/opt/zabov/zabov"]

4
Dockerfile.arm32v7
View File

@ -1,4 +1,4 @@
FROM arm32v7/golang:1.14.1 AS builder
FROM arm64v8/golang:1.15.6 AS builder
RUN apt install git -y
RUN mkdir -p /go/src/zabov
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov
@ -10,8 +10,10 @@ FROM arm32v7/debian:latest
RUN apt update
RUN apt upgrade -y
RUN apt install ca-certificates -y
RUN apt install tzdata -y
RUN mkdir -p /opt/zabov
WORKDIR /opt/zabov
COPY --from=builder /go/src/zabov /opt/zabov
EXPOSE 53/udp
ENV TZ Europe/Rome
ENTRYPOINT ["/opt/zabov/zabov"]

4
Dockerfile.arm64v8
View File

@ -1,4 +1,4 @@
FROM arm64v8/golang:1.14.1 AS builder
FROM arm64v8/golang:1.15.6 AS builder
RUN apt install git -y
RUN mkdir -p /go/src/zabov
RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov
@ -10,8 +10,10 @@ FROM arm64v8/debian:latest
RUN apt update
RUN apt upgrade -y
RUN apt install ca-certificates -y
RUN apt install tzdata -y
RUN mkdir -p /opt/zabov
WORKDIR /opt/zabov
COPY --from=builder /go/src/zabov /opt/zabov
EXPOSE 53/udp
ENV TZ Europe/Rome
ENTRYPOINT ["/opt/zabov/zabov"]

138
README.md
View File

@ -44,45 +44,151 @@ The second is the format zabov calls "doublefilter" (a file in "/etc/hosts" form
This is why configuration file has two separated items.
The config file should look like:
Minimal config file should look like:
<pre>
{
"zabov": {
"zabov":{
"port":"53",
"proto":"udp",
"ipaddr":"127.0.0.1",
"upstream":"./dns-upstream.txt",
"cachettl": "4",
"killfilettl": "12",
"singlefilters":"./urls-hosts.txt" ,
"doublefilters":"./urls-domains.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
"ipaddr":"0.0.0.0",
"cachettl": 1,
"killfilettl": 12,
"debug:"false"
},
"configs":{
"default":{
"upstream":"./dns-upstream.txt",
"singlefilters":"./urls-domains.txt",
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
},
}
}
</pre>
Where:
Global zabov settings:
- port is the port number. Usually is 53, you can change for docker, if you like
- proto is the protocol. Choices are "udp", "tcp", "tcp/udp"
- ipaddr is the port to listen to. Maybe empty, (which will result in listening to 0.0.0.0) to avoid issues with docker.
- upstream: file containing all DNS we want to query : each line in format IP:PORT
- cachettl: amount of time the cache is kept (in hours)
- killfilettl: refresh time for _killfiles_
- debug: if set to "true" Zabov prints verbose logs, such as config selection and single DNS requests
configs:
- contains multiple zabov configuration dictionaries. "default" configuration name is mandatory
- upstream: file containing all DNS we want to query : each line in format IP:PORT
- singlefilters: name of the file for blacklists following the "singlefilter" schema.(one URL per line)
- doublefilters: name of the file, for blacklists following the "doublefilter" schema.(one URL per line)
- blackholeip: IP address to return when the IP is banned. This is because you may want to avoid MX issues, mail loops on localhost, or you have a web server running on localhost
- hostsfile: path where you keep your local blacklistfile : this is in the format "singlefilter", meaning one domain per line, unlike hosts file.
Advanced configuration includes support for multiple configurations based on IP Source and timetables:
<pre>
{
"zabov":{
"port":"53",
"proto":"udp",
"ipaddr":"0.0.0.0",
"cachettl": 1,
"killfilettl": 12,
"debug":"false",
"timetable":"tt_default"
},
"localresponder":{
"responder":"192.168.178.1:53",
"localdomain":"fritz.box"
},
"ipaliases":{
"pc8":"192.168.178.29",
"localhost":"127.0.0.1"
},
"ipgroups":[
{
"ips":["localhost", "::1", "192.168.178.30", "192.168.178.31", "pc8"],
"cfg":"",
"timetable":"tt_children"
}
],
"timetables":{
"tt_children":{
"tables":[{"times":"00:00-05:00;8:30-12:30;18:30-22:59", "days":"Mo;Tu;We;Th;Fr;Sa;Su"}],
"cfgin":"children_restricted",
"cfgout":"default"
}
"tt_default":{
"tables":[{"times":"8:30-22:30", "days":"Su"}],
"cfgin":"children",
"cfgout":"default"
}
},
"configs":{
"default":{
"upstream":"./dns-upstream.txt",
"singlefilters":"./urls-domains.txt",
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
},
"children":{
"upstream":"./dns-upstream-safe.txt",
"singlefilters":"./urls-domains.txt",
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
},
"children_restricted":{
"upstream":"./dns-upstream-safe.txt",
"singlefilters":"./urls-domains-restricted.txt",
"doublefilters":"./urls-hosts-restricted.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
}
}
}
</pre>
Global zabov settings:
- timetable: sets the global/default timetable. This table will be used for any client that is not already included in an IP group
localresponder:
- allows to set a local DNS to respond for "local" domains. A domain name is handled as "local" if dosen't contains "." (dots) or if it ends with a well known prefix, such as ".local".
Note: the cache is not used for local responder.
- responder: is the local DNS server address in the IP:PORT format.
- localdomain: is the suffix for local domain names. All domains ending with this prefix are resolved by local responder
ipaliases: a dictionary of IPs
- each entry in this dictionary define a domain-alias name and his IP address. It works as replacement of /etc/hosts file.
- each entry is used by Zabov to resolve that names and to replace any value in the ipgroups.ips array.
timetables: a dictionary of timetable dictionaries
- allow to define timetables in the format "time-ranges" and "days-of-week"
- tables: contain an array of dictionaries, each defining a time rule.
- each table is a dictinary containing "time" and "days" values
- time: is a string in the form "start:time1-stop:time1;start:time2-stop:time2..."
- days: is a string containing semicolon separated day names to apply the rule such as "Mo;Tu;We;Th;Fr"
- days names are: "Mo", "Tu" "We", "Th", "Fr", "Sa", "Su"
- empty value means all week-days
You can define complex time rules using more than one entry in this dictionay
- cfgin: is the name of the configuration to apply if current time is "inside" the timetable
- cfgout: is the name of the configuration to apply if current time is "outside" the timetable
ipgroups: an array of ipgroup dictionaries
- let you define a set of IP addresses that shall use a configuration other than "default"
- ips: is an array of strings, each containing an ip address or a name defined in the "ipaliases" config branch
- cfg: is a string containing the name of the configuration to be used for this group; ignored if timetable is also defined
- timetable: is a string containing the name of the tiemtable to be aplied to this group
# DOCKER
Multistage Dockerfiles are provided for AMD64, ARMv7, ARM64V8
NOTE: you shall use TZ env var to change docker image timezone. TZ defaults to CET.
# TODO:
- ~~caching~~

49
adlist_hosts.go
View File

@ -16,9 +16,12 @@ func init() {
}
//DoubleIndexFilter puts the domains inside file
func DoubleIndexFilter(durl string) error {
func DoubleIndexFilter(durl string, configs stringarray) error {
fmt.Println("Retrieving HostFile from: ", durl)
fmt.Println("DoubleIndexFilter: Retrieving HostFile from: ", durl)
// resets malformed HostLines for url
setstatsvalue("Malformed HostLines "+durl, 0)
var err error
@ -48,6 +51,9 @@ func DoubleIndexFilter(durl string) error {
line := scanner.Text()
if len(line) == 0 || strings.TrimSpace(line)[0] == '#' {
continue
}
h := strings.FieldsFunc(line, splitter)
if h == nil {
@ -59,7 +65,8 @@ func DoubleIndexFilter(durl string) error {
}
if net.ParseIP(h[0]) != nil {
DomainKill(h[1], durl)
DomainKill(h[1], durl, configs)
// fmt.Println("MATCH: ", h[1])
numLines++
@ -76,20 +83,44 @@ func DoubleIndexFilter(durl string) error {
}
func getDoubleFilters() {
func getDoubleFilters(urls urlsMap) {
s := fileByLines(ZabovDoubleBL)
for _, a := range s {
DoubleIndexFilter(a)
fmt.Println("getDoubleFilters: downloading all urls:", len(urls))
for url, configs := range urls {
DoubleIndexFilter(url, configs)
}
fmt.Println("getDoubleFilters: DONE!")
}
func downloadDoubleThread() {
fmt.Println("Starting updater of DOUBLE lists, each (hours):", ZabovKillTTL)
time.Sleep(2 * time.Second) // wait for local DNS server up & running (may be our DNS)
_urls := urlsMap{}
for {
getDoubleFilters()
fmt.Println("downloadDoubleThread: collecting urls from all configs...")
for config := range ZabovConfigs {
ZabovDoubleBL := ZabovConfigs[config].ZabovDoubleBL
if len(ZabovDoubleBL) == 0 {
continue
}
s := fileByLines(ZabovDoubleBL)
for _, v := range s {
if len(v) == 0 || strings.TrimSpace(v)[0] == '#' {
continue
}
configs := _urls[v]
if configs == nil {
configs = stringarray{}
_urls[v] = configs
}
configs = append(configs, config)
_urls[v] = configs
}
}
getDoubleFilters(_urls)
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
}

51
adlist_single.go
View File

@ -14,10 +14,13 @@ func init() {
}
//SingleIndexFilter puts the domains inside file
func SingleIndexFilter(durl string) error {
func SingleIndexFilter(durl string, configs stringarray) error {
fmt.Println("Retrieving DomainFile from: ", durl)
// resets malformed HostLines for url
setstatsvalue("Malformed DomainLines "+durl, 0)
var err error
// Get the data
@ -46,6 +49,9 @@ func SingleIndexFilter(durl string) error {
line := scanner.Text()
if len(line) == 0 || strings.TrimSpace(line)[0] == '#' {
continue
}
h := strings.FieldsFunc(line, splitter)
if h == nil {
@ -57,7 +63,9 @@ func SingleIndexFilter(durl string) error {
}
if !strings.Contains(h[0], "#") {
DomainKill(h[0], durl)
DomainKill(h[0], durl, configs)
// fmt.Println("MATCH: ", h[1])
numLines++
} else {
@ -73,20 +81,47 @@ func SingleIndexFilter(durl string) error {
}
func getSingleFilters() {
func getSingleFilters(urls urlsMap) {
s := fileByLines(ZabovSingleBL)
for _, a := range s {
SingleIndexFilter(a)
fmt.Println("getSingleFilters: downloading all urls:", len(urls))
for url, configs := range urls {
SingleIndexFilter(url, configs)
}
fmt.Println("getSingleFilters: DONE!")
}
func downloadThread() {
fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL)
time.Sleep(2 * time.Second) // wait for local DNS server up & running (may be our DNS)
_urls := urlsMap{}
for {
getSingleFilters()
fmt.Println("downloadThread: collecting urls from all configs...")
for config := range ZabovConfigs {
ZabovSingleBL := ZabovConfigs[config].ZabovSingleBL
if len(ZabovSingleBL) == 0 {
continue
}
s := fileByLines(ZabovSingleBL)
for _, v := range s {
if len(v) == 0 || strings.TrimSpace(v)[0] == '#' {
continue
}
configs := _urls[v]
if configs == nil {
configs = stringarray{}
_urls[v] = configs
}
configs = append(configs, config)
_urls[v] = configs
}
}
getSingleFilters(_urls)
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
}

19
config.json
View File

@ -1,15 +1,18 @@
{
"zabov": {
"zabov":{
"port":"53",
"proto":"udp",
"ipaddr":"0.0.0.0",
"upstream":"./dns-upstream.txt" ,
"cachettl": 1,
"killfilettl": 12,
"singlefilters":"./urls-domains.txt" ,
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
"killfilettl": 12
},
"configs":{
"default":{
"upstream":"./dns-upstream.txt",
"singlefilters":"./urls-domains.txt",
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
}
}
}

57
config.sample.json Normal file
View File

@ -0,0 +1,57 @@
{
"zabov":{
"port":"53",
"proto":"udp",
"ipaddr":"0.0.0.0",
"cachettl": 1,
"killfilettl": 12,
"debug":"true",
"debugdbpath":"./logs",
"timetable":""
},
"localresponder":{
"responder":"192.168.1.1:53",
"localdomain":".local"
},
"ipaliases":{
"pc8":"192.168.1.2",
},
"ipgroups":[
{
"ips":["pc8"],
"cfg":"",
"timetable":"tt_children"
}
],
"timetables":{
"tt_children":{
"tables":[{"times":"9:30-11:30", "days":"Mo;Tu;We;Th;Fr;Sa"}],
"cfgin":"children_restricted",
"cfgout":"children"
}
},
"configs":{
"default":{
"upstream":"./dns-upstream.txt",
"singlefilters":"./urls-domains-updated.txt",
"doublefilters":"./urls-hosts-normal.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local-normal.txt"
},
"children":{
"upstream":"./dns-familyscreen.txt",
"singlefilters":"./urls-domains-updated.txt",
"doublefilters":"./urls-hosts-nofb.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local-normal.txt"
},
"children_restricted":{
"upstream":"./dns-familyscreen.txt",
"singlefilters":"./urls-domains-updated.txt",
"doublefilters":"./urls-hosts-nofb.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local-restricted.txt"
}
}
}

26
dns_client.go
View File

@ -12,7 +12,8 @@ import (
//ForwardQuery forwards the query to the upstream server
//first server to answer wins
func ForwardQuery(query *dns.Msg) *dns.Msg {
//accepts config name to select the UP DNS source list
func ForwardQuery(query *dns.Msg, config string, nocache bool) *dns.Msg {
go incrementStats("ForwardQueries", 1)
@ -23,12 +24,14 @@ func ForwardQuery(query *dns.Msg) *dns.Msg {
fqdn := strings.TrimRight(query.Question[0].Name, ".")
lfqdn := fmt.Sprintf("%d", query.Question[0].Qtype) + "." + fqdn
if cached := GetDomainFromCache(lfqdn); cached != nil {
go incrementStats("CacheHit", 1)
cached.SetReply(query)
cached.Authoritative = true
return cached
if !nocache {
if cached := GetDomainFromCache(lfqdn); cached != nil {
go incrementStats("CacheHit", 1)
cached.SetReply(query)
cached.Authoritative = true
return cached
}
}
c := new(dns.Client)
@ -45,7 +48,7 @@ func ForwardQuery(query *dns.Msg) *dns.Msg {
continue
}
d := oneTimeDNS()
d := oneTimeDNS(config)
in, _, err := c.Exchange(query, d)
if err != nil {
@ -78,13 +81,18 @@ func init() {
}
func oneTimeDNS() (dns string) {
func oneTimeDNS(config string) (dns string) {
rand.Seed(time.Now().Unix())
upl := ZabovDNSArray
upl := ZabovConfigs[config].ZabovDNSArray
if len(upl) < 1 {
if len(ZabovLocalResponder) > 0 {
fmt.Println("No DNS defined, fallback to local responder:", ZabovLocalResponder)
return ZabovLocalResponder
}
fmt.Println("No DNS defined, using default 127.0.0.53:53. Hope it works!")
return "127.0.0.53:53"
}

283
dns_handler.go
View File

@ -1,44 +1,315 @@
package main
import (
"fmt"
"log"
"net"
"os"
"path"
"strings"
"time"
"github.com/miekg/dns"
)
var reqTypes map[uint16]string
var weekdays []string
type logItem struct {
clientIP string
name string
reqType uint16
config string
timetable string
killed string
}
// logChannel used by logging thread
var logChannel chan logItem
func init() {
weekdays = []string{"Su", "Mo", "Tu", "We", "Th", "Fr", "Sa"}
if len(ZabovDebugDBPath) > 0 {
os.MkdirAll(ZabovDebugDBPath, 0755)
}
reqTypes = map[uint16]string{
dns.TypeNone: "TypeNone",
dns.TypeA: "TypeA",
dns.TypeNS: "TypeNS",
dns.TypeMD: "TypeMD",
dns.TypeMF: "TypeMF",
dns.TypeCNAME: "TypeCNAME",
dns.TypeSOA: "TypeSOA",
dns.TypeMB: "TypeMB",
dns.TypeMG: "TypeMG",
dns.TypeMR: "TypeMR",
dns.TypeNULL: "TypeNULL",
dns.TypePTR: "TypePTR",
dns.TypeHINFO: "TypeHINFO",
dns.TypeMINFO: "TypeMINFO",
dns.TypeMX: "TypeMX",
dns.TypeTXT: "TypeTXT",
dns.TypeRP: "TypeRP",
dns.TypeAFSDB: "TypeAFSDB",
dns.TypeX25: "TypeX25",
dns.TypeISDN: "TypeISDN",
dns.TypeRT: "TypeRT",
dns.TypeNSAPPTR: "TypeNSAPPTR",
dns.TypeSIG: "TypeSIG",
dns.TypeKEY: "TypeKEY",
dns.TypePX: "TypePX",
dns.TypeGPOS: "TypeGPOS",
dns.TypeAAAA: "TypeAAAA",
dns.TypeLOC: "TypeLOC",
dns.TypeNXT: "TypeNXT",
dns.TypeEID: "TypeEID",
dns.TypeNIMLOC: "TypeNIMLOC",
dns.TypeSRV: "TypeSRV",
dns.TypeATMA: "TypeATMA",
dns.TypeNAPTR: "TypeNAPTR",
dns.TypeKX: "TypeKX",
dns.TypeCERT: "TypeCERT",
dns.TypeDNAME: "TypeDNAME",
dns.TypeOPT: "TypeOPT",
dns.TypeAPL: "TypeAPL",
dns.TypeDS: "TypeDS",
dns.TypeSSHFP: "TypeSSHFP",
dns.TypeRRSIG: "TypeRRSIG",
dns.TypeNSEC: "TypeNSEC",
dns.TypeDNSKEY: "TypeDNSKEY",
dns.TypeDHCID: "TypeDHCID",
dns.TypeNSEC3: "TypeNSEC3",
dns.TypeNSEC3PARAM: "TypeNSEC3PARAM",
dns.TypeTLSA: "TypeTLSA",
dns.TypeSMIMEA: "TypeSMIMEA",
dns.TypeHIP: "TypeHIP",
dns.TypeNINFO: "TypeNINFO",
dns.TypeRKEY: "TypeRKEY",
dns.TypeTALINK: "TypeTALINK",
dns.TypeCDS: "TypeCDS",
dns.TypeCDNSKEY: "TypeCDNSKEY",
dns.TypeOPENPGPKEY: "TypeOPENPGPKEY",
dns.TypeCSYNC: "TypeCSYNC",
dns.TypeSPF: "TypeSPF",
dns.TypeUINFO: "TypeUINFO",
dns.TypeUID: "TypeUID",
dns.TypeGID: "TypeGID",
dns.TypeUNSPEC: "TypeUNSPEC",
dns.TypeNID: "TypeNID",
dns.TypeL32: "TypeL32",
dns.TypeL64: "TypeL64",
dns.TypeLP: "TypeLP",
dns.TypeEUI48: "TypeEUI48",
dns.TypeEUI64: "TypeEUI64",
dns.TypeURI: "TypeURI",
dns.TypeCAA: "TypeCAA",
dns.TypeAVC: "TypeAVC",
dns.TypeTKEY: "TypeTKEY",
dns.TypeTSIG: "TypeTSIG",
dns.TypeIXFR: "TypeIXFR",
dns.TypeAXFR: "TypeAXFR",
dns.TypeMAILB: "TypeMAILB",
dns.TypeMAILA: "TypeMAILA",
dns.TypeANY: "TypeANY",
dns.TypeTA: "TypeTA",
dns.TypeDLV: "TypeDLV",
dns.TypeReserved: "TypeReserved"}
fmt.Println("Local Time:", getLocalTime().Format(time.ANSIC))
if len(ZabovDebugDBPath) > 0 {
logChannel = make(chan logItem, 1024)
go logWriteThread()
}
}
func logWriteThread() {
for item := range logChannel {
var header string
d := time.Now().Format("2006-01-02")
logpath := path.Join(ZabovDebugDBPath, strings.Replace(item.clientIP, ":", "_", -1)+"-"+d+".log")
_, err1 := os.Stat(logpath)
if os.IsNotExist(err1) {
header = strings.Join([]string{"time", "clientIP", "name", "reqType", "config", "timetable", "killed"}, "\t")
}
f, err := os.OpenFile(logpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err == nil {
reqTypeName, err := reqTypes[item.reqType]
if !err {
reqTypeName = fmt.Sprintf("%d", item.reqType)
}
ct := time.Now().Format(time.RFC3339)
log := strings.Join([]string{ct, item.clientIP, strings.TrimRight(item.name, "."), reqTypeName, item.config, item.timetable, item.killed}, "\t")
if len(header) > 0 {
f.Write([]byte(header))
f.Write([]byte("\n"))
}
f.Write([]byte(log))
f.Write([]byte("\n"))
f.Close()
}
}
}
func logQuery(clientIP string, name string, reqType uint16, config string, timetable string, killed string) {
if len(ZabovDebugDBPath) > 0 {
k := logItem{clientIP: clientIP, name: name, reqType: reqType, config: config, timetable: timetable, killed: killed}
logChannel <- k
}
}
func getLocalTime() time.Time {
return time.Now().Local()
}
func confFromTimeTable(timetable string) string {
tt := ZabovTimetables[timetable]
if tt == nil {
if ZabovDebug {
log.Println("confFromTimeTable: return default")
}
return "default"
}
for _, ttentry := range tt.table {
now := getLocalTime()
nowHour := now.Hour()
nowMinute := now.Minute()
weekday := weekdays[now.Weekday()]
if ttentry.days == nil || len(ttentry.days) == 0 || ttentry.days[weekday] || ttentry.days[strings.ToLower(weekday)] {
for _, t := range ttentry.times {
if (nowHour > t.start.hour || (nowHour == t.start.hour && nowMinute >= t.start.minute)) &&
(nowHour < t.stop.hour || (nowHour == t.stop.hour && nowMinute <= t.stop.minute)) {
go incrementStats("TIMETABLE IN: "+timetable, 1)
if ZabovDebug {
log.Println("confFromTimeTable: return IN", tt.cfgin)
}
return tt.cfgin
}
}
}
}
go incrementStats("TIMETABLE OUT: "+timetable, 1)
if ZabovDebug {
log.Println("confFromTimeTable: return OUT", tt.cfgout)
}
return tt.cfgout
}
func confFromIP(clientIP net.IP) (string, string) {
for _, ipgroup := range ZabovIPGroups {
for _, ip := range ipgroup.ips {
if clientIP.Equal(ip) {
if len(ipgroup.timetable) > 0 {
return confFromTimeTable(ipgroup.timetable), ipgroup.timetable
}
if ZabovDebug {
log.Println("confFromIP: ipgroup.cfg", ipgroup.cfg)
}
return ipgroup.cfg, ""
}
}
}
if len(ZabovDefaultTimetable) > 0 {
return confFromTimeTable(ZabovDefaultTimetable), ZabovDefaultTimetable
}
if ZabovDebug {
log.Println("confFromIP: return default")
}
return "default", ""
}
func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
go incrementStats("TotalQueries", 1)
remIP, _, e := net.SplitHostPort(w.RemoteAddr().String())
if e != nil {
go incrementStats("CLIENT ERROR: "+remIP, 1)
} else {
go incrementStats("CLIENT: "+remIP, 1)
}
msg := dns.Msg{}
msg.SetReply(r)
switch r.Question[0].Qtype {
config, timetable := confFromIP(net.ParseIP(remIP))
if ZabovDebug {
log.Println("REQUEST:", remIP, config)
}
ZabovConfig := ZabovConfigs[config]
QType := r.Question[0].Qtype
switch QType {
case dns.TypeA:
msg.Authoritative = true
domain := msg.Question[0].Name
fqdn := strings.TrimRight(domain, ".")
if domainInKillfile(fqdn) {
if ZabovDebug {
log.Println("TypeA: fqdn:", fqdn)
}
if len(ZabovIPAliases[fqdn]) > 0 {
config = "__aliases__"
msg.Answer = append(msg.Answer, &dns.A{
Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
A: net.ParseIP(ZabovIPAliases[fqdn]),
})
go logQuery(remIP, fqdn, QType, config, timetable, "alias")
break
}
if len(ZabovLocalResponder) > 0 {
if !strings.Contains(fqdn, ".") ||
(len(ZabovLocalDomain) > 0 && strings.HasSuffix(fqdn, ZabovLocalDomain)) {
config = "__localresponder__"
ret := ForwardQuery(r, config, true)
w.WriteMsg(ret)
go logQuery(remIP, fqdn, QType, config, timetable, "localresponder")
break
}
}
if domainInKillfile(fqdn, config) {
go incrementStats("Killed", 1)
msg.Answer = append(msg.Answer, &dns.A{
Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
A: net.ParseIP(ZabovAddBL),
A: net.ParseIP(ZabovConfig.ZabovAddBL),
})
go logQuery(remIP, fqdn, QType, config, timetable, "killed")
} else {
ret := ForwardQuery(r)
go logQuery(remIP, fqdn, QType, config, timetable, "forwarded")
ret := ForwardQuery(r, config, false)
w.WriteMsg(ret)
}
default:
ret := ForwardQuery(r)
case dns.TypePTR:
if ZabovDebug {
log.Println("TypePTR: Name:", msg.Question[0].Name)
}
if len(ZabovLocalResponder) > 0 {
// if set use local responder for reverse lookup (suffix ".in-addr.arpa.")
config = "__localresponder__"
}
ret := ForwardQuery(r, config, true)
w.WriteMsg(ret)
go logQuery(remIP, msg.Question[0].Name, QType, config, timetable, "localresponder")
default:
ret := ForwardQuery(r, config, false)
w.WriteMsg(ret)
if len(ZabovDebugDBPath) > 0 {
go logQuery(remIP, msg.Question[0].Name, QType, config, timetable, "forwarded")
}
}
go incrementStats("CONFIG: "+config, 1)
w.WriteMsg(&msg)
}

54
hostfile.go
View File

@ -4,33 +4,55 @@ import (
"bufio"
"fmt"
"os"
"strings"
)
func init() {
fmt.Println("Ingesting local hosts file")
ingestLocalBlacklist()
ingestLocalBlacklists()
}
func ingestLocalBlacklist() {
file, err := os.Open(ZabovHostsFile)
if err != nil {
fmt.Println(err.Error())
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
d := scanner.Text()
DomainKill(d, ZabovHostsFile)
incrementStats("Blacklist", 1)
func ingestLocalBlacklists() {
fmt.Println("ingestLocalBlacklist: collecting urls from all configs...")
_files := urlsMap{}
for config := range ZabovConfigs {
ZabovHostsFile := ZabovConfigs[config].ZabovHostsFile
if len(ZabovHostsFile) == 0 {
continue
}
configs := _files[ZabovHostsFile]
if configs == nil {
configs = stringarray{}
_files[ZabovHostsFile] = configs
}
configs = append(configs, config)
_files[ZabovHostsFile] = configs
}
if err := scanner.Err(); err != nil {
fmt.Println(err.Error())
for ZabovHostsFile, configs := range _files {
file, err := os.Open(ZabovHostsFile)
if err != nil {
fmt.Println(err.Error())
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
d := scanner.Text()
if len(d) == 0 || strings.TrimSpace(d)[0] == '#' {
continue
}
DomainKill(d, ZabovHostsFile, configs)
incrementStats("Blacklist", 1)
}
if err := scanner.Err(); err != nil {
fmt.Println(err.Error())
}
}
}

89
main.go
View File

@ -2,6 +2,7 @@ package main
import (
"log"
"net"
"github.com/miekg/dns"
)
@ -9,32 +10,84 @@ import (
//MyDNS is my dns server
var MyDNS *dns.Server
//ZabovUpDNS keeps the name of upstream DNSs
var ZabovUpDNS string
//ZabovSingleBL list of urls returning a file with just names of domains
var ZabovSingleBL string
//ZabovDoubleBL list of urls returning a file with IP<space>domain
var ZabovDoubleBL string
//ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1
var ZabovAddBL string
//ZabovCacheTTL is the amount of hours we cache records of DNS
//ZabovCacheTTL is the amount of hours we cache records of DNS (global)
var ZabovCacheTTL int
//ZabovKillTTL is the amount of hours we cache the killfile
//ZabovKillTTL is the amount of hours we cache the killfile (global)
var ZabovKillTTL int
//ZabovHostsFile is the file we use to keep our hosts
var ZabovHostsFile string
//ZabovLocalResponder is the default DNS server for local domains (global)
var ZabovLocalResponder string
//ZabovDNSArray is the array containing all the DNS we mention
var ZabovDNSArray []string
//ZabovLocalDomain is the default local domain (global)
var ZabovLocalDomain string
//ZabovDefaultTimetable is the default timetable, applied to any client that is not already in any IP Group (global)
var ZabovDefaultTimetable string
//ZabovDebug activate more logging if set to true (global)
var ZabovDebug bool
//ZabovDebugDBPath path to store debug query logs: activate logging of each single query in a csv like file (global)
var ZabovDebugDBPath string
type handler struct{}
// ZabovConfig contains all Zabov configs
type ZabovConfig struct {
ZabovSingleBL string // json:singlefilters -> ZabovSingleBL list of urls returning a file with just names of domains