Merge pull request 'master' (#1) from bloved/zabov:master into master

Reviewed-on: #1

00.database.go

@@ -7,12 +7,12 @@ import (
 	"github.com/syndtr/goleveldb/leveldb"
 )
 
-//MyZabovKDB is the storage where we'll put domains to block
+//MyZabovCDB is the storage where we'll put domains to cache (global for all configs)
-var MyZabovKDB *leveldb.DB
-
-//MyZabovCDB is the storage where we'll put domains to cache
 var MyZabovCDB *leveldb.DB
 
+//MyZabovKDBs is the storage where we'll put domains to block (one for each config)
+var MyZabovKDBs map[string]*leveldb.DB
+
 func init() {
 
 	var err error
@@ -21,13 +21,6 @@ func init() {
 
 	os.MkdirAll("./db", 0755)
 
-	MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil)
-	if err != nil {
-		fmt.Println("Cannot create Killfile db: ", err.Error())
-	} else {
-		fmt.Println("Killfile DB created")
-	}
-
 	MyZabovCDB, err = leveldb.OpenFile("./db/cache", nil)
 	if err != nil {
 		fmt.Println("Cannot create Cache db: ", err.Error())
@@ -35,4 +28,21 @@ func init() {
 		fmt.Println("Cache DB created")
 	}
 
+	MyZabovKDBs = map[string]*leveldb.DB{}
+}
+
+// ZabovCreateKDB creates Kill DBs
+func ZabovCreateKDB(conf string) {
+	var err error
+
+	dbname := "./db/killfile_" + conf
+	KDB, err := leveldb.OpenFile(dbname, nil)
+	if err != nil {
+		fmt.Println("Cannot create Killfile db: ", err.Error())
+	} else {
+		fmt.Println("Killfile DB created:", dbname)
+	}
+
+	MyZabovKDBs[conf] = KDB
+
 }

01.conf.go

@@ -5,30 +5,19 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net"
 	"os"
+	"strconv"
+	"strings"
 
 	"github.com/miekg/dns"
 )
 
+type stringarray []string
+type urlsMap map[string]stringarray
+
 func init() {
-
+	var MyConfRaw interface{}
-	//ZabovConf describes the Json we use for configuration
-	type ZabovConf struct {
-		Zabov struct {
-			Port          string `json:"port"`
-			Proto         string `json:"proto"`
-			Ipaddr        string `json:"ipaddr"`
-			Upstream      string `json:"upstream"`
-			Cachettl      int    `json:"cachettl"`
-			Killfilettl   int    `json:"killfilettl"`
-			Singlefilters string `json:"singlefilters"`
-			Doublefilters string `json:"doublefilters"`
-			Blackholeip   string `json:"blackholeip"`
-			Hostsfile     string `json:"hostsfile"`
-		} `json:"zabov"`
-	}
-
-	var MyConf ZabovConf
 
 	file, err := ioutil.ReadFile("config.json")
 
@@ -37,26 +26,53 @@ func init() {
 		os.Exit(1)
 	}
 
-	err = json.Unmarshal([]byte(file), &MyConf)
+	err = json.Unmarshal([]byte(file), &MyConfRaw)
 
 	if err != nil {
-		log.Println("Cannot marshal json: ", err.Error())
+		log.Println("Cannot unmarshal json: ", err.Error())
 		os.Exit(1)
 	}
 
 	// now we read configuration file
 	fmt.Println("Reading configuration file...")
 
-	ZabovPort := MyConf.Zabov.Port
+	MyConf := MyConfRaw.(map[string]interface{})
-	ZabovType := MyConf.Zabov.Proto
+
-	ZabovAddr := MyConf.Zabov.Ipaddr
+	//******************************
-	ZabovUpDNS = MyConf.Zabov.Upstream
+	// zabov section (global config)
-	ZabovSingleBL = MyConf.Zabov.Singlefilters
+	//******************************
-	ZabovDoubleBL = MyConf.Zabov.Doublefilters
+	zabov := MyConf["zabov"].(map[string]interface{})
-	ZabovAddBL = MyConf.Zabov.Blackholeip
+
-	ZabovCacheTTL = MyConf.Zabov.Cachettl
+	ZabovPort := zabov["port"].(string)
-	ZabovKillTTL = MyConf.Zabov.Killfilettl
+	ZabovType := zabov["proto"].(string)
-	ZabovHostsFile = MyConf.Zabov.Hostsfile
+	ZabovAddr := zabov["ipaddr"].(string)
+
+	ZabovCacheTTL = int(zabov["cachettl"].(float64))
+	ZabovKillTTL = int(zabov["killfilettl"].(float64))
+
+	if zabov["debug"] != nil {
+		ZabovDebug = zabov["debug"].(string) == "true"
+	}
+	if zabov["debugdbpath"] != nil {
+		ZabovDebugDBPath = (zabov["debugdbpath"].(string))
+	}
+
+	if MyConf["configs"] == nil {
+		log.Println("configs not set: you shall set at least 'default' config")
+		os.Exit(1)
+	}
+
+	configs := MyConf["configs"].(map[string]interface{})
+
+	if len(configs) == 0 {
+		log.Println("you shall set at least 'default' config")
+		os.Exit(1)
+	}
+
+	if configs["default"] == nil {
+		log.Println("'default' config is required")
+		os.Exit(1)
+	}
 
 	zabovString := ZabovAddr + ":" + ZabovPort
 
@@ -64,6 +80,203 @@ func init() {
 	MyDNS.Addr = zabovString
 	MyDNS.Net = ZabovType
 
-	ZabovDNSArray = fileByLines(ZabovUpDNS)
+	ZabovConfigs = map[string]*ZabovConfig{}
+	ZabovIPGroups = []ZabovIPGroup{}
+	ZabovTimetables = map[string]*ZabovTimetable{}
+	ZabovIPAliases = map[string]string{}
+
+	//*******************
+	// IP aliases section
+	//*******************
+	if MyConf["ipaliases"] != nil {
+		IPAliasesRaw := MyConf["ipaliases"].(map[string]interface{})
+
+		for alias, ip := range IPAliasesRaw {
+			fmt.Println("IP Alias:", alias, ip)
+			ZabovIPAliases[alias] = ip.(string)
+		}
+	}
+
+	//****************
+	// configs section
+	//****************
+	for name, v := range configs {
+		fmt.Println("evaluaing config name:", name)
+		confRaw := v.(map[string]interface{})
+		var conf ZabovConfig
+		conf.ZabovUpDNS = confRaw["upstream"].(string)
+		conf.ZabovSingleBL = confRaw["singlefilters"].(string)
+		conf.ZabovDoubleBL = confRaw["doublefilters"].(string)
+		conf.ZabovAddBL = confRaw["blackholeip"].(string)
+		conf.ZabovHostsFile = confRaw["hostsfile"].(string)
+
+		conf.ZabovDNSArray = fileByLines(conf.ZabovUpDNS)
+		ZabovConfigs[name] = &conf
+
+	}
+
+	// default config is mandatory
+	ZabovConfigs["default"].references++
+
+	//*******************
+	// timetables section
+	//*******************
+	if MyConf["timetables"] != nil {
+		timetables := MyConf["timetables"].(map[string]interface{})
+
+		for name, v := range timetables {
+			fmt.Println("evaluaing timetable name:", name)
+			timetableRaw := v.(map[string]interface{})
+			var timetable ZabovTimetable
+
+			timetable.cfgin = timetableRaw["cfgin"].(string)
+			timetable.cfgout = timetableRaw["cfgout"].(string)
+
+			if timetable.cfgin == "" {
+				timetable.cfgin = "default"
+			}
+			if timetable.cfgout == "" {
+				timetable.cfgout = "default"
+			}
+
+			refConfig, ok := ZabovConfigs[timetable.cfgin]
+			if !ok {
+				log.Println("timetable: inexistent cfgin:", timetable.cfgin)
+				os.Exit(1)
+			}
+
+			refConfig.references++
+			refConfig, ok = ZabovConfigs[timetable.cfgout]
+			if !ok {
+				log.Println("timetable: inexistent cfgout:", timetable.cfgout)
+				os.Exit(1)
+			}
+			refConfig.references++
+
+			tables := timetableRaw["tables"].([]interface{})
+
+			for i := range tables {
+				table := tables[i].(map[string]interface{})
+				var ttEntry ZabovTimetableEntry
+				ttEntry.times = []*ZabovTimeRange{}
+				for _, tRaw := range strings.Split(table["times"].(string), ";") {
+					tRawArr := strings.Split(tRaw, "-")
+					if len(tRawArr) > 1 {
+						startArr := strings.Split(tRawArr[0], ":")
+						stopArr := strings.Split(tRawArr[1], ":")
+
+						if len(startArr) > 1 && len(stopArr) > 1 {
+							hourStart, _ := strconv.Atoi(startArr[0])
+							minuteStart, _ := strconv.Atoi(startArr[1])
+							start := ZabovTime{hour: hourStart, minute: minuteStart}
+
+							hourStop, _ := strconv.Atoi(stopArr[0])
+							minuteStop, _ := strconv.Atoi(stopArr[1])
+							stop := ZabovTime{hour: hourStop, minute: minuteStop}
+							t := ZabovTimeRange{start: start, stop: stop}
+							ttEntry.times = append(ttEntry.times, &t)
+						}
+					}
+
+				}
+
+				ttEntry.days = map[string]bool{}
+				for _, day := range strings.Split(table["days"].(string), ";") {
+					ttEntry.days[day] = true
+				}
+
+				timetable.table = append(timetable.table, &ttEntry)
+			}
+			ZabovTimetables[name] = &timetable
+		}
+	}
+
+	//******************
+	// IP groups section
+	//******************
+	if MyConf["ipgroups"] != nil {
+		IPGroups := MyConf["ipgroups"].([]interface{})
+
+		fmt.Println("evaluating IP Groups: ", len(IPGroups))
+		for i := range IPGroups {
+			fmt.Println("evaluating IP Group n.", i)
+			var groupStruct ZabovIPGroup
+			groupMap := IPGroups[i].(map[string]interface{})
+			IPsRaw := groupMap["ips"].([]interface{})
+			groupStruct.ips = []net.IP{}
+			for x := range IPsRaw {
+				ipRaw := IPsRaw[x].(string)
+				ip := net.ParseIP(ipRaw)
+				fmt.Println("adding IP ", ipRaw)
+
+				alias, ok := ZabovIPAliases[ipRaw]
+				if ok {
+					fmt.Println("IP alias: ", ipRaw, alias)
+					ip = net.ParseIP(alias)
+				}
+				groupStruct.ips = append(groupStruct.ips, ip)
+			}
+			groupStruct.cfg = groupMap["cfg"].(string)
+			groupStruct.timetable = groupMap["timetable"].(string)
+			if len(groupStruct.cfg) > 0 {
+				refConfig, ok := ZabovConfigs[groupStruct.cfg]
+				if !ok {
+					log.Println("ipgroups: inexistent cfg:", groupStruct.cfg)
+					os.Exit(1)
+				} else {
+					refConfig.references++
+				}
+			}
+			fmt.Println("cfg:", groupStruct.cfg)
+			fmt.Println("timetable:", groupStruct.timetable)
+			_, ok := ZabovTimetables[groupStruct.timetable]
+			if !ok {
+				log.Println("inexistent timetable:", groupStruct.timetable)
+				os.Exit(1)
+			}
+			ZabovIPGroups = append(ZabovIPGroups, groupStruct)
+		}
+	}
+
+	if zabov["timetable"] != nil {
+		ZabovDefaultTimetable = zabov["timetable"].(string)
+		_, ok := ZabovTimetables[ZabovDefaultTimetable]
+		if !ok {
+			log.Println("inexistent timetable:", ZabovDefaultTimetable)
+			os.Exit(1)
+		}
+	}
+
+	//************************
+	// Local responser section
+	//************************
+	if MyConf["localresponder"] != nil {
+		localresponder := MyConf["localresponder"].(map[string]interface{})
+
+		if localresponder != nil {
+			if localresponder["responder"] != nil {
+				ZabovLocalResponder = localresponder["responder"].(string)
+				if len(ZabovLocalResponder) > 0 {
+					local := ZabovConfig{ZabovDNSArray: []string{ZabovLocalResponder}, references: 1}
+					ZabovConfigs["__localresponder__"] = &local
+					fmt.Println("ZabovLocalResponder:", ZabovLocalResponder)
+				}
+			}
+			if localresponder["localdomain"] != nil {
+				ZabovLocalDomain = localresponder["localdomain"].(string)
+			}
+		}
+	}
+	//******************************************
+	// clearing unused configs to save resources
+	//******************************************
+	for name, conf := range ZabovConfigs {
+		if conf.references == 0 {
+			log.Println("WARNING: disabling unused configuration:", name)
+			delete(ZabovConfigs, name)
+		} else {
+			ZabovCreateKDB(name)
+		}
+	}
 
 }

01.killfile.go

@@ -5,11 +5,10 @@ import (
 	"strings"
 )
 
-var zabovKbucket = []byte("killfile")
-
 type killfileItem struct {
-	Kdomain string
+	Kdomain  string
-	Ksource string
+	Ksource  string
+	Kconfigs stringarray
 }
 
 var bChannel chan killfileItem
@@ -27,16 +26,25 @@ func bWriteThread() {
 
 	for item := range bChannel {
 
-		writeInKillfile(item.Kdomain, item.Ksource)
+		alreadyInSomeDB := false
-		incrementStats("BL domains from "+item.Ksource, 1)
+
-		incrementStats("TOTAL", 1)
+		for _, config := range item.Kconfigs {
+			if !alreadyInSomeDB {
+				alreadyInSomeDB = domainInKillfile(item.Kdomain, config)
+			}
+			writeInKillfile(item.Kdomain, item.Ksource, config)
+		}
+		if !alreadyInSomeDB {
+			incrementStats("BL domains from "+item.Ksource, 1)
+			incrementStats("TOTAL", 1)
+		}
 
 	}
 
 }
 
 //DomainKill stores a domain name inside the killfile
-func DomainKill(s, durl string) {
+func DomainKill(s, durl string, configs stringarray) {
 
 	if len(s) > 2 {
 
@@ -46,6 +54,7 @@ func DomainKill(s, durl string) {
 
 		k.Kdomain = s
 		k.Ksource = durl
+		k.Kconfigs = configs
 
 		bChannel <- k
 
@@ -53,11 +62,12 @@ func DomainKill(s, durl string) {
 
 }
 
-func writeInKillfile(key, value string) {
+func writeInKillfile(key, value string, config string) {
 
 	stK := []byte(key)
 	stV := []byte(value)
 
+	MyZabovKDB := MyZabovKDBs[config]
 	err := MyZabovKDB.Put(stK, stV, nil)
 	if err != nil {
 		fmt.Println("Cannot write to Killfile DB: ", err.Error())
@@ -65,10 +75,11 @@ func writeInKillfile(key, value string) {
 
 }
 
-func domainInKillfile(domain string) bool {
+func domainInKillfile(domain string, config string) bool {
 
 	s := strings.ToLower(domain)
 
+	MyZabovKDB := MyZabovKDBs[config]
 	has, err := MyZabovKDB.Has([]byte(s), nil)
 	if err != nil {
 		fmt.Println("Cannot read from Killfile DB: ", err.Error())

01.stats.go

@@ -81,7 +81,12 @@ func statsThread() {
 		case "INC":
 			ZabovStats[item.Payload] += item.Number
 		case "SET":
-			ZabovStats[item.Payload] = item.Number
+			if item.Number == 0 {
+
+				delete(ZabovStats, item.Payload)
+			} else {
+				ZabovStats[item.Payload] = item.Number
+			}
 		case "PRI":
 			statsPrint()
 		}

Dockerfile.amd64

@@ -1,4 +1,4 @@
-FROM golang:1.14.1 AS builder
+FROM arm64v8/golang:1.15.6 AS builder
 RUN apt install git -y
 RUN mkdir -p /go/src/zabov
 RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov 
@@ -10,9 +10,10 @@ FROM debian:latest
 RUN apt update
 RUN apt upgrade -y
 RUN apt install ca-certificates -y
+RUN apt install tzdata -y
 RUN mkdir -p /opt/zabov
 WORKDIR  /opt/zabov
 COPY --from=builder /go/src/zabov /opt/zabov 
 EXPOSE 53/udp
+ENV TZ Europe/Rome
 ENTRYPOINT ["/opt/zabov/zabov"]
-

Dockerfile.arm32v7

@@ -1,4 +1,4 @@
-FROM arm32v7/golang:1.14.1 AS builder
+FROM arm64v8/golang:1.15.6 AS builder
 RUN apt install git -y
 RUN mkdir -p /go/src/zabov
 RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov 
@@ -10,8 +10,10 @@ FROM arm32v7/debian:latest
 RUN apt update
 RUN apt upgrade -y
 RUN apt install ca-certificates -y
+RUN apt install tzdata -y
 RUN mkdir -p /opt/zabov
 WORKDIR  /opt/zabov
 COPY --from=builder /go/src/zabov /opt/zabov 
 EXPOSE 53/udp
+ENV TZ Europe/Rome
 ENTRYPOINT ["/opt/zabov/zabov"]

Dockerfile.arm64v8

@@ -1,4 +1,4 @@
-FROM arm64v8/golang:1.14.1 AS builder
+FROM arm64v8/golang:1.15.6 AS builder
 RUN apt install git -y
 RUN mkdir -p /go/src/zabov
 RUN git clone https://git.keinpfusch.net/loweel/zabov /go/src/zabov 
@@ -10,8 +10,10 @@ FROM arm64v8/debian:latest
 RUN apt update
 RUN apt upgrade -y
 RUN apt install ca-certificates -y
+RUN apt install tzdata -y
 RUN mkdir -p /opt/zabov
 WORKDIR  /opt/zabov
 COPY --from=builder /go/src/zabov /opt/zabov 
 EXPOSE 53/udp
+ENV TZ Europe/Rome
 ENTRYPOINT ["/opt/zabov/zabov"]

README.md

@@ -44,45 +44,151 @@ The second is the format zabov calls "doublefilter" (a file in "/etc/hosts" form
 
 This is why configuration file has two separated items.
 
-The config file should look like:
+Minimal config file should look like:
 
 <pre>
 {
-    "zabov": {  
+    "zabov":{
         "port":"53", 
         "proto":"udp", 
-        "ipaddr":"127.0.0.1",
+        "ipaddr":"0.0.0.0",
-        "upstream":"./dns-upstream.txt",
+        "cachettl": 1,
-        "cachettl": "4",
+        "killfilettl": 12,
-        "killfilettl": "12",
+        "debug:"false"
-        "singlefilters":"./urls-hosts.txt" ,
+    },
-        "doublefilters":"./urls-domains.txt", 
+    "configs":{
-        "blackholeip":"127.0.0.1",
+        "default":{
-        "hostsfile":"./urls-local.txt"
+            "upstream":"./dns-upstream.txt",
+            "singlefilters":"./urls-domains.txt",
+            "doublefilters":"./urls-hosts.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local.txt"
+        },
     }
-
 }
-
-
-
 </pre>
 
-Where:
+Global zabov settings:
 
 - port is the port number. Usually is 53, you can change for docker, if you like
 - proto is the protocol. Choices are "udp", "tcp", "tcp/udp"
 - ipaddr is the port to listen to. Maybe empty, (which will result in listening to 0.0.0.0) to avoid issues with docker.
-- upstream: file containing all DNS we want to query :  each line in format IP:PORT
 - cachettl: amount of time the cache is kept (in hours)
 - killfilettl: refresh time for _killfiles_
+- debug: if set to "true" Zabov prints verbose logs, such as config selection and single DNS requests
+
+configs:
+- contains multiple zabov configuration dictionaries. "default" configuration name is mandatory
+- upstream: file containing all DNS we want to query :  each line in format IP:PORT
 - singlefilters: name of the file  for blacklists following the "singlefilter" schema.(one URL per line)
 - doublefilters: name of the file, for blacklists following the "doublefilter" schema.(one URL per line)
 - blackholeip: IP address to return when the IP is banned. This is because you may want to avoid MX issues, mail loops on localhost, or you have a web server running on localhost
 - hostsfile: path where you keep your local blacklistfile : this is in the format "singlefilter", meaning one domain per line, unlike hosts file.
 
+
+Advanced configuration includes support for multiple configurations based on IP Source and timetables:
+<pre>
+{
+    "zabov":{
+        "port":"53", 
+        "proto":"udp", 
+        "ipaddr":"0.0.0.0",
+        "cachettl": 1,
+        "killfilettl": 12,
+        "debug":"false",
+        "timetable":"tt_default"
+    },
+    "localresponder":{
+        "responder":"192.168.178.1:53",
+        "localdomain":"fritz.box"
+    },
+    "ipaliases":{
+        "pc8":"192.168.178.29",
+        "localhost":"127.0.0.1"
+    },
+    "ipgroups":[
+        {
+            "ips":["localhost", "::1", "192.168.178.30", "192.168.178.31", "pc8"],
+            "cfg":"",
+            "timetable":"tt_children"
+        }
+    ],
+    "timetables":{
+        "tt_children":{
+            "tables":[{"times":"00:00-05:00;8:30-12:30;18:30-22:59", "days":"Mo;Tu;We;Th;Fr;Sa;Su"}],
+            "cfgin":"children_restricted",
+            "cfgout":"default"
+        }
+        "tt_default":{
+            "tables":[{"times":"8:30-22:30", "days":"Su"}],
+            "cfgin":"children",
+            "cfgout":"default"
+        }
+    },
+    "configs":{
+        "default":{
+            "upstream":"./dns-upstream.txt",
+            "singlefilters":"./urls-domains.txt",
+            "doublefilters":"./urls-hosts.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local.txt"
+        },
+        "children":{
+            "upstream":"./dns-upstream-safe.txt",
+            "singlefilters":"./urls-domains.txt",
+            "doublefilters":"./urls-hosts.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local.txt"
+        },
+        "children_restricted":{
+            "upstream":"./dns-upstream-safe.txt",
+            "singlefilters":"./urls-domains-restricted.txt",
+            "doublefilters":"./urls-hosts-restricted.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local.txt"
+        }
+    }
+}
+</pre>
+
+Global zabov settings:
+
+- timetable: sets the global/default timetable. This table will be used for any client that is not already included in an IP group
+
+localresponder:
+  - allows to set a local DNS to respond for "local" domains. A domain name is handled as "local" if dosen't contains "." (dots) or if it ends with a well known prefix, such as ".local".
+  Note: the cache is not used for local responder.
+  - responder: is the local DNS server address in the IP:PORT format.
+  - localdomain: is the suffix for local domain names. All domains ending with this prefix are resolved by local responder
+
+ipaliases: a dictionary of IPs
+  - each entry in this dictionary define a domain-alias name and his IP address. It works as replacement of  /etc/hosts file.
+  - each entry is used by Zabov to resolve that names and to replace any value in the ipgroups.ips array.
+
+timetables: a dictionary of timetable dictionaries
+  - allow to define timetables in the format "time-ranges" and "days-of-week"
+  - tables: contain an array of dictionaries, each defining a time rule.
+    - each table is a dictinary containing "time" and "days" values
+    - time: is a string in the form "start:time1-stop:time1;start:time2-stop:time2..."
+    - days: is a string containing semicolon separated day names to apply the rule such as "Mo;Tu;We;Th;Fr"
+      - days names are: "Mo", "Tu" "We", "Th", "Fr", "Sa", "Su"
+      - empty value means all week-days
+    You can define complex time rules using more than one entry in this dictionay
+  - cfgin: is the name of the configuration to apply if current time is "inside" the timetable
+  - cfgout: is the name of the configuration to apply if current time is "outside" the timetable
+  
+ipgroups: an array of ipgroup dictionaries
+  - let you define a set of IP addresses that shall use a configuration other than "default"
+  - ips: is an array of strings, each containing an ip address or a name defined in the "ipaliases" config branch
+  - cfg: is a string containing the name of the configuration to be used for this group; ignored if timetable is also defined
+  - timetable: is a string containing the name of the tiemtable to be aplied to this group
+
+
 # DOCKER
 Multistage Dockerfiles are provided for AMD64, ARMv7, ARM64V8
 
+NOTE: you shall use TZ env var to change docker image timezone. TZ defaults to CET.
+
 # TODO:
 
 - ~~caching~~

adlist_hosts.go

@@ -16,9 +16,12 @@ func init() {
 }
 
 //DoubleIndexFilter puts the domains inside file
-func DoubleIndexFilter(durl string) error {
+func DoubleIndexFilter(durl string, configs stringarray) error {
 
-	fmt.Println("Retrieving HostFile from: ", durl)
+	fmt.Println("DoubleIndexFilter: Retrieving HostFile from: ", durl)
+
+	// resets malformed HostLines for url
+	setstatsvalue("Malformed HostLines "+durl, 0)
 
 	var err error
 
@@ -48,6 +51,9 @@ func DoubleIndexFilter(durl string) error {
 
 		line := scanner.Text()
 
+		if len(line) == 0 || strings.TrimSpace(line)[0] == '#' {
+			continue
+		}
 		h := strings.FieldsFunc(line, splitter)
 
 		if h == nil {
@@ -59,7 +65,8 @@ func DoubleIndexFilter(durl string) error {
 		}
 
 		if net.ParseIP(h[0]) != nil {
-			DomainKill(h[1], durl)
+
+			DomainKill(h[1], durl, configs)
 
 			// fmt.Println("MATCH: ", h[1])
 			numLines++
@@ -76,20 +83,44 @@ func DoubleIndexFilter(durl string) error {
 
 }
 
-func getDoubleFilters() {
+func getDoubleFilters(urls urlsMap) {
 
-	s := fileByLines(ZabovDoubleBL)
+	fmt.Println("getDoubleFilters: downloading all urls:", len(urls))
-
+	for url, configs := range urls {
-	for _, a := range s {
+		DoubleIndexFilter(url, configs)
-		DoubleIndexFilter(a)
 	}
+	fmt.Println("getDoubleFilters: DONE!")
 
 }
 
 func downloadDoubleThread() {
 	fmt.Println("Starting updater of DOUBLE lists, each (hours):", ZabovKillTTL)
+	time.Sleep(2 * time.Second) // wait for local DNS server up & running (may be our DNS)
+	_urls := urlsMap{}
+
 	for {
-		getDoubleFilters()
+		fmt.Println("downloadDoubleThread: collecting urls from all configs...")
+		for config := range ZabovConfigs {
+			ZabovDoubleBL := ZabovConfigs[config].ZabovDoubleBL
+			if len(ZabovDoubleBL) == 0 {
+				continue
+			}
+			s := fileByLines(ZabovDoubleBL)
+			for _, v := range s {
+				if len(v) == 0 || strings.TrimSpace(v)[0] == '#' {
+					continue
+				}
+				configs := _urls[v]
+				if configs == nil {
+					configs = stringarray{}
+					_urls[v] = configs
+				}
+				configs = append(configs, config)
+				_urls[v] = configs
+			}
+		}
+
+		getDoubleFilters(_urls)
 		time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
 	}
 

adlist_single.go

@@ -14,10 +14,13 @@ func init() {
 }
 
 //SingleIndexFilter puts the domains inside file
-func SingleIndexFilter(durl string) error {
+func SingleIndexFilter(durl string, configs stringarray) error {
 
 	fmt.Println("Retrieving DomainFile from: ", durl)
 
+	// resets malformed HostLines for url
+	setstatsvalue("Malformed DomainLines "+durl, 0)
+
 	var err error
 
 	// Get the data
@@ -46,6 +49,9 @@ func SingleIndexFilter(durl string) error {
 
 		line := scanner.Text()
 
+		if len(line) == 0 || strings.TrimSpace(line)[0] == '#' {
+			continue
+		}
 		h := strings.FieldsFunc(line, splitter)
 
 		if h == nil {
@@ -57,7 +63,9 @@ func SingleIndexFilter(durl string) error {
 		}
 
 		if !strings.Contains(h[0], "#") {
-			DomainKill(h[0], durl)
+
+			DomainKill(h[0], durl, configs)
+
 			// fmt.Println("MATCH: ", h[1])
 			numLines++
 		} else {
@@ -73,20 +81,47 @@ func SingleIndexFilter(durl string) error {
 
 }
 
-func getSingleFilters() {
+func getSingleFilters(urls urlsMap) {
 
-	s := fileByLines(ZabovSingleBL)
+	fmt.Println("getSingleFilters: downloading all urls:", len(urls))
-
+	for url, configs := range urls {
-	for _, a := range s {
+		SingleIndexFilter(url, configs)
-		SingleIndexFilter(a)
 	}
+	fmt.Println("getSingleFilters: DONE!")
 
 }
 
 func downloadThread() {
 	fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL)
+	time.Sleep(2 * time.Second) // wait for local DNS server up & running (may be our DNS)
+
+	_urls := urlsMap{}
+
 	for {
-		getSingleFilters()
+		fmt.Println("downloadThread: collecting urls from all configs...")
+		for config := range ZabovConfigs {
+			ZabovSingleBL := ZabovConfigs[config].ZabovSingleBL
+
+			if len(ZabovSingleBL) == 0 {
+				continue
+			}
+
+			s := fileByLines(ZabovSingleBL)
+			for _, v := range s {
+				if len(v) == 0 || strings.TrimSpace(v)[0] == '#' {
+					continue
+				}
+				configs := _urls[v]
+				if configs == nil {
+					configs = stringarray{}
+					_urls[v] = configs
+				}
+				configs = append(configs, config)
+				_urls[v] = configs
+			}
+		}
+
+		getSingleFilters(_urls)
 		time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
 	}
 

config.json

@@ -1,15 +1,18 @@
 {
-    "zabov": {  
+    "zabov":{
         "port":"53", 
         "proto":"udp", 
         "ipaddr":"0.0.0.0",
-        "upstream":"./dns-upstream.txt"  ,
         "cachettl": 1,
-        "killfilettl": 12,
+        "killfilettl": 12
-        "singlefilters":"./urls-domains.txt" ,
+    },
-        "doublefilters":"./urls-hosts.txt", 
+    "configs":{
-        "blackholeip":"127.0.0.1",
+        "default":{
-        "hostsfile":"./urls-local.txt"
+            "upstream":"./dns-upstream.txt",
+            "singlefilters":"./urls-domains.txt",
+            "doublefilters":"./urls-hosts.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local.txt"
+        }
     }
-
 }

config.sample.json

@@ -0,0 +1,57 @@
+{
+    "zabov":{
+        "port":"53", 
+        "proto":"udp", 
+        "ipaddr":"0.0.0.0",
+        "cachettl": 1,
+        "killfilettl": 12,
+        "debug":"true",
+        "debugdbpath":"./logs",
+        "timetable":""
+    },
+    "localresponder":{
+        "responder":"192.168.1.1:53",
+        "localdomain":".local"
+    },
+    "ipaliases":{
+        "pc8":"192.168.1.2",
+    },
+    "ipgroups":[
+        {
+            "ips":["pc8"],
+            "cfg":"",
+            "timetable":"tt_children"
+        }
+    ],
+    "timetables":{
+        "tt_children":{
+            "tables":[{"times":"9:30-11:30", "days":"Mo;Tu;We;Th;Fr;Sa"}],
+            "cfgin":"children_restricted",
+            "cfgout":"children"
+        }
+    },
+    "configs":{
+        "default":{
+            "upstream":"./dns-upstream.txt",
+            "singlefilters":"./urls-domains-updated.txt",
+            "doublefilters":"./urls-hosts-normal.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local-normal.txt"
+        },
+        "children":{
+            "upstream":"./dns-familyscreen.txt",
+            "singlefilters":"./urls-domains-updated.txt",
+            "doublefilters":"./urls-hosts-nofb.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local-normal.txt"
+        },
+        "children_restricted":{
+            "upstream":"./dns-familyscreen.txt",
+            "singlefilters":"./urls-domains-updated.txt",
+            "doublefilters":"./urls-hosts-nofb.txt", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"./urls-local-restricted.txt"
+        }
+    }
+
+}

dns_client.go

@@ -12,7 +12,8 @@ import (
 
 //ForwardQuery forwards the query to the upstream server
 //first server to answer wins
-func ForwardQuery(query *dns.Msg) *dns.Msg {
+//accepts config name to select the UP DNS source list
+func ForwardQuery(query *dns.Msg, config string, nocache bool) *dns.Msg {
 
 	go incrementStats("ForwardQueries", 1)
 
@@ -23,12 +24,14 @@ func ForwardQuery(query *dns.Msg) *dns.Msg {
 	fqdn := strings.TrimRight(query.Question[0].Name, ".")
 
 	lfqdn := fmt.Sprintf("%d", query.Question[0].Qtype) + "." + fqdn
-	if cached := GetDomainFromCache(lfqdn); cached != nil {
+	if !nocache {
-		go incrementStats("CacheHit", 1)
+		if cached := GetDomainFromCache(lfqdn); cached != nil {
-		cached.SetReply(query)
+			go incrementStats("CacheHit", 1)
-		cached.Authoritative = true
+			cached.SetReply(query)
-		return cached
+			cached.Authoritative = true
+			return cached
 
+		}
 	}
 
 	c := new(dns.Client)
@@ -45,7 +48,7 @@ func ForwardQuery(query *dns.Msg) *dns.Msg {
 			continue
 		}
 
-		d := oneTimeDNS()
+		d := oneTimeDNS(config)
 
 		in, _, err := c.Exchange(query, d)
 		if err != nil {
@@ -78,13 +81,18 @@ func init() {
 
 }
 
-func oneTimeDNS() (dns string) {
+func oneTimeDNS(config string) (dns string) {
 
 	rand.Seed(time.Now().Unix())
 
-	upl := ZabovDNSArray
+	upl := ZabovConfigs[config].ZabovDNSArray
 
 	if len(upl) < 1 {
+
+		if len(ZabovLocalResponder) > 0 {
+			fmt.Println("No DNS defined, fallback to local responder:", ZabovLocalResponder)
+			return ZabovLocalResponder
+		}
 		fmt.Println("No DNS defined, using default 127.0.0.53:53. Hope it works!")
 		return "127.0.0.53:53"
 	}

dns_handler.go

@@ -1,44 +1,315 @@
 package main
 
 import (
+	"fmt"
+	"log"
 	"net"
+	"os"
+	"path"
 	"strings"
+	"time"
 
 	"github.com/miekg/dns"
 )
 
+var reqTypes map[uint16]string
+
+var weekdays []string
+
+type logItem struct {
+	clientIP  string
+	name      string
+	reqType   uint16
+	config    string
+	timetable string
+	killed    string
+}
+
+// logChannel used by logging thread
+var logChannel chan logItem
+
+func init() {
+
+	weekdays = []string{"Su", "Mo", "Tu", "We", "Th", "Fr", "Sa"}
+
+	if len(ZabovDebugDBPath) > 0 {
+		os.MkdirAll(ZabovDebugDBPath, 0755)
+	}
+
+	reqTypes = map[uint16]string{
+		dns.TypeNone:       "TypeNone",
+		dns.TypeA:          "TypeA",
+		dns.TypeNS:         "TypeNS",
+		dns.TypeMD:         "TypeMD",
+		dns.TypeMF:         "TypeMF",
+		dns.TypeCNAME:      "TypeCNAME",
+		dns.TypeSOA:        "TypeSOA",
+		dns.TypeMB:         "TypeMB",
+		dns.TypeMG:         "TypeMG",
+		dns.TypeMR:         "TypeMR",
+		dns.TypeNULL:       "TypeNULL",
+		dns.TypePTR:        "TypePTR",
+		dns.TypeHINFO:      "TypeHINFO",
+		dns.TypeMINFO:      "TypeMINFO",
+		dns.TypeMX:         "TypeMX",
+		dns.TypeTXT:        "TypeTXT",
+		dns.TypeRP:         "TypeRP",
+		dns.TypeAFSDB:      "TypeAFSDB",
+		dns.TypeX25:        "TypeX25",
+		dns.TypeISDN:       "TypeISDN",
+		dns.TypeRT:         "TypeRT",
+		dns.TypeNSAPPTR:    "TypeNSAPPTR",
+		dns.TypeSIG:        "TypeSIG",
+		dns.TypeKEY:        "TypeKEY",
+		dns.TypePX:         "TypePX",
+		dns.TypeGPOS:       "TypeGPOS",
+		dns.TypeAAAA:       "TypeAAAA",
+		dns.TypeLOC:        "TypeLOC",
+		dns.TypeNXT:        "TypeNXT",
+		dns.TypeEID:        "TypeEID",
+		dns.TypeNIMLOC:     "TypeNIMLOC",
+		dns.TypeSRV:        "TypeSRV",
+		dns.TypeATMA:       "TypeATMA",
+		dns.TypeNAPTR:      "TypeNAPTR",
+		dns.TypeKX:         "TypeKX",
+		dns.TypeCERT:       "TypeCERT",
+		dns.TypeDNAME:      "TypeDNAME",
+		dns.TypeOPT:        "TypeOPT",
+		dns.TypeAPL:        "TypeAPL",
+		dns.TypeDS:         "TypeDS",
+		dns.TypeSSHFP:      "TypeSSHFP",
+		dns.TypeRRSIG:      "TypeRRSIG",
+		dns.TypeNSEC:       "TypeNSEC",
+		dns.TypeDNSKEY:     "TypeDNSKEY",
+		dns.TypeDHCID:      "TypeDHCID",
+		dns.TypeNSEC3:      "TypeNSEC3",
+		dns.TypeNSEC3PARAM: "TypeNSEC3PARAM",
+		dns.TypeTLSA:       "TypeTLSA",
+		dns.TypeSMIMEA:     "TypeSMIMEA",
+		dns.TypeHIP:        "TypeHIP",
+		dns.TypeNINFO:      "TypeNINFO",
+		dns.TypeRKEY:       "TypeRKEY",
+		dns.TypeTALINK:     "TypeTALINK",
+		dns.TypeCDS:        "TypeCDS",
+		dns.TypeCDNSKEY:    "TypeCDNSKEY",
+		dns.TypeOPENPGPKEY: "TypeOPENPGPKEY",
+		dns.TypeCSYNC:      "TypeCSYNC",
+		dns.TypeSPF:        "TypeSPF",
+		dns.TypeUINFO:      "TypeUINFO",
+		dns.TypeUID:        "TypeUID",
+		dns.TypeGID:        "TypeGID",
+		dns.TypeUNSPEC:     "TypeUNSPEC",
+		dns.TypeNID:        "TypeNID",
+		dns.TypeL32:        "TypeL32",
+		dns.TypeL64:        "TypeL64",
+		dns.TypeLP:         "TypeLP",
+		dns.TypeEUI48:      "TypeEUI48",
+		dns.TypeEUI64:      "TypeEUI64",
+		dns.TypeURI:        "TypeURI",
+		dns.TypeCAA:        "TypeCAA",
+		dns.TypeAVC:        "TypeAVC",
+		dns.TypeTKEY:       "TypeTKEY",
+		dns.TypeTSIG:       "TypeTSIG",
+		dns.TypeIXFR:       "TypeIXFR",
+		dns.TypeAXFR:       "TypeAXFR",
+		dns.TypeMAILB:      "TypeMAILB",
+		dns.TypeMAILA:      "TypeMAILA",
+		dns.TypeANY:        "TypeANY",
+		dns.TypeTA:         "TypeTA",
+		dns.TypeDLV:        "TypeDLV",
+		dns.TypeReserved:   "TypeReserved"}
+
+	fmt.Println("Local Time:", getLocalTime().Format(time.ANSIC))
+
+	if len(ZabovDebugDBPath) > 0 {
+		logChannel = make(chan logItem, 1024)
+		go logWriteThread()
+	}
+}
+
+func logWriteThread() {
+	for item := range logChannel {
+		var header string
+		d := time.Now().Format("2006-01-02")
+		logpath := path.Join(ZabovDebugDBPath, strings.Replace(item.clientIP, ":", "_", -1)+"-"+d+".log")
+
+		_, err1 := os.Stat(logpath)
+		if os.IsNotExist(err1) {
+			header = strings.Join([]string{"time", "clientIP", "name", "reqType", "config", "timetable", "killed"}, "\t")
+		}
+		f, err := os.OpenFile(logpath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err == nil {
+			reqTypeName, err := reqTypes[item.reqType]
+			if !err {
+				reqTypeName = fmt.Sprintf("%d", item.reqType)
+			}
+			ct := time.Now().Format(time.RFC3339)
+			log := strings.Join([]string{ct, item.clientIP, strings.TrimRight(item.name, "."), reqTypeName, item.config, item.timetable, item.killed}, "\t")
+			if len(header) > 0 {
+				f.Write([]byte(header))
+				f.Write([]byte("\n"))
+			}
+			f.Write([]byte(log))
+			f.Write([]byte("\n"))
+			f.Close()
+		}
+	}
+}
+
+func logQuery(clientIP string, name string, reqType uint16, config string, timetable string, killed string) {
+	if len(ZabovDebugDBPath) > 0 {
+		k := logItem{clientIP: clientIP, name: name, reqType: reqType, config: config, timetable: timetable, killed: killed}
+
+		logChannel <- k
+
+	}
+}
+
+func getLocalTime() time.Time {
+	return time.Now().Local()
+}
+
+func confFromTimeTable(timetable string) string {
+	tt := ZabovTimetables[timetable]
+	if tt == nil {
+		if ZabovDebug {
+			log.Println("confFromTimeTable: return default")
+		}
+		return "default"
+	}
+	for _, ttentry := range tt.table {
+		now := getLocalTime()
+
+		nowHour := now.Hour()
+		nowMinute := now.Minute()
+		weekday := weekdays[now.Weekday()]
+		if ttentry.days == nil || len(ttentry.days) == 0 || ttentry.days[weekday] || ttentry.days[strings.ToLower(weekday)] {
+			for _, t := range ttentry.times {
+
+				if (nowHour > t.start.hour || (nowHour == t.start.hour && nowMinute >= t.start.minute)) &&
+					(nowHour < t.stop.hour || (nowHour == t.stop.hour && nowMinute <= t.stop.minute)) {
+					go incrementStats("TIMETABLE IN: "+timetable, 1)
+					if ZabovDebug {
+						log.Println("confFromTimeTable: return IN", tt.cfgin)
+					}
+					return tt.cfgin
+				}
+			}
+		}
+	}
+	go incrementStats("TIMETABLE OUT: "+timetable, 1)
+	if ZabovDebug {
+		log.Println("confFromTimeTable: return OUT", tt.cfgout)
+	}
+	return tt.cfgout
+}
+
+func confFromIP(clientIP net.IP) (string, string) {
+	for _, ipgroup := range ZabovIPGroups {
+		for _, ip := range ipgroup.ips {
+			if clientIP.Equal(ip) {
+				if len(ipgroup.timetable) > 0 {
+					return confFromTimeTable(ipgroup.timetable), ipgroup.timetable
+				}
+				if ZabovDebug {
+					log.Println("confFromIP: ipgroup.cfg", ipgroup.cfg)
+				}
+				return ipgroup.cfg, ""
+			}
+		}
+	}
+	if len(ZabovDefaultTimetable) > 0 {
+		return confFromTimeTable(ZabovDefaultTimetable), ZabovDefaultTimetable
+	}
+
+	if ZabovDebug {
+		log.Println("confFromIP: return default")
+	}
+	return "default", ""
+}
 func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 	go incrementStats("TotalQueries", 1)
 
 	remIP, _, e := net.SplitHostPort(w.RemoteAddr().String())
 	if e != nil {
+		go incrementStats("CLIENT ERROR: "+remIP, 1)
+	} else {
 		go incrementStats("CLIENT: "+remIP, 1)
 	}
 
 	msg := dns.Msg{}
 	msg.SetReply(r)
 
-	switch r.Question[0].Qtype {
+	config, timetable := confFromIP(net.ParseIP(remIP))
+
+	if ZabovDebug {
+		log.Println("REQUEST:", remIP, config)
+	}
+	ZabovConfig := ZabovConfigs[config]
+	QType := r.Question[0].Qtype
+	switch QType {
 	case dns.TypeA:
 		msg.Authoritative = true
 		domain := msg.Question[0].Name
 		fqdn := strings.TrimRight(domain, ".")
 
-		if domainInKillfile(fqdn) {
+		if ZabovDebug {
+			log.Println("TypeA: fqdn:", fqdn)
+		}
+
+		if len(ZabovIPAliases[fqdn]) > 0 {
+			config = "__aliases__"
+			msg.Answer = append(msg.Answer, &dns.A{
+				Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
+				A:   net.ParseIP(ZabovIPAliases[fqdn]),
+			})
+			go logQuery(remIP, fqdn, QType, config, timetable, "alias")
+			break
+		}
+		if len(ZabovLocalResponder) > 0 {
+			if !strings.Contains(fqdn, ".") ||
+				(len(ZabovLocalDomain) > 0 && strings.HasSuffix(fqdn, ZabovLocalDomain)) {
+				config = "__localresponder__"
+				ret := ForwardQuery(r, config, true)
+				w.WriteMsg(ret)
+				go logQuery(remIP, fqdn, QType, config, timetable, "localresponder")
+				break
+			}
+
+		}
+		if domainInKillfile(fqdn, config) {
 			go incrementStats("Killed", 1)
 
 			msg.Answer = append(msg.Answer, &dns.A{
 				Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
-				A:   net.ParseIP(ZabovAddBL),
+				A:   net.ParseIP(ZabovConfig.ZabovAddBL),
 			})
+			go logQuery(remIP, fqdn, QType, config, timetable, "killed")
 		} else {
-			ret := ForwardQuery(r)
+			go logQuery(remIP, fqdn, QType, config, timetable, "forwarded")
+			ret := ForwardQuery(r, config, false)
 			w.WriteMsg(ret)
 		}
-	default:
+	case dns.TypePTR:
-		ret := ForwardQuery(r)
+		if ZabovDebug {
+			log.Println("TypePTR: Name:", msg.Question[0].Name)
+		}
+
+		if len(ZabovLocalResponder) > 0 {
+			// if set use local responder for reverse lookup (suffix ".in-addr.arpa.")
+			config = "__localresponder__"
+		}
+		ret := ForwardQuery(r, config, true)
 		w.WriteMsg(ret)
+		go logQuery(remIP, msg.Question[0].Name, QType, config, timetable, "localresponder")
+	default:
+		ret := ForwardQuery(r, config, false)
+		w.WriteMsg(ret)
+		if len(ZabovDebugDBPath) > 0 {
+			go logQuery(remIP, msg.Question[0].Name, QType, config, timetable, "forwarded")
+		}
 	}
+	go incrementStats("CONFIG: "+config, 1)
 	w.WriteMsg(&msg)
 
 }

hostfile.go

@@ -4,33 +4,55 @@ import (
 	"bufio"
 	"fmt"
 	"os"
+	"strings"
 )
 
 func init() {
 
 	fmt.Println("Ingesting local hosts file")
-	ingestLocalBlacklist()
+	ingestLocalBlacklists()
 
 }
 
-func ingestLocalBlacklist() {
+func ingestLocalBlacklists() {
-
-	file, err := os.Open(ZabovHostsFile)
-	if err != nil {
-		fmt.Println(err.Error())
-	}
-	defer file.Close()
-
-	scanner := bufio.NewScanner(file)
-	for scanner.Scan() {
-		d := scanner.Text()
-		DomainKill(d, ZabovHostsFile)
-		incrementStats("Blacklist", 1)
 
+	fmt.Println("ingestLocalBlacklist: collecting urls from all configs...")
+	_files := urlsMap{}
+	for config := range ZabovConfigs {
+		ZabovHostsFile := ZabovConfigs[config].ZabovHostsFile
+		if len(ZabovHostsFile) == 0 {
+			continue
+		}
+		configs := _files[ZabovHostsFile]
+		if configs == nil {
+			configs = stringarray{}
+			_files[ZabovHostsFile] = configs
+		}
+		configs = append(configs, config)
+		_files[ZabovHostsFile] = configs
 	}
 
-	if err := scanner.Err(); err != nil {
+	for ZabovHostsFile, configs := range _files {
-		fmt.Println(err.Error())
+		file, err := os.Open(ZabovHostsFile)
+		if err != nil {
+			fmt.Println(err.Error())
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			d := scanner.Text()
+			if len(d) == 0 || strings.TrimSpace(d)[0] == '#' {
+				continue
+			}
+			DomainKill(d, ZabovHostsFile, configs)
+			incrementStats("Blacklist", 1)
+
+		}
+
+		if err := scanner.Err(); err != nil {
+			fmt.Println(err.Error())
+		}
 	}
 
 }

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"log"
+	"net"
 
 	"github.com/miekg/dns"
 )
@@ -9,32 +10,84 @@ import (
 //MyDNS is my dns server
 var MyDNS *dns.Server
 
-//ZabovUpDNS keeps the name of upstream DNSs
+//ZabovCacheTTL is the amount of hours we cache records of DNS (global)
-var ZabovUpDNS string
-
-//ZabovSingleBL list of urls returning a file with just names of domains
-var ZabovSingleBL string
-
-//ZabovDoubleBL list of urls returning a file with  IP<space>domain
-var ZabovDoubleBL string
-
-//ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1
-var ZabovAddBL string
-
-//ZabovCacheTTL is the amount of hours we cache records of DNS
 var ZabovCacheTTL int
 
-//ZabovKillTTL is the amount of hours we cache the killfile
+//ZabovKillTTL is the amount of hours we cache the killfile (global)
 var ZabovKillTTL int
 
-//ZabovHostsFile is the file we use to keep our hosts
+//ZabovLocalResponder is the default DNS server for local domains (global)
-var ZabovHostsFile string
+var ZabovLocalResponder string
 
-//ZabovDNSArray is the array containing all the DNS we mention
+//ZabovLocalDomain is the default local domain (global)
-var ZabovDNSArray []string
+var ZabovLocalDomain string
+
+//ZabovDefaultTimetable is the default timetable, applied to any client that is not already in any IP Group (global)
+var ZabovDefaultTimetable string
+
+//ZabovDebug activate more logging if set to true (global)
+var ZabovDebug bool
+
+//ZabovDebugDBPath path to store debug query logs: activate logging of each single query in a csv like file (global)
+var ZabovDebugDBPath string
 
 type handler struct{}
 
+// ZabovConfig contains all Zabov configs
+type ZabovConfig struct {
+	ZabovSingleBL  string   // json:singlefilters -> ZabovSingleBL list of urls returning a file with just names of domains
+	ZabovDoubleBL  string   // json:doublefilters -> ZabovDoubleBL list of urls returning a file with  IP<space>domain
+	ZabovAddBL     string   // json:blackholeip  -> ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1
+	ZabovHostsFile string   // json:hostsfile -> ZabovHostsFile is the file we use to keep our hosts
+	ZabovUpDNS     string   // json:upstream -> ZabovUpDNS keeps the name of upstream DNSs
+	ZabovDNSArray  []string // contains all the DNS we mention, parsed from ZabovUpDNS file
+	references     int      // contains references to this config; if zero, config shall be removed
+}
+
+// ZabovConfigs contains all Zabov configs
+var ZabovConfigs map[string]*ZabovConfig
+
+// ZabovIPGroup contains Zabov groups of IPs
+type ZabovIPGroup struct {
+	ips       []net.IP // IPs in this group
+	cfg       string   // config name to be used if there is no timetable
+	timetable string   // timetable name to be used for this group; timetable SHALL reference to config name to use
+}
+
+// ZabovIPGroups contains an array of all Zabov groups of IP rules
+var ZabovIPGroups []ZabovIPGroup
+
+// ZabovTime contains Zabov single time
+type ZabovTime struct {
+	hour   int
+	minute int
+}
+
+// ZabovTimeRange contains Zabov single time range
+type ZabovTimeRange struct {
+	start ZabovTime
+	stop  ZabovTime
+}
+
+// ZabovTimetableEntry contains Zabov single time table entry
+type ZabovTimetableEntry struct {
+	times []*ZabovTimeRange
+	days  map[string]bool
+}
+
+// ZabovTimetable contains a Zabov time table
+type ZabovTimetable struct {
+	table  []*ZabovTimetableEntry
+	cfgin  string // configuration name to be used if "inside" timetable
+	cfgout string // configuration name to be used if "outiside" timetable
+}
+
+// ZabovTimetables contains all Zabov time tables, by name
+var ZabovTimetables map[string]*ZabovTimetable
+
+// ZabovIPAliases contains an array of all Zabov IP aliases
+var ZabovIPAliases map[string]string
+
 func main() {
 
 	MyDNS.Handler = &handler{}

urls-domains.txt

@@ -1,12 +1,8 @@
-https://mirror1.malwaredomains.com/files/justdomains
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/adaway.org/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/adblock-nocoin-list/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/adguard-simplified/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/anudeepnd-adservers/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/antipopads/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-ad/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/digitalside-threat-intel/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-malvertising/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-malware/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/disconnect.me-tracking/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/easylist/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/easyprivacy/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/eth-phishing-detect/list.txt
@@ -14,24 +10,28 @@ https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.2o7ne
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.dead/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.risk/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/fademind-add.spam/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/gfrogeye-firstparty-trackers/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/hostsvn/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/kadhosts/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomainlist.com/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/lightswitch05-ads-and-tracking/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomains.com-immortaldomains/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/malwaredomains.com-justdomains/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/matomo.org-spammers/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/mitchellkrogza-badd-boyz-hosts/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/pgl.yoyo.org/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/ransomwaretracker.abuse.ch/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/phishing.army/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/socram8888-notonmyshift/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/someonewhocares.org/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/spam404.com/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/stevenblack/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-abuse/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-badware/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/ublock-privacy/list.txt
+https://raw.githubusercontent.com/hectorm/hmirror/master/data/urlhaus/list.txt
 https://raw.githubusercontent.com/hectorm/hmirror/master/data/winhelp2002.mvps.org/list.txt
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/zerodot1-coinblockerlists-browser/list.txt
+
-https://raw.githubusercontent.com/hectorm/hmirror/master/data/zeustracker.abuse.ch/list.txt
+
-https://raw.githubusercontent.com/CHEF-KOCH/Audio-fingerprint-pages/master/AudioFp.txt
-https://raw.githubusercontent.com/CHEF-KOCH/Canvas-fingerprinting-pages/master/Canvas.txt
-https://raw.githubusercontent.com/CHEF-KOCH/WebRTC-tracking/master/WebRTC.txt
-https://raw.githubusercontent.com/CHEF-KOCH/CKs-FilterList/master/Anti-Corp/hosts/NSABlocklist.txt
 https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-blocklist.txt
 https://gitlab.com/quidsup/notrack-blocklists/raw/master/notrack-malware.txt
 https://www.stopforumspam.com/downloads/toxic_domains_whole.txt
+
+https://mirror.cedia.org.ec/malwaredomains/immortal_domains.txt