From b6dfee64a6c8527237f5ddc13d8868e95dfd928c Mon Sep 17 00:00:00 2001 From: bloved Date: Tue, 12 Jan 2021 00:04:34 +0100 Subject: [PATCH] - WIP: - new json configuration: added multiple configs, ip groups/ip aliases and timetables - added multiple configurations: - each configuration has his own upstream, singlefilters, doublefilters, blackholeip hostsfile - cache DB is global to all configs - BL downloader and parser is optimized: each BL source is downloaded/parsed only once - TODO: - implement configuration selection based on source IPs and timetables - unused code cleanup --- 00.database.go | 40 +++++++++--- 01.conf.go | 163 ++++++++++++++++++++++++++++++++++++++--------- 01.killfile.go | 20 +++--- adlist_hosts.go | 37 ++++++++--- adlist_single.go | 35 +++++++--- config.json | 49 ++++++++++++-- dns_client.go | 8 ++- dns_handler.go | 10 ++- hostfile.go | 47 +++++++++----- main.go | 47 ++++++++++++++ 10 files changed, 362 insertions(+), 94 deletions(-) diff --git a/00.database.go b/00.database.go index 3f5e82a..62cdd6c 100644 --- a/00.database.go +++ b/00.database.go @@ -7,12 +7,15 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) -//MyZabovKDB is the storage where we'll put domains to block -var MyZabovKDB *leveldb.DB +//MyZabovKDB is the storage where we'll put domains to block (obsolete) +//var MyZabovKDB *leveldb.DB -//MyZabovCDB is the storage where we'll put domains to cache +//MyZabovCDB is the storage where we'll put domains to cache (global for all configs) var MyZabovCDB *leveldb.DB +//MyZabovKDBs is the storage where we'll put domains to block (one for each config) +var MyZabovKDBs map[string]*leveldb.DB + func init() { var err error @@ -20,13 +23,13 @@ func init() { os.RemoveAll("./db") os.MkdirAll("./db", 0755) - - MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil) - if err != nil { - fmt.Println("Cannot create Killfile db: ", err.Error()) - } else { - fmt.Println("Killfile DB created") - } + /* + MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil) + if err != nil { + fmt.Println("Cannot create Killfile db: ", err.Error()) + } else { + fmt.Println("Killfile DB created") + }*/ MyZabovCDB, err = leveldb.OpenFile("./db/cache", nil) if err != nil { @@ -35,4 +38,21 @@ func init() { fmt.Println("Cache DB created") } + MyZabovKDBs = map[string]*leveldb.DB{} +} + +// ZabovCreateKDB creates Kill DBs +func ZabovCreateKDB(conf string) { + var err error + + dbname := "./db/killfile_" + conf + KDB, err := leveldb.OpenFile(dbname, nil) + if err != nil { + fmt.Println("Cannot create Killfile db: ", err.Error()) + } else { + fmt.Println("Killfile DB created") + } + + MyZabovKDBs[conf] = KDB + } diff --git a/01.conf.go b/01.conf.go index 45d1cf8..efdbb96 100644 --- a/01.conf.go +++ b/01.conf.go @@ -6,29 +6,16 @@ import ( "io/ioutil" "log" "os" + "strings" "github.com/miekg/dns" ) +type stringarray []string +type urlsMap map[string]stringarray + func init() { - - //ZabovConf describes the Json we use for configuration - type ZabovConf struct { - Zabov struct { - Port string `json:"port"` - Proto string `json:"proto"` - Ipaddr string `json:"ipaddr"` - Upstream string `json:"upstream"` - Cachettl int `json:"cachettl"` - Killfilettl int `json:"killfilettl"` - Singlefilters string `json:"singlefilters"` - Doublefilters string `json:"doublefilters"` - Blackholeip string `json:"blackholeip"` - Hostsfile string `json:"hostsfile"` - } `json:"zabov"` - } - - var MyConf ZabovConf + var MyConfRaw interface{} file, err := ioutil.ReadFile("config.json") @@ -37,26 +24,35 @@ func init() { os.Exit(1) } - err = json.Unmarshal([]byte(file), &MyConf) + err = json.Unmarshal([]byte(file), &MyConfRaw) if err != nil { - log.Println("Cannot marshal json: ", err.Error()) + log.Println("Cannot unmarshal json: ", err.Error()) os.Exit(1) } // now we read configuration file fmt.Println("Reading configuration file...") - ZabovPort := MyConf.Zabov.Port - ZabovType := MyConf.Zabov.Proto - ZabovAddr := MyConf.Zabov.Ipaddr - ZabovUpDNS = MyConf.Zabov.Upstream - ZabovSingleBL = MyConf.Zabov.Singlefilters - ZabovDoubleBL = MyConf.Zabov.Doublefilters - ZabovAddBL = MyConf.Zabov.Blackholeip - ZabovCacheTTL = MyConf.Zabov.Cachettl - ZabovKillTTL = MyConf.Zabov.Killfilettl - ZabovHostsFile = MyConf.Zabov.Hostsfile + MyConf := MyConfRaw.(map[string]interface{}) + + zabov := MyConf["zabov"].(map[string]interface{}) + + ZabovPort := zabov["port"].(string) + ZabovType := zabov["proto"].(string) + ZabovAddr := zabov["ipaddr"].(string) + ZabovCacheTTL = int(zabov["cachettl"].(float64)) + ZabovKillTTL = int(zabov["killfilettl"].(float64)) + + configs := MyConf["configs"].(map[string]interface{}) + + defaultConf := configs["default"].(map[string]interface{}) + + ZabovUpDNS = defaultConf["upstream"].(string) + ZabovSingleBL = defaultConf["singlefilters"].(string) + ZabovDoubleBL = defaultConf["doublefilters"].(string) + ZabovAddBL = defaultConf["blackholeip"].(string) + ZabovHostsFile = defaultConf["hostsfile"].(string) zabovString := ZabovAddr + ":" + ZabovPort @@ -66,4 +62,111 @@ func init() { ZabovDNSArray = fileByLines(ZabovUpDNS) + ZabovConfigs = map[string]ZabovConfig{} + ZabovIPGroups = []ZabovIPGroup{} + ZabovTimetables = map[string]ZabovTimetable{} + ZabovIPAliases = map[string]string{} + ZabovDNSArrays = map[string][]string{} + IPAliasesRaw := MyConf["ipaliases"].(map[string]interface{}) + + for alias, ip := range IPAliasesRaw { + fmt.Println("IP Alias:", alias, ip) + ZabovIPAliases[alias] = ip.(string) + } + + for name, v := range configs { + fmt.Println("evaluaing config name:", name) + confRaw := v.(map[string]interface{}) + var conf ZabovConfig + conf.ZabovUpDNS = confRaw["upstream"].(string) + conf.ZabovSingleBL = confRaw["singlefilters"].(string) + conf.ZabovDoubleBL = confRaw["doublefilters"].(string) + conf.ZabovAddBL = confRaw["blackholeip"].(string) + conf.ZabovHostsFile = confRaw["hostsfile"].(string) + + ZabovDNSArrays[name] = fileByLines(conf.ZabovUpDNS) + ZabovConfigs[name] = conf + if name == "default" { + ZabovConfigDefault = conf + } + ZabovCreateKDB(name) + } + + timetables := MyConf["timetables"].(map[string]interface{}) + + for name, v := range timetables { + fmt.Println("evaluaing timetable name:", name) + timetableRaw := v.(map[string]interface{}) + var timetable ZabovTimetable + + timetable.cfgin = timetableRaw["cfgin"].(string) + timetable.cfgout = timetableRaw["cfgout"].(string) + + if timetable.cfgin == "" { + timetable.cfgin = "default" + } + if timetable.cfgout == "" { + timetable.cfgout = "default" + } + + _, ok := ZabovConfigs[timetable.cfgin] + if !ok { + log.Println("inexistent cfgin:", timetable.cfgin) + os.Exit(1) + } + + _, ok = ZabovConfigs[timetable.cfgout] + if !ok { + log.Println("inexistent cfgout:", timetable.cfgout) + os.Exit(1) + } + + tables := timetableRaw["tables"].([]interface{}) + + for i := range tables { + table := tables[i].(map[string]interface{}) + var ttEntry ZabovTimetableEntry + ttEntry.times = strings.Split(table["times"].(string), ";") + ttEntry.days = strings.Split(table["days"].(string), ";") + timetable.table = append(timetable.table, ttEntry) + } + ZabovTimetables[name] = timetable + } + + IPGroups := MyConf["ipgroups"].([]interface{}) + + fmt.Println("evaluating IP Groups: ", len(IPGroups)) + for i := range IPGroups { + fmt.Println("evaluating IP Group n.", i) + var groupStruct ZabovIPGroup + groupMap := IPGroups[i].(map[string]interface{}) + IPsRaw := groupMap["ips"].([]interface{}) + groupStruct.ips = []string{} + for x := range IPsRaw { + ip := IPsRaw[x].(string) + fmt.Println("adding IP ", ip) + + alias, ok := ZabovIPAliases[ip] + if ok { + fmt.Println("IP alias: ", ip, alias) + ip = alias + } + groupStruct.ips = append(groupStruct.ips, ip) + } + groupStruct.cfg = groupMap["cfg"].(string) + groupStruct.timetable = groupMap["timetable"].(string) + fmt.Println("cfg:", groupStruct.cfg) + fmt.Println("timetable:", groupStruct.timetable) + _, ok := ZabovTimetables[groupStruct.timetable] + if !ok { + log.Println("inexistent timetable:", groupStruct.timetable) + os.Exit(1) + } + ZabovIPGroups = append(ZabovIPGroups, groupStruct) + } + fmt.Println("ZabovConfigs:", ZabovConfigs) + fmt.Println("ZabovTimetables:", ZabovTimetables) + fmt.Println("ZabovIPAliases:", ZabovIPAliases) + fmt.Println("ZabovIPGroups:", ZabovIPGroups) + } diff --git a/01.killfile.go b/01.killfile.go index b5d914b..60f6071 100644 --- a/01.killfile.go +++ b/01.killfile.go @@ -5,11 +5,10 @@ import ( "strings" ) -var zabovKbucket = []byte("killfile") - type killfileItem struct { - Kdomain string - Ksource string + Kdomain string + Ksource string + Kconfigs stringarray } var bChannel chan killfileItem @@ -27,7 +26,9 @@ func bWriteThread() { for item := range bChannel { - writeInKillfile(item.Kdomain, item.Ksource) + for _, config := range item.Kconfigs { + writeInKillfile(item.Kdomain, item.Ksource, config) + } incrementStats("BL domains from "+item.Ksource, 1) incrementStats("TOTAL", 1) @@ -36,7 +37,7 @@ func bWriteThread() { } //DomainKill stores a domain name inside the killfile -func DomainKill(s, durl string) { +func DomainKill(s, durl string, configs stringarray) { if len(s) > 2 { @@ -46,6 +47,7 @@ func DomainKill(s, durl string) { k.Kdomain = s k.Ksource = durl + k.Kconfigs = configs bChannel <- k @@ -53,11 +55,12 @@ func DomainKill(s, durl string) { } -func writeInKillfile(key, value string) { +func writeInKillfile(key, value string, config string) { stK := []byte(key) stV := []byte(value) + MyZabovKDB := MyZabovKDBs[config] err := MyZabovKDB.Put(stK, stV, nil) if err != nil { fmt.Println("Cannot write to Killfile DB: ", err.Error()) @@ -65,10 +68,11 @@ func writeInKillfile(key, value string) { } -func domainInKillfile(domain string) bool { +func domainInKillfile(domain string, config string) bool { s := strings.ToLower(domain) + MyZabovKDB := MyZabovKDBs[config] has, err := MyZabovKDB.Has([]byte(s), nil) if err != nil { fmt.Println("Cannot read from Killfile DB: ", err.Error()) diff --git a/adlist_hosts.go b/adlist_hosts.go index 5e9150d..fc45b74 100644 --- a/adlist_hosts.go +++ b/adlist_hosts.go @@ -16,9 +16,9 @@ func init() { } //DoubleIndexFilter puts the domains inside file -func DoubleIndexFilter(durl string) error { +func DoubleIndexFilter(durl string, configs stringarray) error { - fmt.Println("Retrieving HostFile from: ", durl) + fmt.Println("DoubleIndexFilter: Retrieving HostFile from: ", durl) var err error @@ -59,7 +59,8 @@ func DoubleIndexFilter(durl string) error { } if net.ParseIP(h[0]) != nil { - DomainKill(h[1], durl) + + DomainKill(h[1], durl, configs) // fmt.Println("MATCH: ", h[1]) numLines++ @@ -76,20 +77,38 @@ func DoubleIndexFilter(durl string) error { } -func getDoubleFilters() { +func getDoubleFilters(urls urlsMap) { - s := fileByLines(ZabovDoubleBL) - - for _, a := range s { - DoubleIndexFilter(a) + fmt.Println("getDoubleFilters: downloading all urls:", len(urls)) + for url, configs := range urls { + DoubleIndexFilter(url, configs) } } func downloadDoubleThread() { fmt.Println("Starting updater of DOUBLE lists, each (hours):", ZabovKillTTL) + + _urls := urlsMap{} + for { - getDoubleFilters() + fmt.Println("downloadDoubleThread: collecting urls from all configs...") + for config := range ZabovConfigs { + ZabovDoubleBL := ZabovConfigs[config].ZabovDoubleBL + + s := fileByLines(ZabovDoubleBL) + for _, v := range s { + configs := _urls[v] + if configs == nil { + configs = stringarray{} + _urls[v] = configs + } + configs = append(configs, config) + _urls[v] = configs + } + } + + getDoubleFilters(_urls) time.Sleep(time.Duration(ZabovKillTTL) * time.Hour) } diff --git a/adlist_single.go b/adlist_single.go index 6945c5e..bbdd3fc 100644 --- a/adlist_single.go +++ b/adlist_single.go @@ -14,7 +14,7 @@ func init() { } //SingleIndexFilter puts the domains inside file -func SingleIndexFilter(durl string) error { +func SingleIndexFilter(durl string, configs stringarray) error { fmt.Println("Retrieving DomainFile from: ", durl) @@ -57,7 +57,9 @@ func SingleIndexFilter(durl string) error { } if !strings.Contains(h[0], "#") { - DomainKill(h[0], durl) + + DomainKill(h[0], durl, configs) + // fmt.Println("MATCH: ", h[1]) numLines++ } else { @@ -73,20 +75,37 @@ func SingleIndexFilter(durl string) error { } -func getSingleFilters() { +func getSingleFilters(urls urlsMap) { - s := fileByLines(ZabovSingleBL) - - for _, a := range s { - SingleIndexFilter(a) + fmt.Println("getSingleFilters: downloading all urls:", len(urls)) + for url, configs := range urls { + SingleIndexFilter(url, configs) } } func downloadThread() { fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL) + _urls := urlsMap{} + for { - getSingleFilters() + fmt.Println("downloadThread: collecting urls from all configs...") + for config := range ZabovConfigs { + ZabovSingleBL := ZabovConfigs[config].ZabovSingleBL + + s := fileByLines(ZabovSingleBL) + for _, v := range s { + configs := _urls[v] + if configs == nil { + configs = stringarray{} + _urls[v] = configs + } + configs = append(configs, config) + _urls[v] = configs + } + } + + getSingleFilters(_urls) time.Sleep(time.Duration(ZabovKillTTL) * time.Hour) } diff --git a/config.json b/config.json index 79394d9..c104a8c 100644 --- a/config.json +++ b/config.json @@ -1,15 +1,50 @@ { - "zabov": { + "zabov":{ "port":"53", "proto":"udp", "ipaddr":"0.0.0.0", - "upstream":"./dns-upstream.txt" , "cachettl": 1, - "killfilettl": 12, - "singlefilters":"./urls-domains.txt" , - "doublefilters":"./urls-hosts.txt", - "blackholeip":"127.0.0.1", - "hostsfile":"./urls-local.txt" + "killfilettl": 12 + }, + "ipaliases":{ + "pc8":"192.168.178.29" + }, + "ipgroups":[ + { + "ips":["192.168.178.30/32", "192.168.178.31/32", "pc8"], + "cfg":"", + "timetable":"tt_children" + } + ], + "timetables":{ + "tt_children":{ + "tables":[{"times":"8:30-12:30;18:30-22:30", "days":"Mo;Tu;We;Th;Fr;Sa;Su"}], + "cfgin":"children_restricted", + "cfgout":"children" + } + }, + "configs":{ + "default":{ + "upstream":"./dns-upstream.txt", + "singlefilters":"./urls-domains.txt", + "doublefilters":"./urls-hosts.txt", + "blackholeip":"127.0.0.1", + "hostsfile":"./urls-local.txt" + }, + "children":{ + "upstream":"./dns-upstream.txt", + "singlefilters":"./urls-domains.txt", + "doublefilters":"./urls-hosts.txt", + "blackholeip":"127.0.0.1", + "hostsfile":"./urls-local.txt" + }, + "children_restricted":{ + "upstream":"./dns-upstream.txt", + "singlefilters":"./urls-domains.txt", + "doublefilters":"./urls-hosts.txt", + "blackholeip":"127.0.0.1", + "hostsfile":"./urls-local.txt" + } } } diff --git a/dns_client.go b/dns_client.go index 6aa97a0..7ce761c 100644 --- a/dns_client.go +++ b/dns_client.go @@ -12,7 +12,8 @@ import ( //ForwardQuery forwards the query to the upstream server //first server to answer wins -func ForwardQuery(query *dns.Msg) *dns.Msg { +//accepts config name to select the UP DNS source list +func ForwardQuery(query *dns.Msg, config string) *dns.Msg { go incrementStats("ForwardQueries", 1) @@ -45,7 +46,7 @@ func ForwardQuery(query *dns.Msg) *dns.Msg { continue } - d := oneTimeDNS() + d := oneTimeDNS(config) in, _, err := c.Exchange(query, d) if err != nil { @@ -78,10 +79,11 @@ func init() { } -func oneTimeDNS() (dns string) { +func oneTimeDNS(config string) (dns string) { rand.Seed(time.Now().Unix()) + ZabovDNSArray := ZabovDNSArrays[config] upl := ZabovDNSArray if len(upl) < 1 { diff --git a/dns_handler.go b/dns_handler.go index 9cf0fb1..9bff55d 100644 --- a/dns_handler.go +++ b/dns_handler.go @@ -12,19 +12,23 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { remIP, _, e := net.SplitHostPort(w.RemoteAddr().String()) if e != nil { + go incrementStats("CLIENT ERROR: "+remIP, 1) + } else { go incrementStats("CLIENT: "+remIP, 1) } msg := dns.Msg{} msg.SetReply(r) + config := "default" // TODO: get config from client IP & timetable + switch r.Question[0].Qtype { case dns.TypeA: msg.Authoritative = true domain := msg.Question[0].Name fqdn := strings.TrimRight(domain, ".") - if domainInKillfile(fqdn) { + if domainInKillfile(fqdn, config) { go incrementStats("Killed", 1) msg.Answer = append(msg.Answer, &dns.A{ @@ -32,11 +36,11 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { A: net.ParseIP(ZabovAddBL), }) } else { - ret := ForwardQuery(r) + ret := ForwardQuery(r, config) w.WriteMsg(ret) } default: - ret := ForwardQuery(r) + ret := ForwardQuery(r, config) w.WriteMsg(ret) } w.WriteMsg(&msg) diff --git a/hostfile.go b/hostfile.go index 51e4508..a2946ff 100644 --- a/hostfile.go +++ b/hostfile.go @@ -9,28 +9,43 @@ import ( func init() { fmt.Println("Ingesting local hosts file") - ingestLocalBlacklist() + ingestLocalBlacklists() } -func ingestLocalBlacklist() { - - file, err := os.Open(ZabovHostsFile) - if err != nil { - fmt.Println(err.Error()) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - d := scanner.Text() - DomainKill(d, ZabovHostsFile) - incrementStats("Blacklist", 1) +func ingestLocalBlacklists() { + fmt.Println("ingestLocalBlacklist: collecting urls from all configs...") + _files := urlsMap{} + for config := range ZabovConfigs { + ZabovHostsFile := ZabovConfigs[config].ZabovHostsFile + configs := _files[ZabovHostsFile] + if configs == nil { + configs = stringarray{} + _files[ZabovHostsFile] = configs + } + configs = append(configs, config) + _files[ZabovHostsFile] = configs } - if err := scanner.Err(); err != nil { - fmt.Println(err.Error()) + for ZabovHostsFile, configs := range _files { + file, err := os.Open(ZabovHostsFile) + if err != nil { + fmt.Println(err.Error()) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + d := scanner.Text() + DomainKill(d, ZabovHostsFile, configs) + incrementStats("Blacklist", 1) + + } + + if err := scanner.Err(); err != nil { + fmt.Println(err.Error()) + } } } diff --git a/main.go b/main.go index aaf1ebc..a9178f5 100644 --- a/main.go +++ b/main.go @@ -35,6 +35,53 @@ var ZabovDNSArray []string type handler struct{} +//ZabovDNSArrays contains the arrays containing all the DNS we mention +var ZabovDNSArrays map[string][]string + +// ZabovConfig contains all Zabov configs +type ZabovConfig struct { + ZabovUpDNS string // json:upstream -> ZabovDNSArray + ZabovSingleBL string // json:singlefilters + ZabovDoubleBL string // json:doublefilters + ZabovAddBL string // json:blackholeip + ZabovHostsFile string // json:hostsfile +} + +// ZabovConfigs contains all Zabov configs +var ZabovConfigs map[string]ZabovConfig + +// ZabovConfigDefault contains only "default" config +var ZabovConfigDefault ZabovConfig + +// ZabovIPGroup contains Zabov groups of IPs +type ZabovIPGroup struct { + ips []string // IPs in this group + cfg string // config name to be used if there is no timetable + timetable string // timetable name to be used for this group; timetable SHALL reference to config name to use +} + +// ZabovIPGroups contains an array of all Zabov groups of IP rules +var ZabovIPGroups []ZabovIPGroup + +// ZabovTimetableEntry contains Zabov single time table entry +type ZabovTimetableEntry struct { + times []string + days []string +} + +// ZabovTimetable contains a Zabov time table +type ZabovTimetable struct { + table []ZabovTimetableEntry + cfgin string // configuration name to be used if "inside" timetable + cfgout string // configuration name to be used if "outiside" timetable +} + +// ZabovTimetables contains all Zabov time tables, by name +var ZabovTimetables map[string]ZabovTimetable + +// ZabovIPAliases contains an array of all Zabov IP aliases +var ZabovIPAliases map[string]string + func main() { MyDNS.Handler = &handler{}