- new json configuration: added multiple configs, ip groups/ip aliases and timetables
  - added multiple configurations:
    - each configuration has his own upstream, singlefilters, doublefilters, blackholeip hostsfile
    - cache DB is global to all configs
    - BL downloader and parser is optimized: each BL source is downloaded/parsed only once
- TODO:
  - implement configuration selection based on source IPs and timetables
  - unused code cleanup
remotes/1680050961956510080/tmp_refs/heads/master
bloved 2021-01-12 00:04:34 +01:00
parent 5c5e4e4417
commit b6dfee64a6
10 changed files with 362 additions and 94 deletions

View File

@ -7,12 +7,15 @@ import (
"github.com/syndtr/goleveldb/leveldb"
)
//MyZabovKDB is the storage where we'll put domains to block
var MyZabovKDB *leveldb.DB
//MyZabovKDB is the storage where we'll put domains to block (obsolete)
//var MyZabovKDB *leveldb.DB
//MyZabovCDB is the storage where we'll put domains to cache
//MyZabovCDB is the storage where we'll put domains to cache (global for all configs)
var MyZabovCDB *leveldb.DB
//MyZabovKDBs is the storage where we'll put domains to block (one for each config)
var MyZabovKDBs map[string]*leveldb.DB
func init() {
var err error
@ -20,13 +23,13 @@ func init() {
os.RemoveAll("./db")
os.MkdirAll("./db", 0755)
MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil)
if err != nil {
fmt.Println("Cannot create Killfile db: ", err.Error())
} else {
fmt.Println("Killfile DB created")
}
/*
MyZabovKDB, err = leveldb.OpenFile("./db/killfile", nil)
if err != nil {
fmt.Println("Cannot create Killfile db: ", err.Error())
} else {
fmt.Println("Killfile DB created")
}*/
MyZabovCDB, err = leveldb.OpenFile("./db/cache", nil)
if err != nil {
@ -35,4 +38,21 @@ func init() {
fmt.Println("Cache DB created")
}
MyZabovKDBs = map[string]*leveldb.DB{}
}
// ZabovCreateKDB creates Kill DBs
func ZabovCreateKDB(conf string) {
var err error
dbname := "./db/killfile_" + conf
KDB, err := leveldb.OpenFile(dbname, nil)
if err != nil {
fmt.Println("Cannot create Killfile db: ", err.Error())
} else {
fmt.Println("Killfile DB created")
}
MyZabovKDBs[conf] = KDB
}

View File

@ -6,29 +6,16 @@ import (
"io/ioutil"
"log"
"os"
"strings"
"github.com/miekg/dns"
)
type stringarray []string
type urlsMap map[string]stringarray
func init() {
//ZabovConf describes the Json we use for configuration
type ZabovConf struct {
Zabov struct {
Port string `json:"port"`
Proto string `json:"proto"`
Ipaddr string `json:"ipaddr"`
Upstream string `json:"upstream"`
Cachettl int `json:"cachettl"`
Killfilettl int `json:"killfilettl"`
Singlefilters string `json:"singlefilters"`
Doublefilters string `json:"doublefilters"`
Blackholeip string `json:"blackholeip"`
Hostsfile string `json:"hostsfile"`
} `json:"zabov"`
}
var MyConf ZabovConf
var MyConfRaw interface{}
file, err := ioutil.ReadFile("config.json")
@ -37,26 +24,35 @@ func init() {
os.Exit(1)
}
err = json.Unmarshal([]byte(file), &MyConf)
err = json.Unmarshal([]byte(file), &MyConfRaw)
if err != nil {
log.Println("Cannot marshal json: ", err.Error())
log.Println("Cannot unmarshal json: ", err.Error())
os.Exit(1)
}
// now we read configuration file
fmt.Println("Reading configuration file...")
ZabovPort := MyConf.Zabov.Port
ZabovType := MyConf.Zabov.Proto
ZabovAddr := MyConf.Zabov.Ipaddr
ZabovUpDNS = MyConf.Zabov.Upstream
ZabovSingleBL = MyConf.Zabov.Singlefilters
ZabovDoubleBL = MyConf.Zabov.Doublefilters
ZabovAddBL = MyConf.Zabov.Blackholeip
ZabovCacheTTL = MyConf.Zabov.Cachettl
ZabovKillTTL = MyConf.Zabov.Killfilettl
ZabovHostsFile = MyConf.Zabov.Hostsfile
MyConf := MyConfRaw.(map[string]interface{})
zabov := MyConf["zabov"].(map[string]interface{})
ZabovPort := zabov["port"].(string)
ZabovType := zabov["proto"].(string)
ZabovAddr := zabov["ipaddr"].(string)
ZabovCacheTTL = int(zabov["cachettl"].(float64))
ZabovKillTTL = int(zabov["killfilettl"].(float64))
configs := MyConf["configs"].(map[string]interface{})
defaultConf := configs["default"].(map[string]interface{})
ZabovUpDNS = defaultConf["upstream"].(string)
ZabovSingleBL = defaultConf["singlefilters"].(string)
ZabovDoubleBL = defaultConf["doublefilters"].(string)
ZabovAddBL = defaultConf["blackholeip"].(string)
ZabovHostsFile = defaultConf["hostsfile"].(string)
zabovString := ZabovAddr + ":" + ZabovPort
@ -66,4 +62,111 @@ func init() {
ZabovDNSArray = fileByLines(ZabovUpDNS)
ZabovConfigs = map[string]ZabovConfig{}
ZabovIPGroups = []ZabovIPGroup{}
ZabovTimetables = map[string]ZabovTimetable{}
ZabovIPAliases = map[string]string{}
ZabovDNSArrays = map[string][]string{}
IPAliasesRaw := MyConf["ipaliases"].(map[string]interface{})
for alias, ip := range IPAliasesRaw {
fmt.Println("IP Alias:", alias, ip)
ZabovIPAliases[alias] = ip.(string)
}
for name, v := range configs {
fmt.Println("evaluaing config name:", name)
confRaw := v.(map[string]interface{})
var conf ZabovConfig
conf.ZabovUpDNS = confRaw["upstream"].(string)
conf.ZabovSingleBL = confRaw["singlefilters"].(string)
conf.ZabovDoubleBL = confRaw["doublefilters"].(string)
conf.ZabovAddBL = confRaw["blackholeip"].(string)
conf.ZabovHostsFile = confRaw["hostsfile"].(string)
ZabovDNSArrays[name] = fileByLines(conf.ZabovUpDNS)
ZabovConfigs[name] = conf
if name == "default" {
ZabovConfigDefault = conf
}
ZabovCreateKDB(name)
}
timetables := MyConf["timetables"].(map[string]interface{})
for name, v := range timetables {
fmt.Println("evaluaing timetable name:", name)
timetableRaw := v.(map[string]interface{})
var timetable ZabovTimetable
timetable.cfgin = timetableRaw["cfgin"].(string)
timetable.cfgout = timetableRaw["cfgout"].(string)
if timetable.cfgin == "" {
timetable.cfgin = "default"
}
if timetable.cfgout == "" {
timetable.cfgout = "default"
}
_, ok := ZabovConfigs[timetable.cfgin]
if !ok {
log.Println("inexistent cfgin:", timetable.cfgin)
os.Exit(1)
}
_, ok = ZabovConfigs[timetable.cfgout]
if !ok {
log.Println("inexistent cfgout:", timetable.cfgout)
os.Exit(1)
}
tables := timetableRaw["tables"].([]interface{})
for i := range tables {
table := tables[i].(map[string]interface{})
var ttEntry ZabovTimetableEntry
ttEntry.times = strings.Split(table["times"].(string), ";")
ttEntry.days = strings.Split(table["days"].(string), ";")
timetable.table = append(timetable.table, ttEntry)
}
ZabovTimetables[name] = timetable
}
IPGroups := MyConf["ipgroups"].([]interface{})
fmt.Println("evaluating IP Groups: ", len(IPGroups))
for i := range IPGroups {
fmt.Println("evaluating IP Group n.", i)
var groupStruct ZabovIPGroup
groupMap := IPGroups[i].(map[string]interface{})
IPsRaw := groupMap["ips"].([]interface{})
groupStruct.ips = []string{}
for x := range IPsRaw {
ip := IPsRaw[x].(string)
fmt.Println("adding IP ", ip)
alias, ok := ZabovIPAliases[ip]
if ok {
fmt.Println("IP alias: ", ip, alias)
ip = alias
}
groupStruct.ips = append(groupStruct.ips, ip)
}
groupStruct.cfg = groupMap["cfg"].(string)
groupStruct.timetable = groupMap["timetable"].(string)
fmt.Println("cfg:", groupStruct.cfg)
fmt.Println("timetable:", groupStruct.timetable)
_, ok := ZabovTimetables[groupStruct.timetable]
if !ok {
log.Println("inexistent timetable:", groupStruct.timetable)
os.Exit(1)
}
ZabovIPGroups = append(ZabovIPGroups, groupStruct)
}
fmt.Println("ZabovConfigs:", ZabovConfigs)
fmt.Println("ZabovTimetables:", ZabovTimetables)
fmt.Println("ZabovIPAliases:", ZabovIPAliases)
fmt.Println("ZabovIPGroups:", ZabovIPGroups)
}

View File

@ -5,11 +5,10 @@ import (
"strings"
)
var zabovKbucket = []byte("killfile")
type killfileItem struct {
Kdomain string
Ksource string
Kdomain string
Ksource string
Kconfigs stringarray
}
var bChannel chan killfileItem
@ -27,7 +26,9 @@ func bWriteThread() {
for item := range bChannel {
writeInKillfile(item.Kdomain, item.Ksource)
for _, config := range item.Kconfigs {
writeInKillfile(item.Kdomain, item.Ksource, config)
}
incrementStats("BL domains from "+item.Ksource, 1)
incrementStats("TOTAL", 1)
@ -36,7 +37,7 @@ func bWriteThread() {
}
//DomainKill stores a domain name inside the killfile
func DomainKill(s, durl string) {
func DomainKill(s, durl string, configs stringarray) {
if len(s) > 2 {
@ -46,6 +47,7 @@ func DomainKill(s, durl string) {
k.Kdomain = s
k.Ksource = durl
k.Kconfigs = configs
bChannel <- k
@ -53,11 +55,12 @@ func DomainKill(s, durl string) {
}
func writeInKillfile(key, value string) {
func writeInKillfile(key, value string, config string) {
stK := []byte(key)
stV := []byte(value)
MyZabovKDB := MyZabovKDBs[config]
err := MyZabovKDB.Put(stK, stV, nil)
if err != nil {
fmt.Println("Cannot write to Killfile DB: ", err.Error())
@ -65,10 +68,11 @@ func writeInKillfile(key, value string) {
}
func domainInKillfile(domain string) bool {
func domainInKillfile(domain string, config string) bool {
s := strings.ToLower(domain)
MyZabovKDB := MyZabovKDBs[config]
has, err := MyZabovKDB.Has([]byte(s), nil)
if err != nil {
fmt.Println("Cannot read from Killfile DB: ", err.Error())

View File

@ -16,9 +16,9 @@ func init() {
}
//DoubleIndexFilter puts the domains inside file
func DoubleIndexFilter(durl string) error {
func DoubleIndexFilter(durl string, configs stringarray) error {
fmt.Println("Retrieving HostFile from: ", durl)
fmt.Println("DoubleIndexFilter: Retrieving HostFile from: ", durl)
var err error
@ -59,7 +59,8 @@ func DoubleIndexFilter(durl string) error {
}
if net.ParseIP(h[0]) != nil {
DomainKill(h[1], durl)
DomainKill(h[1], durl, configs)
// fmt.Println("MATCH: ", h[1])
numLines++
@ -76,20 +77,38 @@ func DoubleIndexFilter(durl string) error {
}
func getDoubleFilters() {
func getDoubleFilters(urls urlsMap) {
s := fileByLines(ZabovDoubleBL)
for _, a := range s {
DoubleIndexFilter(a)
fmt.Println("getDoubleFilters: downloading all urls:", len(urls))
for url, configs := range urls {
DoubleIndexFilter(url, configs)
}
}
func downloadDoubleThread() {
fmt.Println("Starting updater of DOUBLE lists, each (hours):", ZabovKillTTL)
_urls := urlsMap{}
for {
getDoubleFilters()
fmt.Println("downloadDoubleThread: collecting urls from all configs...")
for config := range ZabovConfigs {
ZabovDoubleBL := ZabovConfigs[config].ZabovDoubleBL
s := fileByLines(ZabovDoubleBL)
for _, v := range s {
configs := _urls[v]
if configs == nil {
configs = stringarray{}
_urls[v] = configs
}
configs = append(configs, config)
_urls[v] = configs
}
}
getDoubleFilters(_urls)
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
}

View File

@ -14,7 +14,7 @@ func init() {
}
//SingleIndexFilter puts the domains inside file
func SingleIndexFilter(durl string) error {
func SingleIndexFilter(durl string, configs stringarray) error {
fmt.Println("Retrieving DomainFile from: ", durl)
@ -57,7 +57,9 @@ func SingleIndexFilter(durl string) error {
}
if !strings.Contains(h[0], "#") {
DomainKill(h[0], durl)
DomainKill(h[0], durl, configs)
// fmt.Println("MATCH: ", h[1])
numLines++
} else {
@ -73,20 +75,37 @@ func SingleIndexFilter(durl string) error {
}
func getSingleFilters() {
func getSingleFilters(urls urlsMap) {
s := fileByLines(ZabovSingleBL)
for _, a := range s {
SingleIndexFilter(a)
fmt.Println("getSingleFilters: downloading all urls:", len(urls))
for url, configs := range urls {
SingleIndexFilter(url, configs)
}
}
func downloadThread() {
fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL)
_urls := urlsMap{}
for {
getSingleFilters()
fmt.Println("downloadThread: collecting urls from all configs...")
for config := range ZabovConfigs {
ZabovSingleBL := ZabovConfigs[config].ZabovSingleBL
s := fileByLines(ZabovSingleBL)
for _, v := range s {
configs := _urls[v]
if configs == nil {
configs = stringarray{}
_urls[v] = configs
}
configs = append(configs, config)
_urls[v] = configs
}
}
getSingleFilters(_urls)
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
}

View File

@ -1,15 +1,50 @@
{
"zabov": {
"zabov":{
"port":"53",
"proto":"udp",
"ipaddr":"0.0.0.0",
"upstream":"./dns-upstream.txt" ,
"cachettl": 1,
"killfilettl": 12,
"singlefilters":"./urls-domains.txt" ,
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
"killfilettl": 12
},
"ipaliases":{
"pc8":"192.168.178.29"
},
"ipgroups":[
{
"ips":["192.168.178.30/32", "192.168.178.31/32", "pc8"],
"cfg":"",
"timetable":"tt_children"
}
],
"timetables":{
"tt_children":{
"tables":[{"times":"8:30-12:30;18:30-22:30", "days":"Mo;Tu;We;Th;Fr;Sa;Su"}],
"cfgin":"children_restricted",
"cfgout":"children"
}
},
"configs":{
"default":{
"upstream":"./dns-upstream.txt",
"singlefilters":"./urls-domains.txt",
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
},
"children":{
"upstream":"./dns-upstream.txt",
"singlefilters":"./urls-domains.txt",
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
},
"children_restricted":{
"upstream":"./dns-upstream.txt",
"singlefilters":"./urls-domains.txt",
"doublefilters":"./urls-hosts.txt",
"blackholeip":"127.0.0.1",
"hostsfile":"./urls-local.txt"
}
}
}

View File

@ -12,7 +12,8 @@ import (
//ForwardQuery forwards the query to the upstream server
//first server to answer wins
func ForwardQuery(query *dns.Msg) *dns.Msg {
//accepts config name to select the UP DNS source list
func ForwardQuery(query *dns.Msg, config string) *dns.Msg {
go incrementStats("ForwardQueries", 1)
@ -45,7 +46,7 @@ func ForwardQuery(query *dns.Msg) *dns.Msg {
continue
}
d := oneTimeDNS()
d := oneTimeDNS(config)
in, _, err := c.Exchange(query, d)
if err != nil {
@ -78,10 +79,11 @@ func init() {
}
func oneTimeDNS() (dns string) {
func oneTimeDNS(config string) (dns string) {
rand.Seed(time.Now().Unix())
ZabovDNSArray := ZabovDNSArrays[config]
upl := ZabovDNSArray
if len(upl) < 1 {

View File

@ -12,19 +12,23 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
remIP, _, e := net.SplitHostPort(w.RemoteAddr().String())
if e != nil {
go incrementStats("CLIENT ERROR: "+remIP, 1)
} else {
go incrementStats("CLIENT: "+remIP, 1)
}
msg := dns.Msg{}
msg.SetReply(r)
config := "default" // TODO: get config from client IP & timetable
switch r.Question[0].Qtype {
case dns.TypeA:
msg.Authoritative = true
domain := msg.Question[0].Name
fqdn := strings.TrimRight(domain, ".")
if domainInKillfile(fqdn) {
if domainInKillfile(fqdn, config) {
go incrementStats("Killed", 1)
msg.Answer = append(msg.Answer, &dns.A{
@ -32,11 +36,11 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
A: net.ParseIP(ZabovAddBL),
})
} else {
ret := ForwardQuery(r)
ret := ForwardQuery(r, config)
w.WriteMsg(ret)
}
default:
ret := ForwardQuery(r)
ret := ForwardQuery(r, config)
w.WriteMsg(ret)
}
w.WriteMsg(&msg)

View File

@ -9,28 +9,43 @@ import (
func init() {
fmt.Println("Ingesting local hosts file")
ingestLocalBlacklist()
ingestLocalBlacklists()
}
func ingestLocalBlacklist() {
file, err := os.Open(ZabovHostsFile)
if err != nil {
fmt.Println(err.Error())
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
d := scanner.Text()
DomainKill(d, ZabovHostsFile)
incrementStats("Blacklist", 1)
func ingestLocalBlacklists() {
fmt.Println("ingestLocalBlacklist: collecting urls from all configs...")
_files := urlsMap{}
for config := range ZabovConfigs {
ZabovHostsFile := ZabovConfigs[config].ZabovHostsFile
configs := _files[ZabovHostsFile]
if configs == nil {
configs = stringarray{}
_files[ZabovHostsFile] = configs
}
configs = append(configs, config)
_files[ZabovHostsFile] = configs
}
if err := scanner.Err(); err != nil {
fmt.Println(err.Error())
for ZabovHostsFile, configs := range _files {
file, err := os.Open(ZabovHostsFile)
if err != nil {
fmt.Println(err.Error())
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
d := scanner.Text()
DomainKill(d, ZabovHostsFile, configs)
incrementStats("Blacklist", 1)
}
if err := scanner.Err(); err != nil {
fmt.Println(err.Error())
}
}
}

47
main.go
View File

@ -35,6 +35,53 @@ var ZabovDNSArray []string
type handler struct{}
//ZabovDNSArrays contains the arrays containing all the DNS we mention
var ZabovDNSArrays map[string][]string
// ZabovConfig contains all Zabov configs
type ZabovConfig struct {
ZabovUpDNS string // json:upstream -> ZabovDNSArray
ZabovSingleBL string // json:singlefilters
ZabovDoubleBL string // json:doublefilters
ZabovAddBL string // json:blackholeip
ZabovHostsFile string // json:hostsfile
}
// ZabovConfigs contains all Zabov configs
var ZabovConfigs map[string]ZabovConfig
// ZabovConfigDefault contains only "default" config
var ZabovConfigDefault ZabovConfig
// ZabovIPGroup contains Zabov groups of IPs
type ZabovIPGroup struct {
ips []string // IPs in this group
cfg string // config name to be used if there is no timetable
timetable string // timetable name to be used for this group; timetable SHALL reference to config name to use
}
// ZabovIPGroups contains an array of all Zabov groups of IP rules
var ZabovIPGroups []ZabovIPGroup
// ZabovTimetableEntry contains Zabov single time table entry
type ZabovTimetableEntry struct {
times []string
days []string
}
// ZabovTimetable contains a Zabov time table
type ZabovTimetable struct {
table []ZabovTimetableEntry
cfgin string // configuration name to be used if "inside" timetable
cfgout string // configuration name to be used if "outiside" timetable
}
// ZabovTimetables contains all Zabov time tables, by name
var ZabovTimetables map[string]ZabovTimetable
// ZabovIPAliases contains an array of all Zabov IP aliases
var ZabovIPAliases map[string]string
func main() {
MyDNS.Handler = &handler{}