fixed debug log (ParallelResolve)

new setting ZabovParallelQueries: max number of parallel queries

00.database.go

@@ -13,6 +13,9 @@ var MyZabovCDB *leveldb.DB
 //MyZabovKDBs is the storage where we'll put domains to block (one for each config)
 var MyZabovKDBs map[string]*leveldb.DB
 
+//MyZabovWLDBs is the storage where we'll put domains to whitelist (one for each config)
+var MyZabovWLDBs map[string]*leveldb.DB
+
 func init() {
 
 	var err error
@@ -29,6 +32,7 @@ func init() {
 	}
 
 	MyZabovKDBs = map[string]*leveldb.DB{}
+	MyZabovWLDBs = map[string]*leveldb.DB{}
 }
 
 // ZabovCreateKDB creates Kill DBs
@@ -46,3 +50,19 @@ func ZabovCreateKDB(conf string) {
 	MyZabovKDBs[conf] = KDB
 
 }
+
+// ZabovCreateWLDB creates Whitelist DBs
+func ZabovCreateWLDB(conf string) {
+	var err error
+
+	dbname := "./db/whitelist_" + conf
+	WLDB, err := leveldb.OpenFile(dbname, nil)
+	if err != nil {
+		fmt.Println("Cannot create whitelist db: ", err.Error())
+	} else {
+		fmt.Println("whitelist DB created:", dbname)
+	}
+
+	MyZabovWLDBs[conf] = WLDB
+
+}

01.conf.go

@@ -13,10 +13,13 @@ import (
 	"github.com/miekg/dns"
 )
 
+var localresponderConfigName string
+
 type stringarray []string
 type urlsMap map[string]stringarray
 
 func init() {
+	localresponderConfigName = "__localresponder__"
 	var MyConfRaw interface{}
 
 	file, err := ioutil.ReadFile("config.json")
@@ -107,10 +110,42 @@ func init() {
 		conf.ZabovUpDNS = confRaw["upstream"].(string)
 		conf.ZabovSingleBL = confRaw["singlefilters"].(string)
 		conf.ZabovDoubleBL = confRaw["doublefilters"].(string)
-		conf.ZabovAddBL = confRaw["blackholeip"].(string)
-		conf.ZabovHostsFile = confRaw["hostsfile"].(string)
+		conf.ZabovAddBL = net.ParseIP(confRaw["blackholeip"].(string))
+		if confRaw["hostsfile"] != nil {
+			conf.ZabovHostsFile = confRaw["hostsfile"].(string)
+		}
 
-		conf.ZabovDNSArray = fileByLines(conf.ZabovUpDNS)
+		if confRaw["whitelist"] != nil {
+			conf.ZabovWhiteList = confRaw["whitelist"].(string)
+		}
+
+		if confRaw["cache"] != nil {
+			conf.ZabovCache = confRaw["cache"].(bool)
+		} else {
+			conf.ZabovCache = true
+		}
+		ZabovDNSArray := fileByLines(conf.ZabovUpDNS)
+
+		conf.ZabovDNSArray = []string{}
+
+		for _, value := range ZabovDNSArray {
+			tvalue := strings.TrimSpace(value)
+			if tvalue != "" && !strings.HasPrefix(tvalue, "#") {
+				conf.ZabovDNSArray = append(conf.ZabovDNSArray, tvalue)
+			}
+		}
+
+		if confRaw["parallelqueries"] != nil {
+			conf.ZabovParallelQueries = int(confRaw["parallelqueries"].(float64))
+		}
+		if conf.ZabovParallelQueries > len(conf.ZabovDNSArray) {
+			conf.ZabovParallelQueries = len(conf.ZabovDNSArray)
+		}
+		if conf.ZabovParallelQueries < 1 {
+			conf.ZabovParallelQueries = 1
+		}
+
+		fmt.Println("ZabovParallelQueries:", conf.ZabovParallelQueries)
 		ZabovConfigs[name] = &conf
 
 	}
@@ -216,8 +251,16 @@ func init() {
 				}
 				groupStruct.ips = append(groupStruct.ips, ip)
 			}
-			groupStruct.cfg = groupMap["cfg"].(string)
-			groupStruct.timetable = groupMap["timetable"].(string)
+			if groupMap["cfg"] != nil {
+				groupStruct.cfg = groupMap["cfg"].(string)
+			}
+			if groupMap["timetable"] != nil {
+				groupStruct.timetable = groupMap["timetable"].(string)
+			}
+			if len(groupStruct.cfg) == 0 && len(groupStruct.timetable) == 0 {
+				log.Println("ip group error: specify cfg or timetable")
+				os.Exit(1)
+			}
 			if len(groupStruct.cfg) > 0 {
 				refConfig, ok := ZabovConfigs[groupStruct.cfg]
 				if !ok {
@@ -226,14 +269,17 @@ func init() {
 				} else {
 					refConfig.references++
 				}
+				fmt.Println("cfg:", groupStruct.cfg)
 			}
-			fmt.Println("cfg:", groupStruct.cfg)
-			fmt.Println("timetable:", groupStruct.timetable)
-			_, ok := ZabovTimetables[groupStruct.timetable]
-			if !ok {
-				log.Println("inexistent timetable:", groupStruct.timetable)
-				os.Exit(1)
+			if len(groupStruct.timetable) > 0 {
+				fmt.Println("timetable:", groupStruct.timetable)
+				_, ok := ZabovTimetables[groupStruct.timetable]
+				if !ok {
+					log.Println("inexistent timetable:", groupStruct.timetable)
+					os.Exit(1)
+				}
 			}
+
 			ZabovIPGroups = append(ZabovIPGroups, groupStruct)
 		}
 	}
@@ -248,7 +294,7 @@ func init() {
 	}
 
 	//************************
-	// Local responser section
+	// Local responder section
 	//************************
 	if MyConf["localresponder"] != nil {
 		localresponder := MyConf["localresponder"].(map[string]interface{})
@@ -257,8 +303,9 @@ func init() {
 			if localresponder["responder"] != nil {
 				ZabovLocalResponder = localresponder["responder"].(string)
 				if len(ZabovLocalResponder) > 0 {
-					local := ZabovConfig{ZabovDNSArray: []string{ZabovLocalResponder}, references: 1}
-					ZabovConfigs["__localresponder__"] = &local
+					local := ZabovConfig{ZabovDNSArray: []string{ZabovLocalResponder},
+						references: 1, ZabovParallelQueries: 1}
+					ZabovConfigs[localresponderConfigName] = &local
 					fmt.Println("ZabovLocalResponder:", ZabovLocalResponder)
 				}
 			}
@@ -276,6 +323,7 @@ func init() {
 			delete(ZabovConfigs, name)
 		} else {
 			ZabovCreateKDB(name)
+			ZabovCreateWLDB(name)
 		}
 	}
 

01.killfile.go

@@ -11,20 +11,23 @@ type killfileItem struct {
 	Kconfigs stringarray
 }
 
-var bChannel chan killfileItem
+var bKillChannel chan killfileItem
+var bWhiteListChannel chan killfileItem
 
 func init() {
 
-	bChannel = make(chan killfileItem, 1024)
+	bKillChannel = make(chan killfileItem, 1024)
+	bWhiteListChannel = make(chan killfileItem, 1024)
 	fmt.Println("Initializing kill channel engine.")
 
-	go bWriteThread()
+	go bWriteKillThread()
+	go bWriteWhiteListThread()
 
 }
 
-func bWriteThread() {
+func bWriteKillThread() {
 
-	for item := range bChannel {
+	for item := range bKillChannel {
 
 		alreadyInSomeDB := false
 
@@ -56,7 +59,7 @@ func DomainKill(s, durl string, configs stringarray) {
 		k.Ksource = durl
 		k.Kconfigs = configs
 
-		bChannel <- k
+		bKillChannel <- k
 
 	}
 
@@ -88,3 +91,69 @@ func domainInKillfile(domain string, config string) bool {
 	return has
 
 }
+
+func bWriteWhiteListThread() {
+
+	for item := range bWhiteListChannel {
+
+		alreadyInSomeDB := false
+
+		for _, config := range item.Kconfigs {
+			if !alreadyInSomeDB {
+				alreadyInSomeDB = domainInWhiteListfile(item.Kdomain, config)
+			}
+			writeInWhiteListfile(item.Kdomain, item.Ksource, config)
+		}
+		if !alreadyInSomeDB {
+			incrementStats("WL domains from "+item.Ksource, 1)
+			incrementStats("WL TOTAL", 1)
+		}
+
+	}
+
+}
+
+//DomainWhiteList stores a domain name inside the killfile
+func DomainWhiteList(s, durl string, configs stringarray) {
+
+	if len(s) > 2 {
+
+		s = strings.ToLower(s)
+
+		var k killfileItem
+
+		k.Kdomain = s
+		k.Ksource = durl
+		k.Kconfigs = configs
+
+		bWhiteListChannel <- k
+
+	}
+
+}
+
+func writeInWhiteListfile(key, value string, config string) {
+
+	stK := []byte(key)
+	stV := []byte(value)
+
+	MyZabovWLDB := MyZabovWLDBs[config]
+	err := MyZabovWLDB.Put(stK, stV, nil)
+	if err != nil {
+		fmt.Println("Cannot write to Whitelist DB: ", err.Error())
+	}
+
+}
+func domainInWhiteListfile(domain string, config string) bool {
+
+	s := strings.ToLower(domain)
+
+	MyZabovWLDB := MyZabovWLDBs[config]
+	has, err := MyZabovWLDB.Has([]byte(s), nil)
+	if err != nil {
+		fmt.Println("Cannot read from Whitelist DB: ", err.Error())
+	}
+
+	return has
+
+}

02.cache.go

@@ -13,7 +13,7 @@ import (
 
 type cacheItem struct {
 	Query []byte
-	Date  time.Time
+	ExpireDate  time.Time
 }
 
 //DomainCache stores a domain name inside the cache
@@ -28,7 +28,17 @@ func DomainCache(s string, resp *dns.Msg) {
 	if err != nil {
 		fmt.Println("Problems packing the response: ", err.Error())
 	}
-	domain2cache.Date = time.Now()
+	if resp.Rcode == dns.RcodeSuccess{
+		// on success stores response normally
+		domain2cache.ExpireDate = time.Now().Add((time.Duration(ZabovCacheTTL) * time.Hour))
+	}else
+	{
+		// on failure stores response for a very short time
+		if ZabovDebug {
+			fmt.Println("DomainCache(): DNS error Rcode: ", resp.Rcode, s, "cache time reduced to 10 seconds...")
+		}
+		domain2cache.ExpireDate = time.Now().Add((time.Duration(10) * time.Second))
+	}
 
 	err = enc.Encode(domain2cache)
 
@@ -65,7 +75,7 @@ func GetDomainFromCache(s string) *dns.Msg {
 
 	conf, errDB = MyZabovCDB.Get([]byte(s), nil)
 	if errDB != nil {
-		fmt.Println("Cant READ DB :" , errDB.Error() )
+		fmt.Println("Cant READ DB:" , errDB.Error() )
 		return nil
 	}
 
@@ -77,7 +87,10 @@ func GetDomainFromCache(s string) *dns.Msg {
 		return nil
 	}
 
-	if time.Since(record.Date) > (time.Duration(ZabovCacheTTL) * time.Hour) {
+	if time.Now().After(record.ExpireDate) {
+		if ZabovDebug {
+			fmt.Println("GetDomainFromCache(): entry expired:", s)
+		}
 		return nil
 	}
 

LICENSE

@@ -1,14 +1,289 @@
-    Copyright (C) 2020  loweel@keinpfusch.net
+Copyright (C) 2020  loweel@keinpfusch.net
 
-    This program is free software: you can redistribute it and/or modify
-    it under the terms of the GNU General Public License as published by
-    the Free Software Foundation, either version 3 of the License, or
-    (at your option) any later version.
+                          EUROPEAN UNION PUBLIC LICENCE v. 1.2
+                      EUPL © the European Union 2007, 2016
 
-    This program is distributed in the hope that it will be useful,
-    but WITHOUT ANY WARRANTY; without even the implied warranty of
-    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-    GNU General Public License for more details.
+This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined
+below) which is provided under the terms of this Licence. Any use of the Work,
+other than as authorised under this Licence is prohibited (to the extent such
+use is covered by a right of the copyright holder of the Work).
 
-    You should have received a copy of the GNU General Public License
-    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+The Work is provided under the terms of this Licence when the Licensor (as
+defined below) has placed the following notice immediately following the
+copyright notice for the Work:
+
+        Licensed under the EUPL
+
+or has expressed by any other means his willingness to license under the EUPL.
+
+1. Definitions
+
+In this Licence, the following terms have the following meaning:
+
+- ‘The Licence’: this Licence.
+
+- ‘The Original Work’: the work or software distributed or communicated by the
+  Licensor under this Licence, available as Source Code and also as Executable
+  Code as the case may be.
+
+- ‘Derivative Works’: the works or software that could be created by the
+  Licensee, based upon the Original Work or modifications thereof. This Licence
+  does not define the extent of modification or dependence on the Original Work
+  required in order to classify a work as a Derivative Work; this extent is
+  determined by copyright law applicable in the country mentioned in Article 15.
+
+- ‘The Work’: the Original Work or its Derivative Works.
+
+- ‘The Source Code’: the human-readable form of the Work which is the most
+  convenient for people to study and modify.
+
+- ‘The Executable Code’: any code which has generally been compiled and which is
+  meant to be interpreted by a computer as a program.
+
+- ‘The Licensor’: the natural or legal person that distributes or communicates
+  the Work under the Licence.
+
+- ‘Contributor(s)’: any natural or legal person who modifies the Work under the
+  Licence, or otherwise contributes to the creation of a Derivative Work.
+
+- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of
+  the Work under the terms of the Licence.
+
+- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending,
+  renting, distributing, communicating, transmitting, or otherwise making
+  available, online or offline, copies of the Work or providing access to its
+  essential functionalities at the disposal of any other natural or legal
+  person.
+
+2. Scope of the rights granted by the Licence
+
+The Licensor hereby grants You a worldwide, royalty-free, non-exclusive,
+sublicensable licence to do the following, for the duration of copyright vested
+in the Original Work:
+
+- use the Work in any circumstance and for all usage,
+- reproduce the Work,
+- modify the Work, and make Derivative Works based upon the Work,
+- communicate to the public, including the right to make available or display
+  the Work or copies thereof to the public and perform publicly, as the case may
+  be, the Work,
+- distribute the Work or copies thereof,
+- lend and rent the Work or copies thereof,
+- sublicense rights in the Work or copies thereof.
+
+Those rights can be exercised on any media, supports and formats, whether now
+known or later invented, as far as the applicable law permits so.
+
+In the countries where moral rights apply, the Licensor waives his right to
+exercise his moral right to the extent allowed by law in order to make effective
+the licence of the economic rights here above listed.
+
+The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to
+any patents held by the Licensor, to the extent necessary to make use of the
+rights granted on the Work under this Licence.
+
+3. Communication of the Source Code
+
+The Licensor may provide the Work either in its Source Code form, or as
+Executable Code. If the Work is provided as Executable Code, the Licensor
+provides in addition a machine-readable copy of the Source Code of the Work
+along with each copy of the Work that the Licensor distributes or indicates, in
+a notice following the copyright notice attached to the Work, a repository where
+the Source Code is easily and freely accessible for as long as the Licensor
+continues to distribute or communicate the Work.
+
+4. Limitations on copyright
+
+Nothing in this Licence is intended to deprive the Licensee of the benefits from
+any exception or limitation to the exclusive rights of the rights owners in the
+Work, of the exhaustion of those rights or of other applicable limitations
+thereto.
+
+5. Obligations of the Licensee
+
+The grant of the rights mentioned above is subject to some restrictions and
+obligations imposed on the Licensee. Those obligations are the following:
+
+Attribution right: The Licensee shall keep intact all copyright, patent or
+trademarks notices and all notices that refer to the Licence and to the
+disclaimer of warranties. The Licensee must include a copy of such notices and a
+copy of the Licence with every copy of the Work he/she distributes or
+communicates. The Licensee must cause any Derivative Work to carry prominent
+notices stating that the Work has been modified and the date of modification.
+
+Copyleft clause: If the Licensee distributes or communicates copies of the
+Original Works or Derivative Works, this Distribution or Communication will be
+done under the terms of this Licence or of a later version of this Licence
+unless the Original Work is expressly distributed only under this version of the
+Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee
+(becoming Licensor) cannot offer or impose any additional terms or conditions on
+the Work or Derivative Work that alter or restrict the terms of the Licence.
+
+Compatibility clause: If the Licensee Distributes or Communicates Derivative
+Works or copies thereof based upon both the Work and another work licensed under
+a Compatible Licence, this Distribution or Communication can be done under the
+terms of this Compatible Licence. For the sake of this clause, ‘Compatible
+Licence’ refers to the licences listed in the appendix attached to this Licence.
+Should the Licensee's obligations under the Compatible Licence conflict with
+his/her obligations under this Licence, the obligations of the Compatible
+Licence shall prevail.
+
+Provision of Source Code: When distributing or communicating copies of the Work,
+the Licensee will provide a machine-readable copy of the Source Code or indicate
+a repository where this Source will be easily and freely available for as long
+as the Licensee continues to distribute or communicate the Work.
+
+Legal Protection: This Licence does not grant permission to use the trade names,
+trademarks, service marks, or names of the Licensor, except as required for
+reasonable and customary use in describing the origin of the Work and
+reproducing the content of the copyright notice.
+
+6. Chain of Authorship
+
+The original Licensor warrants that the copyright in the Original Work granted
+hereunder is owned by him/her or licensed to him/her and that he/she has the
+power and authority to grant the Licence.
+
+Each Contributor warrants that the copyright in the modifications he/she brings
+to the Work are owned by him/her or licensed to him/her and that he/she has the
+power and authority to grant the Licence.
+
+Each time You accept the Licence, the original Licensor and subsequent
+Contributors grant You a licence to their contributions to the Work, under the
+terms of this Licence.
+
+7. Disclaimer of Warranty
+
+The Work is a work in progress, which is continuously improved by numerous
+Contributors. It is not a finished work and may therefore contain defects or
+‘bugs’ inherent to this type of development.
+
+For the above reason, the Work is provided under the Licence on an ‘as is’ basis
+and without warranties of any kind concerning the Work, including without
+limitation merchantability, fitness for a particular purpose, absence of defects
+or errors, accuracy, non-infringement of intellectual property rights other than
+copyright as stated in Article 6 of this Licence.
+
+This disclaimer of warranty is an essential part of the Licence and a condition
+for the grant of any rights to the Work.
+
+8. Disclaimer of Liability
+
+Except in the cases of wilful misconduct or damages directly caused to natural
+persons, the Licensor will in no event be liable for any direct or indirect,
+material or moral, damages of any kind, arising out of the Licence or of the use
+of the Work, including without limitation, damages for loss of goodwill, work
+stoppage, computer failure or malfunction, loss of data or any commercial
+damage, even if the Licensor has been advised of the possibility of such damage.
+However, the Licensor will be liable under statutory product liability laws as
+far such laws apply to the Work.
+
+9. Additional agreements
+
+While distributing the Work, You may choose to conclude an additional agreement,
+defining obligations or services consistent with this Licence. However, if
+accepting obligations, You may act only on your own behalf and on your sole
+responsibility, not on behalf of the original Licensor or any other Contributor,
+and only if You agree to indemnify, defend, and hold each Contributor harmless
+for any liability incurred by, or claims asserted against such Contributor by
+the fact You have accepted any warranty or additional liability.
+
+10. Acceptance of the Licence
+
+The provisions of this Licence can be accepted by clicking on an icon ‘I agree’
+placed under the bottom of a window displaying the text of this Licence or by
+affirming consent in any other similar way, in accordance with the rules of
+applicable law. Clicking on that icon indicates your clear and irrevocable
+acceptance of this Licence and all of its terms and conditions.
+
+Similarly, you irrevocably accept this Licence and all of its terms and
+conditions by exercising any rights granted to You by Article 2 of this Licence,
+such as the use of the Work, the creation by You of a Derivative Work or the
+Distribution or Communication by You of the Work or copies thereof.
+
+11. Information to the public
+
+In case of any Distribution or Communication of the Work by means of electronic
+communication by You (for example, by offering to download the Work from a
+remote location) the distribution channel or media (for example, a website) must
+at least provide to the public the information requested by the applicable law
+regarding the Licensor, the Licence and the way it may be accessible, concluded,
+stored and reproduced by the Licensee.
+
+12. Termination of the Licence
+
+The Licence and the rights granted hereunder will terminate automatically upon
+any breach by the Licensee of the terms of the Licence.
+
+Such a termination will not terminate the licences of any person who has
+received the Work from the Licensee under the Licence, provided such persons
+remain in full compliance with the Licence.
+
+13. Miscellaneous
+
+Without prejudice of Article 9 above, the Licence represents the complete
+agreement between the Parties as to the Work.
+
+If any provision of the Licence is invalid or unenforceable under applicable
+law, this will not affect the validity or enforceability of the Licence as a
+whole. Such provision will be construed or reformed so as necessary to make it
+valid and enforceable.
+
+The European Commission may publish other linguistic versions or new versions of
+this Licence or updated versions of the Appendix, so far this is required and
+reasonable, without reducing the scope of the rights granted by the Licence. New
+versions of the Licence will be published with a unique version number.
+
+All linguistic versions of this Licence, approved by the European Commission,
+have identical value. Parties can take advantage of the linguistic version of
+their choice.
+
+14. Jurisdiction
+
+Without prejudice to specific agreement between parties,
+
+- any litigation resulting from the interpretation of this License, arising
+  between the European Union institutions, bodies, offices or agencies, as a
+  Licensor, and any Licensee, will be subject to the jurisdiction of the Court
+  of Justice of the European Union, as laid down in article 272 of the Treaty on
+  the Functioning of the European Union,
+
+- any litigation arising between other parties and resulting from the
+  interpretation of this License, will be subject to the exclusive jurisdiction
+  of the competent court where the Licensor resides or conducts its primary
+  business.
+
+15. Applicable Law
+
+Without prejudice to specific agreement between parties,
+
+- this Licence shall be governed by the law of the European Union Member State
+  where the Licensor has his seat, resides or has his registered office,
+
+- this licence shall be governed by Belgian law if the Licensor has no seat,
+  residence or registered office inside a European Union Member State.
+
+Appendix
+
+‘Compatible Licences’ according to Article 5 EUPL are:
+
+- GNU General Public License (GPL) v. 2, v. 3
+- GNU Affero General Public License (AGPL) v. 3
+- Open Software License (OSL) v. 2.1, v. 3.0
+- Eclipse Public License (EPL) v. 1.0
+- CeCILL v. 2.0, v. 2.1
+- Mozilla Public Licence (MPL) v. 2
+- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3
+- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for
+  works other than software
+- European Union Public Licence (EUPL) v. 1.1, v. 1.2
+- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong
+  Reciprocity (LiLiQ-R+).
+
+The European Commission may update this Appendix to later versions of the above
+licences without producing a new version of the EUPL, as long as they provide
+the rights granted in Article 2 of this Licence and protect the covered Source
+Code from exclusive appropriation.
+
+All other changes or additions to this Appendix require the production of a new
+EUPL version.

README.md

@@ -62,7 +62,9 @@ Minimal config file should look like:
             "singlefilters":"./urls-domains.txt",
             "doublefilters":"./urls-hosts.txt", 
             "blackholeip":"127.0.0.1",
-            "hostsfile":"./urls-local.txt"
+            "hostsfile":"./urls-local.txt",
+            "whitelist":"./whitelist.txt",
+            "cache":true
         },
     }
 }
@@ -83,8 +85,9 @@ configs:
 - singlefilters: name of the file  for blacklists following the "singlefilter" schema.(one URL per line)
 - doublefilters: name of the file, for blacklists following the "doublefilter" schema.(one URL per line)
 - blackholeip: IP address to return when the IP is banned. This is because you may want to avoid MX issues, mail loops on localhost, or you have a web server running on localhost
-- hostsfile: path where you keep your local blacklistfile : this is in the format "singlefilter", meaning one domain per line, unlike hosts file.
-
+- hostsfile: path where you keep your local blacklist file : this is in the format "singlefilter", meaning one domain per line, unlike hosts file.
+- whitelist: path where you keep your local whitelist file : this is in the format "singlefilter", meaning one domain per line, unlike hosts file.
+- cache: if set to false disable the cache for this configuration. Boolean, defaults true
 
 Advanced configuration includes support for multiple configurations based on IP Source and timetables:
 <pre>
@@ -104,6 +107,7 @@ Advanced configuration includes support for multiple configurations based on IP
     },
     "ipaliases":{
         "pc8":"192.168.178.29",
+        "lg-tv":"192.168.178.10",
         "localhost":"127.0.0.1"
     },
     "ipgroups":[
@@ -111,6 +115,11 @@ Advanced configuration includes support for multiple configurations based on IP
             "ips":["localhost", "::1", "192.168.178.30", "192.168.178.31", "pc8"],
             "cfg":"",
             "timetable":"tt_children"
+        },
+        {
+            "ips":["lg-tv"],
+            "cfg":"tv",
+            "timetable":""
         }
     ],
     "timetables":{
@@ -131,21 +140,33 @@ Advanced configuration includes support for multiple configurations based on IP
             "singlefilters":"./urls-domains.txt",
             "doublefilters":"./urls-hosts.txt", 
             "blackholeip":"127.0.0.1",
-            "hostsfile":"./urls-local.txt"
+            "hostsfile":"./urls-local.txt",
+            "whitelist":"./whitelist.txt",
         },
         "children":{
             "upstream":"./dns-upstream-safe.txt",
             "singlefilters":"./urls-domains.txt",
             "doublefilters":"./urls-hosts.txt", 
             "blackholeip":"127.0.0.1",
-            "hostsfile":"./urls-local.txt"
+            "hostsfile":"./urls-local.txt",
+            "whitelist":"./whitelist.txt",
         },
         "children_restricted":{
             "upstream":"./dns-upstream-safe.txt",
             "singlefilters":"./urls-domains-restricted.txt",
             "doublefilters":"./urls-hosts-restricted.txt", 
             "blackholeip":"127.0.0.1",
-            "hostsfile":"./urls-local.txt"
+            "hostsfile":"./urls-local.txt",
+            "whitelist":"./whitelist.txt",
+        },
+        "tv":{
+            "upstream":"./dns-upstream.txt",
+            "singlefilters":"",
+            "doublefilters":"", 
+            "blackholeip":"127.0.0.1",
+            "hostsfile":"",
+            "whitelist":"",
+            "cache":false
         }
     }
 }

config.sample.json

@@ -36,7 +36,9 @@
             "singlefilters":"./urls-domains-updated.txt",
             "doublefilters":"./urls-hosts-normal.txt", 
             "blackholeip":"127.0.0.1",
-            "hostsfile":"./urls-local-normal.txt"
+            "hostsfile":"./urls-local-normal.txt",
+            "whitelist":"./urls-wl-normal.txt",
+            "parallelqueries":4
         },
         "children":{
             "upstream":"./dns-familyscreen.txt",

dns-upstream.txt

@@ -225,7 +225,6 @@
 212.89.128.28:53
 213.133.116.14:53
 213.166.247.100:53
-217.243.173.82:53
 217.5.182.118:53
 217.7.80.40:53
 217.7.81.136:53
@@ -362,7 +361,6 @@
 85.214.62.160:53
 85.93.91.101:53
 87.106.63.208:53
-87.118.126.225:53
 88.198.37.146:53
 88.99.66.18:53
 89.163.150.209:53

dns_client.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"log"
 	"time"
 
 	"math/rand"
@@ -10,11 +11,84 @@ import (
 	"github.com/miekg/dns"
 )
 
+func SingleResolve(query *dns.Msg, d string, lfqdn string, ch chan *dns.Msg) {
+	c := new(dns.Client)
+
+	c.ReadTimeout = 500 * time.Millisecond
+	c.WriteTimeout = 500 * time.Millisecond
+
+	in, _, err := c.Exchange(query, d)
+	if err != nil {
+		fmt.Printf("SingleResolve: Problem with DNS %s : %s\n", d, err.Error())
+		go incrementStats("DNS Problems "+d, 1)
+		ch <- nil
+	} else {
+		go incrementStats(d, 1)
+		Rcode := in.MsgHdr.Rcode
+		in.SetReply(query)
+		in.MsgHdr.Rcode = Rcode
+		in.Authoritative = true
+		in.Compress = true
+		go DomainCache(lfqdn, in)
+		if ZabovDebug {
+			log.Println("SingleResolve: OK:", d, lfqdn)
+		}
+		ch <- in
+	}
+}
+
+func ParallelResolve(query *dns.Msg, config string, lfqdn string) *dns.Msg {
+
+	var resCur *dns.Msg
+	var res *dns.Msg
+	ch := make(chan *dns.Msg)
+
+	dnss := oneTimeDNS(config, ZabovConfigs[config].ZabovParallelQueries)
+
+	for _, d := range dnss {
+		if ZabovDebug {
+			log.Println("ParallelResolve: running SingleResolve on:", d, lfqdn)
+		}
+		go SingleResolve(query, d, lfqdn, ch)
+	}
+
+	if ZabovDebug {
+		log.Println("ParallelResolve: wait for results...")
+	}
+	for range dnss {
+		resCur = <-ch
+		if resCur != nil {
+
+			if res == nil {
+				if ZabovDebug {
+					log.Println("ParallelResolve: got first result!")
+				}
+				res = resCur
+			} else if resCur.Rcode == dns.RcodeSuccess {
+				if ZabovDebug {
+					log.Println("ParallelResolve: got next result, RcodeSuccess, replacing previous...")
+				}
+				res = resCur
+			} else {
+				if ZabovDebug {
+					log.Println("ParallelResolve: got next result, discarding...")
+				}
+			}
+			if res.Rcode == dns.RcodeSuccess {
+				break
+			}
+		}
+	}
+	return res
+}
+
 //ForwardQuery forwards the query to the upstream server
 //first server to answer wins
 //accepts config name to select the UP DNS source list
 func ForwardQuery(query *dns.Msg, config string, nocache bool) *dns.Msg {
-
+	if ZabovDebug {
+		log.Println("ForwardQuery: nocache", nocache)
+	}
 	go incrementStats("ForwardQueries", 1)
 
 	r := new(dns.Msg)
@@ -27,41 +101,32 @@ func ForwardQuery(query *dns.Msg, config string, nocache bool) *dns.Msg {
 	if !nocache {
 		if cached := GetDomainFromCache(lfqdn); cached != nil {
 			go incrementStats("CacheHit", 1)
+			Rcode := cached.MsgHdr.Rcode
 			cached.SetReply(query)
+			cached.MsgHdr.Rcode = Rcode
 			cached.Authoritative = true
+			if ZabovDebug {
+				log.Println("ForwardQuery: CacheHit")
+			}
+			cached.Compress = true
 			return cached
-
 		}
 	}
 
-	c := new(dns.Client)
-
-	c.ReadTimeout = 500 * time.Millisecond
-	c.WriteTimeout = 500 * time.Millisecond
-
 	for {
 		// round robin with retry
 
-		if !NetworkUp {
+		// local responder should always be available also if no internet connection
+		if !NetworkUp && localresponderConfigName != config {
 			time.Sleep(10 * time.Second)
 			go incrementStats("Network Problems ", 1)
 			continue
 		}
 
-		d := oneTimeDNS(config)
+		in := ParallelResolve(query, config, lfqdn)
 
-		in, _, err := c.Exchange(query, d)
-		if err != nil {
-			fmt.Printf("Problem with DNS %s : %s\n", d, err.Error())
-			go incrementStats("DNS Problems "+d, 1)
-			continue
-		} else {
-			go incrementStats(d, 1)
-			in.SetReply(query)
-			in.Authoritative = true
-			go DomainCache(lfqdn, in)
+		if in != nil {
 			return in
-
 		}
 
 	}
@@ -81,9 +146,12 @@ func init() {
 
 }
 
-func oneTimeDNS(config string) (dns string) {
+func oneTimeDNS(config string, count int) (dns []string) {
 
-	rand.Seed(time.Now().Unix())
+	if count == 0 {
+		count = 1
+	}
+	rand.Seed(time.Now().UnixNano())
 
 	upl := ZabovConfigs[config].ZabovDNSArray
 
@@ -91,16 +159,27 @@ func oneTimeDNS(config string) (dns string) {
 
 		if len(ZabovLocalResponder) > 0 {
 			fmt.Println("No DNS defined, fallback to local responder:", ZabovLocalResponder)
-			return ZabovLocalResponder
+			return []string{ZabovLocalResponder}
 		}
 		fmt.Println("No DNS defined, using default 127.0.0.53:53. Hope it works!")
-		return "127.0.0.53:53"
+		return []string{"127.0.0.53:53"}
 	}
 
 	n := rand.Intn(128*len(upl)) % len(upl)
 
-	dns = upl[n]
+	res := []string{}
+	for i := 0; i < count; i++ {
+		res = append(res, upl[(n+i)%len(upl)])
+	}
 
-	return
+	check := make(map[string]int)
+	for _, val := range res {
+		check[val] = 1
+	}
 
+	res = []string{}
+	for d, _ := range check {
+		res = append(res, d)
+	}
+	return res
 }

dns_handler.go

@@ -269,7 +269,7 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 		if len(ZabovLocalResponder) > 0 {
 			if !strings.Contains(fqdn, ".") ||
 				(len(ZabovLocalDomain) > 0 && strings.HasSuffix(fqdn, ZabovLocalDomain)) {
-				config = "__localresponder__"
+				config = localresponderConfigName
 				ret := ForwardQuery(r, config, true)
 				w.WriteMsg(ret)
 				go logQuery(remIP, fqdn, QType, config, timetable, "localresponder")
@@ -277,17 +277,17 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 			}
 
 		}
-		if domainInKillfile(fqdn, config) {
+		if !domainInWhiteListfile(fqdn, config) && domainInKillfile(fqdn, config) {
 			go incrementStats("Killed", 1)
 
 			msg.Answer = append(msg.Answer, &dns.A{
 				Hdr: dns.RR_Header{Name: domain, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60},
-				A:   net.ParseIP(ZabovConfig.ZabovAddBL),
+				A:   ZabovConfig.ZabovAddBL,
 			})
 			go logQuery(remIP, fqdn, QType, config, timetable, "killed")
 		} else {
 			go logQuery(remIP, fqdn, QType, config, timetable, "forwarded")
-			ret := ForwardQuery(r, config, false)
+			ret := ForwardQuery(r, config, !ZabovConfig.ZabovCache)
 			w.WriteMsg(ret)
 		}
 	case dns.TypePTR:
@@ -297,13 +297,13 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) {
 
 		if len(ZabovLocalResponder) > 0 {
 			// if set use local responder for reverse lookup (suffix ".in-addr.arpa.")
-			config = "__localresponder__"
+			config = localresponderConfigName
 		}
 		ret := ForwardQuery(r, config, true)
 		w.WriteMsg(ret)
 		go logQuery(remIP, msg.Question[0].Name, QType, config, timetable, "localresponder")
 	default:
-		ret := ForwardQuery(r, config, false)
+		ret := ForwardQuery(r, config, !ZabovConfig.ZabovCache)
 		w.WriteMsg(ret)
 		if len(ZabovDebugDBPath) > 0 {
 			go logQuery(remIP, msg.Question[0].Name, QType, config, timetable, "forwarded")

go.mod

@@ -1,8 +1,11 @@
 module zabov
 
-go 1.13
+go 1.15
 
 require (
-	github.com/miekg/dns v1.1.27
+	github.com/golang/snappy v0.0.4
+	github.com/miekg/dns v1.1.43
 	github.com/syndtr/goleveldb v1.0.0
+	golang.org/x/net v0.0.0-20210916014120-12bc252f5db8
+	golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
 )

go.sum

@@ -1,35 +1,33 @@
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
-github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
+github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg=
+github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE=
 github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8=
-golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
-golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g=
-golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8 h1:/6y1LfuqNuQdHAm0jjtPtgRcxIxjVZgm5OTu8/QhZvk=
+golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M=
-golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210915083310-ed5796bab164 h1:7ZDGnxgHAMw7thfC5bEos0RDAccZKxioiWBhfIe+tvw=
+golang.org/x/sys v0.0.0-20210915083310-ed5796bab164/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=

hostfile.go

@@ -11,28 +11,29 @@ func init() {
 
 	fmt.Println("Ingesting local hosts file")
 	ingestLocalBlacklists()
+	ingestLocalWhiteLists()
 
 }
 
 func ingestLocalBlacklists() {
 
 	fmt.Println("ingestLocalBlacklist: collecting urls from all configs...")
-	_files := urlsMap{}
+	_HostsFiles := urlsMap{}
 	for config := range ZabovConfigs {
 		ZabovHostsFile := ZabovConfigs[config].ZabovHostsFile
 		if len(ZabovHostsFile) == 0 {
 			continue
 		}
-		configs := _files[ZabovHostsFile]
+		configs := _HostsFiles[ZabovHostsFile]
 		if configs == nil {
 			configs = stringarray{}
-			_files[ZabovHostsFile] = configs
+			_HostsFiles[ZabovHostsFile] = configs
 		}
 		configs = append(configs, config)
-		_files[ZabovHostsFile] = configs
+		_HostsFiles[ZabovHostsFile] = configs
 	}
 
-	for ZabovHostsFile, configs := range _files {
+	for ZabovHostsFile, configs := range _HostsFiles {
 		file, err := os.Open(ZabovHostsFile)
 		if err != nil {
 			fmt.Println(err.Error())
@@ -57,6 +58,49 @@ func ingestLocalBlacklists() {
 
 }
 
+func ingestLocalWhiteLists() {
+
+	fmt.Println("ingestLocalWhiteLists: collecting urls from all configs...")
+	_WhiteListFiles := urlsMap{}
+	for config := range ZabovConfigs {
+		ZabovWhiteList := ZabovConfigs[config].ZabovWhiteList
+		if len(ZabovWhiteList) == 0 {
+			continue
+		}
+		configs := _WhiteListFiles[ZabovWhiteList]
+		if configs == nil {
+			configs = stringarray{}
+			_WhiteListFiles[ZabovWhiteList] = configs
+		}
+		configs = append(configs, config)
+		_WhiteListFiles[ZabovWhiteList] = configs
+	}
+
+	for ZabovWhiteList, configs := range _WhiteListFiles {
+		file, err := os.Open(ZabovWhiteList)
+		if err != nil {
+			fmt.Println(err.Error())
+		}
+		defer file.Close()
+
+		scanner := bufio.NewScanner(file)
+		for scanner.Scan() {
+			d := scanner.Text()
+			if len(d) == 0 || strings.TrimSpace(d)[0] == '#' {
+				continue
+			}
+			DomainWhiteList(d, ZabovWhiteList, configs)
+			incrementStats("WhiteList", 1)
+
+		}
+
+		if err := scanner.Err(); err != nil {
+			fmt.Println(err.Error())
+		}
+	}
+
+}
+
 func fileByLines(filename string) (blurls []string) {
 
 	file, err := os.Open(filename)

main.go

@@ -35,13 +35,16 @@ type handler struct{}
 
 // ZabovConfig contains all Zabov configs
 type ZabovConfig struct {
-	ZabovSingleBL  string   // json:singlefilters -> ZabovSingleBL list of urls returning a file with just names of domains
-	ZabovDoubleBL  string   // json:doublefilters -> ZabovDoubleBL list of urls returning a file with  IP<space>domain
-	ZabovAddBL     string   // json:blackholeip  -> ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1
-	ZabovHostsFile string   // json:hostsfile -> ZabovHostsFile is the file we use to keep our hosts
-	ZabovUpDNS     string   // json:upstream -> ZabovUpDNS keeps the name of upstream DNSs
-	ZabovDNSArray  []string // contains all the DNS we mention, parsed from ZabovUpDNS file
-	references     int      // contains references to this config; if zero, config shall be removed
+	ZabovSingleBL        string   // json:singlefilters -> ZabovSingleBL list of urls returning a file with just names of domains
+	ZabovDoubleBL        string   // json:doublefilters -> ZabovDoubleBL list of urls returning a file with  IP<space>domain
+	ZabovAddBL           net.IP   // json:blackholeip  -> ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1
+	ZabovHostsFile       string   // json:hostsfile -> ZabovHostsFile is the file we use to keep our hosts
+	ZabovWhiteList       string   // json:hostsfile -> ZabovWhiteList is the file we use to keep white listed hosts
+	ZabovUpDNS           string   // json:upstream -> ZabovUpDNS keeps the name of upstream DNSs
+	ZabovDNSArray        []string // contains all the DNS we mention, parsed from ZabovUpDNS file
+	ZabovCache           bool     // allows to disable cache
+	ZabovParallelQueries int      // contains max number of parallel queries to multiple DNS servers
+	references           int      // contains references to this config; if zero, config shall be removed
 }
 
 // ZabovConfigs contains all Zabov configs

stats.py

@@ -0,0 +1,95 @@
+#!/bin/env python3
+
+import os
+import glob
+import argparse
+
+
+parser = argparse.ArgumentParser(description='Zabov logs analyzer')
+
+parser.add_argument('--ip', dest="ip", metavar='IP', type=str,
+                    help='filter by source IP (substring to match). Default: any')
+                    
+parser.add_argument('--action', dest="action", metavar='action', type=str, default="killed",
+                    help='filter action (substring to match): killed|forwarded|any. Default: killed')
+                    
+parser.add_argument('--config', dest="config", metavar='name', type=str,
+                    help='filter by config name (substring to match). Default: any')
+                    
+parser.add_argument('--timetable', dest="timetable", metavar='name', type=str,
+                    help='filter by timetable name (substring to match). Default: any')
+
+parser.add_argument('--reqtype', dest="reqtype", metavar='TypeA', type=str, default="TypeA",
+                    help='filter by reqtype name (substring to match): TypeA|TypeAAAA|TypeMX|...')
+                    
+parser.add_argument('--domain', dest="domain", metavar='name', type=str,
+                    help='filter by domain name (substring to match). Default: all')
+                    
+parser.add_argument('--min-entries', dest="minentries", metavar='100', type=int, default=0,
+                    help='filter output by minimum number of entries. Default: any')
+
+parser.add_argument('--logs-path', dest="logs", metavar='path', type=str, default="./config/logs",
+                    help='Zabov logs path')
+                    
+                    
+                    
+args = parser.parse_args()
+
+
+timetables = {}
+configs = {}
+
+killed = {}
+for x in glob.glob(os.path.join(args.logs, "*.log")):
+    #print (x)
+    f = open(x, "r")
+    f.readline()
+    if args.reqtype:
+        args.reqtype = args.reqtype.lower()
+    if args.domain:
+        args.domain = args.domain.lower()
+        
+    for line in f.readlines():
+        linel = line.strip().lower()
+        fields = linel.split("\t")
+        
+        timetables[fields[5]] = timetables.get(fields[5], 0) +1
+        configs[fields[4]] = configs.get(fields[4], 0) +1
+        
+        ok = all((not args.action or fields[6].find(args.action)>=0 or fields[6] == "any", \
+            not args.timetable or fields[5].find(args.timetable)>=0 or fields[5] == "any", \
+            not args.config or fields[4].find(args.config)>=0 or fields[4] == "any", \
+            not args.ip or fields[1].find(args.ip)>=0 or fields[1] == "any",\
+            not args.domain or fields[2].find(args.domain)>=0 or fields[2] == "any", \
+            not args.reqtype or fields[3].find(args.reqtype)>=0 ))
+        if ok:
+            killed[fields[2]] = killed.get(fields[2], 0) +1
+
+killed_sorted = {key: value for key, value in sorted(killed.items(), key=lambda item: item[1], reverse=True)}
+
+total_queries_filtered = 0
+total_domain_filtered = 0
+total_queries = 0
+for k in killed_sorted.keys():
+    if args.minentries == 0 or killed[k] >= args.minentries:
+        print (k, killed[k])
+        total_queries_filtered += killed[k]
+        total_domain_filtered+=1
+    total_queries += killed[k]
+
+print("")
+print("TOTAL domains (filtered):", total_domain_filtered )
+print("TOTAL queries (filtred):", total_queries_filtered )
+print("TOTAL domains:", len(killed_sorted.keys()) )
+print("TOTAL queries:", total_queries )
+
+
+timetables = {key: value for key, value in sorted(timetables.items(), key=lambda item: item[0], reverse=False)}
+configs = {key: value for key, value in sorted(configs.items(), key=lambda item: item[0], reverse=False)}
+
+print("all available timetables:")
+for k in timetables.keys():
+    print(" '%s': %d items" % (k, timetables[k], ))
+print("all available configs:")
+for k in configs.keys():
+    print(" '%s': %d items" % (k, configs[k], ))

vendor/github.com/golang/snappy/AUTHORS

@@ -8,8 +8,11 @@
 
 # Please keep the list sorted.
 
+Amazon.com, Inc
 Damian Gryski <dgryski@gmail.com>
+Eric Buth <eric@topos.com>
 Google Inc.
 Jan Mercl <0xjnml@gmail.com>
+Klaus Post <klauspost@gmail.com>
 Rodolfo Carvalho <rhcarvalho@gmail.com>
 Sebastien Binet <seb.binet@gmail.com>

vendor/github.com/golang/snappy/CONTRIBUTORS

@@ -26,9 +26,13 @@
 
 # Please keep the list sorted.
 
+Alex Legg <alexlegg@google.com>
 Damian Gryski <dgryski@gmail.com>
+Eric Buth <eric@topos.com>
 Jan Mercl <0xjnml@gmail.com>
+Jonathan Swinney <jswinney@amazon.com>
 Kai Backman <kaib@golang.org>
+Klaus Post <klauspost@gmail.com>
 Marc-Antoine Ruel <maruel@chromium.org>
 Nigel Tao <nigeltao@golang.org>
 Rob Pike <r@golang.org>

vendor/github.com/golang/snappy/decode.go

@@ -52,6 +52,8 @@ const (
 // Otherwise, a newly allocated slice will be returned.
 //
 // The dst and src must not overlap. It is valid to pass a nil dst.
+//
+// Decode handles the Snappy block format, not the Snappy stream format.
 func Decode(dst, src []byte) ([]byte, error) {
 	dLen, s, err := decodedLen(src)
 	if err != nil {
@@ -83,6 +85,8 @@ func NewReader(r io.Reader) *Reader {
 }
 
 // Reader is an io.Reader that can read Snappy-compressed bytes.
+//
+// Reader handles the Snappy stream format, not the Snappy block format.
 type Reader struct {
 	r       io.Reader
 	err     error
@@ -114,32 +118,23 @@ func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) {
 	return true
 }
 
-// Read satisfies the io.Reader interface.
-func (r *Reader) Read(p []byte) (int, error) {
-	if r.err != nil {
-		return 0, r.err
-	}
-	for {
-		if r.i < r.j {
-			n := copy(p, r.decoded[r.i:r.j])
-			r.i += n
-			return n, nil
-		}
+func (r *Reader) fill() error {
+	for r.i >= r.j {
 		if !r.readFull(r.buf[:4], true) {
-			return 0, r.err
+			return r.err
 		}
 		chunkType := r.buf[0]
 		if !r.readHeader {
 			if chunkType != chunkTypeStreamIdentifier {
 				r.err = ErrCorrupt
-				return 0, r.err
+				return r.err
 			}
 			r.readHeader = true
 		}
 		chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16
 		if chunkLen > len(r.buf) {
 			r.err = ErrUnsupported
-			return 0, r.err
+			return r.err
 		}
 
 		// The chunk types are specified at
@@ -149,11 +144,11 @@ func (r *Reader) Read(p []byte) (int, error) {
 			// Section 4.2. Compressed data (chunk type 0x00).
 			if chunkLen < checksumSize {
 				r.err = ErrCorrupt
-				return 0, r.err
+				return r.err
 			}
 			buf := r.buf[:chunkLen]
 			if !r.readFull(buf, false) {
-				return 0, r.err
+				return r.err
 			}
 			checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24
 			buf = buf[checksumSize:]
@@ -161,19 +156,19 @@ func (r *Reader) Read(p []byte) (int, error) {
 			n, err := D