From ef9f912cf71ce0c077d0a4dd88f42a6859f77226 Mon Sep 17 00:00:00 2001 From: bloved Date: Tue, 11 May 2021 10:32:48 +0200 Subject: [PATCH] - added simple stats python script - added WhiteList database - updated license to EUPL --- 00.database.go | 20 ++++ 01.conf.go | 9 +- 01.killfile.go | 81 +++++++++++++- LICENSE | 297 +++++++++++++++++++++++++++++++++++++++++++++++-- dns_handler.go | 2 +- hostfile.go | 54 ++++++++- main.go | 1 + stats.py | 95 ++++++++++++++++ 8 files changed, 535 insertions(+), 24 deletions(-) create mode 100644 stats.py diff --git a/00.database.go b/00.database.go index ac4889b..8d3d8ec 100644 --- a/00.database.go +++ b/00.database.go @@ -13,6 +13,9 @@ var MyZabovCDB *leveldb.DB //MyZabovKDBs is the storage where we'll put domains to block (one for each config) var MyZabovKDBs map[string]*leveldb.DB +//MyZabovWLDBs is the storage where we'll put domains to whitelist (one for each config) +var MyZabovWLDBs map[string]*leveldb.DB + func init() { var err error @@ -29,6 +32,7 @@ func init() { } MyZabovKDBs = map[string]*leveldb.DB{} + MyZabovWLDBs = map[string]*leveldb.DB{} } // ZabovCreateKDB creates Kill DBs @@ -46,3 +50,19 @@ func ZabovCreateKDB(conf string) { MyZabovKDBs[conf] = KDB } + +// ZabovCreateWLDB creates Whitelist DBs +func ZabovCreateWLDB(conf string) { + var err error + + dbname := "./db/whitelist_" + conf + WLDB, err := leveldb.OpenFile(dbname, nil) + if err != nil { + fmt.Println("Cannot create whitelist db: ", err.Error()) + } else { + fmt.Println("whitelist DB created:", dbname) + } + + MyZabovWLDBs[conf] = WLDB + +} diff --git a/01.conf.go b/01.conf.go index 937c334..7411968 100644 --- a/01.conf.go +++ b/01.conf.go @@ -111,7 +111,13 @@ func init() { conf.ZabovSingleBL = confRaw["singlefilters"].(string) conf.ZabovDoubleBL = confRaw["doublefilters"].(string) conf.ZabovAddBL = net.ParseIP(confRaw["blackholeip"].(string)) - conf.ZabovHostsFile = confRaw["hostsfile"].(string) + if confRaw["hostsfile"] != nil { + conf.ZabovHostsFile = confRaw["hostsfile"].(string) + } + + if confRaw["whitelist"] != nil { + conf.ZabovWhiteList = confRaw["whitelist"].(string) + } if confRaw["cache"] != nil { conf.ZabovCache = confRaw["cache"].(bool) @@ -295,6 +301,7 @@ func init() { delete(ZabovConfigs, name) } else { ZabovCreateKDB(name) + ZabovCreateWLDB(name) } } diff --git a/01.killfile.go b/01.killfile.go index 8ad4b18..364a220 100644 --- a/01.killfile.go +++ b/01.killfile.go @@ -11,20 +11,23 @@ type killfileItem struct { Kconfigs stringarray } -var bChannel chan killfileItem +var bKillChannel chan killfileItem +var bWhiteListChannel chan killfileItem func init() { - bChannel = make(chan killfileItem, 1024) + bKillChannel = make(chan killfileItem, 1024) + bWhiteListChannel = make(chan killfileItem, 1024) fmt.Println("Initializing kill channel engine.") - go bWriteThread() + go bWriteKillThread() + go bWriteWhiteListThread() } -func bWriteThread() { +func bWriteKillThread() { - for item := range bChannel { + for item := range bKillChannel { alreadyInSomeDB := false @@ -56,7 +59,7 @@ func DomainKill(s, durl string, configs stringarray) { k.Ksource = durl k.Kconfigs = configs - bChannel <- k + bKillChannel <- k } @@ -88,3 +91,69 @@ func domainInKillfile(domain string, config string) bool { return has } + +func bWriteWhiteListThread() { + + for item := range bWhiteListChannel { + + alreadyInSomeDB := false + + for _, config := range item.Kconfigs { + if !alreadyInSomeDB { + alreadyInSomeDB = domainInWhiteListfile(item.Kdomain, config) + } + writeInWhiteListfile(item.Kdomain, item.Ksource, config) + } + if !alreadyInSomeDB { + incrementStats("WL domains from "+item.Ksource, 1) + incrementStats("WL TOTAL", 1) + } + + } + +} + +//DomainWhiteList stores a domain name inside the killfile +func DomainWhiteList(s, durl string, configs stringarray) { + + if len(s) > 2 { + + s = strings.ToLower(s) + + var k killfileItem + + k.Kdomain = s + k.Ksource = durl + k.Kconfigs = configs + + bWhiteListChannel <- k + + } + +} + +func writeInWhiteListfile(key, value string, config string) { + + stK := []byte(key) + stV := []byte(value) + + MyZabovWLDB := MyZabovWLDBs[config] + err := MyZabovWLDB.Put(stK, stV, nil) + if err != nil { + fmt.Println("Cannot write to Whitelist DB: ", err.Error()) + } + +} +func domainInWhiteListfile(domain string, config string) bool { + + s := strings.ToLower(domain) + + MyZabovWLDB := MyZabovWLDBs[config] + has, err := MyZabovWLDB.Has([]byte(s), nil) + if err != nil { + fmt.Println("Cannot read from Whitelist DB: ", err.Error()) + } + + return has + +} diff --git a/LICENSE b/LICENSE index fd3b63f..a2dc14d 100644 --- a/LICENSE +++ b/LICENSE @@ -1,14 +1,289 @@ - Copyright (C) 2020 loweel@keinpfusch.net +Copyright (C) 2020 loweel@keinpfusch.net - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. + EUROPEAN UNION PUBLIC LICENCE v. 1.2 + EUPL © the European Union 2007, 2016 - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. +This European Union Public Licence (the ‘EUPL’) applies to the Work (as defined +below) which is provided under the terms of this Licence. Any use of the Work, +other than as authorised under this Licence is prohibited (to the extent such +use is covered by a right of the copyright holder of the Work). - You should have received a copy of the GNU General Public License - along with this program. If not, see . \ No newline at end of file +The Work is provided under the terms of this Licence when the Licensor (as +defined below) has placed the following notice immediately following the +copyright notice for the Work: + + Licensed under the EUPL + +or has expressed by any other means his willingness to license under the EUPL. + +1. Definitions + +In this Licence, the following terms have the following meaning: + +- ‘The Licence’: this Licence. + +- ‘The Original Work’: the work or software distributed or communicated by the + Licensor under this Licence, available as Source Code and also as Executable + Code as the case may be. + +- ‘Derivative Works’: the works or software that could be created by the + Licensee, based upon the Original Work or modifications thereof. This Licence + does not define the extent of modification or dependence on the Original Work + required in order to classify a work as a Derivative Work; this extent is + determined by copyright law applicable in the country mentioned in Article 15. + +- ‘The Work’: the Original Work or its Derivative Works. + +- ‘The Source Code’: the human-readable form of the Work which is the most + convenient for people to study and modify. + +- ‘The Executable Code’: any code which has generally been compiled and which is + meant to be interpreted by a computer as a program. + +- ‘The Licensor’: the natural or legal person that distributes or communicates + the Work under the Licence. + +- ‘Contributor(s)’: any natural or legal person who modifies the Work under the + Licence, or otherwise contributes to the creation of a Derivative Work. + +- ‘The Licensee’ or ‘You’: any natural or legal person who makes any usage of + the Work under the terms of the Licence. + +- ‘Distribution’ or ‘Communication’: any act of selling, giving, lending, + renting, distributing, communicating, transmitting, or otherwise making + available, online or offline, copies of the Work or providing access to its + essential functionalities at the disposal of any other natural or legal + person. + +2. Scope of the rights granted by the Licence + +The Licensor hereby grants You a worldwide, royalty-free, non-exclusive, +sublicensable licence to do the following, for the duration of copyright vested +in the Original Work: + +- use the Work in any circumstance and for all usage, +- reproduce the Work, +- modify the Work, and make Derivative Works based upon the Work, +- communicate to the public, including the right to make available or display + the Work or copies thereof to the public and perform publicly, as the case may + be, the Work, +- distribute the Work or copies thereof, +- lend and rent the Work or copies thereof, +- sublicense rights in the Work or copies thereof. + +Those rights can be exercised on any media, supports and formats, whether now +known or later invented, as far as the applicable law permits so. + +In the countries where moral rights apply, the Licensor waives his right to +exercise his moral right to the extent allowed by law in order to make effective +the licence of the economic rights here above listed. + +The Licensor grants to the Licensee royalty-free, non-exclusive usage rights to +any patents held by the Licensor, to the extent necessary to make use of the +rights granted on the Work under this Licence. + +3. Communication of the Source Code + +The Licensor may provide the Work either in its Source Code form, or as +Executable Code. If the Work is provided as Executable Code, the Licensor +provides in addition a machine-readable copy of the Source Code of the Work +along with each copy of the Work that the Licensor distributes or indicates, in +a notice following the copyright notice attached to the Work, a repository where +the Source Code is easily and freely accessible for as long as the Licensor +continues to distribute or communicate the Work. + +4. Limitations on copyright + +Nothing in this Licence is intended to deprive the Licensee of the benefits from +any exception or limitation to the exclusive rights of the rights owners in the +Work, of the exhaustion of those rights or of other applicable limitations +thereto. + +5. Obligations of the Licensee + +The grant of the rights mentioned above is subject to some restrictions and +obligations imposed on the Licensee. Those obligations are the following: + +Attribution right: The Licensee shall keep intact all copyright, patent or +trademarks notices and all notices that refer to the Licence and to the +disclaimer of warranties. The Licensee must include a copy of such notices and a +copy of the Licence with every copy of the Work he/she distributes or +communicates. The Licensee must cause any Derivative Work to carry prominent +notices stating that the Work has been modified and the date of modification. + +Copyleft clause: If the Licensee distributes or communicates copies of the +Original Works or Derivative Works, this Distribution or Communication will be +done under the terms of this Licence or of a later version of this Licence +unless the Original Work is expressly distributed only under this version of the +Licence — for example by communicating ‘EUPL v. 1.2 only’. The Licensee +(becoming Licensor) cannot offer or impose any additional terms or conditions on +the Work or Derivative Work that alter or restrict the terms of the Licence. + +Compatibility clause: If the Licensee Distributes or Communicates Derivative +Works or copies thereof based upon both the Work and another work licensed under +a Compatible Licence, this Distribution or Communication can be done under the +terms of this Compatible Licence. For the sake of this clause, ‘Compatible +Licence’ refers to the licences listed in the appendix attached to this Licence. +Should the Licensee's obligations under the Compatible Licence conflict with +his/her obligations under this Licence, the obligations of the Compatible +Licence shall prevail. + +Provision of Source Code: When distributing or communicating copies of the Work, +the Licensee will provide a machine-readable copy of the Source Code or indicate +a repository where this Source will be easily and freely available for as long +as the Licensee continues to distribute or communicate the Work. + +Legal Protection: This Licence does not grant permission to use the trade names, +trademarks, service marks, or names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the copyright notice. + +6. Chain of Authorship + +The original Licensor warrants that the copyright in the Original Work granted +hereunder is owned by him/her or licensed to him/her and that he/she has the +power and authority to grant the Licence. + +Each Contributor warrants that the copyright in the modifications he/she brings +to the Work are owned by him/her or licensed to him/her and that he/she has the +power and authority to grant the Licence. + +Each time You accept the Licence, the original Licensor and subsequent +Contributors grant You a licence to their contributions to the Work, under the +terms of this Licence. + +7. Disclaimer of Warranty + +The Work is a work in progress, which is continuously improved by numerous +Contributors. It is not a finished work and may therefore contain defects or +‘bugs’ inherent to this type of development. + +For the above reason, the Work is provided under the Licence on an ‘as is’ basis +and without warranties of any kind concerning the Work, including without +limitation merchantability, fitness for a particular purpose, absence of defects +or errors, accuracy, non-infringement of intellectual property rights other than +copyright as stated in Article 6 of this Licence. + +This disclaimer of warranty is an essential part of the Licence and a condition +for the grant of any rights to the Work. + +8. Disclaimer of Liability + +Except in the cases of wilful misconduct or damages directly caused to natural +persons, the Licensor will in no event be liable for any direct or indirect, +material or moral, damages of any kind, arising out of the Licence or of the use +of the Work, including without limitation, damages for loss of goodwill, work +stoppage, computer failure or malfunction, loss of data or any commercial +damage, even if the Licensor has been advised of the possibility of such damage. +However, the Licensor will be liable under statutory product liability laws as +far such laws apply to the Work. + +9. Additional agreements + +While distributing the Work, You may choose to conclude an additional agreement, +defining obligations or services consistent with this Licence. However, if +accepting obligations, You may act only on your own behalf and on your sole +responsibility, not on behalf of the original Licensor or any other Contributor, +and only if You agree to indemnify, defend, and hold each Contributor harmless +for any liability incurred by, or claims asserted against such Contributor by +the fact You have accepted any warranty or additional liability. + +10. Acceptance of the Licence + +The provisions of this Licence can be accepted by clicking on an icon ‘I agree’ +placed under the bottom of a window displaying the text of this Licence or by +affirming consent in any other similar way, in accordance with the rules of +applicable law. Clicking on that icon indicates your clear and irrevocable +acceptance of this Licence and all of its terms and conditions. + +Similarly, you irrevocably accept this Licence and all of its terms and +conditions by exercising any rights granted to You by Article 2 of this Licence, +such as the use of the Work, the creation by You of a Derivative Work or the +Distribution or Communication by You of the Work or copies thereof. + +11. Information to the public + +In case of any Distribution or Communication of the Work by means of electronic +communication by You (for example, by offering to download the Work from a +remote location) the distribution channel or media (for example, a website) must +at least provide to the public the information requested by the applicable law +regarding the Licensor, the Licence and the way it may be accessible, concluded, +stored and reproduced by the Licensee. + +12. Termination of the Licence + +The Licence and the rights granted hereunder will terminate automatically upon +any breach by the Licensee of the terms of the Licence. + +Such a termination will not terminate the licences of any person who has +received the Work from the Licensee under the Licence, provided such persons +remain in full compliance with the Licence. + +13. Miscellaneous + +Without prejudice of Article 9 above, the Licence represents the complete +agreement between the Parties as to the Work. + +If any provision of the Licence is invalid or unenforceable under applicable +law, this will not affect the validity or enforceability of the Licence as a +whole. Such provision will be construed or reformed so as necessary to make it +valid and enforceable. + +The European Commission may publish other linguistic versions or new versions of +this Licence or updated versions of the Appendix, so far this is required and +reasonable, without reducing the scope of the rights granted by the Licence. New +versions of the Licence will be published with a unique version number. + +All linguistic versions of this Licence, approved by the European Commission, +have identical value. Parties can take advantage of the linguistic version of +their choice. + +14. Jurisdiction + +Without prejudice to specific agreement between parties, + +- any litigation resulting from the interpretation of this License, arising + between the European Union institutions, bodies, offices or agencies, as a + Licensor, and any Licensee, will be subject to the jurisdiction of the Court + of Justice of the European Union, as laid down in article 272 of the Treaty on + the Functioning of the European Union, + +- any litigation arising between other parties and resulting from the + interpretation of this License, will be subject to the exclusive jurisdiction + of the competent court where the Licensor resides or conducts its primary + business. + +15. Applicable Law + +Without prejudice to specific agreement between parties, + +- this Licence shall be governed by the law of the European Union Member State + where the Licensor has his seat, resides or has his registered office, + +- this licence shall be governed by Belgian law if the Licensor has no seat, + residence or registered office inside a European Union Member State. + +Appendix + +‘Compatible Licences’ according to Article 5 EUPL are: + +- GNU General Public License (GPL) v. 2, v. 3 +- GNU Affero General Public License (AGPL) v. 3 +- Open Software License (OSL) v. 2.1, v. 3.0 +- Eclipse Public License (EPL) v. 1.0 +- CeCILL v. 2.0, v. 2.1 +- Mozilla Public Licence (MPL) v. 2 +- GNU Lesser General Public Licence (LGPL) v. 2.1, v. 3 +- Creative Commons Attribution-ShareAlike v. 3.0 Unported (CC BY-SA 3.0) for + works other than software +- European Union Public Licence (EUPL) v. 1.1, v. 1.2 +- Québec Free and Open-Source Licence — Reciprocity (LiLiQ-R) or Strong + Reciprocity (LiLiQ-R+). + +The European Commission may update this Appendix to later versions of the above +licences without producing a new version of the EUPL, as long as they provide +the rights granted in Article 2 of this Licence and protect the covered Source +Code from exclusive appropriation. + +All other changes or additions to this Appendix require the production of a new +EUPL version. diff --git a/dns_handler.go b/dns_handler.go index 9f06219..fe12b9f 100644 --- a/dns_handler.go +++ b/dns_handler.go @@ -277,7 +277,7 @@ func (mydns *handler) ServeDNS(w dns.ResponseWriter, r *dns.Msg) { } } - if domainInKillfile(fqdn, config) { + if !domainInWhiteListfile(fqdn, config) && domainInKillfile(fqdn, config) { go incrementStats("Killed", 1) msg.Answer = append(msg.Answer, &dns.A{ diff --git a/hostfile.go b/hostfile.go index 63def50..b89dd84 100644 --- a/hostfile.go +++ b/hostfile.go @@ -11,28 +11,29 @@ func init() { fmt.Println("Ingesting local hosts file") ingestLocalBlacklists() + ingestLocalWhiteLists() } func ingestLocalBlacklists() { fmt.Println("ingestLocalBlacklist: collecting urls from all configs...") - _files := urlsMap{} + _HostsFiles := urlsMap{} for config := range ZabovConfigs { ZabovHostsFile := ZabovConfigs[config].ZabovHostsFile if len(ZabovHostsFile) == 0 { continue } - configs := _files[ZabovHostsFile] + configs := _HostsFiles[ZabovHostsFile] if configs == nil { configs = stringarray{} - _files[ZabovHostsFile] = configs + _HostsFiles[ZabovHostsFile] = configs } configs = append(configs, config) - _files[ZabovHostsFile] = configs + _HostsFiles[ZabovHostsFile] = configs } - for ZabovHostsFile, configs := range _files { + for ZabovHostsFile, configs := range _HostsFiles { file, err := os.Open(ZabovHostsFile) if err != nil { fmt.Println(err.Error()) @@ -57,6 +58,49 @@ func ingestLocalBlacklists() { } +func ingestLocalWhiteLists() { + + fmt.Println("ingestLocalWhiteLists: collecting urls from all configs...") + _WhiteListFiles := urlsMap{} + for config := range ZabovConfigs { + ZabovWhiteList := ZabovConfigs[config].ZabovWhiteList + if len(ZabovWhiteList) == 0 { + continue + } + configs := _WhiteListFiles[ZabovWhiteList] + if configs == nil { + configs = stringarray{} + _WhiteListFiles[ZabovWhiteList] = configs + } + configs = append(configs, config) + _WhiteListFiles[ZabovWhiteList] = configs + } + + for ZabovWhiteList, configs := range _WhiteListFiles { + file, err := os.Open(ZabovWhiteList) + if err != nil { + fmt.Println(err.Error()) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + d := scanner.Text() + if len(d) == 0 || strings.TrimSpace(d)[0] == '#' { + continue + } + DomainWhiteList(d, ZabovWhiteList, configs) + incrementStats("WhiteList", 1) + + } + + if err := scanner.Err(); err != nil { + fmt.Println(err.Error()) + } + } + +} + func fileByLines(filename string) (blurls []string) { file, err := os.Open(filename) diff --git a/main.go b/main.go index b57fb60..618ba40 100644 --- a/main.go +++ b/main.go @@ -39,6 +39,7 @@ type ZabovConfig struct { ZabovDoubleBL string // json:doublefilters -> ZabovDoubleBL list of urls returning a file with IPdomain ZabovAddBL net.IP // json:blackholeip -> ZabovAddBL is the IP we want to send all the clients to. Usually is 127.0.0.1 ZabovHostsFile string // json:hostsfile -> ZabovHostsFile is the file we use to keep our hosts + ZabovWhiteList string // json:hostsfile -> ZabovWhiteList is the file we use to keep white listed hosts ZabovUpDNS string // json:upstream -> ZabovUpDNS keeps the name of upstream DNSs ZabovDNSArray []string // contains all the DNS we mention, parsed from ZabovUpDNS file ZabovCache bool // allows to disable cache diff --git a/stats.py b/stats.py new file mode 100644 index 0000000..83a1bcb --- /dev/null +++ b/stats.py @@ -0,0 +1,95 @@ +#!/bin/env python3 + +import os +import glob +import argparse + + +parser = argparse.ArgumentParser(description='Zabov logs analyzer') + +parser.add_argument('--ip', dest="ip", metavar='IP', type=str, + help='filter by source IP (substring to match). Default: any') + +parser.add_argument('--action', dest="action", metavar='action', type=str, default="killed", + help='filter action (substring to match): killed|forwarded|any. Default: killed') + +parser.add_argument('--config', dest="config", metavar='name', type=str, + help='filter by config name (substring to match). Default: any') + +parser.add_argument('--timetable', dest="timetable", metavar='name', type=str, + help='filter by timetable name (substring to match). Default: any') + +parser.add_argument('--reqtype', dest="reqtype", metavar='TypeA', type=str, default="TypeA", + help='filter by reqtype name (substring to match): TypeA|TypeAAAA|TypeMX|...') + +parser.add_argument('--domain', dest="domain", metavar='name', type=str, + help='filter by domain name (substring to match). Default: all') + +parser.add_argument('--min-entries', dest="minentries", metavar='100', type=int, default=0, + help='filter output by minimum number of entries. Default: any') + +parser.add_argument('--logs-path', dest="logs", metavar='path', type=str, default="./config/logs", + help='Zabov logs path') + + + +args = parser.parse_args() + + +timetables = {} +configs = {} + +killed = {} +for x in glob.glob(os.path.join(args.logs, "*.log")): + #print (x) + f = open(x, "r") + f.readline() + if args.reqtype: + args.reqtype = args.reqtype.lower() + if args.domain: + args.domain = args.domain.lower() + + for line in f.readlines(): + linel = line.strip().lower() + fields = linel.split("\t") + + timetables[fields[5]] = timetables.get(fields[5], 0) +1 + configs[fields[4]] = configs.get(fields[4], 0) +1 + + ok = all((not args.action or fields[6].find(args.action)>=0 or fields[6] == "any", \ + not args.timetable or fields[5].find(args.timetable)>=0 or fields[5] == "any", \ + not args.config or fields[4].find(args.config)>=0 or fields[4] == "any", \ + not args.ip or fields[1].find(args.ip)>=0 or fields[1] == "any",\ + not args.domain or fields[2].find(args.domain)>=0 or fields[2] == "any", \ + not args.reqtype or fields[3].find(args.reqtype)>=0 )) + if ok: + killed[fields[2]] = killed.get(fields[2], 0) +1 + +killed_sorted = {key: value for key, value in sorted(killed.items(), key=lambda item: item[1], reverse=True)} + +total_queries_filtered = 0 +total_domain_filtered = 0 +total_queries = 0 +for k in killed_sorted.keys(): + if args.minentries == 0 or killed[k] >= args.minentries: + print (k, killed[k]) + total_queries_filtered += killed[k] + total_domain_filtered+=1 + total_queries += killed[k] + +print("") +print("TOTAL domains (filtered):", total_domain_filtered ) +print("TOTAL queries (filtred):", total_queries_filtered ) +print("TOTAL domains:", len(killed_sorted.keys()) ) +print("TOTAL queries:", total_queries ) + + +timetables = {key: value for key, value in sorted(timetables.items(), key=lambda item: item[0], reverse=False)} +configs = {key: value for key, value in sorted(configs.items(), key=lambda item: item[0], reverse=False)} + +print("all available timetables:") +for k in timetables.keys(): + print(" '%s': %d items" % (k, timetables[k], )) +print("all available configs:") +for k in configs.keys(): + print(" '%s': %d items" % (k, configs[k], ))