forked from loweel/zabov
129 lines
2.5 KiB
Go
129 lines
2.5 KiB
Go
package main
|
|
|
|
import (
|
|
"bufio"
|
|
"errors"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
func init() {
|
|
go downloadThread()
|
|
}
|
|
|
|
//SingleIndexFilter puts the domains inside file
|
|
func SingleIndexFilter(durl string, configs stringarray) error {
|
|
|
|
fmt.Println("Retrieving DomainFile from: ", durl)
|
|
|
|
// resets malformed HostLines for url
|
|
setstatsvalue("Malformed DomainLines "+durl, 0)
|
|
|
|
var err error
|
|
|
|
// Get the data
|
|
resp, err := http.Get(durl)
|
|
if err != nil {
|
|
fmt.Println("HTTP Problem: ", err)
|
|
return err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode == 200 { // OK
|
|
fmt.Println(durl + " Response: OK")
|
|
} else {
|
|
fmt.Println("Server <"+durl+"> returned status code: ", resp.StatusCode)
|
|
return errors.New("Server <" + durl + "> returned status code: " + resp.Status)
|
|
}
|
|
|
|
scanner := bufio.NewScanner(resp.Body)
|
|
splitter := func(c rune) bool {
|
|
return c == ' ' || c == '\t'
|
|
}
|
|
|
|
var numLines int64
|
|
|
|
for scanner.Scan() {
|
|
|
|
line := scanner.Text()
|
|
|
|
if len(line) == 0 || strings.TrimSpace(line)[0] == '#' {
|
|
continue
|
|
}
|
|
h := strings.FieldsFunc(line, splitter)
|
|
|
|
if h == nil {
|
|
continue
|
|
}
|
|
|
|
if len(h) < 1 {
|
|
continue
|
|
}
|
|
|
|
if !strings.Contains(h[0], "#") {
|
|
|
|
DomainKill(h[0], durl, configs)
|
|
|
|
// fmt.Println("MATCH: ", h[1])
|
|
numLines++
|
|
} else {
|
|
incrementStats("Malformed DomainLines "+durl, 1)
|
|
// fmt.Println("Malformed line: <" + line + ">")
|
|
}
|
|
|
|
}
|
|
|
|
fmt.Println("Finished to parse: "+durl+" ,number of lines", numLines)
|
|
|
|
return err
|
|
|
|
}
|
|
|
|
func getSingleFilters(urls urlsMap) {
|
|
|
|
fmt.Println("getSingleFilters: downloading all urls:", len(urls))
|
|
for url, configs := range urls {
|
|
SingleIndexFilter(url, configs)
|
|
}
|
|
fmt.Println("getSingleFilters: DONE!")
|
|
|
|
}
|
|
|
|
func downloadThread() {
|
|
fmt.Println("Starting updater of SINGLE lists, each (hours): ", ZabovKillTTL)
|
|
time.Sleep(2 * time.Second) // wait for local DNS server up & running (may be our DNS)
|
|
|
|
_urls := urlsMap{}
|
|
|
|
for {
|
|
fmt.Println("downloadThread: collecting urls from all configs...")
|
|
for config := range ZabovConfigs {
|
|
ZabovSingleBL := ZabovConfigs[config].ZabovSingleBL
|
|
|
|
if len(ZabovSingleBL) == 0 {
|
|
continue
|
|
}
|
|
|
|
s := fileByLines(ZabovSingleBL)
|
|
for _, v := range s {
|
|
if len(v) == 0 || strings.TrimSpace(v)[0] == '#' {
|
|
continue
|
|
}
|
|
configs := _urls[v]
|
|
if configs == nil {
|
|
configs = stringarray{}
|
|
_urls[v] = configs
|
|
}
|
|
configs = append(configs, config)
|
|
_urls[v] = configs
|
|
}
|
|
}
|
|
|
|
getSingleFilters(_urls)
|
|
time.Sleep(time.Duration(ZabovKillTTL) * time.Hour)
|
|
}
|
|
|
|
}
|