update

Author: jamiesun

Corefile

@@ -1,25 +1,14 @@
 . {
-    dnssrc locals.conf {
-        expire 30s
-        path_reload 3s
-        max_fails 0
-        health_check 30s
-        to 114.114.114.114 223.5.5.5
-        policy round_robin
-        bootstrap 172.21.66.1
+    datahub {
         debug
+        geoip_path data/geoip.dat
+        geosite_path data/geosite.dat
+        geoip_cache cn hk jp google apple
+        geosite_cache cn hk jp private apple
+        geodat_upgrade_url http://xxxx.com
+        geodat_upgrade_cron 0 30 0 * * *
+        keyword_table cn data/keyword_cn.txt
+        reload @every 3s
     }
 
-    dnssrc 172.21.66.137 192.168.0.1/24 {
-        expire 1s
-        path_reload 3s
-        max_fails 0
-        health_check 30s
-        to json-doh://dns.google/resolve
-        to 1.1.1.1
-        to 9.9.9.9
-        policy round_robin
-        bootstrap 172.21.66.1
-        debug
-    }
 }

data/keyword_cn.txt

@@ -0,0 +1 @@
+cn:baidu

go.mod

@@ -3,11 +3,14 @@ module github.com/ca17/datahub
 go 1.16
 
 require (
+	github.com/allegro/bigcache v1.2.1
 	github.com/c-robinson/iplib v1.0.3
+	github.com/ca17/dnssrc v0.0.2
 	github.com/coredns/caddy v1.1.1
 	github.com/coredns/coredns v1.8.6
-	github.com/m13253/dns-over-https/v2 v2.3.0
-	github.com/mdlayher/netlink v1.4.2
+	github.com/digineo/go-ipset/v2 v2.2.1 // indirect
+	github.com/golang/protobuf v1.5.2
 	github.com/miekg/dns v1.1.43
-	github.com/prometheus/client_golang v1.11.0
+	github.com/robfig/cron/v3 v3.0.1
+	google.golang.org/protobuf v1.27.1
 )

go.sum

@@ -1,7 +1,7 @@
 package main
 
 import (
-	_ "github.com/ca17/dnssrc/plugin/dnssrc"
+	_ "github.com/ca17/datahub/plugin/datahub"
 	"github.com/coredns/coredns/core/dnsserver"
 	_ "github.com/coredns/coredns/core/plugin"
 	"github.com/coredns/coredns/coremain"

main.go

@@ -0,0 +1,81 @@
+package datahub
+
+import (
+	"net"
+	"strings"
+
+	"github.com/c-robinson/iplib"
+	"github.com/ca17/datahub/plugin/pkg/loader"
+	"github.com/ca17/datahub/plugin/pkg/v2data"
+)
+
+// LoadGeoIPFromDAT 载入 GEOIP 数据表
+func (dh *Datahub) LoadGeoIPFromDAT(tag string) (*v2data.GeoIP, error) {
+	return loader.LoadGeoIPFromDAT(dh.geoipPath, tag)
+}
+
+// LoadGeoSiteFromDAT 载入 GEOSITE 表
+func (dh *Datahub) LoadGeoSiteFromDAT(country string) (*v2data.GeoSite, error) {
+	return loader.LoadGeoSiteFromDAT(dh.geositePath, country)
+}
+
+func (dh *Datahub) MatchGeoip(tag string, ip net.IP) bool {
+	tag = strings.ToUpper(tag)
+	dh.nlmLock.RLock()
+	defer dh.nlmLock.RUnlock()
+	inet := iplib.NewNet(ip, 32)
+	if list, ok := dh.geoipNetListMap[tag]; ok {
+		return list.MatchNet(inet)
+	}
+	return false
+}
+
+func (dh *Datahub) MatchGeoNet(tag string, net iplib.Net) bool {
+	tag = strings.ToUpper(tag)
+	dh.nlmLock.RLock()
+	defer dh.nlmLock.RUnlock()
+	if list, ok := dh.geoipNetListMap[tag]; ok {
+		return list.MatchNet(net)
+	}
+	return false
+}
+
+// MatchGeosite 匹配 Geosite 域名
+func (dh *Datahub) MatchGeosite(matchType, tag string, name string) bool {
+	tag = strings.ToUpper(tag)
+	dh.dlmLock.RLock()
+	defer dh.dlmLock.RUnlock()
+	if list, ok := dh.geositeDoaminListMap[tag]; ok {
+		return list.Match(matchType, name)
+	}
+	return false
+}
+
+func (dh *Datahub) MatchKeyword(tag string, name string) bool {
+	tag = strings.ToUpper(tag)
+	dh.ktLock.RLock()
+	defer dh.ktLock.RUnlock()
+	if list := dh.getKeywordTableByTag(tag); list!=nil {
+		return list.Match(name)
+	}
+	return false
+}
+
+// MixMatch 混合模式匹配域名
+func (dh *Datahub) MixMatch(tag string, name string) bool {
+	tag = strings.ToUpper(tag)
+
+	if list := dh.getDomainListByTag(tag); list!=nil {
+		if list.MixMatch(name) {
+			return true
+		}
+	}
+
+	if list := dh.getKeywordTableByTag(tag); list!=nil {
+		if list.Match(name) {
+			return true
+		}
+	}
+
+	return false
+}

plugin/datahub/dataapi.go

@@ -0,0 +1,174 @@
+package datahub
+
+import (
+	"context"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/allegro/bigcache"
+	"github.com/c-robinson/iplib"
+	"github.com/ca17/datahub/plugin/pkg/datatable"
+	"github.com/ca17/datahub/plugin/pkg/loader"
+	"github.com/ca17/datahub/plugin/pkg/netutils"
+	"github.com/ca17/datahub/plugin/pkg/v2data"
+	"github.com/coredns/coredns/plugin"
+	"github.com/miekg/dns"
+	"github.com/robfig/cron/v3"
+)
+
+var cronParser = cron.NewParser(
+	cron.SecondOptional | cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow | cron.Descriptor,
+)
+
+type Datahub struct {
+	nlmLock              sync.RWMutex
+	dlmLock              sync.RWMutex
+	ktLock               sync.RWMutex
+	Next                 plugin.Handler
+	geoipCacheTags       []string
+	geositeCacheTags     []string
+	geoipNetListMap      map[string]*netutils.NetList
+	geositeDoaminListMap map[string]*netutils.DomainList
+	keywordTableMap      map[string]*datatable.DataTable
+	geoipPath            string
+	geositePath          string
+	geodatUpgradeUrl     string
+	geodatUpgradeCron    string
+	sched                *cron.Cron
+	matchCache           *bigcache.BigCache
+	reloadCron           string
+	debug                bool
+}
+
+func (dh *Datahub) ServeDNS(ctx context.Context, w dns.ResponseWriter, r *dns.Msg) (int, error) {
+	return plugin.NextOrFailure(dh.Name(), dh.Next, ctx, w, r)
+}
+
+func (dh *Datahub) Name() string { return "datahub" }
+
+func NewDatahub() *Datahub {
+	mc, _ := bigcache.NewBigCache(bigcache.DefaultConfig(time.Second * 3600))
+	return &Datahub{
+		nlmLock:              sync.RWMutex{},
+		dlmLock:              sync.RWMutex{},
+		geoipNetListMap:      make(map[string]*netutils.NetList),
+		geositeDoaminListMap: make(map[string]*netutils.DomainList),
+		keywordTableMap:      make(map[string]*datatable.DataTable),
+		matchCache:           mc,
+		sched:                cron.New(cron.WithParser(cronParser)),
+	}
+}
+
+func (dh *Datahub) getDomainListByTag(tag string) *netutils.DomainList{
+	dh.dlmLock.RLock()
+	defer dh.dlmLock.RUnlock()
+	return dh.geositeDoaminListMap[tag]
+}
+
+func (dh *Datahub) getNetListByTag(tag string) *netutils.NetList{
+	dh.nlmLock.RLock()
+	defer dh.nlmLock.RUnlock()
+	return dh.geoipNetListMap[tag]
+}
+
+func (dh *Datahub) getKeywordTableByTag(tag string) *datatable.DataTable{
+	dh.ktLock.RLock()
+	defer dh.ktLock.RUnlock()
+	return dh.keywordTableMap[tag]
+}
+
+// 根据 tag 从 geoip.dat 加载 geoip 数据
+func (dh *Datahub) reloadGeoipNetListByTag(tags []string, cache bool) error {
+	if !cache {
+		loader.RemoveCache(dh.geoipPath)
+	}
+	tagitems, err := loader.LoadGeoIPFromDATByTags(dh.geoipPath, tags)
+	if err != nil {
+		return err
+	}
+	dh.nlmLock.Lock()
+	defer dh.nlmLock.Unlock()
+	for _, dataitems := range tagitems {
+		var nets []iplib.Net
+		for _, data := range dataitems.GetCidr() {
+			_net := iplib.NewNet(data.GetIp(), int(data.GetPrefix()))
+			nets = append(nets, _net)
+		}
+
+		dh.geoipNetListMap[dataitems.GetCountryCode()] = netutils.NewNetList(nets)
+
+	}
+	return nil
+}
+
+// 根据 tag 从 geosite.dat 加载 geosite 数据
+func (dh *Datahub) reloadGeositeDmoainListByTag(tags []string, cache bool) error {
+	if !cache {
+		loader.RemoveCache(dh.geositePath)
+	}
+	tagitems, err := loader.LoadGeoSiteFromDATByTags(dh.geositePath, tags)
+	if err != nil {
+		return err
+	}
+	dh.dlmLock.Lock()
+	defer dh.dlmLock.Unlock()
+	for _, dataitems := range tagitems {
+		var sites []string
+		var regexs []string
+		for _, data := range dataitems.GetDomain() {
+			switch data.Type {
+			case v2data.Domain_Full, v2data.Domain_Domain:
+				sites = append(sites, data.GetValue())
+			case v2data.Domain_Regex:
+				regexs = append(regexs, data.GetValue())
+			}
+		}
+		dmlist := netutils.NewDomainList()
+		dmlist.InitDomainData(netutils.MatchFullType, sites)
+		dmlist.InitDomainData(netutils.MatchRegexType, regexs)
+		dh.geositeDoaminListMap[dataitems.GetCountryCode()] = dmlist
+	}
+	return nil
+}
+
+func (dh *Datahub) parseKeywordTableByTag(tag string, from string) error {
+	tag = strings.ToUpper(tag)
+	table, err := datatable.NewFromArgs(datatable.DateTypeKeywordTable, tag, from )
+	if err != nil {
+		return err
+	}
+	table.LoadAll()
+	dh.ktLock.Lock()
+	defer dh.ktLock.Unlock()
+	dh.keywordTableMap[tag] = table
+	return nil
+}
+
+func (dh *Datahub) OnStartup() error {
+	dh.startSched()
+	return nil
+}
+
+func (dh *Datahub) OnShutdown() error {
+	dh.stopSched()
+	return nil
+}
+
+func (dh *Datahub) debugPrint() {
+	log.Info("geoip_path ", dh.geositePath)
+	log.Info("geosite_path ", dh.geositePath)
+	log.Info("geoip_cache ", dh.geoipCacheTags)
+	log.Info("geosite_cache ", dh.geositeCacheTags)
+	log.Info("geodat_upgrade_url ", dh.geodatUpgradeUrl)
+	log.Info("geodat_upgrade_cron ", dh.geodatUpgradeCron)
+	for k, v := range dh.geoipNetListMap {
+		log.Infof("geoip_cache %s total %d", k, v.Len())
+	}
+	for k, v := range dh.geositeDoaminListMap {
+		log.Infof("geosite_cache %s full_domain:%d regex_domain:%d", k, v.FullLen(), v.RegexLen())
+	}
+	for k, v := range dh.keywordTableMap {
+		log.Infof("keyword_table %s total %d",k, v.Len())
+	}
+}