reorganized everything and made bootstrap nodes accessible

bootstrap-nodes.txt

@@ -0,0 +1,3 @@
+router.bittorrent.com:6881
+dht.transmissionbt.com:6881
+dht.libtorrent.org:25401

cache/cache.go

@@ -0,0 +1,29 @@
+package cache
+
+import (
+	"dhtc/db"
+	"encoding/hex"
+	"fmt"
+	mapset "github.com/deckarep/golang-set/v2"
+	"github.com/ostafen/clover/v2"
+	"github.com/ostafen/clover/v2/query"
+	"github.com/rs/zerolog/log"
+)
+
+var InfoHashCache = mapset.NewSet[[20]byte]()
+
+func PopulateInfoHashCacheFromDatabase(database clover.DB) {
+	all, _ := database.FindAll(query.NewQuery(db.TorrentTable))
+	for _, d := range all {
+		ih := d.Get("InfoHash").(string)
+		h, err := hex.DecodeString(ih)
+		if len(h) != 20 || err != nil {
+			fmt.Print("x")
+			continue
+		}
+		h20 := (*[20]byte)(h)
+		InfoHashCache.Add(*h20)
+	}
+
+	log.Debug().Msgf("info hash cache size %d elements\n", InfoHashCache.Cardinality())
+}

config/configuration.go

@@ -0,0 +1,57 @@
+package config
+
+import (
+	"flag"
+	"time"
+)
+
+type Configuration struct {
+	DbName       string
+	Address      string
+	MaxNeighbors uint
+	MaxLeeches   int
+	DrainTimeout time.Duration
+
+	TelegramToken    string
+	TelegramUsername string
+
+	SafeMode bool
+
+	CrawlerThreads int
+
+	EnableBlacklist bool
+	NameBlacklist   string
+	FileBlacklist   string
+
+	Statistics bool
+
+	BootstrapNodeFile string
+}
+
+func ParseArguments() *Configuration {
+	config := Configuration{}
+
+	flag.StringVar(&config.DbName, "database", "dhtdb", "database name")
+	flag.StringVar(&config.Address, "address", ":4200", "address to run on")
+	flag.UintVar(&config.MaxNeighbors, "MaxNeighbors", 1000, "max. indexer neighbors")
+	flag.IntVar(&config.MaxLeeches, "MaxLeeches", 256, "max. leeches")
+	flag.DurationVar(&config.DrainTimeout, "DrainTimeout", 5*time.Second, "drain timeout")
+
+	flag.StringVar(&config.TelegramToken, "TelegramToken", "", "bot token for notifications")
+	flag.StringVar(&config.TelegramUsername, "TelegramUsername", "", "username to send notifications to")
+
+	flag.BoolVar(&config.SafeMode, "SafeMode", false, "start with safe mode enabled")
+	flag.IntVar(&config.CrawlerThreads, "CrawlerThreads", 5, "dht crawler threads")
+
+	flag.BoolVar(&config.EnableBlacklist, "EnableBlacklist", false, "enable blacklists")
+	flag.StringVar(&config.NameBlacklist, "NameBlacklist", "", "blacklist for torrent names")
+	flag.StringVar(&config.FileBlacklist, "FileBlacklist", "", "blacklist for file names")
+
+	flag.BoolVar(&config.Statistics, "Statistics", false, "enable Statistics (dashboard)")
+
+	flag.StringVar(&config.BootstrapNodeFile, "BootstrapNodeFile", "bootstrap-nodes.txt", "bootstrap nodes to use")
+
+	flag.Parse()
+
+	return &config
+}

db/blacklist.go

@@ -0,0 +1,92 @@
+package db
+
+import (
+	dhtcclient "dhtc/dhtc-client"
+	"fmt"
+	"github.com/ostafen/clover/v2"
+	"github.com/ostafen/clover/v2/document"
+	"github.com/ostafen/clover/v2/query"
+	"regexp"
+)
+
+func IsInBlacklist(db *clover.DB, filter string, entryType string) bool {
+	result, _ := db.FindAll(query.NewQuery(BlacklistTable).Where(query.Field("Filter").Eq(filter).And(query.Field("Type").Eq(entryType))))
+	return len(result) > 0
+}
+
+func AddToBlacklist(db *clover.DB, filters []string, entryType string) bool {
+	rVal := false
+
+	for _, filter := range filters {
+		if !IsInBlacklist(db, filter, entryType) {
+			fmt.Printf("Is not in blacklist yet: %s\n", filter)
+			doc := document.NewDocument()
+			doc.Set("Type", entryType)
+			doc.Set("Filter", filter)
+			_, err := db.InsertOne(BlacklistTable, doc)
+			if err != nil {
+				fmt.Println(err)
+				rVal = true
+				break
+			}
+		}
+	}
+
+	return rVal
+}
+
+func IsFileBlacklisted(md dhtcclient.Metadata, filter *regexp.Regexp) bool {
+	rVal := false
+	for i := 0; i < len(md.Files); i++ {
+		rVal = filter.MatchString(md.Files[i].Path)
+		if rVal {
+			break
+		}
+	}
+	return rVal
+}
+
+func IsBlacklisted(db *clover.DB, md dhtcclient.Metadata) bool {
+	all, _ := db.FindAll(query.NewQuery(BlacklistTable).MatchFunc(func(doc *document.Document) bool {
+		filterStr := doc.Get("Filter").(string)
+		filter := regexp.MustCompile(filterStr)
+
+		switch doc.Get("Type").(string) {
+		case "0":
+			return filter.MatchString(md.Name)
+		case "1":
+			return IsFileBlacklisted(md, filter)
+		}
+
+		return false
+	}))
+	return len(all) > 0
+}
+
+func DeleteBlacklistItem(db *clover.DB, itemId string) bool {
+	fmt.Println("Removing", itemId)
+	return db.DeleteById(BlacklistTable, itemId) == nil
+}
+
+func GetBlacklistTypeFromStrInt(entryType string) string {
+	switch entryType {
+	case "0":
+		return "Name"
+	case "1":
+		return "File name"
+	}
+	return "Unknown"
+}
+
+func GetBlacklistEntries(db *clover.DB) []BlacklistEntry {
+	all, _ := db.FindAll(query.NewQuery(BlacklistTable))
+	rVal := make([]BlacklistEntry, len(all))
+	for i, value := range all {
+		rVal[i] = BlacklistEntry{
+			value.ObjectId(),
+			value.Get("Filter").(string),
+			GetBlacklistTypeFromStrInt(value.Get("Type").(string)),
+		}
+	}
+	return rVal
+}

db/db.go

@@ -0,0 +1,76 @@
+package db
+
+import (
+	"dhtc/config"
+	dhtcclient "dhtc/dhtc-client"
+	"encoding/hex"
+	"fmt"
+	"github.com/ostafen/clover/v2"
+	"github.com/ostafen/clover/v2/document"
+	"github.com/ostafen/clover/v2/query"
+	"math/rand"
+)
+
+const WatchTable = "watches"
+const BlacklistTable = "blacklist"
+const TorrentTable = "torrents"
+
+func GetInfoHashCount(db *clover.DB) int {
+	vals, _ := db.FindAll(query.NewQuery(TorrentTable))
+	return len(vals)
+}
+
+func FindBy(db *clover.DB, key string, searchType string, searchInput string) []MetaData {
+	values, _ := db.FindAll(query.NewQuery(TorrentTable).MatchFunc(func(doc *document.Document) bool {
+		return Matches(doc, key, searchType, searchInput)
+	}))
+	return Documents2MetaData(values)
+}
+
+func GetNRandomEntries(db *clover.DB, N int) []MetaData {
+	count, _ := db.Count(query.NewQuery(TorrentTable))
+	if count < N {
+		N = count
+	}
+
+	all, _ := db.FindAll(query.NewQuery(TorrentTable))
+	rVal := make([]*document.Document, N)
+	for i := 0; i < N; i++ {
+		rVal[i] = all[rand.Intn(count)]
+	}
+	return Documents2MetaData(rVal)
+}
+
+func InsertMetadata(config *config.Configuration, db *clover.DB, md dhtcclient.Metadata) bool {
+	if config.EnableBlacklist && IsBlacklisted(db, md) {
+		fmt.Printf("Blacklisted: %s\n", md.Name)
+		return false
+	}
+
+	doc := document.NewDocument()
+	doc.Set("Name", md.Name)
+	doc.Set("InfoHash", hex.EncodeToString(md.InfoHash))
+	doc.Set("Files", md.Files)
+	doc.Set("DiscoveredOn", md.DiscoveredOn)
+	doc.Set("TotalSize", md.TotalSize)
+	_, err := db.InsertOne(TorrentTable, doc)
+	if err != nil {
+		return false
+	} else {
+		return true
+	}
+}
+
+func OpenDatabase(config *config.Configuration) *clover.DB {
+	db, err := clover.Open(config.DbName)
+
+	if err != nil {
+		fmt.Println("Error:", err)
+	}
+
+	_ = db.CreateCollection(TorrentTable)
+	_ = db.CreateCollection(WatchTable)
+	_ = db.CreateCollection(BlacklistTable)
+
+	return db
+}

db/helper.go

@@ -0,0 +1,81 @@
+package db
+
+import (
+	"fmt"
+	"github.com/ostafen/clover/v2/document"
+	"strings"
+	"time"
+)
+
+func Document2MetaData(value *document.Document) MetaData {
+	return MetaData{
+		value.Get("Name").(string),
+		value.Get("InfoHash").(string),
+		time.Unix(value.Get("DiscoveredOn").(int64), 0).Format(time.RFC822),
+		value.Get("TotalSize").(uint64),
+		value.Get("Files").([]interface{}),
+	}
+}
+
+func Documents2MetaData(values []*document.Document) []MetaData {
+	rVal := make([]MetaData, len(values))
+	for i, value := range values {
+		rVal[i] = Document2MetaData(value)
+	}
+	return rVal
+}
+
+func MatchString(searchType string, x string, y string) bool {
+	rVal := false
+	switch searchType {
+	case "contains":
+		rVal = strings.Contains(strings.ToLower(x), strings.ToLower(y))
+	case "equals":
+		rVal = strings.ToLower(x) == strings.ToLower(y)
+	case "startswith":
+		rVal = strings.HasPrefix(strings.ToLower(x), strings.ToLower(y))
+	case "endswith":
+		rVal = strings.HasSuffix(strings.ToLower(x), strings.ToLower(y))
+	default:
+		rVal = false
+	}
+	return rVal
+}
+
+func HasMatchingFile(fileList []interface{}, searchType string, searchInput string) bool {
+	rVal := false
+	for _, item := range fileList {
+		for key, value := range item.(map[string]interface{}) {
+			if key == "path" {
+				rVal = MatchString(searchType, value.(string), searchInput)
+			}
+		}
+	}
+	return rVal
+}
+
+func FoundOnDate(date time.Time, searchInput string) bool {
+	parsedInput, err := time.Parse("2006.01.02", searchInput)
+	if err != nil {
+		fmt.Println(err)
+		return false
+	}
+	endDate := parsedInput.AddDate(0, 0, 1)
+	return date.After(parsedInput) && date.Before(endDate)
+}
+
+func Matches(doc *document.Document, key string, searchType string, searchInput string) bool {
+	rVal := doc.Has(key)
+	if rVal {
+		value := doc.Get(key)
+
+		if key == "Files" {
+			rVal = HasMatchingFile(value.([]interface{}), searchType, searchInput)
+		} else if key == "DiscoveredOn" {
+			rVal = FoundOnDate(time.Unix(int64(value.(float64)), 0), searchInput)
+		} else {
+			rVal = MatchString(searchType, value.(string), searchInput)
+		}
+	}
+	return rVal
+}

db/types.go

@@ -0,0 +1,22 @@
+package db
+
+type WatchEntry struct {
+	Id        string
+	Key       string
+	MatchType string
+	Content   string
+}
+
+type BlacklistEntry struct {
+	Id     string
+	Filter string
+	Type   string
+}
+
+type MetaData struct {
+	Name         string
+	InfoHash     string
+	DiscoveredOn string
+	TotalSize    uint64
+	Files        []interface{}
+}