MB-58901: BM25 related constructs and API changes

.github/workflows/tests.yml

@@ -8,7 +8,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.19.x, 1.20.x, 1.21.x]
+        go-version: [1.20.x, 1.21.x, 1.22.x]
         platform: [ubuntu-latest, macos-latest, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:

go.mod

@@ -1,3 +1,3 @@
 module github.com/blevesearch/bleve_index_api
 
-go 1.20
+go 1.21

index.go

@@ -240,6 +240,7 @@ type FieldDict interface {
 	Next() (*DictEntry, error)
 	Close() error
 
+	Cardinality() int
 	BytesRead() uint64
 }
 

indexing_options.go

@@ -24,6 +24,21 @@ const (
 	SkipFreqNorm
 )
 
+const (
+	BM25Scoring  = "bm25"
+	TFIDFScoring = "tfidf"
+)
+
+// Scoring model indicates the algorithm used to rank documents fetched
+// for a query performed on a text field.
+const DefaultScoringModel = TFIDFScoring
+
+// Supported similarity models
+var SupportedScoringModels = map[string]struct{}{
+	BM25Scoring:  {},
+	TFIDFScoring: {},
+}
+
 func (o FieldIndexingOptions) IsIndexed() bool {
 	return o&IndexField != 0
 }

vector.go

@@ -37,10 +37,10 @@ const (
 	CosineSimilarity = "cosine"
 )
 
-const DefaultSimilarityMetric = EuclideanDistance
+const DefaultVectorSimilarityMetric = EuclideanDistance
 
 // Supported similarity metrics for vector fields
-var SupportedSimilarityMetrics = map[string]struct{}{
+var SupportedVectorSimilarityMetrics = map[string]struct{}{
 	EuclideanDistance: {},
 	InnerProduct:      {},
 	CosineSimilarity:  {},