Handle zero chunk size

.github/workflows/tests.yml

@@ -2,6 +2,7 @@ on:
   push:
     branches:
     - master
+    - v15.x
     - v14.x
     - v13.x
     - v12.x
@@ -12,7 +13,7 @@ jobs:
   test:
     strategy:
       matrix:
-        go-version: [1.18.x, 1.19.x, 1.20.x]
+        go-version: [1.20.x, 1.21.x, 1.22.x]
         platform: [ubuntu-latest, macos-latest]
     runs-on: ${{ matrix.platform }}
     steps:

build.go

@@ -24,8 +24,8 @@ import (
 	"github.com/blevesearch/vellum"
 )
 
-const Version uint32 = 15
-
+const Version uint32 = 16
+const IndexSectionsVersion uint32 = 16
 const Type string = "zap"
 
 const fieldNotUninverted = math.MaxUint64
@@ -98,7 +98,7 @@ func persistSegmentBaseToWriter(sb *SegmentBase, w io.Writer) (int, error) {
 		return 0, err
 	}
 
-	err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset,
+	err = persistFooter(sb.numDocs, sb.storedIndexOffset, sb.fieldsIndexOffset, sb.sectionsIndexOffset,
 		sb.docValueOffset, sb.chunkMode, sb.memCRC, br)
 	if err != nil {
 		return 0, err
@@ -159,25 +159,33 @@ func persistStoredFieldValues(fieldID int,
 
 func InitSegmentBase(mem []byte, memCRC uint32, chunkMode uint32,
 	fieldsMap map[string]uint16, fieldsInv []string, numDocs uint64,
-	storedIndexOffset uint64, fieldsIndexOffset uint64, docValueOffset uint64,
-	dictLocs []uint64) (*SegmentBase, error) {
+	storedIndexOffset uint64, dictLocs []uint64,
+	sectionsIndexOffset uint64) (*SegmentBase, error) {
 	sb := &SegmentBase{
-		mem:               mem,
-		memCRC:            memCRC,
-		chunkMode:         chunkMode,
-		fieldsMap:         fieldsMap,
-		fieldsInv:         fieldsInv,
-		numDocs:           numDocs,
-		storedIndexOffset: storedIndexOffset,
-		fieldsIndexOffset: fieldsIndexOffset,
-		docValueOffset:    docValueOffset,
-		dictLocs:          dictLocs,
-		fieldDvReaders:    make(map[uint16]*docValueReader),
-		fieldFSTs:         make(map[uint16]*vellum.FST),
+		mem:                 mem,
+		memCRC:              memCRC,
+		chunkMode:           chunkMode,
+		fieldsMap:           fieldsMap,
+		numDocs:             numDocs,
+		storedIndexOffset:   storedIndexOffset,
+		fieldsIndexOffset:   sectionsIndexOffset,
+		sectionsIndexOffset: sectionsIndexOffset,
+		fieldDvReaders:      make([]map[uint16]*docValueReader, len(segmentSections)),
+		docValueOffset:      0, // docValueOffsets identified automatically by the section
+		dictLocs:            dictLocs,
+		fieldFSTs:           make(map[uint16]*vellum.FST),
+		vecIndexCache:       newVectorIndexCache(),
 	}
 	sb.updateSize()
 
-	err := sb.loadDvReaders()
+	// load the data/section starting offsets for each field
+	// by via the sectionsIndexOffset as starting point.
+	err := sb.loadFieldsNew()
+	if err != nil {
+		return nil, err
+	}
+
+	err = sb.loadDvReaders()
 	if err != nil {
 		return nil, err
 	}

chunk.go

@@ -15,6 +15,7 @@
 package zap
 
 import (
+	"errors"
 	"fmt"
 )
 
@@ -26,6 +27,13 @@ var LegacyChunkMode uint32 = 1024
 // be used by default.
 var DefaultChunkMode uint32 = 1026
 
+var ErrChunkSizeZero = errors.New("chunk size is zero")
+
+// getChunkSize returns the chunk size for the given chunkMode, cardinality, and
+// maxDocs.
+//
+// In error cases, the returned chunk size will be 0. Caller can differentiate
+// between a valid chunk size of 0 and an error by checking for ErrChunkSizeZero.
 func getChunkSize(chunkMode uint32, cardinality uint64, maxDocs uint64) (uint64, error) {
 	switch {
 	// any chunkMode <= 1024 will always chunk with chunkSize=chunkMode
@@ -46,6 +54,9 @@ func getChunkSize(chunkMode uint32, cardinality uint64, maxDocs uint64) (uint64,
 		// chunk-size items.
 		// no attempt is made to tweak any other case
 		if cardinality <= 1024 {
+			if maxDocs == 0 {
+				return 0, ErrChunkSizeZero
+			}
 			return maxDocs, nil
 		}
 		return 1024, nil
@@ -61,6 +72,9 @@ func getChunkSize(chunkMode uint32, cardinality uint64, maxDocs uint64) (uint64,
 		// 2.  convert to chunkSize, dividing into maxDocs
 		numChunks := (cardinality / 1024) + 1
 		chunkSize := maxDocs / numChunks
+		if chunkSize == 0 {
+			return 0, ErrChunkSizeZero
+		}
 		return chunkSize, nil
 	}
 	return 0, fmt.Errorf("unknown chunk mode %d", chunkMode)

cmd/zap/cmd/dict.go

@@ -21,7 +21,7 @@ import (
 
 	"github.com/RoaringBitmap/roaring"
 	"github.com/blevesearch/vellum"
-	zap "github.com/blevesearch/zapx/v15"
+	zap "github.com/blevesearch/zapx/v16"
 	"github.com/spf13/cobra"
 )