field_plcount.log

=== RUN   TestPosingsListSanity
PWD:  /Users/shaad/bleve_issue/zapx
PRJECT_ROOT:  /Users/shaad/bleve_issue
FIELD_NAME:  category_id
PL_COUNT:  959934
FIELD_NAME:  product_code
PL_COUNT:  250472
FIELD_NAME:  product_current_quantity
PL_COUNT:  959935
FIELD_NAME:  product_desc
FIELD_NAME:  product_id
PL_COUNT:  959935
FIELD_NAME:  product_name
PL_COUNT:  495197
FIELD_NAME:  product_promo
FIELD_NAME:  product_remark
PL_COUNT:  1910930
FIELD_NAME:  product_status
PL_COUNT:  932634
FIELD_NAME:  productidentifierid
PL_COUNT:  909778
FIELD_NAME:  qr_code
FIELD_NAME:  sequence_no
PL_COUNT:  959929
--- PASS: TestPosingsListSanity (1.02s)
PASS
ok  	github.com/blevesearch/zapx/v16	1.517s

my_test.go

package zap

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	segment "github.com/blevesearch/scorch_segment_api/v2"
	"github.com/blevesearch/vellum"
	"github.com/davecgh/go-spew/spew"
)

func openSegment(t *testing.T) segment.Segment {
	// zap plugin instance for testing only
	var zapPlugin = &ZapPlugin{}

	pwd, err := os.Getwd()
	if err != nil {
		t.Fatal(err)
	}
	fmt.Println("PWD: ", pwd)

	projectRoot := pwd
	// get index of last /

	idx := strings.LastIndex(projectRoot, "/")
	projectRoot = projectRoot[:idx]

	fmt.Println("PRJECT_ROOT: ", projectRoot)
	zapFilePath := filepath.Join(projectRoot, "moss", "store", "00000000000e.zap")

	segment, err := zapPlugin.Open(zapFilePath)
	if err != nil {
		t.Fatal(err)
	}

	return segment
}

func TestIssue209(t *testing.T) {
	segment := openSegment(t)
	defer func() {
		cerr := segment.Close()
		if cerr != nil {
			t.Fatalf("error closing segment: %v", cerr)
		}
	}()

	zapSegment, ok := segment.(*Segment)
	if !ok {
		t.Fatalf("segment is not SegmentBase, it is %T", segment)
	}

	fmt.Println("chunkMode: ", zapSegment.chunkMode)
	fmt.Println("fieldsMap: ", spew.Sdump(zapSegment.fieldsMap))
	fmt.Println("fieldsInv: ", spew.Sdump(zapSegment.fieldsInv))
	fmt.Println("fieldsSectionsMap: ", spew.Sdump(zapSegment.fieldsSectionsMap))
	fmt.Println("numDocs: ", zapSegment.numDocs)
	fmt.Println("storedIndexOffset: ", zapSegment.storedIndexOffset)
	fmt.Println("fieldsIndexOffset: ", zapSegment.fieldsIndexOffset)
	fmt.Println("sectionsIndexOffset: ", zapSegment.sectionsIndexOffset)
	fmt.Println("docValueOffset: ", zapSegment.docValueOffset)
	fmt.Println("dictLocs: ", spew.Sdump(zapSegment.dictLocs))
	fmt.Println("fieldDvReaders: ", spew.Sdump(zapSegment.fieldDvReaders))
	fmt.Println("fieldDvNames: ", spew.Sdump(zapSegment.fieldDvNames))
	fmt.Println("size: ", zapSegment.size)
}

func TestPosingsListSanity(t *testing.T) {
	segment := openSegment(t)
	defer func() {
		cerr := segment.Close()
		if cerr != nil {
			t.Fatalf("error closing segment: %v", cerr)
		}
	}()

	zapSegment, ok := segment.(*Segment)
	if !ok {
		t.Fatalf("segment is not SegmentBase, it is %T", segment)
	}

	for i := 2; i < len(zapSegment.fieldsInv); i++ {
		fieldName := zapSegment.fieldsInv[i]
		fmt.Println("FIELD_NAME: ", fieldName)

		td, err := segment.Dictionary(fieldName)
		if err != nil {
			t.Fatal(fmt.Errorf("error getting dictionary for field %s: %v", fieldName, err))
		}

		d, ok := td.(*Dictionary)
		if !ok {
			t.Fatalf("dictionary is not Dictionary, it is %T", td)
		}

		itr, err := d.fst.Iterator(nil, nil)
		if err != nil {
			if err == vellum.ErrIteratorDone {
				continue
			}
			t.Fatal(fmt.Errorf("error creating iterator for field %s: %v", fieldName, err))
		}
		defer itr.Close()

		var plCount uint64
		for err := itr.Next(); err == nil; err = itr.Next() {
			key, _ := itr.Current()

			_, exist, err := d.fst.Get(key)
			if err != nil {
				t.Fatal(fmt.Errorf("error getting postings offset for key %s: %v", key, err))
			}
			if !exist {
				continue
			}

			pl, err := td.PostingsList(key, nil, nil)
			if err != nil {
				t.Fatal(fmt.Errorf("error getting postings list for key %s: %v", key, err))
			}

			plCount += pl.Count()
		}
		fmt.Println("PL_COUNT: ", plCount)
	}
}