refactor: small changes + unit tests

filter.go

@@ -89,7 +89,7 @@ func (c *filterBuilder[T]) Or(filters ...Filter) Filter {
 }
 
 func (c *filterBuilder[T]) Eq(index string, key string) Filter {
-	// fetch the IndexBlock and store it in the result set
+	// fetch the IndexBlock and index it in the result set
 	index_ := IndexName(index).Normalize()
 	idx, ok := c.indexes[index_]
 	if !ok {

filter_setup_test.go → filter_test.go

@@ -3,23 +3,25 @@ package ethwal
 import (
 	"context"
 	"fmt"
+	"math"
 	"os"
 	"path"
+	"testing"
 
 	"github.com/0xsequence/ethkit/go-ethereum/common"
-	"github.com/0xsequence/ethkit/go-ethereum/common/math"
 	"github.com/0xsequence/ethwal/storage"
 	"github.com/0xsequence/ethwal/storage/local"
+	"github.com/stretchr/testify/assert"
 )
 
 var (
 	indexTestDir = ".tmp/ethwal_index_test"
 )
 
-func setupMockData[T any](subDir string, indexGenerator func(fs storage.FS) Indexes[T], blockGenerator func() []Block[T]) (*IndexBuilder[T], Indexes[T], storage.FS, func(), error) {
+func setupMockData[T any](subDir string, indexGenerator func(fs storage.FS) Indexes[T], blockGenerator func() []Block[T]) (*Indexer[T], Indexes[T], storage.FS, func(), error) {
 	fs := local.NewLocalFS(path.Join(indexTestDir, subDir))
 	indexes := indexGenerator(fs)
-	indexBuilder, err := NewIndexBuilder(context.Background(), indexes)
+	indexBuilder, err := NewIndexer(context.Background(), indexes)
 	if err != nil {
 		return nil, nil, nil, nil, err
 	}
@@ -248,3 +250,142 @@ func indexAll(block Block[[]int]) (toIndex bool, indexValueMap map[IndexedValue]
 func indexNone(block Block[[]int]) (toIndex bool, indexValueMap map[IndexedValue][]uint16, err error) {
 	return false, nil, nil
 }
+
+func TestMaxMagicCompoundID(t *testing.T) {
+	id := NewIndexCompoundID(uint64(math.Exp2(48)-1), math.MaxUint16)
+	assert.Equal(t, uint64(math.Exp2(48)-1), id.BlockNumber())
+	assert.Equal(t, uint16(math.MaxUint16), id.DataIndex())
+}
+
+func TestIntMixFiltering(t *testing.T) {
+	_, indexes, _, cleanup, err := setupMockData("int_mix", generateMixedIntIndexes, generateMixedIntBlocks)
+	assert.NoError(t, err)
+	defer cleanup()
+
+	f, err := NewFilterBuilder(indexes)
+	assert.NoError(t, err)
+	assert.NotNil(t, f)
+
+	onlyEvenFilter := f.Eq("only_even", "true")
+	onlyOddFilter := f.Eq("only_odd", "true")
+	oddFilter := f.Eq("odd_even", "odd")
+	numbersIdxs := []string{
+		"121",
+		"123",
+		"125",
+		"999",
+		"777",
+		"333",
+		"555",
+		"111",
+	}
+	var numberFilter Filter
+	for _, number := range numbersIdxs {
+		if numberFilter == nil {
+			numberFilter = f.Eq("all", number)
+		} else {
+			numberFilter = f.Or(numberFilter, f.Eq("all", number))
+		}
+	}
+
+	onlyEvenResults := onlyEvenFilter.Eval()
+	assert.Len(t, onlyEvenResults.Bitmap().ToArray(), 20)
+	for _, id := range onlyEvenResults.Bitmap().ToArray() {
+		block, _ := IndexCompoundID(id).Split()
+		assert.True(t, block <= 20)
+	}
+
+	onlyOddResults := onlyOddFilter.Eval()
+	assert.Len(t, onlyOddResults.Bitmap().ToArray(), 20+20)
+	for _, id := range onlyOddResults.Bitmap().ToArray() {
+		block, _ := IndexCompoundID(id).Split()
+		assert.True(t, (block > 20 && block < 41) || (block > 50 && block < 71))
+	}
+
+	numberAllResults := numberFilter.Eval()
+	// 20*20
+	assert.Len(t, numberAllResults.Bitmap().ToArray(), 400)
+	for _, id := range numberAllResults.Bitmap().ToArray() {
+		block, _ := IndexCompoundID(id).Split()
+		assert.True(t, block > 50 && block < 71)
+	}
+
+	allNumberAndOdd := f.And(numberFilter, oddFilter)
+	allNumberOddResults := allNumberAndOdd.Eval()
+	assert.ElementsMatch(t, numberAllResults.Bitmap().ToArray(), allNumberOddResults.Bitmap().ToArray())
+}
+
+func TestFiltering(t *testing.T) {
+	_, indexes, _, cleanup, err := setupMockData("int_filtering", generateIntIndexes, generateIntBlocks)
+	assert.NoError(t, err)
+	defer cleanup()
+
+	f, err := NewFilterBuilder(indexes)
+	assert.NoError(t, err)
+	assert.NotNil(t, f)
+	result := f.Or(f.And(f.Eq("all", "1"), f.Eq("all", "2")), f.Eq("all", "3")).Eval()
+	// result should contain block 1, 2, 3
+	assert.Len(t, result.Bitmap().ToArray(), 3)
+	block, _ := result.Next()
+	assert.Equal(t, uint64(1), block)
+	block, _ = result.Next()
+	assert.Equal(t, uint64(2), block)
+	block, _ = result.Next()
+	assert.Equal(t, uint64(3), block)
+
+	result = f.And(f.Eq("all", "1"), f.Eq("all", "2")).Eval()
+	// result should contain block 1
+	assert.Len(t, result.Bitmap().ToArray(), 1)
+	block, _ = result.Next()
+	assert.Equal(t, uint64(1), block)
+}
+
+func TestLowestIndexedBlockNum(t *testing.T) {
+	indexer, indexes, fs, cleanup, err := setupMockData("int_indexing_num", generateIntIndexes, generateIntBlocks)
+	assert.NoError(t, err)
+	defer cleanup()
+
+	blockNum := indexer.BlockNum()
+	assert.Equal(t, uint64(99), blockNum)
+
+	for _, i := range indexes {
+		i.numBlocksIndexed = nil
+		block, err := i.LastBlockNumIndexed(context.Background())
+		assert.NoError(t, err)
+		assert.Equal(t, uint64(99), block)
+	}
+
+	indexes = generateIntIndexes(fs)
+	indexer, err = NewIndexer(context.Background(), indexes)
+	assert.NoError(t, err)
+	lowestBlockIndexed := indexer.BlockNum()
+	assert.Equal(t, uint64(99), lowestBlockIndexed)
+
+	// add another filter...
+	// indexes["odd_even"] = NewIndex("odd_even", indexOddEvenBlocks)
+	// setup fresh objects
+	indexes["odd_even"] = NewIndex("odd_even", indexOddEvenBlocks, fs)
+	indexer, err = NewIndexer(context.Background(), indexes)
+	assert.NoError(t, err)
+	lowestBlockIndexed = indexer.BlockNum()
+	assert.Equal(t, uint64(0), lowestBlockIndexed)
+	blocks := generateIntBlocks()
+	for _, block := range blocks[:50] {
+		err = indexer.Index(context.Background(), block)
+		assert.NoError(t, err)
+	}
+	err = indexer.Flush(context.Background())
+	assert.NoError(t, err)
+	lowestBlockIndexed = indexer.BlockNum()
+	assert.Equal(t, uint64(49), lowestBlockIndexed)
+
+	// index more blocks
+	for _, block := range blocks[50:] {
+		err = indexer.Index(context.Background(), block)
+		assert.NoError(t, err)
+	}
+	err = indexer.Flush(context.Background())
+	assert.NoError(t, err)
+	lowestBlockIndexed = indexer.BlockNum()
+	assert.Equal(t, uint64(99), lowestBlockIndexed)
+}

index.go

@@ -51,10 +51,10 @@ func (i IndexName) Normalize() IndexName {
 	return IndexName(strings.ToLower(string(i)))
 }
 
-// IndexedValue is the value of an index.
+// IndexedValue is the indexed value of an index.
 type IndexedValue string
 
-// IndexUpdate is a map of index values to bitmaps.
+// IndexUpdate is a map of indexed values and their corresponding bitmaps.
 type IndexUpdate struct {
 	Data         map[IndexedValue]*roaring64.Bitmap
 	LastBlockNum uint64
@@ -190,7 +190,7 @@ func (i *Index[T]) Store(ctx context.Context, indexUpdate *IndexUpdate) error {
 
 	err = i.storeLastBlockNumIndexed(ctx, indexUpdate.LastBlockNum)
 	if err != nil {
-		return fmt.Errorf("failed to store number of blocks indexed: %w", err)
+		return fmt.Errorf("failed to index number of blocks indexed: %w", err)
 	}
 
 	return nil
@@ -201,7 +201,7 @@ func (i *Index[T]) LastBlockNumIndexed(ctx context.Context) (uint64, error) {
 		return i.numBlocksIndexed.Load(), nil
 	}
 
-	file, err := i.fs.Open(ctx, lastBlockNumIndexedPath(string(i.name)), nil)
+	file, err := i.fs.Open(ctx, indexedBlockNumFilePath(string(i.name)), nil)
 	if err != nil {
 		// file doesn't exist
 		return 0, nil
@@ -236,7 +236,7 @@ func (i *Index[T]) storeLastBlockNumIndexed(ctx context.Context, numBlocksIndexe
 		return nil
 	}
 
-	file, err := i.fs.Create(ctx, lastBlockNumIndexedPath(string(i.name)), nil)
+	file, err := i.fs.Create(ctx, indexedBlockNumFilePath(string(i.name)), nil)
 	if err != nil {
 		return fmt.Errorf("failed to open IndexBlock file: %w", err)
 	}
@@ -258,6 +258,10 @@ func (i *Index[T]) storeLastBlockNumIndexed(ctx context.Context, numBlocksIndexe
 	return nil
 }
 
+func indexedBlockNumFilePath(index string) string {
+	return fmt.Sprintf("%s/%s", index, "indexed")
+}
+
 func indexPath(index string, indexValue string) string {
 	hash := sha256.Sum224([]byte(indexValue))
 	return fmt.Sprintf("%s/%06d/%06d/%06d/%s",
@@ -268,7 +272,3 @@ func indexPath(index string, indexValue string) string {
 		fmt.Sprintf("%s.idx", indexValue),                                // filename
 	)
 }
-
-func lastBlockNumIndexedPath(index string) string {
-	return fmt.Sprintf("%s/%s", index, "indexed")
-}