From 2966f4faa5c1fa95761452d87d325ce30537e79d Mon Sep 17 00:00:00 2001 From: Ryo Kanbayashi Date: Fri, 23 Aug 2024 13:19:42 +0900 Subject: [PATCH] updated bltree-go-for-embedding lib and modified testcases which uses the lib. --- lib/container/btree/bltree_wrapper.go | 41 ------------------- lib/container/btree/btree_iterator.go | 7 ++-- lib/container/btree/btree_test.go | 32 +++++++-------- .../btree_index_executor_test.go | 1 + lib/go.mod | 2 +- lib/go.sum | 4 +- lib/storage/index/btree_index.go | 28 ++++++------- server/go.mod | 2 +- server/go.sum | 4 +- 9 files changed, 40 insertions(+), 81 deletions(-) delete mode 100644 lib/container/btree/bltree_wrapper.go diff --git a/lib/container/btree/bltree_wrapper.go b/lib/container/btree/bltree_wrapper.go deleted file mode 100644 index a2512153..00000000 --- a/lib/container/btree/bltree_wrapper.go +++ /dev/null @@ -1,41 +0,0 @@ -package btree - -import ( - "github.com/ryogrid/SamehadaDB/lib/types" - blink_tree "github.com/ryogrid/bltree-go-for-embedding" -) - -type BLTreeWrapper struct { - *blink_tree.BLTree - bufMgr *blink_tree.BufMgr -} - -func NewBLTreeWrapper(bltree *blink_tree.BLTree, bufMgr *blink_tree.BufMgr) *BLTreeWrapper { - return &BLTreeWrapper{bltree, bufMgr} -} - -func (bltw *BLTreeWrapper) GetValue(key *types.Value) uint64 { - // TODO: (SDB) need to implement this - panic("Not implemented yet") -} - -func (bltw *BLTreeWrapper) Insert(key *types.Value, value uint64) (err error) { - // TODO: (SDB) need to implement this - panic("Not implemented yet") -} - -func (bltw *BLTreeWrapper) Remove(key *types.Value, value uint64) (isDeleted_ bool) { - // TODO: (SDB) need to implement this - panic("Not implemented yet") -} - -func (bltw *BLTreeWrapper) Iterator(rangeStartKey *types.Value, rangeEndKey *types.Value) *BTreeIterator { - // TODO: (SDB) need to implement this - panic("Not implemented yet") -} - -// call this at shutdown of the system -// to write out the state and allocated pages of the container to BPM -func (bltw *BLTreeWrapper) WriteOutContainerStateToBPM() { - bltw.bufMgr.Close() -} diff --git a/lib/container/btree/btree_iterator.go b/lib/container/btree/btree_iterator.go index 4f2a9f06..13840682 100644 --- a/lib/container/btree/btree_iterator.go +++ b/lib/container/btree/btree_iterator.go @@ -7,10 +7,11 @@ import ( "github.com/ryogrid/SamehadaDB/lib/storage/page" "github.com/ryogrid/SamehadaDB/lib/storage/page/skip_list_page" "github.com/ryogrid/SamehadaDB/lib/types" + blink_tree "github.com/ryogrid/bltree-go-for-embedding" ) type BTreeIterator struct { - bltw *BLTreeWrapper + bltr *blink_tree.BLTree bpm *buffer.BufferPoolManager curNode *skip_list_page.SkipListBlockPage curEntry *index_common.IndexEntry @@ -21,7 +22,7 @@ type BTreeIterator struct { curEntryIdx int32 } -func NewSkipListIterator(bltr *BLTreeWrapper, rangeStartKey *types.Value, rangeEndKey *types.Value) *BTreeIterator { +func NewSkipListIterator(bltr *blink_tree.BLTree, rangeStartKey *types.Value, rangeEndKey *types.Value) *BTreeIterator { ret := new(BTreeIterator) // TODO: (SDB) need to implement this @@ -41,7 +42,7 @@ func NewSkipListIterator(bltr *BLTreeWrapper, rangeStartKey *types.Value, rangeE return ret } -func (itr *BTreeIterator) initRIDList(bltw *BLTreeWrapper) { +func (itr *BTreeIterator) initRIDList(bltr *blink_tree.BLTree) { // TODO: (SDB) need to implement this panic("Not implemented yet") } diff --git a/lib/container/btree/btree_test.go b/lib/container/btree/btree_test.go index 273e1568..ea805d63 100644 --- a/lib/container/btree/btree_test.go +++ b/lib/container/btree/btree_test.go @@ -112,7 +112,7 @@ func TestBLTree_deleteMany_embedding(t *testing.T) { } for i := range keys { - if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk { + if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk { t.Errorf("InsertKey() = %v, want %v", err, blink_tree.BLTErrOk) } if i%2 == 0 { @@ -128,8 +128,8 @@ func TestBLTree_deleteMany_embedding(t *testing.T) { t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i]) } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) } } } @@ -154,7 +154,7 @@ func TestBLTree_deleteAll_embedding(t *testing.T) { } for i := range keys { - if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk { + if err := bltree.InsertKey(keys[i], 0, [blink_tree.BtId]byte{0, 0, 0, 0, 0, 0, 0, 0}, true); err != blink_tree.BLTErrOk { t.Errorf("InsertKey() = %v, want %v", err, blink_tree.BLTErrOk) } } @@ -216,8 +216,8 @@ func TestBLTree_deleteManyConcurrently_embedding(t *testing.T) { panic("FindKey() != -1") } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) panic("FindKey() != 6") } } @@ -245,8 +245,8 @@ func TestBLTree_deleteManyConcurrently_embedding(t *testing.T) { t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i]) } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) } } } @@ -330,8 +330,8 @@ func TestBLTree_deleteInsertRangeScanConcurrently_embedding(t *testing.T) { } rangeScanCheck(keys[i]) } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) panic("FindKey() != 6") } rangeScanCheck(keys[i]) @@ -363,8 +363,8 @@ func TestBLTree_deleteInsertRangeScanConcurrently_embedding(t *testing.T) { } } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) } } } @@ -428,8 +428,8 @@ func TestBLTree_deleteManyConcurrentlyShuffle_embedding(t *testing.T) { panic("FindKey() != -1") } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) panic("FindKey() != 6") } } @@ -457,8 +457,8 @@ func TestBLTree_deleteManyConcurrentlyShuffle_embedding(t *testing.T) { t.Errorf("FindKey() = %v, want %v, key %v", found, -1, keys[i]) } } else { - if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 6 { - t.Errorf("FindKey() = %v, want %v, key %v", found, 6, keys[i]) + if found, _, _ := bltree.FindKey(keys[i], blink_tree.BtId); found != 8 { + t.Errorf("FindKey() = %v, want %v, key %v", found, 8, keys[i]) } } } diff --git a/lib/execution/executors/executor_test/btree_index_executor_test.go b/lib/execution/executors/executor_test/btree_index_executor_test.go index 577d2114..67f7ffa1 100644 --- a/lib/execution/executors/executor_test/btree_index_executor_test.go +++ b/lib/execution/executors/executor_test/btree_index_executor_test.go @@ -95,6 +95,7 @@ func testKeyDuplicateInsertDeleteWithBTreeIndex[T float32 | int32 | string](t *t testingpkg.Assert(t, len(result) == 0, "duplicated key point scan got illegal results.") txnMgr.Commit(c, txn) + shi.Shutdown(samehada.ShutdownPatternCloseFiles) } func TestKeyDuplicateInsertDeleteWithBTreeIndexInt(t *testing.T) { diff --git a/lib/go.mod b/lib/go.mod index c4771b60..af9cf18f 100644 --- a/lib/go.mod +++ b/lib/go.mod @@ -12,7 +12,7 @@ require ( github.com/notEpsilon/go-pair v0.0.0-20221220200415-e91ef28c6c0b github.com/pingcap/parser v0.0.0-20200623164729-3a18f1e5dceb github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c - github.com/ryogrid/bltree-go-for-embedding v1.0.2 + github.com/ryogrid/bltree-go-for-embedding v1.0.3 github.com/spaolacci/murmur3 v1.1.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 ) diff --git a/lib/go.sum b/lib/go.sum index d828bd85..9bc6ff4e 100644 --- a/lib/go.sum +++ b/lib/go.sum @@ -448,8 +448,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryogrid/bltree-go-for-embedding v1.0.2 h1:GquFei9pmPy4GK9YBGwXITcJFCnRjWTX+BDpaIVxJLI= -github.com/ryogrid/bltree-go-for-embedding v1.0.2/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= +github.com/ryogrid/bltree-go-for-embedding v1.0.3 h1:EvhpPa8UeD+BD/hM46qbqbWCQMLRAqHE0A4l/hywamU= +github.com/ryogrid/bltree-go-for-embedding v1.0.3/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= diff --git a/lib/storage/index/btree_index.go b/lib/storage/index/btree_index.go index 9e7a969d..efbc1ba9 100644 --- a/lib/storage/index/btree_index.go +++ b/lib/storage/index/btree_index.go @@ -16,27 +16,30 @@ import ( ) type BTreeIndex struct { - container *btree.BLTreeWrapper + container *blink_tree.BLTree metadata *IndexMetadata // idx of target column on table col_idx uint32 log_manager *recovery.LogManager // UpdateEntry only get Write lock updateMtx sync.RWMutex + // for call of Close method .... + bufMgr *blink_tree.BufMgr } -func NewBTreeIndex(metadata *IndexMetadata, buffer_pool_manager *buffer.BufferPoolManager, col_idx uint32, log_manager *recovery.LogManager) *BTreeIndex { +func NewBTreeIndex(metadata *IndexMetadata, buffer_pool_manager *buffer.BufferPoolManager, col_idx uint32, log_manager *recovery.LogManager, lastPageZeroId *int32) *BTreeIndex { ret := new(BTreeIndex) ret.metadata = metadata // BTreeIndex uses special technique to support key duplication with SkipList supporting unique key only // for the thechnique, key type is fixed to Varchar (comparison is done on dict order as byte array) - bufMgr := blink_tree.NewBufMgr(12, blink_tree.HASH_TABLE_ENTRY_CHAIN_LEN*common.MaxTxnThreadNum*2, btree.NewParentBufMgrImpl(buffer_pool_manager), nil) - ret.container = btree.NewBLTreeWrapper(blink_tree.NewBLTree(bufMgr), bufMgr) + bufMgr := blink_tree.NewBufMgr(12, blink_tree.HASH_TABLE_ENTRY_CHAIN_LEN*common.MaxTxnThreadNum*2, btree.NewParentBufMgrImpl(buffer_pool_manager), lastPageZeroId) + ret.container = blink_tree.NewBLTree(bufMgr) ret.col_idx = col_idx ret.updateMtx = sync.RWMutex{} ret.log_manager = log_manager + ret.bufMgr = bufMgr return ret } @@ -63,11 +66,6 @@ func (btidx *BTreeIndex) deleteEntryInner(key *tuple.Tuple, rid page.RID, txn in convedKeyVal := samehada_util.EncodeValueAndRIDToDicOrderComparableVarchar(&orgKeyVal, &rid) - //revertedOrgKey := samehada_util.ExtractOrgKeyFromDicOrderComparableEncodedVarchar(convedKeyVal, orgKeyVal.ValueType()) - //if !revertedOrgKey.CompareEquals(orgKeyVal) { - // panic("key conversion may fail!") - //} - if isNoLock == false { btidx.updateMtx.RLock() defer btidx.updateMtx.RUnlock() @@ -146,12 +144,12 @@ func (btidx *BTreeIndex) GetTupleSchema() *schema.Schema { func (btidx *BTreeIndex) GetKeyAttrs() []uint32 { return btidx.metadata.GetKeyAttrs() } -//func (slidx *BTreeIndex) GetHeaderPageId() types.PageID { -// return slidx.container.GetHeaderPageId() -//} +func (slidx *BTreeIndex) GetHeaderPageId() types.PageID { + return types.PageID(slidx.bufMgr.GetMappedPPageIdOfPageZero()) +} // call this at shutdown of the system -// to write out the state and allocated pages of the container to BPM -func (btidx *BTreeIndex) Close() { - btidx.container.WriteOutContainerStateToBPM() +// to write out the state and allocated pages of the BLTree container to BPM +func (btidx *BTreeIndex) WriteOutContainerStateToBPM() { + btidx.bufMgr.Close() } diff --git a/server/go.mod b/server/go.mod index c5cd7567..55274b90 100644 --- a/server/go.mod +++ b/server/go.mod @@ -30,7 +30,7 @@ require ( github.com/pingcap/tidb v1.1.0-beta.0.20200630082100-328b6d0a955c // indirect github.com/pingcap/tipb v0.0.0-20200522051215-f31a15d98fce // indirect github.com/remyoudompheng/bigfft v0.0.0-20190728182440-6a916e37a237 // indirect - github.com/ryogrid/bltree-go-for-embedding v1.0.2 // indirect + github.com/ryogrid/bltree-go-for-embedding v1.0.3 // indirect github.com/shirou/gopsutil v2.19.10+incompatible // indirect github.com/sirupsen/logrus v1.6.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/server/go.sum b/server/go.sum index ddb6bbca..e25a07be 100644 --- a/server/go.sum +++ b/server/go.sum @@ -450,8 +450,8 @@ github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/ryogrid/bltree-go-for-embedding v1.0.2 h1:GquFei9pmPy4GK9YBGwXITcJFCnRjWTX+BDpaIVxJLI= -github.com/ryogrid/bltree-go-for-embedding v1.0.2/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= +github.com/ryogrid/bltree-go-for-embedding v1.0.3 h1:EvhpPa8UeD+BD/hM46qbqbWCQMLRAqHE0A4l/hywamU= +github.com/ryogrid/bltree-go-for-embedding v1.0.3/go.mod h1:IjwQznZH7W6JZiFGk4vwDbNgLbgPODUzjlRgKQgCIFE= github.com/sasha-s/go-deadlock v0.2.0/go.mod h1:StQn567HiB1fF2yJ44N9au7wOhrPS3iZqiDbRupzT10= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sergi/go-diff v1.0.1-0.20180205163309-da645544ed44/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=