Skip to content

Commit

Permalink
COMPLETE project
Browse files Browse the repository at this point in the history
  • Loading branch information
@Metafora072
Metafora072 committed Aug 18, 2024
1 parent 4fd24ac commit 494b836
Show file tree
Hide file tree
Showing 13 changed files with 1,237 additions and 36 deletions.
14 changes: 14 additions & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,20 @@ project2b:
# $(TEST_CLEAN)


project2_multiTestsWithErrors:
$(TEST_CLEAN)
PASS_COUNT=0; \
for i in {1..30}; do \
RESULT=$$($(GOTEST) ./kv/test_raftstore -run ^$(TEST_NAME)$$ | grep -v "info" || true); \
echo "$$RESULT"; \
PASS_COUNT=$$(($$PASS_COUNT + $$(echo "$$RESULT" | grep -c "PASS"))); \
done; \
HALF_PASS_COUNT=$$(($$PASS_COUNT / 2)); \
GREEN='\033[0;32m'; \
NC='\033[0m'; \
echo -e "$$GREEN TOTAL tests: 30 $$NC"; \
echo -e "$$GREEN PASS tests: $$HALF_PASS_COUNT $$NC"
$(TEST_CLEAN)

project2c:
$(TEST_CLEAN)
Expand Down
6 changes: 5 additions & 1 deletion kv/raftstore/peer_msg_handler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func (d *peerMsgHandler) HandleRaftReady() {
// 调用 d.Send() 方法将 Ready 中的 Msg 发送出去
d.Send(d.ctx.trans, ready.Messages)

// 处理待 apply 的日志

if len(ready.CommittedEntries) > 0 {
entries := ready.CommittedEntries

Expand Down Expand Up @@ -435,6 +435,10 @@ func (d *peerMsgHandler) processAdminRequest(entry *pb.Entry,request *raft_cmdpb
meta.WriteRegionState(writeBatch,d.Region(),rspb.PeerState_Normal)
meta.WriteRegionState(writeBatch,newRegion,rspb.PeerState_Normal)

// TODO handle usage: test timed out after 10m0s
d.SizeDiffHint = 0
d.ApproximateSize = new(uint64)

// 根据参考文档提示，调用 createPeer 方法创建当前 store 上的 newRegion Peer，注册到 router，并启动
newRegionPeer, err := createPeer(d.storeID(),d.ctx.cfg,d.ctx.schedulerTaskSender,d.ctx.engine,newRegion)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion kv/raftstore/peer_storage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -406,7 +406,7 @@ func (ps *PeerStorage) SaveReadyState(ready *raft.Ready) (*ApplySnapResult, erro
var err error
var applySnapResult *ApplySnapResult
// 通过 raft.isEmptySnap() 方法判断是否存在 Snapshot，如果有，则调用ApplySnapshot() 方法应用；
// Snap TODO
// TODO apply snapshot
if raft.IsEmptySnap(&ready.Snapshot) == false {
kvWriteBatch := &engine_util.WriteBatch{}
applySnapResult,err = ps.ApplySnapshot(&ready.Snapshot,kvWriteBatch,writeBatch)
Expand Down
Binary file added note/TinyKV/doc/assets/KV系统逻辑结构.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added note/TinyKV/doc/assets/Project2Bpeer层结构图.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added note/TinyKV/doc/assets/Project2简化的cluster结构.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added note/TinyKV/doc/assets/project3逻辑结构.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added note/TinyKV/doc/assets/raft层三个模块之间的关系图.png
View file
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
1,189 changes: 1,183 additions & 6 deletions note/TinyKV/doc/项目总结.md
View file

Large diffs are not rendered by default.

38 changes: 18 additions & 20 deletions note/project3B.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -102,8 +102,6 @@
4. 更新 metaStore 中的 region 信息
5. 调用 removePeerCache 方法更新 peerCache
处理完AddNode/RemoveNode之后，我们根据参考文档提示，调用 raft.RawNode 的 ApplyConfChange()，目的是更新 raft 层的配置信息。
之后处理 proposal，即找到相应的回调，存入操作的执行结果（resp）。
Expand Down Expand Up @@ -173,24 +171,24 @@ d.Region().RegionEpoch.Version++

## 测试点反馈记录

| 测试点类型 | 测试点名称 | 测试情况 / 30TESTS | 错误类型 |
| :------------: | :----------------------------------------------------------: | :---------------------------------: | :----------------------------------------: |
| TransferLeader | TestTransferLeader3B | <font color=Green>**10PASS**</font> | |
| ConfChange | TestBasicConfChange3B | <font color=Green>**10PASS**</font> | |
| ConfChange | TestConfChangeRemoveLeader3B | <font color=Green>**10PASS**</font> | |
| ConfChange | TestConfChangeRecover3B | <font color=Green>**10PASS**</font> | |
| ConfChange | TestConfChangeRecoverManyClients3B | <font color=Green>**10PASS**</font> | |
| ConfChange | TestConfChangeUnreliable3B | <font color=Red>**0~2FAIL**</font> | request timeout |
| ConfChange | TestConfChangeUnreliableRecover3B | <font color=Red>**0~2FAIL**</font> | request timeout |
| ConfChange | TestConfChangeSnapshotUnreliableRecover3B | <font color=Red>**0~2FAIL**</font> | request timeout |
| ConfChange | TestConfChangeSnapshotUnreliableRecoverConcurrentPartition3B | <font color=Green>**10PASS**</font> | |
| Split | TestOneSplit3B | <font color=Green>**10PASS**</font> | |
| Split | TestSplitRecover3B | <font color=Green>**10PASS**</font> | |
| Split | TestSplitRecoverManyClients3B | <font color=Red>**10FAIL**</font> | test timed out after 10m0s |
| Split | TestSplitUnreliable3B | <font color=Red>**1FAIL**</font> | request timeout |
| Split | TestSplitUnreliableRecover3B | <font color=Green>**30PASS**</font> | |
| Split | TestSplitConfChangeSnapshotUnreliableRecover3B | <font color=Red>**2FAIL**</font> | request timeout |
| Split | TestSplitConfChangeSnapshotUnreliableRecoverConcurrentPartition3B | <font color=Green>**30PASS**</font> | request timeout key is not in region |
| 测试点类型 | 测试点名称 | 测试情况 / 30TESTS | 错误类型 | 调试记录 |
| :------------: | :----------------------------------------------------------: | :---------------------------------: | :---------------------------------------: | :----------------------------------------------------------: |
| TransferLeader | TestTransferLeader3B | <font color=Green>**30PASS**</font> | | |
| ConfChange | TestBasicConfChange3B | <font color=Green>**30PASS**</font> | | |
| ConfChange | TestConfChangeRemoveLeader3B | <font color=Green>**30PASS**</font> | | |
| ConfChange | TestConfChangeRecover3B | <font color=Green>**30PASS**</font> | | |
| ConfChange | TestConfChangeRecoverManyClients3B | <font color=Green>**30PASS**</font> | | |
| ConfChange | TestConfChangeUnreliable3B | <font color=Green>**30PASS**</font> | request timeout | raft的handlePropose方法将leadTransferee判断提到方法开始处，解决 |
| ConfChange | TestConfChangeUnreliableRecover3B | <font color=Green>**30PASS**</font> | request timeout | raft的handlePropose方法将leadTransferee判断提到方法开始处，解决 |
| ConfChange | TestConfChangeSnapshotUnreliableRecover3B | <font color=Green>**30PASS**</font> | request timeout | raft的handlePropose方法将leadTransferee判断提到方法开始处，解决 |
| ConfChange | TestConfChangeSnapshotUnreliableRecoverConcurrentPartition3B | <font color=Green>**30PASS**</font> | | |
| Split | TestOneSplit3B | <font color=Green>**30PASS**</font> | | |
| Split | TestSplitRecover3B | <font color=Green>**30PASS**</font> | | |
| Split | TestSplitRecoverManyClients3B | <font color=Red>**1FAIL**</font> | test timed out after 10m0s | 在split的时候更新approximateSize和Sizediffhint这俩字段，FAIL率大大降低（10->1） |
| Split | TestSplitUnreliable3B | <font color=Green>**30PASS**</font> | request timeout | raft的handlePropose方法将leadTransferee判断提到方法开始处，解决 |
| Split | TestSplitUnreliableRecover3B | <font color=Green>**30PASS**</font> | | snap复制指针，split判断peer不相等，解决key not in region |
| Split | TestSplitConfChangeSnapshotUnreliableRecover3B | <font color=Red>**2FAIL**</font> | request timeout & region is not split | |
| Split | TestSplitConfChangeSnapshotUnreliableRecoverConcurrentPartition3B | <font color=Green>**30PASS**</font> | | |

## 遇到的问题

Expand Down
2 changes: 2 additions & 0 deletions raft/log.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,8 @@ func newLog(storage Storage) *RaftLog {
applied: firstIndex - 1,
stabled: lastIndex,
entries: entries,
// TODO project2C
pendingSnapshot: nil,
dummyIndex: firstIndex,
}

Expand Down
15 changes: 10 additions & 5 deletions raft/raft.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,8 @@ func (r *Raft) tick() {
case StateLeader:
r.heartbeatElapsed++
if r.heartbeatElapsed >= r.heartbeatTimeout {
// TODO judge usage. reset heartbeatElapsed
// r.heartbeatElapsed = 0
// leader该发送心跳包
r.Step(pb.Message{
MsgType: pb.MessageType_MsgBeat,
Expand Down Expand Up @@ -759,6 +761,11 @@ func (r *Raft) handleHeartBeatResponse(m pb.Message) {

// handlePropose Leader 追加从上层应用接收到的新日志，并广播给 Follower
func (r *Raft) handlePropose(m pb.Message) {
// leader 处于领导权禅让，停止接收新的请求
if r.leadTransferee != None {
return
}

for idx := range m.Entries {
// 设置新的日志的索引和日期
m.Entries[idx].Term = r.Term
Expand All @@ -770,11 +777,6 @@ func (r *Raft) handlePropose(m pb.Message) {
}
r.RaftLog.appendEntry(m.Entries)

// leader 处于领导权禅让，停止接收新的请求
if r.leadTransferee != None {
return
}

// 更新节点日志复制进度 Progress
r.Prs[r.id].Match = r.RaftLog.LastIndex()
r.Prs[r.id].Next = r.RaftLog.LastIndex() + 1
Expand Down Expand Up @@ -897,6 +899,8 @@ func (r *Raft) handleAppendEntriesResponse(m pb.Message) {
}
return
}

// TODO 需不需要更新 Prs 字段的 match 和 next ？
// 同意同步
if r.Prs[m.From].handleUpdate(m.Index) {
// 更新了 Progress
Expand Down Expand Up @@ -952,6 +956,7 @@ func (r *Raft) handleHeartbeat(m pb.Message) {
} else {
// 成为发送心跳包的跟随者，信任其是 Leader
r.becomeFollower(m.Term, m.From)
// TODO：judge usage。重置 electionElapsed
}

r.msgs = append(r.msgs, message)
Expand Down
7 changes: 4 additions & 3 deletions raft/rawnode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,10 +220,11 @@ func (rn *RawNode) HasReady() bool {
hasReady = true
}

// TODO judge usage. needn't check this condition
// 检查 SoftState 是否有更新
if rn.isSoftStateUpdate() {
hasReady = true
}
//if rn.isSoftStateUpdate() {
// hasReady = true
//}

// 有需要持久化的条目
if len(rn.Raft.RaftLog.unstableEntries()) > 0 {
Expand Down

0 comments on commit 494b836

Please sign in to comment.