Testnet shard reduction (#4301)

* reduce shard of testnet to 2
* remove harmony inner nodes in shard2 and shard3
* check shardID in VerifyCrossLink
* get NumShards by epoch instead of hardcodeed shards
* add Localnet in NewGenesisSpec
* do mayTestnetShardReduction in the block before the last block
* fix localnet epoch calculation
* not use old committee if shard reduction happens
* fix preLastBlock calculation
* do testnet shard reduction when propose new blocks
* enable 2s block period at epoch 0
* call IntermediateRoot once in the end
* skip inactive&baned validators when testnet shard reduction happens
* update ShardReductionEpoch to epoch 486

core/genesis.go

@@ -113,6 +113,8 @@ func NewGenesisSpec(netType nodeconfig.NetworkType, shardID uint32) *Genesis {
 		chainConfig = *params.PartnerChainConfig
 	case nodeconfig.Stressnet:
 		chainConfig = *params.StressnetChainConfig
+	case nodeconfig.Localnet:
+		chainConfig = *params.LocalnetChainConfig
 	default: // all other types share testnet config
 		chainConfig = *params.TestChainConfig
 	}

core/state_processor.go

@@ -31,9 +31,11 @@ import (
 	"github.com/harmony-one/harmony/core/state"
 	"github.com/harmony-one/harmony/core/types"
 	"github.com/harmony-one/harmony/core/vm"
+	nodeconfig "github.com/harmony-one/harmony/internal/configs/node"
 	"github.com/harmony-one/harmony/internal/params"
 	"github.com/harmony-one/harmony/internal/utils"
 	"github.com/harmony-one/harmony/shard"
+	"github.com/harmony-one/harmony/staking/effective"
 	"github.com/harmony-one/harmony/staking/slash"
 	staking "github.com/harmony-one/harmony/staking/types"
 	"github.com/pkg/errors"
@@ -175,6 +177,10 @@ func (p *StateProcessor) Process(
 		}
 	}
 
+	if err := MayTestnetShardReduction(p.bc, statedb, header); err != nil {
+		return nil, nil, nil, nil, 0, nil, statedb, err
+	}
+
 	// Finalize the block, applying any consensus engine specific extras (e.g. block rewards)
 	sigsReady := make(chan bool)
 	go func() {
@@ -425,3 +431,50 @@ func StakingToMessage(
 	msg.SetType(types.StakingTypeMap[stkType])
 	return msg, nil
 }
+
+// MayTestnetShardReduction handles the change in the number of Shards. It will mark the affected validator as inactive.
+// This function does not handle all cases, only for ShardNum from 4 to 2.
+func MayTestnetShardReduction(bc ChainContext, statedb *state.DB, header *block.Header) error {
+	isBeaconChain := header.ShardID() == shard.BeaconChainShardID
+	isLastBlock := shard.Schedule.IsLastBlock(header.Number().Uint64())
+	isTestnet := nodeconfig.GetDefaultConfig().GetNetworkType() == nodeconfig.Testnet
+	if !(isTestnet && isBeaconChain && isLastBlock) {
+		return nil
+	}
+	curInstance := shard.Schedule.InstanceForEpoch(header.Epoch())
+	nextEpoch := big.NewInt(header.Epoch().Int64() + 1)
+	nextInstance := shard.Schedule.InstanceForEpoch(nextEpoch)
+	curNumShards := curInstance.NumShards()
+	nextNumShards := nextInstance.NumShards()
+
+	if curNumShards == nextNumShards {
+		return nil
+	}
+
+	if curNumShards != 4 && nextNumShards != 2 {
+		return errors.New("can only handle the reduction from 4 to 2")
+	}
+	addresses, err := bc.ReadValidatorList()
+	if err != nil {
+		return err
+	}
+	for _, address := range addresses {
+		validator, err := statedb.ValidatorWrapper(address, true, false)
+		if err != nil {
+			return err
+		}
+		if validator.Status == effective.Inactive || validator.Status == effective.Banned {
+			continue
+		}
+		for _, pubKey := range validator.SlotPubKeys {
+			curShard := new(big.Int).Mod(pubKey.Big(), big.NewInt(int64(curNumShards))).Uint64()
+			nextShard := new(big.Int).Mod(pubKey.Big(), big.NewInt(int64(nextNumShards))).Uint64()
+			if curShard >= uint64(nextNumShards) || curShard != nextShard {
+				validator.Status = effective.Inactive
+				break
+			}
+		}
+	}
+	statedb.IntermediateRoot(bc.Config().IsS3(header.Epoch()))
+	return nil
+}

internal/configs/sharding/localnet.go

@@ -19,7 +19,7 @@ type localnetSchedule struct{}
 const (
 	localnetV1Epoch = 1
 
-	localnetEpochBlock1      = 10
+	localnetEpochBlock1      = 5
 	localnetBlocksPerEpoch   = 5
 	localnetBlocksPerEpochV2 = 10
 
@@ -57,27 +57,23 @@ func (ls localnetSchedule) twoSecondsFirstBlock() uint64 {
 }
 
 func (ls localnetSchedule) CalcEpochNumber(blockNum uint64) *big.Int {
-	blocks := ls.BlocksPerEpochOld()
-	var oldEpochNumber int64
-	switch {
-	case blockNum >= localnetEpochBlock1:
-		oldEpochNumber = int64((blockNum-localnetEpochBlock1)/blocks) + 1
-	default:
-		oldEpochNumber = 0
-	}
-
 	firstBlock2s := ls.twoSecondsFirstBlock()
-
 	switch {
-	case params.LocalnetChainConfig.IsTwoSeconds(big.NewInt(oldEpochNumber)):
-		return big.NewInt(int64((blockNum-firstBlock2s)/ls.BlocksPerEpoch() + params.LocalnetChainConfig.TwoSecondsEpoch.Uint64()))
-	default: // genesis
-		return big.NewInt(oldEpochNumber)
+	case blockNum < localnetEpochBlock1:
+		return big.NewInt(0)
+	case blockNum < firstBlock2s:
+		return big.NewInt(int64((blockNum-localnetEpochBlock1)/ls.BlocksPerEpochOld() + 1))
+	default:
+		extra := uint64(0)
+		if firstBlock2s == 0 {
+			blockNum -= localnetEpochBlock1
+			extra = 1
+		}
+		return big.NewInt(int64(extra + (blockNum-firstBlock2s)/ls.BlocksPerEpoch() + params.LocalnetChainConfig.TwoSecondsEpoch.Uint64()))
 	}
 }
 
 func (ls localnetSchedule) IsLastBlock(blockNum uint64) bool {
-	blocks := ls.BlocksPerEpochOld()
 	switch {
 	case blockNum < localnetEpochBlock1-1:
 		return false
@@ -87,25 +83,34 @@ func (ls localnetSchedule) IsLastBlock(blockNum uint64) bool {
 		firstBlock2s := ls.twoSecondsFirstBlock()
 		switch {
 		case blockNum >= firstBlock2s:
+			if firstBlock2s == 0 {
+				blockNum -= localnetEpochBlock1
+			}
 			return ((blockNum-firstBlock2s)%ls.BlocksPerEpoch() == ls.BlocksPerEpoch()-1)
 		default: // genesis
+			blocks := ls.BlocksPerEpochOld()
 			return ((blockNum-localnetEpochBlock1)%blocks == blocks-1)
 		}
 	}
 }
 
 func (ls localnetSchedule) EpochLastBlock(epochNum uint64) uint64 {
-	blocks := ls.BlocksPerEpochOld()
 	switch {
 	case epochNum == 0:
 		return localnetEpochBlock1 - 1
 	default:
-		firstBlock2s := ls.twoSecondsFirstBlock()
 		switch {
 		case params.LocalnetChainConfig.IsTwoSeconds(big.NewInt(int64(epochNum))):
-			return firstBlock2s - 1 + ls.BlocksPerEpoch()*(epochNum-params.LocalnetChainConfig.TwoSecondsEpoch.Uint64()+1)
+			blocks := ls.BlocksPerEpoch()
+			firstBlock2s := ls.twoSecondsFirstBlock()
+			block2s := (1 + epochNum - params.LocalnetChainConfig.TwoSecondsEpoch.Uint64()) * blocks
+			if firstBlock2s == 0 {
+				return block2s - blocks + localnetEpochBlock1 - 1
+			}
+			return firstBlock2s + block2s - 1
 		default: // genesis
-			return localnetEpochBlock1 - 1 + blocks*epochNum
+			blocks := ls.BlocksPerEpochOld()
+			return localnetEpochBlock1 + blocks*epochNum - 1
 		}
 	}
 }

internal/configs/sharding/localnet_test.go

@@ -0,0 +1,50 @@
+package shardingconfig
+
+import (
+	"math/big"
+	"testing"
+
+	"github.com/harmony-one/harmony/internal/params"
+)
+
+func TestLocalnetEpochCalculation(t *testing.T) {
+	check := func(epoch, expected uint64) {
+		if got := LocalnetSchedule.EpochLastBlock(epoch); got != expected {
+			t.Fatalf("wrong EpochLastBlock at epoch %d. TwoSecondsEpoch: %s. expected: %d got: %d.", epoch, params.LocalnetChainConfig.TwoSecondsEpoch.String(), expected, got)
+		}
+		if !LocalnetSchedule.IsLastBlock(expected) {
+			t.Fatalf("%d is not LastBlock. TwoSecondsEpoch: %s", expected, params.LocalnetChainConfig.TwoSecondsEpoch.String())
+		}
+		epochStart := uint64(0)
+		if epoch > 0 {
+			epochStart = LocalnetSchedule.EpochLastBlock(epoch-1) + 1
+		}
+		for blockNo := epochStart; blockNo <= expected; blockNo++ {
+			if isLastBlock := LocalnetSchedule.IsLastBlock(blockNo); isLastBlock != (blockNo == expected) {
+				t.Fatalf("IsLastBlock for %d is wrong. TwoSecondsEpoch: %s. expected %v got %v", blockNo, params.LocalnetChainConfig.TwoSecondsEpoch.String(), blockNo == expected, isLastBlock)
+			}
+			got := LocalnetSchedule.CalcEpochNumber(blockNo).Uint64()
+			if got != epoch {
+				t.Fatalf("CalcEpochNumber for %d is wrong. TwoSecondsEpoch: %s. expected %d got %d", blockNo, params.LocalnetChainConfig.TwoSecondsEpoch.String(), epoch, got)
+			}
+		}
+	}
+	backup := params.LocalnetChainConfig.TwoSecondsEpoch
+	params.LocalnetChainConfig.TwoSecondsEpoch = big.NewInt(0)
+	check(0, localnetEpochBlock1-1)
+	check(1, localnetEpochBlock1+localnetBlocksPerEpochV2-1)
+	check(2, localnetEpochBlock1+localnetBlocksPerEpochV2*2-1)
+
+	params.LocalnetChainConfig.TwoSecondsEpoch = big.NewInt(1)
+	check(0, localnetEpochBlock1-1)
+	check(1, localnetEpochBlock1+localnetBlocksPerEpochV2-1)
+	check(2, localnetEpochBlock1+localnetBlocksPerEpochV2*2-1)
+
+	params.LocalnetChainConfig.TwoSecondsEpoch = big.NewInt(2)
+	check(0, localnetEpochBlock1-1)
+	check(1, localnetEpochBlock1+localnetBlocksPerEpoch-1)
+	check(2, localnetEpochBlock1+localnetBlocksPerEpoch+localnetBlocksPerEpochV2-1)
+	check(3, localnetEpochBlock1+localnetBlocksPerEpoch+localnetBlocksPerEpochV2*2-1)
+
+	params.LocalnetChainConfig.TwoSecondsEpoch = backup
+}

internal/configs/sharding/testnet.go

@@ -12,6 +12,8 @@ import (
 // configuration schedule.
 var TestnetSchedule testnetSchedule
 
+var ShardReductionEpoch = big.NewInt(486)
+
 type testnetSchedule struct{}
 
 const (
@@ -31,6 +33,8 @@ const (
 
 func (ts testnetSchedule) InstanceForEpoch(epoch *big.Int) Instance {
 	switch {
+	case epoch.Cmp(ShardReductionEpoch) >= 0:
+		return testnetV3
 	case params.TestnetChainConfig.IsTestnetNinetyPercent(epoch):
 		return testnetV2
 	case params.TestnetChainConfig.IsStaking(epoch):
@@ -112,4 +116,5 @@ var (
 	testnetV0 = MustNewInstance(4, 8, 8, 0, numeric.OneDec(), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch())
 	testnetV1 = MustNewInstance(4, 30, 8, 0.15, numeric.MustNewDecFromStr("0.70"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch())
 	testnetV2 = MustNewInstance(4, 30, 8, 0.15, numeric.MustNewDecFromStr("0.90"), genesis.TNHarmonyAccounts, genesis.TNFoundationalAccounts, emptyAllowlist, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch())
+	testnetV3 = MustNewInstance(2, 30, 8, 0.15, numeric.MustNewDecFromStr("0.90"), genesis.TNHarmonyAccountsV1, genesis.TNFoundationalAccounts, emptyAllowlist, testnetReshardingEpoch, TestnetSchedule.BlocksPerEpoch())
 )