Skip to content

Commit

Permalink
Update FOCIL branch with master changes (Consensys#9037)
Browse files Browse the repository at this point in the history
* Merge branch master
  • Loading branch information
mehdi-aouadi authored Jan 27, 2025
1 parent 3d1744e commit 4d2ca38
Show file tree
Hide file tree
Showing 75 changed files with 536 additions and 326 deletions.
31 changes: 2 additions & 29 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,35 +7,8 @@
## Unreleased Changes

### Breaking Changes
- `--Xvalidators-builder-registration-default-gas-limit` CLI option is replaced by `--validators-builder-registration-default-gas-limit`
- `--Xp2p-sync-rate-limit` CLI option is removed in favour of `--Xp2p-sync-blocks-rate-limit` and `--Xp2p-sync-blob-sidecars-rate-limit`
- `--Xpeer-rate-limit` CLI options is removed in favour of `--Xpeer-blocks-rate-limit` and `--Xpeer-blob-sidecars-rate-limit`
- With the upgrade of the Prometheus Java Metrics library, there are the following changes:
- Gauge names are not allowed to end with `total`, therefore metrics as `beacon_proposers_data_total` and `beacon_eth1_current_period_votes_total` are dropping the `_total` suffix
- The `_created` timestamps are not returned by default.
- Some JVM metrics have changed name to adhere to the OTEL standard (see the table below), [Teku Detailed Dashboard](https://grafana.com/grafana/dashboards/16737-teku-detailed/) is updated to support both names

| Old Name | New Name |
|---------------------------------|---------------------------------|
| jvm_memory_bytes_committed | jvm_memory_committed_bytes |
| jvm_memory_bytes_init | jvm_memory_init_bytes |
| jvm_memory_bytes_max | jvm_memory_max_bytes |
| jvm_memory_bytes_used | jvm_memory_used_bytes |
| jvm_memory_pool_bytes_committed | jvm_memory_pool_committed_bytes |
| jvm_memory_pool_bytes_init | jvm_memory_pool_init_bytes |
| jvm_memory_pool_bytes_max | jvm_memory_pool_max_bytes |
| jvm_memory_pool_bytes_used | jvm_memory_pool_used_bytes |


### Additions and Improvements
- Default the gas limit to 36 million for externally produced blocks
- Optimized blobs validation pipeline
- Remove delay when fetching blobs from the local EL on block arrival
- New validator metric `validator_next_attestation_slot` to highlight the next slot that a validator is expected to publish an attestation [#8795](https://github.com/Consensys/teku/issues/8795)
- Support for SSZ format in builder API (mev-boost)
- Added the expected gas limit to the 'not honouring the validator gas limit preference' warning message.
- Implemented the ability to have fork based ascii art.
- Applied spec change to alter `GOSSIP_MAX_SIZE` to `MAX_PAYLOAD_SIZE`.

### Bug Fixes
- Fix `--version` command output [#8960](https://github.com/Consensys/teku/issues/8960)
- Fix issue (introduced in `24.12.1`) with peer stability when the upperbound is set to a high number
### Bug Fixes
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ synchronized void fail() {
}

private void reportOutage(final UInt64 outageStartInSeconds) {
STATUS_LOG.eth1ServiceDown(
timeProvider.getTimeInSeconds().minus(outageStartInSeconds).longValue());
STATUS_LOG.eth1ServiceDown(timeProvider.getTimeInSeconds().minus(outageStartInSeconds));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -97,12 +97,9 @@ public SafeFuture<Void> requestBlocksByRange(
@Override
public SafeFuture<Void> requestBlobSidecarsByRange(
final UInt64 startSlot, final UInt64 count, final RpcResponseListener<BlobSidecar> listener) {
final long blobSidecarsCount =
maybeMaxBlobsPerBlock
.map(maxBlobsPerBlock -> maxBlobsPerBlock * count.longValue())
.orElse(0L);
final UInt64 blobSidecarsCount = maybeMaxBlobsPerBlock.map(count::times).orElse(UInt64.ZERO);
return blobSidecarsRateTracker
.approveObjectsRequest(blobSidecarsCount)
.approveObjectsRequest(blobSidecarsCount.longValue())
.map(
requestApproval -> {
LOG.debug("Sending request for approximately {} blob sidecars", blobSidecarsCount);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -241,7 +241,7 @@ public void createUnsignedAttestation_withLatestBlockFromAnOldEpoch(
specContext.assumeIsNotOneOf(SpecMilestone.DENEB);
final UInt64 latestEpoch = UInt64.valueOf(2);
final UInt64 latestSlot = specContext.getSpec().computeStartSlotAtEpoch(latestEpoch).plus(ONE);
final UInt64 targetEpoch = UInt64.valueOf(latestEpoch.longValue() + 3);
final UInt64 targetEpoch = latestEpoch.plus(3);
final UInt64 targetEpochStartSlot = specContext.getSpec().computeStartSlotAtEpoch(targetEpoch);
final UInt64 targetSlot = targetEpochStartSlot.plus(2);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,7 @@
import tech.pegasys.teku.spec.datastructures.operations.SignedVoluntaryExit;
import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateCache;
import tech.pegasys.teku.spec.datastructures.state.versions.electra.PendingPartialWithdrawal;
import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment;
import tech.pegasys.teku.spec.datastructures.type.SszKZGProof;
import tech.pegasys.teku.spec.executionlayer.ExecutionLayerBlockProductionManager;
Expand Down Expand Up @@ -146,7 +147,7 @@ public Function<BeaconBlockBodyBuilder, SafeFuture<Void>> createSelector(
final SszList<SignedVoluntaryExit> voluntaryExits =
voluntaryExitPool.getItemsForBlock(
blockSlotState,
exit -> !exitedValidators.contains(exit.getMessage().getValidatorIndex()),
exit -> voluntaryExitPredicate(blockSlotState, exitedValidators, exit),
exit -> exitedValidators.add(exit.getMessage().getValidatorIndex()));

bodyBuilder
Expand Down Expand Up @@ -204,6 +205,25 @@ public Function<BeaconBlockBodyBuilder, SafeFuture<Void>> createSelector(
};
}

private boolean voluntaryExitPredicate(
final BeaconState blockSlotState,
final Set<UInt64> exitedValidators,
final SignedVoluntaryExit exit) {
final UInt64 validatorIndex = exit.getMessage().getValidatorIndex();
if (exitedValidators.contains(validatorIndex)) {
return false;
}
// if there is a pending withdrawal, the exit is not valid for inclusion in a block.
return blockSlotState
.toVersionElectra()
.map(
beaconStateElectra ->
beaconStateElectra.getPendingPartialWithdrawals().stream()
.map(PendingPartialWithdrawal::getValidatorIndex)
.noneMatch(index -> index == validatorIndex.intValue()))
.orElse(true);
}

private SafeFuture<Void> setExecutionData(
final Optional<ExecutionPayloadContext> executionPayloadContext,
final BeaconBlockBodyBuilder bodyBuilder,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@
import tech.pegasys.teku.spec.datastructures.operations.versions.altair.SignedContributionAndProof;
import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconStateCache;
import tech.pegasys.teku.spec.datastructures.state.beaconstate.versions.electra.MutableBeaconStateElectra;
import tech.pegasys.teku.spec.datastructures.type.SszKZGCommitment;
import tech.pegasys.teku.spec.datastructures.type.SszKZGProof;
import tech.pegasys.teku.spec.executionlayer.ExecutionLayerBlockProductionManager;
Expand Down Expand Up @@ -264,6 +265,39 @@ void shouldNotSelectOperationsWhenNoneAreAvailable() {
assertThat(bodyBuilder.blsToExecutionChanges).isEmpty();
}

@Test
void shouldNotSelectVoluntaryExitWhenValidatorHasPendingWithdrawal() {
final UInt64 slot = UInt64.ONE;
final MutableBeaconStateElectra blockSlotState =
dataStructureUtil.randomBeaconState(slot).toVersionElectra().get().createWritableCopy();
blockSlotState
.getPendingPartialWithdrawals()
.append(dataStructureUtil.randomPendingPartialWithdrawal(1));
final SignedVoluntaryExit voluntaryExit =
dataStructureUtil.randomSignedVoluntaryExit(UInt64.valueOf(1));
final ExecutionPayload randomExecutionPayload = dataStructureUtil.randomExecutionPayload();
final UInt256 blockExecutionValue = dataStructureUtil.randomUInt256();

addToPool(voluntaryExitPool, voluntaryExit);
prepareBlockProductionWithPayload(
randomExecutionPayload,
executionPayloadContext,
blockSlotState,
Optional.of(blockExecutionValue));

safeJoin(
factory
.createSelector(
parentRoot,
blockSlotState,
randaoReveal,
Optional.of(defaultGraffiti),
Optional.empty(),
BlockProductionPerformance.NOOP)
.apply(bodyBuilder));
assertThat(bodyBuilder.voluntaryExits).isEmpty();
}

@Test
void shouldIncludeValidOperations() {
final UInt64 slot = UInt64.valueOf(2);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -598,10 +598,8 @@ public void createAttestationData_shouldFailWhenHeadIsOptimistic() {
final UInt64 slot = spec.computeStartSlotAtEpoch(EPOCH).plus(ONE);
when(chainDataClient.getCurrentSlot()).thenReturn(slot);

final BeaconState state = createStateWithActiveValidators(epochStartSlot);
final SignedBeaconBlock block =
dataStructureUtil.randomSignedBeaconBlock(state.getSlot(), state);
final SignedBlockAndState blockAndState = new SignedBlockAndState(block, state);
final SignedBlockAndState blockAndState =
dataStructureUtil.randomSignedBlockAndState(epochStartSlot);

final SafeFuture<Optional<SignedBlockAndState>> blockAndStateResult =
completedFuture(Optional.of(blockAndState));
Expand All @@ -624,10 +622,9 @@ public void createAttestationData_shouldCreateAttestation() {
final UInt64 slot = spec.computeStartSlotAtEpoch(EPOCH).plus(ONE);
when(chainDataClient.getCurrentSlot()).thenReturn(slot);

final BeaconState state = createStateWithActiveValidators(epochStartSlot);
final SignedBeaconBlock block =
dataStructureUtil.randomSignedBeaconBlock(state.getSlot(), state);
final SignedBlockAndState blockAndState = new SignedBlockAndState(block, state);
dataStructureUtil.randomBlockAndState(epochStartSlot);
final SignedBlockAndState blockAndState =
dataStructureUtil.randomSignedBlockAndState(epochStartSlot);

final SafeFuture<Optional<SignedBlockAndState>> blockAndStateResult =
completedFuture(Optional.of(blockAndState));
Expand All @@ -646,7 +643,10 @@ public void createAttestationData_shouldCreateAttestation() {
assertThat(attestationData)
.isEqualTo(
spec.getGenericAttestationData(
slot, state, block.getMessage(), UInt64.valueOf(committeeIndex)));
slot,
blockAndState.getState(),
blockAndState.getBlock().getMessage(),
UInt64.valueOf(committeeIndex)));
assertThat(attestationData.getSlot()).isEqualTo(slot);
final InOrder inOrder = inOrder(forkChoiceTrigger, chainDataClient);

Expand Down Expand Up @@ -674,11 +674,10 @@ public void createAttestationData_shouldUseCorrectSourceWhenEpochTransitionRequi
// Slot is from before the current epoch, so we need to ensure we process the epoch transition
final UInt64 blockSlot = slot.minus(1);

final BeaconState wrongState = createStateWithActiveValidators(blockSlot);
final BeaconState rightState = createStateWithActiveValidators(slot);
final SignedBeaconBlock block =
dataStructureUtil.randomSignedBeaconBlock(wrongState.getSlot(), wrongState);
final SignedBlockAndState blockAndState = new SignedBlockAndState(block, wrongState);
final SignedBlockAndState blockAndState =
dataStructureUtil.randomSignedBlockAndState(blockSlot);
final SignedBeaconBlock block = blockAndState.getBlock();

final SafeFuture<Optional<SignedBlockAndState>> blockAndStateResult =
completedFuture(Optional.of(blockAndState));
Expand Down
5 changes: 2 additions & 3 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import com.github.jk1.license.filter.LicenseBundleNormalizer
import groovy.json.JsonSlurper
import tech.pegasys.internal.license.reporter.GroupedLicenseHtmlRenderer
import tech.pegasys.teku.depcheck.DepCheckPlugin

import java.text.SimpleDateFormat

import groovy.json.JsonSlurper

import static tech.pegasys.teku.repackage.Repackage.repackage

buildscript {
Expand Down Expand Up @@ -325,7 +324,7 @@ allprojects {
}

def nightly = System.getenv("NIGHTLY") != null
def refTestVersion = nightly ? "nightly" : "v1.5.0-alpha.10"
def refTestVersion = nightly ? "nightly" : "v1.5.0-beta.0"
def blsRefTestVersion = 'v0.1.2'
def slashingProtectionInterchangeRefTestVersion = 'v5.3.0'
def refTestBaseUrl = 'https://github.com/ethereum/consensus-spec-tests/releases/download'
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
"EPOCHS_PER_SLASHINGS_VECTOR" : "8192",
"MIN_SLASHING_PENALTY_QUOTIENT" : "128",
"MAX_BLS_TO_EXECUTION_CHANGES" : "16",
"MAX_PAYLOAD_SIZE" : "10485760",
"GOSSIP_MAX_SIZE" : "10485760",
"DOMAIN_BEACON_ATTESTER" : "0x01000000",
"EPOCHS_PER_SUBNET_SUBSCRIPTION" : "256",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,7 @@ private void verifyBlocksWithReferenceState(
block.getParentRoot().equals(previousArchiveLastBlock.getRoot()),
"First block in archive does not match last block of previous archive.");
}
// TODO should verify signature
// when fully implemented, we would check signature also
++populatedSlots;
}
System.out.println(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -123,15 +123,15 @@ public SyncCommitteeRewardData getSyncCommitteeRewardData(
blockAndMetadata.isExecutionOptimistic(), blockAndMetadata.isFinalized());
return calculateSyncCommitteeRewards(
committeeIndices,
participantReward.longValue(),
participantReward,
block.getBody().getOptionalSyncAggregate(),
rewardData);
}

@VisibleForTesting
SyncCommitteeRewardData calculateSyncCommitteeRewards(
final Map<Integer, Integer> committeeIndices,
final long participantReward,
final UInt64 participantReward,
final Optional<SyncAggregate> maybeAggregate,
final SyncCommitteeRewardData data) {
if (maybeAggregate.isEmpty()) {
Expand All @@ -143,9 +143,9 @@ SyncCommitteeRewardData calculateSyncCommitteeRewards(
committeeIndices.forEach(
(i, key) -> {
if (aggregate.getSyncCommitteeBits().getBit(i)) {
data.increaseReward(key, participantReward);
data.increaseReward(key, participantReward.longValue());
} else {
data.decreaseReward(key, participantReward);
data.decreaseReward(key, participantReward.longValue());
}
});

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,8 @@ public Map<String, String> getConfigMap() {
configAttributes.put("DOMAIN_SELECTION_PROOF", getDomainSelectionProof().toHexString());
configAttributes.put("DOMAIN_AGGREGATE_AND_PROOF", getDomainAggregateAndProof().toHexString());
configAttributes.put("DOMAIN_APPLICATION_BUILDER", getDomainApplicationBuilder().toHexString());
// Backwards compatibility with old phase0 constants, otherwise nodes won't start
configAttributes.put("GOSSIP_MAX_SIZE", configAttributes.get("MAX_PAYLOAD_SIZE"));
getDomainSyncCommittee()
.ifPresent(
committee -> configAttributes.put("DOMAIN_SYNC_COMMITTEE", committee.toHexString()));
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,19 +80,13 @@ public StateSelector genesisSelector() {
@Override
public StateSelector finalizedSelector() {
return () ->
SafeFuture.completedFuture(
client
.getLatestFinalized()
.map(
finalized ->
addMetaData(
finalized.getState(),
// The finalized checkpoint may change because of optimistically
// imported blocks at the head and if the head isn't optimistic, the
// finalized block can't be optimistic.
client.isChainHeadOptimistic(),
true,
true)));
client
.getBestFinalizedState()
.thenApply(
maybeFinalized ->
maybeFinalized.map(
finalized ->
addMetaData(finalized, client.isChainHeadOptimistic(), true, true)));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,9 @@ void checkValidatorsList_shouldAcceptValidInput(
@Test
void calculateSyncCommitteeRewards_shouldNotChangeValuesWhenAggregateEmpty() {
final SyncCommitteeRewardData data = mock(SyncCommitteeRewardData.class);
assertThat(calculator.calculateSyncCommitteeRewards(Map.of(1, 1), 1L, Optional.empty(), data))
assertThat(
calculator.calculateSyncCommitteeRewards(
Map.of(1, 1), UInt64.ONE, Optional.empty(), data))
.isEqualTo(data);
verifyNoMoreInteractions(data);
}
Expand All @@ -222,7 +224,7 @@ void calculateSyncCommitteeRewards_shouldAdjustVRewards() {
final SyncAggregate aggregate = data.randomSyncAggregate(1);
assertThat(
calculator.calculateSyncCommitteeRewards(
Map.of(1, 1, 2, 2), 1L, Optional.of(aggregate), rewardData))
Map.of(1, 1, 2, 2), UInt64.ONE, Optional.of(aggregate), rewardData))
.isEqualTo(rewardData);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlockHeader;
import tech.pegasys.teku.spec.datastructures.blocks.SignedBlockAndState;
import tech.pegasys.teku.spec.datastructures.metadata.StateAndMetaData;
import tech.pegasys.teku.spec.datastructures.state.AnchorPoint;
import tech.pegasys.teku.spec.datastructures.state.beaconstate.BeaconState;
import tech.pegasys.teku.spec.util.DataStructureUtil;
import tech.pegasys.teku.storage.api.StorageQueryChannel;
Expand Down Expand Up @@ -66,8 +65,7 @@ public void headSelector_shouldGetBestState() {

@Test
public void finalizedSelector_shouldGetFinalizedState() {
when(client.getLatestFinalized())
.thenReturn(Optional.of(AnchorPoint.fromInitialState(spec, state)));
when(client.getBestFinalizedState()).thenReturn(SafeFuture.completedFuture(Optional.of(state)));
Optional<StateAndMetaData> result = safeJoin(factory.finalizedSelector().getState());
assertThat(result).contains(withMetaData(state, true));
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ public abstract class Eth2ReferenceTestCase {
.put("light_client/single_merkle_proof", TestExecutor.IGNORE_TESTS)
.put("light_client/sync", TestExecutor.IGNORE_TESTS)
.put("light_client/update_ranking", TestExecutor.IGNORE_TESTS)
.put("light_client/data_collection", TestExecutor.IGNORE_TESTS)
.build();

private static final ImmutableMap<String, TestExecutor> PHASE_0_TEST_TYPES =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ public class ReferenceTestFinder {
TestFork.ALTAIR,
TestFork.BELLATRIX,
TestFork.CAPELLA,
TestFork.DENEB); // TODO: Add Electra fork tests back
TestFork.DENEB,
TestFork.ELECTRA);

@MustBeClosed
public static Stream<TestDefinition> findReferenceTests() throws IOException {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ private void setupResponse(final SafeFuture<Object> response) {
when(delegatedTestClient.testMethod(any()))
.thenAnswer(
__ -> {
stubTimeProvider.advanceTimeByMillis(RESPONSE_DELAY.longValue());
stubTimeProvider.advanceTimeByMillis(RESPONSE_DELAY);
return response;
});
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ DOMAIN_AGGREGATE_AND_PROOF: 0x06000000
# Networking
# ---------------------------------------------------------------
# `10 * 2**20` (= 10485760, 10 MiB)
GOSSIP_MAX_SIZE: 10485760
MAX_PAYLOAD_SIZE: 10485760
# `2**10` (= 1024)
MAX_REQUEST_BLOCKS: 1024
# `2**8` (= 256)
Expand Down
Loading

0 comments on commit 4d2ca38

Please sign in to comment.