diff --git a/docs/design/block-streams.md b/docs/design/block-streams.md index 9ebcdac9170..24044c3b163 100644 --- a/docs/design/block-streams.md +++ b/docs/design/block-streams.md @@ -37,7 +37,8 @@ package com.hedera.mirror.common.domain.transaction; public record BlockItem(Transaction transaction, TransactionResult transactionResult, List transactionOutput, // Note: List may be empty - Optional stateChanges) implements StreamItem {} + List stateChanges // Note: List may be empty +) implements StreamItem {} ``` #### BlockFile diff --git a/hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockFile.java b/hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockFile.java index b6bdc971536..03e41d5ba19 100644 --- a/hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockFile.java +++ b/hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockFile.java @@ -18,10 +18,11 @@ import com.hedera.hapi.block.stream.output.protoc.BlockHeader; import com.hedera.hapi.block.stream.protoc.BlockProof; +import com.hedera.hapi.block.stream.protoc.RecordFileItem; import com.hedera.mirror.common.domain.DigestAlgorithm; import com.hedera.mirror.common.domain.StreamFile; import com.hedera.mirror.common.domain.StreamType; -import com.hederahashgraph.api.proto.java.BlockStreamInfo; +import java.util.ArrayList; import java.util.Collection; import java.util.List; import lombok.AllArgsConstructor; @@ -43,9 +44,6 @@ public class BlockFile implements StreamFile { // Used to generate block hash private BlockProof blockProof; - // Contained within the last StateChange of the block, contains hashes needed to generate the block hash - private BlockStreamInfo blockStreamInfo; - @ToString.Exclude private byte[] bytes; @@ -60,6 +58,8 @@ public class BlockFile implements StreamFile { @ToString.Exclude private String hash; + private Long index; + @Builder.Default @EqualsAndHashCode.Exclude @ToString.Exclude @@ -76,6 +76,8 @@ public class BlockFile implements StreamFile { @ToString.Exclude private String previousHash; + private RecordFileItem recordFileItem; + private Long roundEnd; private Long roundStart; @@ -94,13 +96,55 @@ public String getFileHash() { return null; } - @Override - public Long getIndex() { - return blockHeader.getNumber(); - } - @Override public StreamType getType() { return StreamType.BLOCK; } + + public static BlockFileBuilder builder() { + return new BlockFileBuilder() { + @Override + public BlockFile build() { + prebuild(); + return super.build(); + } + }; + } + + public static class BlockFileBuilder { + + public BlockFileBuilder addItem(BlockItem blockItem) { + if (this.items$value == null) { + items$set = true; + items$value = new ArrayList<>(); + } + + items$value.add(blockItem); + return this; + } + + public BlockFileBuilder onNewRound(long roundNumber) { + if (roundStart == null) { + roundStart = roundNumber; + } + + roundEnd = roundNumber; + return this; + } + + public BlockFileBuilder onNewTransaction(long consensusTimestamp) { + if (consensusStart == null) { + consensusStart = consensusTimestamp; + } + + consensusEnd = consensusTimestamp; + return this; + } + + void prebuild() { + if (count == null) { + count = items$value != null ? (long) items$value.size() : 0; + } + } + } } diff --git a/hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockItem.java b/hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockItem.java index 8319a3ea9bc..3f4b622fffb 100644 --- a/hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockItem.java +++ b/hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockItem.java @@ -22,7 +22,6 @@ import com.hedera.mirror.common.domain.StreamItem; import com.hederahashgraph.api.proto.java.Transaction; import java.util.List; -import java.util.Optional; import lombok.Builder; @Builder(toBuilder = true) @@ -30,5 +29,5 @@ public record BlockItem( Transaction transaction, TransactionResult transactionResult, List transactionOutput, - Optional stateChanges) + List stateChanges) implements StreamItem {} diff --git a/hedera-mirror-common/src/test/java/com/hedera/mirror/common/domain/transaction/BlockFileTest.java b/hedera-mirror-common/src/test/java/com/hedera/mirror/common/domain/transaction/BlockFileTest.java new file mode 100644 index 00000000000..fde3a1cd111 --- /dev/null +++ b/hedera-mirror-common/src/test/java/com/hedera/mirror/common/domain/transaction/BlockFileTest.java @@ -0,0 +1,70 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.common.domain.transaction; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; +import org.junit.jupiter.api.Test; + +class BlockFileTest { + + @Test + void addItem() { + var blockItem = BlockItem.builder().build(); + var blockFile = BlockFile.builder().addItem(blockItem).build(); + assertThat(blockFile.getItems()).containsExactly(blockItem); + } + + @Test + void count() { + var blockFile = BlockFile.builder().build(); + assertThat(blockFile.getCount()).isZero(); + + blockFile = BlockFile.builder().count(10L).build(); + assertThat(blockFile.getCount()).isEqualTo(10L); + + var blockItem = BlockItem.builder().build(); + blockFile = BlockFile.builder().items(List.of(blockItem)).build(); + assertThat(blockFile.getCount()).isEqualTo(1L); + + blockFile = BlockFile.builder().addItem(blockItem).build(); + assertThat(blockFile.getCount()).isEqualTo(1L); + + blockFile = BlockFile.builder().addItem(blockItem).count(5L).build(); + assertThat(blockFile.getCount()).isEqualTo(5L); + } + + @Test + void onNewRound() { + var blockFile = BlockFile.builder().onNewRound(1L).build(); + assertThat(blockFile).returns(1L, BlockFile::getRoundStart).returns(1L, BlockFile::getRoundEnd); + + blockFile = BlockFile.builder().onNewRound(1L).onNewRound(2L).build(); + assertThat(blockFile).returns(1L, BlockFile::getRoundStart).returns(2L, BlockFile::getRoundEnd); + } + + @Test + void onNewTransaction() { + var blockFile = BlockFile.builder().onNewTransaction(1).build(); + assertThat(blockFile).returns(1L, BlockFile::getConsensusStart).returns(1L, BlockFile::getConsensusEnd); + + blockFile = + BlockFile.builder().onNewTransaction(1L).onNewTransaction(2L).build(); + assertThat(blockFile).returns(1L, BlockFile::getConsensusStart).returns(2L, BlockFile::getConsensusEnd); + } +} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/StreamFilename.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/StreamFilename.java index c38d9a1cfbd..cd1da101b64 100644 --- a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/StreamFilename.java +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/StreamFilename.java @@ -108,7 +108,9 @@ private StreamFilename(String path, String filename, String pathSeparator) { // A compressed and uncompressed file can exist simultaneously, so we need uniqueness to not include .gz this.filenameWithoutCompressor = isCompressed() ? removeExtension(this.filename) : this.filename; - this.instant = extractInstant(filename, this.fullExtension, this.sidecarId, this.streamType.getSuffix()); + this.instant = streamType != StreamType.BLOCK + ? extractInstant(filename, this.fullExtension, this.sidecarId, this.streamType.getSuffix()) + : null; var builder = new StringBuilder(); if (!StringUtils.isEmpty(this.path)) { @@ -156,6 +158,14 @@ public static String getFilename(StreamType streamType, FileType fileType, Insta return StringUtils.joinWith(".", StringUtils.join(timestamp, suffix), extension); } + public Instant getInstant() { + if (streamType == StreamType.BLOCK) { + throw new IllegalStateException("BLOCK stream file doesn't have instant in its filename"); + } + + return instant; + } + @SuppressWarnings("java:S3776") private static TypeInfo extractTypeInfo(String filename) { List parts = FILENAME_SPLITTER.splitToList(filename); diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/BlockFileReader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/BlockFileReader.java new file mode 100644 index 00000000000..1e4217145c0 --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/BlockFileReader.java @@ -0,0 +1,23 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.importer.reader.block; + +import com.hedera.mirror.common.domain.transaction.BlockFile; +import com.hedera.mirror.common.domain.transaction.BlockItem; +import com.hedera.mirror.importer.reader.StreamFileReader; + +public interface BlockFileReader extends StreamFileReader {} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/BlockRootHashDigest.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/BlockRootHashDigest.java new file mode 100644 index 00000000000..b3b7108fb9c --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/BlockRootHashDigest.java @@ -0,0 +1,138 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.importer.reader.block; + +import static com.hedera.mirror.common.domain.DigestAlgorithm.SHA_384; + +import com.hedera.hapi.block.stream.protoc.BlockItem; +import com.hedera.mirror.common.util.DomainUtils; +import com.hedera.mirror.importer.exception.StreamFileReaderException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import lombok.NoArgsConstructor; +import lombok.Value; +import lombok.experimental.NonFinal; + +/** + * Calculates a block's root hash per the algorithm defined in HIP-1056. Note both the input merkle tree and the output + * merkle tree are padded with SHA2-384 hash of an empty bytearray to be perfect binary trees. + */ +@NoArgsConstructor +@Value +class BlockRootHashDigest { + + private static final byte[] EMPTY_HASH = createMessageDigest().digest(new byte[0]); + + @NonFinal + private boolean finalized; + + private List inputHashes = new ArrayList<>(); + + private List outputHashes = new ArrayList<>(); + + @NonFinal + private byte[] previousHash; + + @NonFinal + private byte[] startOfBlockStateHash; + + public void addInputBlockItem(BlockItem blockItem) { + inputHashes.add(createMessageDigest().digest(blockItem.toByteArray())); + } + + public void addOutputBlockItem(BlockItem blockItem) { + outputHashes.add(createMessageDigest().digest(blockItem.toByteArray())); + } + + public String digest() { + if (finalized) { + throw new IllegalStateException("Block root hash is already calculated"); + } + + validateHash(previousHash, "previousHash"); + validateHash(startOfBlockStateHash, "startOfBlockStateHash"); + + List leaves = new ArrayList<>(); + leaves.add(previousHash); + leaves.add(getRootHash(inputHashes)); + leaves.add(getRootHash(outputHashes)); + leaves.add(startOfBlockStateHash); + + byte[] rootHash = getRootHash(leaves); + finalized = true; + + return DomainUtils.bytesToHex(rootHash); + } + + public void setPreviousHash(byte[] previousHash) { + validateHash(previousHash, "previousHash"); + this.previousHash = previousHash; + } + + public void setStartOfBlockStateHash(byte[] startOfBlockStateHash) { + validateHash(startOfBlockStateHash, "startOfBlockStateHash"); + this.startOfBlockStateHash = startOfBlockStateHash; + } + + private static MessageDigest createMessageDigest() { + try { + return MessageDigest.getInstance(SHA_384.getName()); + } catch (NoSuchAlgorithmException ex) { + throw new StreamFileReaderException(ex); + } + } + + private static byte[] getRootHash(List leaves) { + if (leaves.isEmpty()) { + return EMPTY_HASH; + } + + // Pad leaves with EMPTY_HASH to the next 2^n to form a perfect binary tree + int size = leaves.size(); + if ((size & (size - 1)) != 0) { + size = Integer.highestOneBit(size) << 1; + while (leaves.size() < size) { + leaves.add(EMPTY_HASH); + } + } + + // Iteratively calculate the parent node hash as h(left | right) to get the root hash in bottom-up fashion + while (size > 1) { + for (int i = 0; i < size; i += 2) { + var digest = createMessageDigest(); + byte[] left = leaves.get(i); + byte[] right = leaves.get(i + 1); + digest.update(left); + digest.update(right); + leaves.set(i >> 1, digest.digest()); + } + + size >>= 1; + } + + return leaves.getFirst(); + } + + private static void validateHash(byte[] hash, String name) { + if (Objects.requireNonNull(hash, "Null " + name).length != SHA_384.getSize()) { + throw new IllegalArgumentException(String.format("%s is not %d bytes", name, SHA_384.getSize())); + } + } +} diff --git a/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/ProtoBlockFileReader.java b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/ProtoBlockFileReader.java new file mode 100644 index 00000000000..d72773985f9 --- /dev/null +++ b/hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/reader/block/ProtoBlockFileReader.java @@ -0,0 +1,253 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.importer.reader.block; + +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.BLOCK_HEADER; +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.BLOCK_PROOF; +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.EVENT_HEADER; +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.EVENT_TRANSACTION; +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.RECORD_FILE; +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.ROUND_HEADER; +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.STATE_CHANGES; +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.TRANSACTION_OUTPUT; +import static com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase.TRANSACTION_RESULT; + +import com.google.protobuf.InvalidProtocolBufferException; +import com.hedera.hapi.block.stream.output.protoc.StateChanges; +import com.hedera.hapi.block.stream.output.protoc.TransactionOutput; +import com.hedera.hapi.block.stream.output.protoc.TransactionResult; +import com.hedera.hapi.block.stream.protoc.Block; +import com.hedera.hapi.block.stream.protoc.BlockItem; +import com.hedera.hapi.block.stream.protoc.BlockItem.ItemCase; +import com.hedera.mirror.common.domain.DigestAlgorithm; +import com.hedera.mirror.common.domain.transaction.BlockFile; +import com.hedera.mirror.common.util.DomainUtils; +import com.hedera.mirror.importer.domain.StreamFileData; +import com.hedera.mirror.importer.exception.InvalidStreamFileException; +import com.hederahashgraph.api.proto.java.BlockHashAlgorithm; +import com.hederahashgraph.api.proto.java.Transaction; +import jakarta.inject.Named; +import jakarta.validation.constraints.NotNull; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import lombok.Value; +import lombok.experimental.NonFinal; + +@Named +public class ProtoBlockFileReader implements BlockFileReader { + + private static final int VERSION = 7; + + @Override + public BlockFile read(StreamFileData streamFileData) { + String filename = streamFileData.getFilename(); + + try (var inputStream = streamFileData.getInputStream()) { + var block = Block.parseFrom(inputStream); + byte[] bytes = streamFileData.getBytes(); + var context = new ReaderContext(block.getItemsList(), filename); + var blockFileBuilder = context.getBlockFile() + .bytes(bytes) + .loadStart(streamFileData.getStreamFilename().getTimestamp()) + .name(filename) + .size(bytes.length) + .version(VERSION); + + var blockItem = context.readBlockItemFor(RECORD_FILE); + if (blockItem != null) { + return blockFileBuilder + .recordFileItem(blockItem.getRecordFile()) + .build(); + } + + readBlockHeader(context); + readRounds(context); + readTrailingStateChanges(context); + readBlockProof(context); + + return blockFileBuilder + .hash(context.getBlockRootHashDigest().digest()) + .build(); + } catch (Exception e) { + if (e instanceof InvalidStreamFileException invalidStreamFileException) { + throw invalidStreamFileException; + } + + throw new InvalidStreamFileException("Failed to read " + filename, e); + } + } + + private Long getTransactionConsensusTimestamp(TransactionResult transactionResult) { + return DomainUtils.timestampInNanosMax(transactionResult.getConsensusTimestamp()); + } + + private void readBlockHeader(ReaderContext context) { + var blockItem = context.readBlockItemFor(BLOCK_HEADER); + if (blockItem == null) { + throw new InvalidStreamFileException("Missing block header in block file " + context.getFilename()); + } + + var blockFileBuilder = context.getBlockFile(); + var blockHeader = blockItem.getBlockHeader(); + + if (blockHeader.getHashAlgorithm().equals(BlockHashAlgorithm.SHA2_384)) { + blockFileBuilder.digestAlgorithm(DigestAlgorithm.SHA_384); + } else { + throw new InvalidStreamFileException(String.format( + "Unsupported hash algorithm %s in block header of block file %s", + blockHeader.getHashAlgorithm(), context.getFilename())); + } + + var previousHash = DomainUtils.toBytes(blockHeader.getPreviousBlockHash()); + blockFileBuilder.blockHeader(blockHeader); + blockFileBuilder.index(blockHeader.getNumber()); + blockFileBuilder.previousHash(DomainUtils.bytesToHex(previousHash)); + context.getBlockRootHashDigest().setPreviousHash(previousHash); + } + + private void readBlockProof(ReaderContext context) { + var blockItem = context.readBlockItemFor(BLOCK_PROOF); + if (blockItem == null) { + throw new InvalidStreamFileException("Missing block proof in file " + context.getFilename()); + } + + var blockProof = blockItem.getBlockProof(); + context.getBlockFile().blockProof(blockProof); + context.getBlockRootHashDigest() + .setStartOfBlockStateHash(DomainUtils.toBytes(blockProof.getStartOfBlockStateRootHash())); + } + + private void readEvents(ReaderContext context) { + while (context.readBlockItemFor(EVENT_HEADER) != null) { + readEventTransactions(context); + } + } + + private void readEventTransactions(ReaderContext context) { + BlockItem protoBlockItem; + while ((protoBlockItem = context.readBlockItemFor(EVENT_TRANSACTION)) != null) { + try { + var eventTransaction = protoBlockItem.getEventTransaction(); + var transaction = eventTransaction.hasApplicationTransaction() + ? Transaction.parseFrom(eventTransaction.getApplicationTransaction()) + : null; + + var transactionResultProtoBlockItem = context.readBlockItemFor(TRANSACTION_RESULT); + if (transactionResultProtoBlockItem == null) { + throw new InvalidStreamFileException( + "Missing transaction result in block file " + context.getFilename()); + } + + var transactionOutputs = new ArrayList(); + while ((protoBlockItem = context.readBlockItemFor(TRANSACTION_OUTPUT)) != null) { + transactionOutputs.add(protoBlockItem.getTransactionOutput()); + } + + var stateChangesList = new ArrayList(); + while ((protoBlockItem = context.readBlockItemFor(STATE_CHANGES)) != null) { + var stateChanges = protoBlockItem.getStateChanges(); + stateChangesList.add(stateChanges); + } + + if (transaction != null) { + var transactionResult = transactionResultProtoBlockItem.getTransactionResult(); + var blockItem = com.hedera.mirror.common.domain.transaction.BlockItem.builder() + .transaction(transaction) + .transactionResult(transactionResult) + .transactionOutput(Collections.unmodifiableList(transactionOutputs)) + .stateChanges(Collections.unmodifiableList(stateChangesList)) + .build(); + context.getBlockFile() + .addItem(blockItem) + .onNewTransaction(getTransactionConsensusTimestamp(transactionResult)); + } + } catch (InvalidProtocolBufferException e) { + throw new InvalidStreamFileException( + "Failed to deserialize Transaction from block file " + context.getFilename(), e); + } + } + } + + private void readRounds(ReaderContext context) { + BlockItem blockItem; + while ((blockItem = context.readBlockItemFor(ROUND_HEADER)) != null) { + context.getBlockFile().onNewRound(blockItem.getRoundHeader().getRoundNumber()); + readEvents(context); + } + } + + /** + * Read trailing state changes. There is no marker to distinguish between transactional and non-transactional + * statechanges. This function reads those trailing non-transactional statechanges without immediately preceding + * transactional statechanges. + * + * @param context - The reader context + */ + private void readTrailingStateChanges(ReaderContext context) { + while (context.readBlockItemFor(STATE_CHANGES) != null) { + // read all trailing statechanges + } + } + + @Value + private static class ReaderContext { + private BlockFile.BlockFileBuilder blockFile; + private List blockItems; + private BlockRootHashDigest blockRootHashDigest; + private String filename; + + @NonFinal + private int index; + + ReaderContext(@NotNull List blockItems, @NotNull String filename) { + this.blockFile = BlockFile.builder(); + this.blockItems = blockItems; + this.blockRootHashDigest = new BlockRootHashDigest(); + this.filename = filename; + } + + /** + * Returns the current block item if it matches the itemCase, and advances the index. If no match, index is not + * changed + * @param itemCase - block item case + * @return The matching block item, or null + */ + public BlockItem readBlockItemFor(ItemCase itemCase) { + if (index >= blockItems.size()) { + return null; + } + + var blockItem = blockItems.get(index); + if (blockItem.getItemCase() != itemCase) { + return null; + } + + index++; + switch (itemCase) { + case EVENT_HEADER, EVENT_TRANSACTION -> blockRootHashDigest.addInputBlockItem(blockItem); + case STATE_CHANGES, TRANSACTION_OUTPUT, TRANSACTION_RESULT -> blockRootHashDigest.addOutputBlockItem( + blockItem); + default -> { + // other block items aren't considered input / output + } + } + + return blockItem; + } + } +} diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/domain/StreamFilenameTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/domain/StreamFilenameTest.java index e584a1fefdb..62b9c95180a 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/domain/StreamFilenameTest.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/domain/StreamFilenameTest.java @@ -19,6 +19,7 @@ import static com.hedera.mirror.importer.domain.StreamFilename.FileType.DATA; import static com.hedera.mirror.importer.domain.StreamFilename.FileType.SIDECAR; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; import static org.junit.jupiter.api.Assertions.assertThrows; import com.hedera.mirror.common.domain.StreamType; @@ -46,6 +47,8 @@ class StreamFilenameTest { "2020-06-03T16_45_00.1Z_Balances.pb.gz, gz, pb, DATA, pb.gz, 2020-06-03T16:45:00.1Z, BALANCE", "2020-06-03T16_45_00.1Z.rcd_sig,, rcd_sig, SIGNATURE, rcd_sig, 2020-06-03T16:45:00.1Z, RECORD", "2020-06-03T16_45_00.1Z.rcd,, rcd, DATA, rcd, 2020-06-03T16:45:00.1Z, RECORD", + "000000000000000000000000000007647866.blk,, blk, DATA, blk,, BLOCK", + "000000000000000000000000000007647866.blk.gz, gz, blk, DATA, blk.gz,, BLOCK" // @formatter:on }) void newStreamFile( @@ -132,6 +135,12 @@ void getFilenameAfter(String filename, String expected) { assertThat(streamFilename.getFilenameAfter()).isEqualTo(expected); } + @Test + void getInstantThrows() { + var streamFilename = StreamFilename.from("000000000000000000000000000007647866.blk.gz"); + assertThatThrownBy(streamFilename::getInstant).isInstanceOf(IllegalStateException.class); + } + @ParameterizedTest @CsvSource({ "2020-06-03T16_45_00.100200345Z.rcd.gz, 1, 2020-06-03T16_45_00.100200345Z_01.rcd.gz", diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/domain/BlockItemBuilder.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/domain/BlockItemBuilder.java index 6b245e1e8e2..453709be044 100644 --- a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/domain/BlockItemBuilder.java +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/parser/domain/BlockItemBuilder.java @@ -30,8 +30,8 @@ import com.hederahashgraph.api.proto.java.Transaction; import com.hederahashgraph.api.proto.java.TransactionBody; import jakarta.inject.Named; +import java.util.Collections; import java.util.List; -import java.util.Optional; import org.springframework.beans.factory.config.ConfigurableBeanFactory; import org.springframework.context.annotation.Scope; @@ -68,7 +68,7 @@ public BlockItemBuilder.Builder cryptoTra transactionBody, transactionResult, List.of(contractCallTransactionOutput, cryptoTransferTransactionOutput), - Optional.empty()); + Collections.emptyList()); } private AssessedCustomFee.Builder assessedCustomFees() { @@ -85,7 +85,7 @@ public class Builder> { private final TransactionBody.Builder transactionBodyWrapper; private final List transactionOutputs; private final TransactionResult transactionResult; - private final Optional stateChanges; + private final List stateChanges; private final BlockItem.BlockItemBuilder blockItemBuilder; private Builder( @@ -93,7 +93,7 @@ private Builder( T transactionBody, TransactionResult transactionResult, List transactionOutputs, - Optional stateChanges) { + List stateChanges) { this.blockItemBuilder = BlockItem.builder(); this.stateChanges = stateChanges; this.type = type; diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/reader/block/BlockRootHashDigestTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/reader/block/BlockRootHashDigestTest.java new file mode 100644 index 00000000000..dc2e251dd6a --- /dev/null +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/reader/block/BlockRootHashDigestTest.java @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.importer.reader.block; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.hedera.hapi.block.stream.output.protoc.BlockHeader; +import com.hedera.hapi.block.stream.output.protoc.StateChanges; +import com.hedera.hapi.block.stream.protoc.BlockItem; +import org.bouncycastle.util.encoders.Hex; +import org.junit.jupiter.api.Test; + +class BlockRootHashDigestTest { + + private static final byte[] EMPTY_HASH = Hex.decode( + "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b"); + + @Test + void digest() { + // given + var subject = new BlockRootHashDigest(); + subject.setPreviousHash(EMPTY_HASH); + subject.setStartOfBlockStateHash(EMPTY_HASH); + subject.addInputBlockItem(BlockItem.newBuilder() + .setBlockHeader(BlockHeader.newBuilder().build()) + .build()); + subject.addOutputBlockItem(BlockItem.newBuilder() + .setStateChanges(StateChanges.newBuilder().build()) + .build()); + + // when + String actual = subject.digest(); + + // then + assertThat(actual) + .isEqualTo( + "4183fa8f91550afb353aaef723a7375e5e8feefc0be04daa8c5d74731c425ddb3d92ef7309f3c71639ad35f7e02e913e"); + + // digest again + assertThatThrownBy(subject::digest).isInstanceOf(IllegalStateException.class); + } + + @Test + void digestWithEmptyInputOutputTrees() { + // given + var subject = new BlockRootHashDigest(); + subject.setPreviousHash(EMPTY_HASH); + subject.setStartOfBlockStateHash(EMPTY_HASH); + + // when + String actual = subject.digest(); + + // then + assertThat(actual) + .isEqualTo( + "f524650830c65a98cda4cbbc9b500c01cfb5aa86225a920b49fe69458ac52aa64e8028a095d5028e363447e27efa31a8"); + } + + @Test + void digestWithPadding() { + // given + var subject = new BlockRootHashDigest(); + subject.setPreviousHash(EMPTY_HASH); + subject.setStartOfBlockStateHash(EMPTY_HASH); + + var inputBlockItem = BlockItem.newBuilder() + .setBlockHeader(BlockHeader.newBuilder().build()) + .build(); + for (int i = 0; i < 3; i++) { + subject.addInputBlockItem(inputBlockItem); + } + + var outputBlockItem = BlockItem.newBuilder() + .setStateChanges(StateChanges.newBuilder().build()) + .build(); + for (int i = 0; i < 11; i++) { + subject.addOutputBlockItem(outputBlockItem); + } + + // when + String actual = subject.digest(); + + // then + assertThat(actual) + .isEqualTo( + "1062c46277c5be0408165dd5eb4aba605b8193066fd66c9f05d92a2ba62150406a897104804e540deb3412657f208f13"); + } + + @Test + void shouldThrowWhenPreviousHashNotSet() { + var subject = new BlockRootHashDigest(); + subject.setStartOfBlockStateHash(EMPTY_HASH); + assertThatThrownBy(subject::digest).isInstanceOf(NullPointerException.class); + } + + @Test + void shouldThrowWhenSetInvalidPreviousHash() { + var subject = new BlockRootHashDigest(); + assertThatThrownBy(() -> subject.setPreviousHash(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> subject.setPreviousHash(new byte[8])).isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldThrowWhenSetInvalidStartOfBlockStateHash() { + var subject = new BlockRootHashDigest(); + assertThatThrownBy(() -> subject.setStartOfBlockStateHash(null)).isInstanceOf(NullPointerException.class); + assertThatThrownBy(() -> subject.setStartOfBlockStateHash(new byte[10])) + .isInstanceOf(IllegalArgumentException.class); + } + + @Test + void shouldThrowWhenStartOfBlockStateHashNotSet() { + var subject = new BlockRootHashDigest(); + subject.setPreviousHash(EMPTY_HASH); + assertThatThrownBy(subject::digest).isInstanceOf(NullPointerException.class); + } +} diff --git a/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/reader/block/ProtoBlockFileReaderTest.java b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/reader/block/ProtoBlockFileReaderTest.java new file mode 100644 index 00000000000..ef420386f49 --- /dev/null +++ b/hedera-mirror-importer/src/test/java/com/hedera/mirror/importer/reader/block/ProtoBlockFileReaderTest.java @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2025 Hedera Hashgraph, LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.hedera.mirror.importer.reader.block; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import com.google.protobuf.ByteString; +import com.hedera.hapi.block.stream.input.protoc.EventHeader; +import com.hedera.hapi.block.stream.input.protoc.RoundHeader; +import com.hedera.hapi.block.stream.output.protoc.BlockHeader; +import com.hedera.hapi.block.stream.output.protoc.TransactionResult; +import com.hedera.hapi.block.stream.protoc.Block; +import com.hedera.hapi.block.stream.protoc.BlockItem; +import com.hedera.hapi.block.stream.protoc.BlockProof; +import com.hedera.hapi.block.stream.protoc.RecordFileItem; +import com.hedera.hapi.platform.event.legacy.EventTransaction; +import com.hedera.mirror.common.domain.DigestAlgorithm; +import com.hedera.mirror.common.domain.transaction.BlockFile; +import com.hedera.mirror.importer.TestUtils; +import com.hedera.mirror.importer.domain.StreamFileData; +import com.hedera.mirror.importer.exception.InvalidStreamFileException; +import com.hederahashgraph.api.proto.java.CryptoTransferTransactionBody; +import com.hederahashgraph.api.proto.java.SignedTransaction; +import com.hederahashgraph.api.proto.java.Transaction; +import com.hederahashgraph.api.proto.java.TransactionBody; +import java.io.ByteArrayOutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Stream; +import lombok.SneakyThrows; +import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.springframework.core.io.ClassPathResource; + +class ProtoBlockFileReaderTest { + + private final ProtoBlockFileReader reader = new ProtoBlockFileReader(); + + @ParameterizedTest(name = "{0}") + @MethodSource("readTestArgumentsProvider") + void read(String filename, StreamFileData streamFileData, BlockFile expected) { + var actual = reader.read(streamFileData); + assertThat(actual) + .usingRecursiveComparison() + .ignoringFields("blockHeader", "blockProof", "items") + .isEqualTo(expected); + assertThat(actual) + .returns(expected.getCount(), a -> (long) a.getItems().size()) + .satisfies(a -> assertThat(a.getBlockHeader()).isNotNull()) + .satisfies(a -> assertThat(a.getBlockProof()).isNotNull()); + } + + @Test + void readRecordFileItem() { + // given + var block = Block.newBuilder() + .addItems(BlockItem.newBuilder() + .setRecordFile(RecordFileItem.getDefaultInstance()) + .build()) + .build(); + byte[] bytes = gzip(block); + var streamFileData = StreamFileData.from("000000000000000000000000000000000001.blk.gz", bytes); + var expected = BlockFile.builder() + .bytes(bytes) + .loadStart(streamFileData.getStreamFilename().getTimestamp()) + .name(streamFileData.getFilename()) + .recordFileItem(RecordFileItem.getDefaultInstance()) + .size(bytes.length) + .version(7) + .build(); + + // when + var actual = reader.read(streamFileData); + + // then + assertThat(actual).isEqualTo(expected); + } + + @Test + void throwWhenMissingBlockHeader() { + var block = Block.newBuilder().addItems(blockProof()).build(); + var streamFileData = StreamFileData.from("000000000000000000000000000000000001.blk.gz", gzip(block)); + assertThatThrownBy(() -> reader.read(streamFileData)) + .isInstanceOf(InvalidStreamFileException.class) + .hasMessageContaining("Missing block header"); + } + + @Test + void throwWhenMissingBlockProof() { + var block = Block.newBuilder().addItems(blockHeader()).build(); + var streamFileData = StreamFileData.from("000000000000000000000000000000000001.blk.gz", gzip(block)); + assertThatThrownBy(() -> reader.read(streamFileData)) + .isInstanceOf(InvalidStreamFileException.class) + .hasMessageContaining("Missing block proof"); + } + + @Test + void throwWhenMissingTransactionResult() { + var roundHeader = BlockItem.newBuilder().setRoundHeader(RoundHeader.getDefaultInstance()); + var eventHeader = BlockItem.newBuilder().setEventHeader(EventHeader.getDefaultInstance()); + var block = Block.newBuilder() + .addItems(blockHeader()) + .addItems(roundHeader) + .addItems(eventHeader) + .addItems(eventTransaction()) + .addItems(blockProof()) + .build(); + var streamFileData = StreamFileData.from("000000000000000000000000000000000001.blk.gz", gzip(block)); + assertThatThrownBy(() -> reader.read(streamFileData)) + .isInstanceOf(InvalidStreamFileException.class) + .hasMessageContaining("Missing transaction result"); + } + + @Test + void thrownWhenTransactionBytesCorrupted() { + var roundHeader = BlockItem.newBuilder().setRoundHeader(RoundHeader.getDefaultInstance()); + var eventHeader = BlockItem.newBuilder().setEventHeader(EventHeader.getDefaultInstance()); + var eventTransaction = BlockItem.newBuilder() + .setEventTransaction(EventTransaction.newBuilder() + .setApplicationTransaction(ByteString.copyFrom(TestUtils.generateRandomByteArray(32)))); + var transactionResult = BlockItem.newBuilder().setTransactionResult(TransactionResult.getDefaultInstance()); + var block = Block.newBuilder() + .addItems(blockHeader()) + .addItems(roundHeader) + .addItems(eventHeader) + .addItems(eventTransaction) + .addItems(eventTransaction) + .addItems(transactionResult) + .addItems(blockProof()) + .build(); + var streamFileData = StreamFileData.from("000000000000000000000000000000000001.blk.gz", gzip(block)); + assertThatThrownBy(() -> reader.read(streamFileData)) + .isInstanceOf(InvalidStreamFileException.class) + .hasMessageContaining("Failed to deserialize Transaction"); + } + + private BlockItem blockHeader() { + return BlockItem.newBuilder() + .setBlockHeader(BlockHeader.newBuilder() + .setPreviousBlockHash(ByteString.copyFrom(TestUtils.generateRandomByteArray(48)))) + .build(); + } + + private BlockItem blockProof() { + return BlockItem.newBuilder() + .setBlockProof(BlockProof.newBuilder() + .setStartOfBlockStateRootHash(ByteString.copyFrom(TestUtils.generateRandomByteArray(48)))) + .build(); + } + + private BlockItem eventTransaction() { + var transaction = Transaction.newBuilder() + .setSignedTransactionBytes(SignedTransaction.newBuilder() + .setBodyBytes(TransactionBody.newBuilder() + .setCryptoTransfer(CryptoTransferTransactionBody.getDefaultInstance()) + .build() + .toByteString()) + .build() + .toByteString()) + .build() + .toByteString(); + return BlockItem.newBuilder() + .setEventTransaction(EventTransaction.newBuilder() + .setApplicationTransaction(transaction) + .build()) + .build(); + } + + @SneakyThrows + private static byte[] gzip(Block block) { + try (var bos = new ByteArrayOutputStream(); + var gos = new GzipCompressorOutputStream(bos)) { + gos.write(block.toByteArray()); + gos.finish(); + return bos.toByteArray(); + } + } + + @SneakyThrows + private static Stream readTestArgumentsProvider() { + List argumentsList = new ArrayList<>(); + + String filename = "000000000000000000000000000007858853.blk.gz"; + long index = 7858853; + long round = index + 1; + var file = new ClassPathResource("data/blockstreams/" + filename).getFile(); + var streamFileData = StreamFileData.from(file); + var expected = BlockFile.builder() + .bytes(streamFileData.getBytes()) + .consensusStart(1736197012160646000L) + .consensusEnd(1736197012160646001L) + .count(2L) + .digestAlgorithm(DigestAlgorithm.SHA_384) + .hash( + "581caa8ab1fad535a0fac97957c5c0cf44c528ee55724353b4bab9093083fda32429f73248bc3128e329bbdfa1967d20") + .index(index) + .loadStart(streamFileData.getStreamFilename().getTimestamp()) + .name(filename) + .previousHash( + "ba1a0222099d542425f6915053b7f15e3b75fd680b0d84ca6d41fbffcd38f8fb5ac6ab6a235e69f7ae23118d1996c7f1") + .roundStart(round) + .roundEnd(round) + .size(streamFileData.getBytes().length) + .version(7) + .build(); + argumentsList.add(Arguments.of(filename, streamFileData, expected)); + + // A block without event transactions, note consensusStart and consensusEnd are both null due to the bug that + // BlockHeader.first_transaction_consensus_time is null + filename = "000000000000000000000000000007858854.blk.gz"; + index = 7858854; + round = index + 1; + file = new ClassPathResource("data/blockstreams/" + filename).getFile(); + streamFileData = StreamFileData.from(file); + // Verifies the calculated hash of the previous block matches the previous hash in this (the next) block file + String previousHash = expected.getHash(); + expected = BlockFile.builder() + .bytes(streamFileData.getBytes()) + .consensusStart(null) + .consensusEnd(null) + .count(0L) + .digestAlgorithm(DigestAlgorithm.SHA_384) + .hash( + "ef32f163bee6553087002310467b970b1de2c8cbec2eab46f0d0c58ff34043d080f43c9e3c759956fda19fc9f5a5966b") + .index(index) + .loadStart(streamFileData.getStreamFilename().getTimestamp()) + .name(filename) + .previousHash(previousHash) + .roundStart(round) + .roundEnd(round) + .size(streamFileData.getBytes().length) + .version(7) + .build(); + argumentsList.add(Arguments.of(filename, streamFileData, expected)); + + return argumentsList.stream(); + } +} diff --git a/hedera-mirror-importer/src/test/resources/data/blockstreams/000000000000000000000000000007858853.blk.gz b/hedera-mirror-importer/src/test/resources/data/blockstreams/000000000000000000000000000007858853.blk.gz new file mode 100644 index 00000000000..b6b4e492e45 Binary files /dev/null and b/hedera-mirror-importer/src/test/resources/data/blockstreams/000000000000000000000000000007858853.blk.gz differ diff --git a/hedera-mirror-importer/src/test/resources/data/blockstreams/000000000000000000000000000007858854.blk.gz b/hedera-mirror-importer/src/test/resources/data/blockstreams/000000000000000000000000000007858854.blk.gz new file mode 100644 index 00000000000..f1e78687f1e Binary files /dev/null and b/hedera-mirror-importer/src/test/resources/data/blockstreams/000000000000000000000000000007858854.blk.gz differ