Skip to content

Commit

Permalink
HIP-1056 Add block file transformer (#10147)
Browse files Browse the repository at this point in the history
- Adds the block file transformer for the cryptotransfer transaction type

Signed-off-by: Edwin Greene <[email protected]>
Signed-off-by: Xin Li <[email protected]>
Co-authored-by: Xin Li <[email protected]>
  • Loading branch information
edwin-greene and xin-hedera authored Jan 22, 2025
1 parent 33efcfc commit f4a74ca
Show file tree
Hide file tree
Showing 18 changed files with 688 additions and 117 deletions.
11 changes: 2 additions & 9 deletions docs/design/block-streams.md
Original file line number Diff line number Diff line change
Expand Up @@ -103,18 +103,11 @@ package com.hedera.mirror.importer.downloader.block;
public class BlockFileTransformer implements StreamFileTransformer<RecordFile, BlockFile> {

/**
* Transforms the block file into a record file and calculates the block hash
* The transformation uses a mapping of block fields to record file fields
* Block items are only iterated once in the transform method
* State changes are accumulated and used for calculating the block hash
*
* If the Block File contains a Wrapped Record File, then convert the Wrapped Record File to a Record File
* Transforms the block file into a record file. The transformation uses a mapping of block fields to record file
* fields. Block items are only iterated once in the transform method.
*/
@Override
public RecordFile transform(BlockFile block);

// The transaction hash will not be included in the block stream output so we will need to calculate it
private byte[] calculateTransactionHash(EventTransaction transaction);
}
```

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import lombok.NoArgsConstructor;
import lombok.Singular;
import lombok.ToString;
import org.apache.commons.lang3.StringUtils;

@Builder(toBuilder = true)
@Data
Expand Down Expand Up @@ -92,7 +93,7 @@ public StreamFile<BlockItem> copy() {

@Override
public String getFileHash() {
return null;
return StringUtils.EMPTY;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@
import com.google.protobuf.Internal;
import com.google.protobuf.UnsafeByteOperations;
import com.hedera.mirror.common.converter.ObjectToStringSerializer;
import com.hedera.mirror.common.domain.DigestAlgorithm;
import com.hedera.mirror.common.domain.entity.EntityId;
import com.hedera.mirror.common.exception.InvalidEntityException;
import com.hedera.mirror.common.exception.ProtobufException;
Expand All @@ -32,6 +33,8 @@
import jakarta.annotation.Nullable;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.List;
import lombok.CustomLog;
Expand Down Expand Up @@ -166,6 +169,14 @@ public static long convertToNanosMax(Instant instant) {
return convertToNanosMax(instant.getEpochSecond(), instant.getNano());
}

public static MessageDigest createSha384Digest() {
try {
return MessageDigest.getInstance(DigestAlgorithm.SHA_384.getName());
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("SHA-384 algorithm not found", e);
}
}

/**
* Pad a byte array with leading zeros to a given length.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1032,7 +1032,7 @@ public DomainWrapper<TokenTransfer, TokenTransfer.TokenTransferBuilder> tokenTra
public DomainWrapper<TopicMessage, TopicMessage.TopicMessageBuilder> topicMessage() {
var transactionId = TransactionID.newBuilder()
.setAccountID(AccountID.newBuilder().setAccountNum(id()))
.setTransactionValidStart(Timestamp.newBuilder().setSeconds(timestamp()))
.setTransactionValidStart(protoTimestamp())
.build()
.toByteArray();
var builder = TopicMessage.builder()
Expand Down Expand Up @@ -1203,6 +1203,14 @@ public void resetTimestamp(long value) {
timestampOffset = value - timestampNoOffset();
}

public Timestamp protoTimestamp() {
long timestamp = timestamp();
return Timestamp.newBuilder()
.setSeconds(timestamp / DomainUtils.NANOS_PER_SECOND)
.setNanos((int) (timestamp % DomainUtils.NANOS_PER_SECOND))
.build();
}

public long timestamp() {
return timestampNoOffset() + timestampOffset;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,82 @@
package com.hedera.mirror.importer.downloader.block;

import com.hedera.mirror.common.domain.transaction.BlockFile;
import com.hedera.mirror.common.domain.transaction.BlockItem;
import com.hedera.mirror.common.domain.transaction.RecordFile;
import com.hedera.mirror.common.domain.transaction.RecordItem;
import com.hedera.mirror.importer.downloader.StreamFileTransformer;
import com.hedera.mirror.importer.downloader.block.transformer.BlockItemTransformerFactory;
import jakarta.inject.Named;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import lombok.RequiredArgsConstructor;
import org.apache.commons.lang3.StringUtils;
import org.springframework.data.util.Version;

@Named
@RequiredArgsConstructor
public class BlockFileTransformer implements StreamFileTransformer<RecordFile, BlockFile> {

private final BlockItemTransformerFactory blockItemTransformerFactory;

@Override
public RecordFile transform(BlockFile blockFile) {
return RecordFile.builder().build();
var blockHeader = blockFile.getBlockHeader();
var hapiProtoVersion = blockHeader.getHapiProtoVersion();
int major = hapiProtoVersion.getMajor();
int minor = hapiProtoVersion.getMinor();
int patch = hapiProtoVersion.getPatch();
var hapiVersion = new Version(major, minor, patch);
var softwareVersion = blockHeader.getSoftwareVersion();
return RecordFile.builder()
.bytes(blockFile.getBytes())
.consensusEnd(blockFile.getConsensusEnd())
.consensusStart(blockFile.getConsensusStart())
.count(blockFile.getCount())
.digestAlgorithm(blockFile.getDigestAlgorithm())
.fileHash(StringUtils.EMPTY)
.hapiVersionMajor(major)
.hapiVersionMinor(minor)
.hapiVersionPatch(patch)
.hash(blockFile.getHash())
.index(blockHeader.getNumber())
.items(getRecordItems(blockFile.getItems(), hapiVersion))
.loadEnd(blockFile.getLoadEnd())
.loadStart(blockFile.getLoadStart())
.name(blockFile.getName())
.nodeId(blockFile.getNodeId())
.previousHash(blockFile.getPreviousHash())
.roundEnd(blockFile.getRoundEnd())
.roundStart(blockFile.getRoundStart())
.size(blockFile.getSize())
.softwareVersionMajor(softwareVersion.getMajor())
.softwareVersionMinor(softwareVersion.getMinor())
.softwareVersionPatch(softwareVersion.getPatch())
.version(blockFile.getVersion())
.build();
}

private List<RecordItem> getRecordItems(Collection<BlockItem> blockItems, Version hapiVersion) {
if (blockItems.isEmpty()) {
return Collections.emptyList();
}

RecordItem previousItem = null;
var recordItems = new ArrayList<RecordItem>(blockItems.size());
for (var blockItem : blockItems) {
var recordItem = RecordItem.builder()
.hapiVersion(hapiVersion)
.previous(previousItem)
.transaction(blockItem.transaction())
.transactionIndex(recordItems.size())
.transactionRecord(blockItemTransformerFactory.getTransactionRecord(blockItem))
.build();
recordItems.add(recordItem);
previousItem = recordItem;
}

return recordItems;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
/*
* Copyright (C) 2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hedera.mirror.importer.downloader.block.transformer;

import static com.hedera.mirror.common.util.DomainUtils.createSha384Digest;

import com.google.protobuf.ByteString;
import com.hedera.mirror.common.domain.transaction.BlockItem;
import com.hedera.mirror.common.util.DomainUtils;
import com.hederahashgraph.api.proto.java.TransactionBody;
import com.hederahashgraph.api.proto.java.TransactionReceipt;
import com.hederahashgraph.api.proto.java.TransactionRecord;
import java.security.MessageDigest;

abstract class AbstractBlockItemTransformer implements BlockItemTransformer {

private static final MessageDigest DIGEST = createSha384Digest();

public TransactionRecord getTransactionRecord(BlockItem blockItem, TransactionBody transactionBody) {
var transactionResult = blockItem.transactionResult();
var receiptBuilder = TransactionReceipt.newBuilder().setStatus(transactionResult.getStatus());
var transactionRecordBuilder = TransactionRecord.newBuilder()
.addAllAutomaticTokenAssociations(transactionResult.getAutomaticTokenAssociationsList())
.addAllPaidStakingRewards(transactionResult.getPaidStakingRewardsList())
.addAllTokenTransferLists(transactionResult.getTokenTransferListsList())
.setConsensusTimestamp(transactionResult.getConsensusTimestamp())
.setParentConsensusTimestamp(transactionResult.getParentConsensusTimestamp())
.setMemo(transactionBody.getMemo())
.setReceipt(receiptBuilder)
.setScheduleRef(transactionResult.getScheduleRef())
.setTransactionFee(transactionResult.getTransactionFeeCharged())
.setTransactionHash(
calculateTransactionHash(blockItem.transaction().getSignedTransactionBytes()))
.setTransactionID(transactionBody.getTransactionID())
.setTransferList(transactionResult.getTransferList());

updateTransactionRecord(blockItem, transactionRecordBuilder);
return transactionRecordBuilder.build();
}

protected void updateTransactionRecord(BlockItem blockItem, TransactionRecord.Builder transactionRecordBuilder) {
// do nothing
}

private ByteString calculateTransactionHash(ByteString signedTransactionBytes) {
return DomainUtils.fromBytes(DIGEST.digest(DomainUtils.toBytes(signedTransactionBytes)));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Copyright (C) 2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hedera.mirror.importer.downloader.block.transformer;

import com.hedera.mirror.common.domain.transaction.BlockItem;
import com.hedera.mirror.common.domain.transaction.TransactionType;
import com.hederahashgraph.api.proto.java.TransactionBody;
import com.hederahashgraph.api.proto.java.TransactionRecord;

interface BlockItemTransformer {

TransactionRecord getTransactionRecord(BlockItem blockItem, TransactionBody transactionBody);

TransactionType getType();
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Copyright (C) 2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hedera.mirror.importer.downloader.block.transformer;

import com.google.protobuf.ByteString;
import com.google.protobuf.InvalidProtocolBufferException;
import com.hedera.mirror.common.domain.transaction.BlockItem;
import com.hedera.mirror.common.domain.transaction.TransactionType;
import com.hedera.mirror.common.exception.ProtobufException;
import com.hederahashgraph.api.proto.java.SignedTransaction;
import com.hederahashgraph.api.proto.java.TransactionBody;
import com.hederahashgraph.api.proto.java.TransactionRecord;
import jakarta.inject.Named;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;

@Named
public class BlockItemTransformerFactory {

private final BlockItemTransformer defaultTransformer;
private final Map<TransactionType, BlockItemTransformer> transformers;

BlockItemTransformerFactory(List<BlockItemTransformer> transformers) {
this.transformers = transformers.stream()
.collect(Collectors.toUnmodifiableMap(BlockItemTransformer::getType, Function.identity()));
this.defaultTransformer = this.transformers.get(TransactionType.UNKNOWN);
}

public TransactionRecord getTransactionRecord(BlockItem blockItem) {
var transactionBody = parse(blockItem.transaction().getSignedTransactionBytes());
var blockItemTransformer = get(transactionBody);
// pass transactionBody for performance
return blockItemTransformer.getTransactionRecord(blockItem, transactionBody);
}

private BlockItemTransformer get(TransactionBody transactionBody) {
var transactionType = TransactionType.of(transactionBody.getDataCase().getNumber());
return transformers.getOrDefault(transactionType, defaultTransformer);
}

private TransactionBody parse(ByteString signedTransactionBytes) {
try {
var signedTransaction = SignedTransaction.parseFrom(signedTransactionBytes);
return TransactionBody.parseFrom(signedTransaction.getBodyBytes());
} catch (InvalidProtocolBufferException e) {
throw new ProtobufException("Error parsing transaction body from signed transaction bytes", e);
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright (C) 2025 Hedera Hashgraph, LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.hedera.mirror.importer.downloader.block.transformer;

import com.hedera.mirror.common.domain.transaction.BlockItem;
import com.hedera.mirror.common.domain.transaction.TransactionType;
import com.hederahashgraph.api.proto.java.TransactionRecord;
import jakarta.inject.Named;

@Named
class CryptoTransferTransformer extends AbstractBlockItemTransformer {

@Override
protected void updateTransactionRecord(BlockItem blockItem, TransactionRecord.Builder transactionRecordBuilder) {
for (var transactionOutput : blockItem.transactionOutput()) {
if (transactionOutput.hasCryptoTransfer()) {
var cryptoTransferOutput = transactionOutput.getCryptoTransfer();
var assessedCustomFees = cryptoTransferOutput.getAssessedCustomFeesList();
transactionRecordBuilder.addAllAssessedCustomFees(assessedCustomFees);
break;
}
}
}

@Override
public TransactionType getType() {
return TransactionType.CRYPTOTRANSFER;
}
}
Loading

0 comments on commit f4a74ca

Please sign in to comment.