Merge branch 'main' into 10167-hip-1056-topic-block-item-to-record-it…

…em-trasnformer

build.gradle.kts

@@ -134,7 +134,7 @@ allprojects {
             property("sonar.issue.ignore.multicriteria.e6.ruleKey", "java:S2970")
             property(
                 "sonar.exclusions",
-                "src/main/java/com/hedera/services/**,src/test/java/com/hedera/services/**"
+                "src/main/java/com/hedera/services/**,src/test/java/com/hedera/services/**",
             )
         }
     }
@@ -204,18 +204,12 @@ spotless {
     }
     format("javascript") {
         endWithNewline()
-        indentWithSpaces(2)
+        leadingTabsToSpaces(2)
         licenseHeader(licenseHeader, "$").updateYearWithLatest(true)
         prettier()
             .npmExecutable(npmExecutable)
             .npmInstallCache(Paths.get("${rootProject.rootDir}", ".gradle", "spotless"))
-            .config(
-                mapOf(
-                    "bracketSpacing" to false,
-                    "printWidth" to 120,
-                    "singleQuote" to true,
-                )
-            )
+            .config(mapOf("bracketSpacing" to false, "printWidth" to 120, "singleQuote" to true))
         target("hedera-mirror-rest/**/*.js", "hedera-mirror-test/**/*.js")
         targetExclude("**/build/**", "**/node_modules/**", "**/__tests__/integration/*.test.js")
     }
@@ -229,7 +223,7 @@ spotless {
             "hedera-mirror-rest/**",
             "hedera-mirror-rosetta/**",
             // Known issue with Java 21: https://github.com/palantir/palantir-java-format/issues/933
-            "hedera-mirror-rest-java/**/EntityServiceImpl.java"
+            "hedera-mirror-rest-java/**/EntityServiceImpl.java",
         )
         toggleOffOn()
     }
@@ -249,23 +243,23 @@ spotless {
     }
     format("miscellaneous") {
         endWithNewline()
-        indentWithSpaces(2)
+        leadingTabsToSpaces(2)
         prettier().npmExecutable(npmExecutable)
         target("**/*.json", "**/*.md", "**/*.yml", "**/*.yaml")
         targetExclude("**/build/**", "**/charts/**", "**/node_modules/**", "**/package-lock.json")
         trimTrailingWhitespace()
     }
     format("proto") {
         endWithNewline()
-        indentWithSpaces(4)
+        leadingTabsToSpaces(4)
         licenseHeader(licenseHeader, "(package|syntax)").updateYearWithLatest(true)
         target("hedera-mirror-protobuf/**/*.proto")
         targetExclude("build/**")
         trimTrailingWhitespace()
     }
     sql {
         endWithNewline()
-        indentWithSpaces()
+        leadingTabsToSpaces()
         target("hedera-mirror-(common|importer|rest)/**/*.sql")
         targetExclude("**/build/**", "**/node_modules/**")
         trimTrailingWhitespace()
@@ -278,7 +272,7 @@ fun replaceVersion(files: String, match: String) {
             "fileset"(
                 "dir" to rootProject.projectDir,
                 "includes" to files,
-                "excludes" to "**/node_modules/"
+                "excludes" to "**/node_modules/",
             )
         }
     }
@@ -295,12 +289,12 @@ tasks.register("release") {
         replaceVersion("gradle.properties", "(?<=^version=).+")
         replaceVersion(
             "hedera-mirror-rest/**/package*.json",
-            "(?<=\"@hashgraph/(check-state-proof|mirror-rest|mirror-monitor)\",\\s{3,7}\"version\": \")[^\"]+"
+            "(?<=\"@hashgraph/(check-state-proof|mirror-rest|mirror-monitor)\",\\s{3,7}\"version\": \")[^\"]+",
         )
         replaceVersion("hedera-mirror-rest/**/openapi.yml", "(?<=^  version: ).+")
         replaceVersion(
             "tools/traffic-replay/log-downloader/package*.json",
-            "(?<=\"@hashgraph/mirror-log-downloader\",\\s{3,7}\"version\": \")[^\"]+"
+            "(?<=\"@hashgraph/mirror-log-downloader\",\\s{3,7}\"version\": \")[^\"]+",
         )
     }
 }

buildSrc/build.gradle.kts

@@ -32,7 +32,7 @@ dependencies {
     implementation("com.github.docker-java:docker-java-core:$dockerJavaVersion")
 
     implementation("com.bmuschko:gradle-docker-plugin:9.4.0")
-    implementation("com.diffplug.spotless:spotless-plugin-gradle:6.25.0")
+    implementation("com.diffplug.spotless:spotless-plugin-gradle:7.0.2")
     implementation("com.gradleup.shadow:shadow-gradle-plugin:8.3.6")
     implementation("com.github.node-gradle:gradle-node-plugin:7.1.0")
     implementation("com.google.protobuf:protobuf-gradle-plugin:0.9.4")

buildSrc/src/main/kotlin/go-conventions.gradle.kts

@@ -68,7 +68,7 @@ tasks.register<Go>("test") {
         "-covermode=atomic",
         "-race",
         "-v",
-        go.pkg
+        go.pkg,
     )
     dependsOn("fix")
 

buildSrc/src/main/kotlin/jooq-conventions.gradle.kts

@@ -81,7 +81,7 @@ tasks.flywayMigrate {
     locations =
         arrayOf(
             "filesystem:../hedera-mirror-importer/src/main/resources/db/migration/v1",
-            "filesystem:../hedera-mirror-importer/src/main/resources/db/migration/common"
+            "filesystem:../hedera-mirror-importer/src/main/resources/db/migration/common",
         )
     password = dbPassword
     placeholders =
@@ -94,7 +94,7 @@ tasks.flywayMigrate {
             "partitionTimeInterval" to "'100 years'",
             "schema" to dbSchema,
             "tempSchema" to "temporary",
-            "topicRunningHashV2AddedTimestamp" to "0"
+            "topicRunningHashV2AddedTimestamp" to "0",
         )
     user = dbUser
 

buildSrc/src/main/kotlin/openapi-conventions.gradle.kts

@@ -37,7 +37,7 @@ openApiGenerate {
             "sourceFolder" to "",
             "supportUrlQuery" to "false",
             "useBeanValidation" to "true",
-            "useJakartaEe" to "true"
+            "useJakartaEe" to "true",
         )
     generateApiTests = false
     generateModelTests = false

docs/configuration.md

@@ -59,6 +59,10 @@ value, it is recommended to only populate overridden properties in the custom `a
 | `hedera.mirror.importer.downloader.balance.writeFiles`                           | false                                                | Whether to write verified stream files to the filesystem.                                                                                                                                                                                                          |
 | `hedera.mirror.importer.downloader.balance.writeSignatures`                      | false                                                | Whether to write verified signature files to the filesystem.                                                                                                                                                                                                       |
 | `hedera.mirror.importer.downloader.batchSize`                                    | 25                                                   | The number of signature files to download per node before downloading the signed files                                                                                                                                                                             |
+| `hedera.mirror.importer.downloader.block.enabled`                                | false                                                | Whether to enable block stream files poller                                                                                                                                                                                                                        |
+| `hedera.mirror.importer.downloader.block.frequency`                              | 100ms                                                | The fixed period between invocations. Can accept duration units like `10s`, `2m`, etc. If not specified, millisecond is implied as the unit.                                                                                                                       |
+| `hedera.mirror.importer.downloader.block.persistBytes`                           | false                                                | Whether to persist the block stream file bytes to the database.                                                                                                                                                                                                    |
+| `hedera.mirror.importer.downloader.block.writeFiles`                             | false                                                | Whether to write verified block stream files to the filesystem.                                                                                                                                                                                                    |
 | `hedera.mirror.importer.downloader.bucketName`                                   |                                                      | The cloud storage bucket name to download streamed files. This value takes priority over network hardcoded bucket names regardless of `hedera.mirror.importer.network` value.                                                                                      |
 | `hedera.mirror.importer.downloader.cloudProvider`                                | S3                                                   | The cloud provider to download files from. Either `GCP`, `LOCAL`, or `S3`.                                                                                                                                                                                         |
 | `hedera.mirror.importer.downloader.consensusRatio`                               | 0.33333333333                                        | The ratio of verified nodes (nodes used to come to consensus on the signature file hash) to total number of nodes available.                                                                                                                                       |
@@ -192,7 +196,7 @@ value, it is recommended to only populate overridden properties in the custom `a
 | `hedera.mirror.importer.topicRunningHashV2AddedTimestamp`                        | Network-based                                        | Unix timestamp (in nanos) of first topic message with v2 as running hash version. Use this config to override the default network based value                                                                                                                      |
 | `hedera.mirror.importer.shard`                                                   | 0                                                    | The default shard number that the component participates in                                                                                                                                                                                                        |
 | `hedera.mirror.importer.startDate`                                               |                                                      | The start date (inclusive) of the data to import. It takes effect 1) if it's set and the date is after the last downloaded file or the database is empty; 2) if it's not set and the database is empty, it defaults to now. Format: YYYY-MM-ddTHH:mm:ss.nnnnnnnnnZ |
-| `hedera.mirror.importer.startBlockNumber`                                        | null                                                 | The block number that will be set as the downloaded stream files starting index.                                                                                                                                                                                   |
+| `hedera.mirror.importer.startBlockNumber`                                        | null                                                 | The block number that will be set as the downloaded stream files starting index. For block stream files, it's the first block to download and ignored if there are existing blocks in database.                                                                    |
 
 ### Transaction and Entity Filtering
 

hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/StreamFile.java

@@ -22,9 +22,10 @@
 
 public interface StreamFile<T extends StreamItem> {
 
-    default void clear() {
+    default StreamFile<T> clear() {
         setBytes(null);
         setItems(List.of());
+        return this;
     }
 
     StreamFile<T> copy();

hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/StreamType.java

@@ -44,9 +44,6 @@ public enum StreamType {
 
     public static final String SIGNATURE_SUFFIX = "_sig";
 
-    private static final String PARSED = "parsed";
-    private static final String SIGNATURES = "signatures";
-
     private final SortedSet<Extension> dataExtensions;
     private final String nodePrefix;
     private final String path;
@@ -78,14 +75,6 @@ public enum StreamType {
                 .collect(ImmutableSortedSet.toImmutableSortedSet(Comparator.naturalOrder()));
     }
 
-    public String getParsed() {
-        return PARSED;
-    }
-
-    public String getSignatures() {
-        return SIGNATURES;
-    }
-
     public boolean isChained() {
         return this != BALANCE;
     }

hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/BlockFile.java

@@ -16,6 +16,8 @@
 
 package com.hedera.mirror.common.domain.transaction;
 
+import static org.apache.commons.lang3.StringUtils.leftPad;
+
 import com.hedera.hapi.block.stream.output.protoc.BlockHeader;
 import com.hedera.hapi.block.stream.protoc.BlockProof;
 import com.hedera.hapi.block.stream.protoc.RecordFileItem;
@@ -38,6 +40,10 @@
 @NoArgsConstructor
 public class BlockFile implements StreamFile<BlockItem> {
 
+    private static final int BASENAME_LENGTH = 36;
+    private static final char BASENAME_PADDING = '0';
+    private static final String COMPRESSED_FILE_SUFFIX = ".blk.gz";
+
     // Contains the block number and the previous block hash
     private BlockHeader blockHeader;
 
@@ -86,6 +92,14 @@ public class BlockFile implements StreamFile<BlockItem> {
 
     private int version;
 
+    public static String getBlockStreamFilename(long blockNumber) {
+        if (blockNumber < 0) {
+            throw new IllegalArgumentException("Block number must be non-negative");
+        }
+
+        return leftPad(Long.toString(blockNumber), BASENAME_LENGTH, BASENAME_PADDING) + COMPRESSED_FILE_SUFFIX;
+    }
+
     @Override
     public StreamFile<BlockItem> copy() {
         return this.toBuilder().build();

hedera-mirror-common/src/main/java/com/hedera/mirror/common/domain/transaction/RecordFile.java

@@ -134,10 +134,11 @@ public class RecordFile implements StreamFile<RecordItem> {
     private int version;
 
     @Override
-    public void clear() {
+    public RecordFile clear() {
         StreamFile.super.clear();
         setLogsBloom(null);
         setSidecars(List.of());
+        return this;
     }
 
     @Override

hedera-mirror-common/src/test/java/com/hedera/mirror/common/domain/transaction/BlockFileTest.java

@@ -17,11 +17,21 @@
 package com.hedera.mirror.common.domain.transaction;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import org.junit.jupiter.api.Test;
 
 class BlockFileTest {
 
+    @Test
+    void getBlockStreamFilename() {
+        assertThat(BlockFile.getBlockStreamFilename(0)).isEqualTo("000000000000000000000000000000000000.blk.gz");
+        assertThat(BlockFile.getBlockStreamFilename(1)).isEqualTo("000000000000000000000000000000000001.blk.gz");
+        assertThatThrownBy(() -> BlockFile.getBlockStreamFilename(-1))
+                .isInstanceOf(IllegalArgumentException.class)
+                .hasMessage("Block number must be non-negative");
+    }
+
     @Test
     void onNewRound() {
         var blockFile = BlockFile.builder().onNewRound(1L).build();

hedera-mirror-graphql/build.gradle.kts

@@ -47,7 +47,7 @@ dependencies {
     runtimeOnly(
         group = "io.netty",
         name = "netty-resolver-dns-native-macos",
-        classifier = "osx-aarch_64"
+        classifier = "osx-aarch_64",
     )
     runtimeOnly("org.postgresql:postgresql")
     testImplementation(project(path = ":common", configuration = "testClasses"))
@@ -74,7 +74,7 @@ generatePojoConf {
                 "java.time.Duration",
                 "",
                 "com.hedera.mirror.graphql.config.GraphQlDuration.INSTANCE",
-                ""
+                "",
             ),
             CustomScalarDefinition("Long", "java.lang.Long", "", "graphql.scalars.GraphQLLong", ""),
             CustomScalarDefinition("Object", "java.lang.Object", "", "graphql.scalars.Object", ""),
@@ -83,7 +83,7 @@ generatePojoConf {
                 "java.time.Instant",
                 "",
                 "com.hedera.mirror.graphql.config.GraphQlTimestamp.INSTANCE",
-                ""
+                "",
             ),
         )
     )

hedera-mirror-grpc/build.gradle.kts

@@ -47,7 +47,7 @@ dependencies {
     runtimeOnly(
         group = "io.netty",
         name = "netty-resolver-dns-native-macos",
-        classifier = "osx-aarch_64"
+        classifier = "osx-aarch_64",
     )
     runtimeOnly("org.postgresql:postgresql")
     testImplementation(project(path = ":common", configuration = "testClasses"))

hedera-mirror-importer/build.gradle.kts

@@ -54,7 +54,7 @@ dependencies {
     runtimeOnly(
         group = "io.netty",
         name = "netty-resolver-dns-native-macos",
-        classifier = "osx-aarch_64"
+        classifier = "osx-aarch_64",
     )
     testImplementation(project(path = ":common", configuration = "testClasses"))
     testImplementation("com.github.vertical-blank:sql-formatter")

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/ImporterProperties.java

@@ -74,6 +74,7 @@ public class ImporterProperties {
 
     private Instant startDate;
 
+    @Min(0)
     private Long startBlockNumber;
 
     private Long topicRunningHashV2AddedTimestamp;

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/config/MetricsExecutionInterceptor.java

@@ -66,7 +66,8 @@ public class MetricsExecutionInterceptor implements ExecutionInterceptor {
     private static final Pattern ENTITY_ID_PATTERN =
             Pattern.compile("/(balance|record)(s_)?(\\d{1,10})\\.\\d{1,10}\\.(\\d{1,10})");
     private static final Pattern SIDECAR_PATTERN = Pattern.compile("Z_\\d{1,2}\\.rcd");
-    private static final Pattern NODE_ID_PATTERN = Pattern.compile("[^/]/(\\d{1,10})/(\\d{1,10})/(balance|record)/");
+    private static final Pattern NODE_ID_PATTERN =
+            Pattern.compile("[^/]/+(\\d{1,10})/(\\d{1,10})/((balance|record)/)?");
 
     private final MeterRegistry meterRegistry;
 
@@ -130,18 +131,18 @@ private UriAttributes getUriAttributes(URI uri) {
         var action = getAction(uriComponent);
 
         Matcher accountIdMatcher = ENTITY_ID_PATTERN.matcher(uriComponent);
-        if (accountIdMatcher.find() && accountIdMatcher.groupCount() == 4) {
+        if (accountIdMatcher.find()) {
             var shard = accountIdMatcher.group(3);
             var nodeId = String.valueOf(Long.parseLong(accountIdMatcher.group(4)) - 3L);
             var streamType = accountIdMatcher.group(1);
             return new UriAttributes(action, nodeId, shard, streamType.toUpperCase());
         }
 
         Matcher nodeIdMatcher = NODE_ID_PATTERN.matcher(uriComponent);
-        if (nodeIdMatcher.find() && nodeIdMatcher.groupCount() == 3) {
+        if (nodeIdMatcher.find()) {
             var shard = nodeIdMatcher.group(1);
             var nodeId = nodeIdMatcher.group(2);
-            var streamType = nodeIdMatcher.group(3);
+            var streamType = nodeIdMatcher.group(4) != null ? nodeIdMatcher.group(4) : StreamType.BLOCK.name();
             return new UriAttributes(action, nodeId, shard, streamType.toUpperCase());
         }
 

hedera-mirror-importer/src/main/java/com/hedera/mirror/importer/domain/StreamFilename.java

@@ -23,6 +23,7 @@
 
 import com.google.common.base.Splitter;
 import com.hedera.mirror.common.domain.StreamType;
+import com.hedera.mirror.common.domain.transaction.BlockFile;
 import com.hedera.mirror.importer.downloader.provider.S3StreamFileProvider;
 import com.hedera.mirror.importer.exception.InvalidStreamFileException;
 import java.time.Instant;
@@ -125,6 +126,10 @@ private StreamFilename(String path, String filename, String pathSeparator) {
         this.filePath = builder.toString();
     }
 
+    public static StreamFilename from(long blockNumber) {
+        return from(BlockFile.getBlockStreamFilename(blockNumber));
+    }
+
     public static StreamFilename from(String filePath) {
         return from(filePath, S3StreamFileProvider.SEPARATOR);
     }