feat: add configurable signer latency correction (#3317)

* feat: add configurable signer latency correction

* coderabbit feedback

* changelog

Author: gartnera

changelog.md

@@ -5,6 +5,8 @@
 ### Features
 
 * [3235](https://github.com/zeta-chain/node/pull/3235) - add /systemtime telemetry endpoint (zetaclient)
+* [3317](https://github.com/zeta-chain/node/pull/3317) - add configurable signer latency correction (zetaclient)
+
 
 ### Tests
 

cmd/zetae2e/local/local.go

@@ -362,7 +362,8 @@ func localE2ETest(cmd *cobra.Command, _ []string) {
 
 	if testAdmin {
 		eg.Go(adminTestRoutine(conf, deployerRunner, verbose,
-			e2etests.TestOperationalFlagsName,
+			e2etests.TestZetaclientSignerOffsetName,
+			e2etests.TestZetaclientRestartHeightName,
 			e2etests.TestWhitelistERC20Name,
 			e2etests.TestPauseZRC20Name,
 			e2etests.TestUpdateBytecodeZRC20Name,

docs/cli/zetacored/cli.md

@@ -13347,34 +13347,35 @@ zetacored tx observer update-operational-flags [flags]
 ### Options
 
 ```
-  -a, --account-number uint      The account number of the signing account (offline mode only)
-      --aux                      Generate aux signer data instead of sending a tx
-  -b, --broadcast-mode string    Transaction broadcasting mode (sync|async) 
-      --chain-id string          The network chain ID
-      --dry-run                  ignore the --gas flag and perform a simulation of a transaction, but don't broadcast it (when enabled, the local Keybase is not accessible)
-      --fee-granter string       Fee granter grants fees for the transaction
-      --fee-payer string         Fee payer pays fees for the transaction instead of deducting from the signer
-      --fees string              Fees to pay along with transaction; eg: 10uatom
-      --file string              Path to a JSON file containing OperationalFlags
-      --from string              Name or address of private key with which to sign
-      --gas string               gas limit to set per-transaction; set to "auto" to calculate sufficient gas automatically. Note: "auto" option doesn't always report accurate results. Set a valid coin value to adjust the result. Can be used instead of "fees". (default 200000)
-      --gas-adjustment float     adjustment factor to be multiplied against the estimate returned by the tx simulation; if the gas limit is set manually this flag is ignored  (default 1)
-      --gas-prices string        Gas prices in decimal format to determine the transaction fee (e.g. 0.1uatom)
-      --generate-only            Build an unsigned transaction and write it to STDOUT (when enabled, the local Keybase only accessed when providing a key name)
-  -h, --help                     help for update-operational-flags
-      --keyring-backend string   Select keyring's backend (os|file|kwallet|pass|test|memory) 
-      --keyring-dir string       The client Keyring directory; if omitted, the default 'home' directory will be used
-      --ledger                   Use a connected Ledger device
-      --node string              [host]:[port] to tendermint rpc interface for this chain 
-      --note string              Note to add a description to the transaction (previously --memo)
-      --offline                  Offline mode (does not allow any online functionality)
-  -o, --output string            Output format (text|json) 
-      --restart-height int       Height for a coordinated zetaclient restart
-  -s, --sequence uint            The sequence number of the signing account (offline mode only)
-      --sign-mode string         Choose sign mode (direct|amino-json|direct-aux), this is an advanced feature
-      --timeout-height uint      Set a block timeout height to prevent the tx from being committed past a certain height
-      --tip string               Tip is the amount that is going to be transferred to the fee payer on the target chain. This flag is only valid when used with --aux, and is ignored if the target chain didn't enable the TipDecorator
-  -y, --yes                      Skip tx broadcasting prompt confirmation
+  -a, --account-number uint                 The account number of the signing account (offline mode only)
+      --aux                                 Generate aux signer data instead of sending a tx
+  -b, --broadcast-mode string               Transaction broadcasting mode (sync|async) 
+      --chain-id string                     The network chain ID
+      --dry-run                             ignore the --gas flag and perform a simulation of a transaction, but don't broadcast it (when enabled, the local Keybase is not accessible)
+      --fee-granter string                  Fee granter grants fees for the transaction
+      --fee-payer string                    Fee payer pays fees for the transaction instead of deducting from the signer
+      --fees string                         Fees to pay along with transaction; eg: 10uatom
+      --file string                         Path to a JSON file containing OperationalFlags
+      --from string                         Name or address of private key with which to sign
+      --gas string                          gas limit to set per-transaction; set to "auto" to calculate sufficient gas automatically. Note: "auto" option doesn't always report accurate results. Set a valid coin value to adjust the result. Can be used instead of "fees". (default 200000)
+      --gas-adjustment float                adjustment factor to be multiplied against the estimate returned by the tx simulation; if the gas limit is set manually this flag is ignored  (default 1)
+      --gas-prices string                   Gas prices in decimal format to determine the transaction fee (e.g. 0.1uatom)
+      --generate-only                       Build an unsigned transaction and write it to STDOUT (when enabled, the local Keybase only accessed when providing a key name)
+  -h, --help                                help for update-operational-flags
+      --keyring-backend string              Select keyring's backend (os|file|kwallet|pass|test|memory) 
+      --keyring-dir string                  The client Keyring directory; if omitted, the default 'home' directory will be used
+      --ledger                              Use a connected Ledger device
+      --node string                         [host]:[port] to tendermint rpc interface for this chain 
+      --note string                         Note to add a description to the transaction (previously --memo)
+      --offline                             Offline mode (does not allow any online functionality)
+  -o, --output string                       Output format (text|json) 
+      --restart-height int                  Height for a coordinated zetaclient restart
+  -s, --sequence uint                       The sequence number of the signing account (offline mode only)
+      --sign-mode string                    Choose sign mode (direct|amino-json|direct-aux), this is an advanced feature
+      --signer-block-time-offset duration   Offset from the zetacore block time to initiate signing
+      --timeout-height uint                 Set a block timeout height to prevent the tx from being committed past a certain height
+      --tip string                          Tip is the amount that is going to be transferred to the fee payer on the target chain. This flag is only valid when used with --aux, and is ignored if the target chain didn't enable the TipDecorator
+  -y, --yes                                 Skip tx broadcasting prompt confirmation
 ```
 
 ### Options inherited from parent commands

docs/openapi/openapi.swagger.yaml

@@ -58110,6 +58110,11 @@ definitions:
         description: |-
           Height for a coordinated zetaclient restart.
           Will be ignored if missed.
+      signer_block_time_offset:
+        type: string
+        description: |-
+          Offset from the zetacore block time to initiate signing.
+          Should be calculated and set based on max(zetaclient_core_block_latency).
     description: Flags for the top-level operation of zetaclient.
   observerPendingNonces:
     type: object

docs/spec/observer/messages.md

@@ -169,7 +169,7 @@ message MsgUpdateGasPriceIncreaseFlags {
 ```proto
 message MsgUpdateOperationalFlags {
 	string creator = 1;
-	OperationalFlags operationalFlags = 2;
+	OperationalFlags operational_flags = 2;
 }
 ```
 

e2e/e2etests/e2etests.go

@@ -132,7 +132,8 @@ const (
 	TestMigrateERC20CustodyFundsName  = "migrate_erc20_custody_funds"
 	TestMigrateTSSName                = "migrate_tss"
 	TestSolanaWhitelistSPLName        = "solana_whitelist_spl"
-	TestOperationalFlagsName          = "operational_flags"
+	TestZetaclientRestartHeightName   = "zetaclient_restart_height"
+	TestZetaclientSignerOffsetName    = "zetaclient_signer_offset"
 
 	/*
 	 Operational tests
@@ -880,10 +881,16 @@ var AllE2ETests = []runner.E2ETest{
 		TestMigrateERC20CustodyFunds,
 	),
 	runner.NewE2ETest(
-		TestOperationalFlagsName,
-		"operational flags functionality",
+		TestZetaclientRestartHeightName,
+		"zetaclient scheduled restart height",
 		[]runner.ArgDefinition{},
-		TestOperationalFlags,
+		TestZetaclientRestartHeight,
+	),
+	runner.NewE2ETest(
+		TestZetaclientSignerOffsetName,
+		"zetaclient signer offset",
+		[]runner.ArgDefinition{},
+		TestZetaclientSignerOffset,
 	),
 	/*
 	 Special tests

e2e/e2etests/test_operational_flags.go

@@ -11,11 +11,13 @@ import (
 )
 
 const (
-	startTimestampMetricName = "zetaclient_last_start_timestamp_seconds"
+	startTimestampMetricName        = "zetaclient_last_start_timestamp_seconds"
+	blockTimeLatencyMetricName      = "zetaclient_core_block_latency"
+	blockTimeLatencySleepMetricName = "zetaclient_core_block_latency_sleep"
 )
 
-// TestOperationalFlags tests the functionality of operations flags.
-func TestOperationalFlags(r *runner.E2ERunner, _ []string) {
+// TestZetaclientRestartHeight tests scheduling a zetaclient restart via operational flags
+func TestZetaclientRestartHeight(r *runner.E2ERunner, _ []string) {
 	_, err := r.Clients.Zetacore.Observer.OperationalFlags(
 		r.Ctx,
 		&observertypes.QueryOperationalFlagsRequest{},
@@ -60,3 +62,46 @@ func TestOperationalFlags(r *runner.E2ERunner, _ []string) {
 
 	require.Greater(r, currentStartTime, originalStartTime+1)
 }
+
+// TestZetaclientSignerOffset tests scheduling a zetaclient restart via operational flags
+func TestZetaclientSignerOffset(r *runner.E2ERunner, _ []string) {
+	startBlockTimeLatencySleep, err := r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencySleepMetricName)
+	require.NoError(r, err)
+	require.InDelta(r, 0, startBlockTimeLatencySleep, .01, "start block time latency should be 0")
+
+	// get starting block time latency.
+	// we need to ensure it's not zero (if zetaclient just finished a restart)
+	var startBlockTimeLatency float64
+	require.Eventually(r, func() bool {
+		startBlockTimeLatency, err = r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencyMetricName)
+		require.NoError(r, err)
+		return startBlockTimeLatency > 1
+	}, time.Second*15, time.Millisecond*100)
+
+	desiredSignerBlockTimeOffset := time.Duration(startBlockTimeLatency*float64(time.Second)) + time.Millisecond*200
+
+	updateMsg := observertypes.NewMsgUpdateOperationalFlags(
+		r.ZetaTxServer.MustGetAccountAddressFromName(utils.OperationalPolicyName),
+		observertypes.OperationalFlags{
+			SignerBlockTimeOffset: &desiredSignerBlockTimeOffset,
+		},
+	)
+
+	_, err = r.ZetaTxServer.BroadcastTx(utils.OperationalPolicyName, updateMsg)
+	require.NoError(r, err)
+
+	operationalFlagsRes, err := r.Clients.Zetacore.Observer.OperationalFlags(
+		r.Ctx,
+		&observertypes.QueryOperationalFlagsRequest{},
+	)
+	require.NoError(r, err)
+	require.InDelta(r, desiredSignerBlockTimeOffset, *(operationalFlagsRes.OperationalFlags.SignerBlockTimeOffset), .01)
+
+	require.Eventually(r, func() bool {
+		blockTimeLatencySleep, err := r.Clients.ZetaclientMetrics.FetchGauge(blockTimeLatencySleepMetricName)
+		if err != nil {
+			return false
+		}
+		return blockTimeLatencySleep > .05
+	}, time.Second*20, time.Second*1)
+}

proto/zetachain/zetacore/observer/operational.proto

@@ -1,11 +1,19 @@
 syntax = "proto3";
 package zetachain.zetacore.observer;
 
+import "gogoproto/gogo.proto";
+import "google/protobuf/duration.proto";
+
 option go_package = "github.com/zeta-chain/node/x/observer/types";
 
 // Flags for the top-level operation of zetaclient.
 message OperationalFlags {
   // Height for a coordinated zetaclient restart.
   // Will be ignored if missed.
   int64 restart_height = 1;
+
+  // Offset from the zetacore block time to initiate signing.
+  // Should be calculated and set based on max(zetaclient_core_block_latency).
+  google.protobuf.Duration signer_block_time_offset = 2
+      [ (gogoproto.stdduration) = true ];
 }

proto/zetachain/zetacore/observer/tx.proto

@@ -143,7 +143,7 @@ message MsgUpdateGasPriceIncreaseFlagsResponse {}
 
 message MsgUpdateOperationalFlags {
   string creator = 1;
-  OperationalFlags operationalFlags = 2 [ (gogoproto.nullable) = false ];
+  OperationalFlags operational_flags = 2 [ (gogoproto.nullable) = false ];
 }
 
 message MsgUpdateOperationalFlagsResponse {}

testutil/sample/observer.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"math/rand"
 	"testing"
+	"time"
 
 	"github.com/cosmos/cosmos-sdk/crypto/keys/ed25519"
 	"github.com/cosmos/cosmos-sdk/testutil/testdata"
@@ -14,6 +15,7 @@ import (
 	"github.com/zeta-chain/node/pkg/chains"
 	"github.com/zeta-chain/node/pkg/cosmos"
 	zetacrypto "github.com/zeta-chain/node/pkg/crypto"
+	"github.com/zeta-chain/node/pkg/ptr"
 	"github.com/zeta-chain/node/x/observer/types"
 )
 
@@ -287,6 +289,7 @@ func GasPriceIncreaseFlags() types.GasPriceIncreaseFlags {
 
 func OperationalFlags() types.OperationalFlags {
 	return types.OperationalFlags{
-		RestartHeight: 1,
+		RestartHeight:         1,
+		SignerBlockTimeOffset: ptr.Ptr(time.Second),
 	}
 }

typescript/zetachain/zetacore/observer/operational_pb.d.ts

@@ -3,7 +3,7 @@
 /* eslint-disable */
 // @ts-nocheck
 
-import type { BinaryReadOptions, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf";
+import type { BinaryReadOptions, Duration, FieldList, JsonReadOptions, JsonValue, PartialMessage, PlainMessage } from "@bufbuild/protobuf";
 import { Message, proto3 } from "@bufbuild/protobuf";
 
 /**
@@ -20,6 +20,14 @@ export declare class OperationalFlags extends Message<OperationalFlags> {
    */
   restartHeight: bigint;
 
+  /**
+   * Offset from the zetacore block time to initiate signing.
+   * Should be calculated and set based on max(zetaclient_core_block_latency).
+   *
+   * @generated from field: google.protobuf.Duration signer_block_time_offset = 2;
+   */
+  signerBlockTimeOffset?: Duration;
+
   constructor(data?: PartialMessage<OperationalFlags>);
 
   static readonly runtime: typeof proto3;

typescript/zetachain/zetacore/observer/tx_pb.d.ts

@@ -694,7 +694,7 @@ export declare class MsgUpdateOperationalFlags extends Message<MsgUpdateOperatio
   creator: string;
 
   /**
-   * @generated from field: zetachain.zetacore.observer.OperationalFlags operationalFlags = 2;
+   * @generated from field: zetachain.zetacore.observer.OperationalFlags operational_flags = 2;
    */
   operationalFlags?: OperationalFlags;
 

x/observer/client/cli/tx_update_operational_flags.go

@@ -13,8 +13,9 @@ import (
 )
 
 const (
-	fileFlag          = "file"
-	restartHeightFlag = "restart-height"
+	fileFlag                  = "file"
+	restartHeightFlag         = "restart-height"
+	signerBlockTimeOffsetFlag = "signer-block-time-offset"
 )
 
 func CmdUpdateOperationalFlags() *cobra.Command {
@@ -33,6 +34,7 @@ func CmdUpdateOperationalFlags() *cobra.Command {
 			flagSet := cmd.Flags()
 			file, _ := flagSet.GetString(fileFlag)
 			restartHeight, _ := flagSet.GetInt64(restartHeightFlag)
+			signerBlockTimeOffset, _ := flagSet.GetDuration(signerBlockTimeOffsetFlag)
 
 			if file != "" {
 				input, err := os.ReadFile(file) // #nosec G304
@@ -45,6 +47,7 @@ func CmdUpdateOperationalFlags() *cobra.Command {
 				}
 			} else {
 				operationalFlags.RestartHeight = restartHeight
+				operationalFlags.SignerBlockTimeOffset = &signerBlockTimeOffset
 			}
 
 			msg := types.NewMsgUpdateOperationalFlags(
@@ -58,6 +61,7 @@ func CmdUpdateOperationalFlags() *cobra.Command {
 
 	cmd.Flags().String(fileFlag, "", "Path to a JSON file containing OperationalFlags")
 	cmd.Flags().Int64(restartHeightFlag, 0, "Height for a coordinated zetaclient restart")
+	cmd.Flags().Duration(signerBlockTimeOffsetFlag, 0, "Offset from the zetacore block time to initiate signing")
 	flags.AddTxFlagsToCmd(cmd)
 
 	return cmd

x/observer/keeper/msg_server_update_operational_flags_test.go

@@ -2,10 +2,12 @@ package keeper_test
 
 import (
 	"testing"
+	"time"
 
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	"github.com/stretchr/testify/require"
 
+	"github.com/zeta-chain/node/pkg/ptr"
 	keepertest "github.com/zeta-chain/node/testutil/keeper"
 	"github.com/zeta-chain/node/testutil/sample"
 	authoritytypes "github.com/zeta-chain/node/x/authority/types"
@@ -26,10 +28,12 @@ func TestMsgServer_UpdateOperationalFlags(t *testing.T) {
 
 		// test initial set
 		restartHeight := int64(100)
+		signerBlockTimeOffset := ptr.Ptr(time.Second)
 		msg := types.MsgUpdateOperationalFlags{
 			Creator: admin,
 			OperationalFlags: types.OperationalFlags{
-				RestartHeight: restartHeight,
+				RestartHeight:         restartHeight,
+				SignerBlockTimeOffset: signerBlockTimeOffset,
 			},
 		}
 		keepertest.MockCheckAuthorization(&authorityMock.Mock, &msg, nil)
@@ -39,13 +43,16 @@ func TestMsgServer_UpdateOperationalFlags(t *testing.T) {
 		operationalFlags, found := k.GetOperationalFlags(ctx)
 		require.True(t, found)
 		require.Equal(t, restartHeight, operationalFlags.RestartHeight)
+		require.Equal(t, signerBlockTimeOffset, operationalFlags.SignerBlockTimeOffset)
 
 		// verify that we can set it again
 		restartHeight = 101
+		signerBlockTimeOffset = ptr.Ptr(time.Second * 2)
 		msg = types.MsgUpdateOperationalFlags{
 			Creator: admin,
 			OperationalFlags: types.OperationalFlags{
-				RestartHeight: restartHeight,
+				RestartHeight:         restartHeight,
+				SignerBlockTimeOffset: signerBlockTimeOffset,
 			},
 		}
 		keepertest.MockCheckAuthorization(&authorityMock.Mock, &msg, nil)
@@ -55,6 +62,7 @@ func TestMsgServer_UpdateOperationalFlags(t *testing.T) {
 		operationalFlags, found = k.GetOperationalFlags(ctx)
 		require.True(t, found)
 		require.Equal(t, restartHeight, operationalFlags.RestartHeight)
+		require.Equal(t, signerBlockTimeOffset, operationalFlags.SignerBlockTimeOffset)
 	})
 
 	t.Run("cannot update operational flags if not authorized", func(t *testing.T) {