Merge pull request #75 from 0xPolygonHermez/feature/73-support_validium

Feature/73 support validium

cmd/run/run_cmd.go

@@ -20,7 +20,7 @@ func RunCmd(cliCtx *cli.Context) error {
 		log.Error("Error creating synchronizer", err)
 		return err
 	}
-	err = sync.Sync(false)
+	err = sync.Sync(true)
 	if err != nil {
 		log.Error("Error syncing", err)
 	} else {

config/config.go

@@ -2,6 +2,8 @@ package config
 
 import (
 	"bytes"
+	"fmt"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -19,6 +21,8 @@ const (
 	FlagYes = "yes"
 	// FlagCfg is the flag for cfg.
 	FlagCfg = "cfg"
+	// EnvPrefix is the prefix for the environment variables.
+	EnvVarPrefix = "ZKEVM_SYNCL1"
 )
 
 type Config struct {
@@ -52,47 +56,81 @@ func Load(ctx *cli.Context) (*Config, error) {
 }
 
 // Load loads the configuration
-func LoadFile(configFilePath string) (*Config, error) {
-	cfg, err := Default()
+func LoadFileFromString(configFileData string, configType string) (*Config, error) {
+	cfg := &Config{}
+	err := loadString(cfg, DefaultValues, "toml", true, EnvVarPrefix, nil)
 	if err != nil {
 		return nil, err
 	}
+	expectedKeys := viper.AllKeys()
+	err = loadString(cfg, configFileData, configType, true, EnvVarPrefix, &expectedKeys)
+	if err != nil {
+		return nil, err
+	}
+	return cfg, nil
+}
 
-	if configFilePath != "" {
-		dirName, fileName := filepath.Split(configFilePath)
-
-		fileExtension := strings.TrimPrefix(filepath.Ext(fileName), ".")
-		fileNameWithoutExtension := strings.TrimSuffix(fileName, "."+fileExtension)
-
-		viper.AddConfigPath(dirName)
-		viper.SetConfigName(fileNameWithoutExtension)
-		viper.SetConfigType(fileExtension)
+// Load loads the configuration
+func LoadFile(configFilePath string) (*Config, error) {
+	_, fileName := filepath.Split(configFilePath)
+	fileExtension := strings.TrimPrefix(filepath.Ext(fileName), ".")
+	configData, err := os.ReadFile(configFilePath)
+	if err != nil {
+		return nil, err
 	}
-	viper.AutomaticEnv()
-	replacer := strings.NewReplacer(".", "_")
-	viper.SetEnvKeyReplacer(replacer)
-	viper.SetEnvPrefix("ZKEVM_SYNCL1")
-	err = viper.ReadInConfig()
+	cfg, err := LoadFileFromString(string(configData), fileExtension)
 	if err != nil {
-		_, ok := err.(viper.ConfigFileNotFoundError)
-		if ok {
-			log.Infof("config file not found %s", configFilePath)
-			return nil, err
-		} else {
-			log.Infof("error reading config file: ", err)
-			return nil, err
-		}
+		return nil, err
 	}
+	return cfg, nil
+}
 
+// Load loads the configuration
+func loadString(cfg *Config, configData string, configType string, allowEnvVars bool, envPrefix string, expectedKeys *[]string) error {
+	viper.SetConfigType(configType)
+	if allowEnvVars {
+		replacer := strings.NewReplacer(".", "_")
+		viper.SetEnvKeyReplacer(replacer)
+		viper.SetEnvPrefix(envPrefix)
+		viper.AutomaticEnv()
+	}
+	err := viper.ReadConfig(bytes.NewBuffer([]byte(configData)))
+	if err != nil {
+		return err
+	}
 	decodeHooks := []viper.DecoderConfigOption{
 		// this allows arrays to be decoded from env var separated by ",", example: MY_VAR="value1,value2,value3"
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(mapstructure.TextUnmarshallerHookFunc(), mapstructure.StringToSliceHookFunc(","))),
 	}
 
 	err = viper.Unmarshal(&cfg, decodeHooks...)
 	if err != nil {
-		return nil, err
+		return err
+	}
+	if expectedKeys != nil {
+		configKeys := viper.AllKeys()
+		err = checkUnknownFieldsOnFile(configKeys, *expectedKeys)
+		if err != nil {
+			return err
+		}
 	}
+	return nil
+}
 
-	return cfg, nil
+func checkUnknownFieldsOnFile(keysOnFile, expectedConfigKeys []string) error {
+	for _, key := range keysOnFile {
+		if !contains(expectedConfigKeys, key) {
+			return fmt.Errorf("unknown field %s on config file", key)
+		}
+	}
+	return nil
+}
+
+func contains(keys []string, key string) bool {
+	for _, k := range keys {
+		if k == key {
+			return true
+		}
+	}
+	return false
 }

config/config_test.go

@@ -0,0 +1,65 @@
+package config_test
+
+import (
+	"testing"
+
+	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/config"
+	"github.com/stretchr/testify/require"
+)
+
+const ConfigFileWithUnknownFieldTest = `
+[Log]
+	Environment = "development" # "production" or "development"
+	Level = "info"
+	Outputs = ["stderr"]
+	ThisIsAnUnknownField = "unknown"
+`
+const ConfigFileOkTest = `
+[Log]
+	Environment = "development" # "production" or "development"
+	Level = "info"
+	Outputs = ["stderr"]
+	#ThisIsAnUnknownField = "unknown"
+`
+
+const ConfigFileValidiumTranslatorTest = `
+[Etherman.Validium.Translator]
+			FullMatchRules = [
+				{Old="http://dataavailability-003.cdk-validium-cardona-03.zkevm.polygon.private:8444", New="https://dataavailability-003-cdk-validium-cardona-03-zkevm.polygondev.tools"},
+				{Old="http://dataavailability-002.cdk-validium-cardona-03.zkevm.polygon.private:8444", New="https://dataavailability-002-cdk-validium-cardona-03-zkevm.polygondev.tools"},
+				{Old="http://dataavailability-001.cdk-validium-cardona-03.zkevm.polygon.private:8444", New="https://dataavailability-001-cdk-validium-cardona-01-zkevm.polygondev.tools"}
+			]
+`
+
+const ConfigFileValidiumTranslatorWrongFieldsInMapTest = `
+[Etherman.Validium.Translator]
+			FullMatchRules = [
+				{NoExpectedField="http://dataavailability-003.cdk-validium-cardona-03.zkevm.polygon.private:8444", New="https://dataavailability-003-cdk-validium-cardona-03-zkevm.polygondev.tools"},
+				{Old="http://dataavailability-002.cdk-validium-cardona-03.zkevm.polygon.private:8444", New="https://dataavailability-002-cdk-validium-cardona-03-zkevm.polygondev.tools"},
+				{Old="http://dataavailability-001.cdk-validium-cardona-03.zkevm.polygon.private:8444", New="https://dataavailability-001-cdk-validium-cardona-01-zkevm.polygondev.tools"}
+			]
+`
+
+func TestLoadConfigUnknownFieldFails(t *testing.T) {
+	fileExtension := "toml"
+	_, err := config.LoadFileFromString(string(ConfigFileWithUnknownFieldTest), fileExtension)
+	require.Error(t, err)
+}
+
+func TestLoadConfigUnknownFieldInMapFails(t *testing.T) {
+	t.Skip("This test is not working as expected, need a deep dive into Viper library")
+	fileExtension := "toml"
+	_, err := config.LoadFileFromString(string(ConfigFileValidiumTranslatorWrongFieldsInMapTest), fileExtension)
+	require.Error(t, err)
+}
+
+func TestLoadConfigCommentedUnknownFieldOk(t *testing.T) {
+	fileExtension := "toml"
+	_, err := config.LoadFileFromString(string(ConfigFileOkTest), fileExtension)
+	require.NoError(t, err)
+}
+func TestLoadConfigValdiumTranslatorOk(t *testing.T) {
+	fileExtension := "toml"
+	_, err := config.LoadFileFromString(string(ConfigFileValidiumTranslatorTest), fileExtension)
+	require.NoError(t, err)
+}

config/default.go

@@ -19,10 +19,20 @@ const DefaultValues = `
 	GenesisBlockNumber = 0
 	SyncUpToBlock = "latest"
 	BlockFinality = "finalized"
+	OverrideStorageCheck = false
 [Etherman]
 	L1URL = "http://localhost:8545"
+	ForkIDChunkSize = 100
+	L1ChainID = 0
+	PararellBlockRequest = false
 	[Etherman.Contracts]
-	GlobalExitRootManagerAddr = "0x2968D6d736178f8FE7393CC33C87f29D9C287e78"
-	RollupManagerAddr = "0xE2EF6215aDc132Df6913C8DD16487aBF118d1764"
-	ZkEVMAddr = "0x89BA0Ed947a88fe43c22Ae305C0713eC8a7Eb361"
+		GlobalExitRootManagerAddr = "0x2968D6d736178f8FE7393CC33C87f29D9C287e78"
+		RollupManagerAddr = "0xE2EF6215aDc132Df6913C8DD16487aBF118d1764"
+		ZkEVMAddr = "0x89BA0Ed947a88fe43c22Ae305C0713eC8a7Eb361"
+	[Etherman.Validium]
+		Enabled = false
+		TrustedSequencerURL = ""
+		DataSourcePriority = ["trusted", "external"]
+		[Etherman.Validium.Translator]
+			FullMatchRules = []
 `

config/default_test.go

@@ -6,10 +6,12 @@ import (
 
 	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/config"
 	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/config/types"
+	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/dataavailability"
 	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/etherman"
 	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/log"
 	storage "github.com/0xPolygonHermez/zkevm-synchronizer-l1/state/storage"
 	syncconfig "github.com/0xPolygonHermez/zkevm-synchronizer-l1/synchronizer/config"
+	"github.com/0xPolygonHermez/zkevm-synchronizer-l1/translator"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/stretchr/testify/require"
 )
@@ -38,12 +40,20 @@ func TestDefault(t *testing.T) {
 			OverrideStorageCheck: false,
 		},
 		Etherman: etherman.Config{
-			L1URL: "http://localhost:8545",
+			L1URL:           "http://localhost:8545",
+			ForkIDChunkSize: 100,
+			L1ChainID:       0,
 			Contracts: etherman.ContractConfig{
 				GlobalExitRootManagerAddr: common.HexToAddress("0x2968D6d736178f8FE7393CC33C87f29D9C287e78"),
 				RollupManagerAddr:         common.HexToAddress("0xE2EF6215aDc132Df6913C8DD16487aBF118d1764"),
 				ZkEVMAddr:                 common.HexToAddress("0x89BA0Ed947a88fe43c22Ae305C0713eC8a7Eb361"),
 			},
+			Validium: etherman.ValidiumConfig{
+				Enabled:             false,
+				TrustedSequencerURL: "",
+				DataSourcePriority:  []dataavailability.DataSourcePriority{dataavailability.Trusted, dataavailability.External},
+				Translator:          translator.Config{},
+			},
 		},
 	}
 	cfg, err := config.Default()

dataavailability/config.go

@@ -0,0 +1,9 @@
+package dataavailability
+
+// DABackendType is the data availability protocol for the CDK
+type DABackendType string
+
+const (
+	// DataAvailabilityCommittee is the DAC protocol backend
+	DataAvailabilityCommittee DABackendType = "DataAvailabilityCommittee"
+)