Feature/#2200 generate json-schema + docs for node config file and network_custom

.github/workflows/jsonschema.yml

@@ -0,0 +1,61 @@
+---
+name: JSON schema
+on:
+  push:
+    branches:
+      - main
+      - master
+      - develop
+      - update-external-dependencies
+      - 'release/**'
+  pull_request:
+
+jobs:
+  json-schema:
+    strategy:
+      matrix:
+        go-version: [ 1.19.x ]
+        goarch: [ "amd64" ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v3
+      # https://github.com/actions/checkout#Checkout-pull-request-HEAD-commit-instead-of-merge-commit
+      # Checkout pull request HEAD commit instead of merge commit
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Install Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+      env:
+        GOARCH: ${{ matrix.goarch }}
+
+    - uses: actions/setup-python@v1
+    - uses: BSFishy/pip-action@v1
+      with:
+        packages: |
+          json-schema-for-humans
+
+    - name: Check if JSON schema and generated doc is up to date
+      run: |
+          EXPECTED_DIFF=""
+          NOT_UPDATED_MSG="JSON Schema is not up to date, run 'make config-doc-gen' before creating the PR"
+
+          echo "Checking if JSON schema is up to date..."
+          make GENERATE_DOC_PATH=/tmp/ config-doc-gen
+          for CHECK_FILE in "node-config-schema.json" "node-config-doc.md" "node-config-doc.html" "custom_network-config-schema.json" "custom_network-config-doc.md" "custom_network-config-doc.html"; do
+            EXPECTED_FILE=tmp/$CHECK_FILE
+            REAL_FILE=docs/config-file/$CHECK_FILE
+            echo "checking $CHECK_FILE ...."
+            diff /tmp/$CHECK_FILE docs/config-file/$CHECK_FILE
+            if [ $? -ne 0  ]; then
+              echo "  FAILED file $CHECK_FILE!"
+              exit 1
+            fi
+            echo "checked $CHECK_FILE OK"
+          done
+
+          echo "Everything up to date"
+

.gitignore

@@ -16,4 +16,6 @@
 .env
 out.dat
 
-cmd/__debug_bin
+cmd/__debug_bin
+
+.venv

Makefile

@@ -20,6 +20,15 @@ LDFLAGS += -X 'github.com/0xPolygonHermez/zkevm-node.GitRev=$(GITREV)'
 LDFLAGS += -X 'github.com/0xPolygonHermez/zkevm-node.GitBranch=$(GITBRANCH)'
 LDFLAGS += -X 'github.com/0xPolygonHermez/zkevm-node.BuildDate=$(DATE)'
 
+# Variables
+VENV           = .venv
+VENV_PYTHON    = $(VENV)/bin/python
+SYSTEM_PYTHON  = $(or $(shell which python3), $(shell which python))
+PYTHON         = $(or $(wildcard $(VENV_PYTHON)), "install_first_venv")
+GENERATE_SCHEMA_DOC = $(VENV)/bin/generate-schema-doc
+GENERATE_DOC_PATH=  "docs/config-file/"
+GENERATE_DOC_TEMPLATES_PATH=  "docs/config-file/templates/"
+
 .PHONY: build
 build: ## Builds the binary locally into ./dist
 	$(GOENVVARS) go build -ldflags "all=$(LDFLAGS)" -o $(GOBIN)/$(GOBINARY) $(GOCMD)
@@ -54,6 +63,52 @@ install-linter: ## Installs the linter
 lint: ## Runs the linter
 	export "GOROOT=$$(go env GOROOT)" && $$(go env GOPATH)/bin/golangci-lint run
 
+
+$(VENV_PYTHON):
+	rm -rf $(VENV)
+	$(SYSTEM_PYTHON) -m venv $(VENV)
+
+venv: $(VENV_PYTHON)
+
+# https://stackoverflow.com/questions/24736146/how-to-use-virtualenv-in-makefile
+.PHONY: install-config-doc-gen
+$(GENERATE_SCHEMA_DOC): $(VENV_PYTHON)
+	$(PYTHON) -m pip install --upgrade pip
+	$(PYTHON) -m pip install json-schema-for-humans
+
+PHONY: config-doc-gen
+config-doc-gen: config-doc-node config-doc-custom_network ## Generate config file's json-schema for node and custom_network  and documentation
+	#
+
+.PHONY: config-doc-node
+config-doc-node: $(GENERATE_SCHEMA_DOC) ## Generate config file's json-schema for node and documentation
+	go run ./cmd generate-json-schema --config-file=node --output=$(GENERATE_DOC_PATH)node-config-schema.json
+	$(GENERATE_SCHEMA_DOC) --config show_breadcrumbs=true \
+							--config footer_show_time=false \
+							--config expand_buttons=true \
+							--config custom_template_path=$(GENERATE_DOC_TEMPLATES_PATH)/js/base.html \
+							$(GENERATE_DOC_PATH)node-config-schema.json \
+							$(GENERATE_DOC_PATH)node-config-doc.html
+	$(GENERATE_SCHEMA_DOC)  --config custom_template_path=$(GENERATE_DOC_TEMPLATES_PATH)/md/base.md \
+							--config footer_show_time=false \
+							$(GENERATE_DOC_PATH)node-config-schema.json \
+							$(GENERATE_DOC_PATH)node-config-doc.md
+
+.PHONY: config-doc-custom_network
+config-doc-custom_network: $(GENERATE_SCHEMA_DOC) ## Generate config file's json-schema for custom_network and documentation
+	go run ./cmd generate-json-schema --config-file=custom_network --output=$(GENERATE_DOC_PATH)custom_network-config-schema.json
+	$(GENERATE_SCHEMA_DOC) --config show_breadcrumbs=true --config footer_show_time=false \
+							--config expand_buttons=true \
+							--config custom_template_path=$(GENERATE_DOC_TEMPLATES_PATH)/js/base.html \
+							$(GENERATE_DOC_PATH)custom_network-config-schema.json \
+							$(GENERATE_DOC_PATH)custom_network-config-doc.html
+	$(GENERATE_SCHEMA_DOC)  --config custom_template_path=$(GENERATE_DOC_TEMPLATES_PATH)/md/base.md \
+							--config footer_show_time=false \
+							--config example_format=JSON \
+							$(GENERATE_DOC_PATH)custom_network-config-schema.json \
+							$(GENERATE_DOC_PATH)custom_network-config-doc.md
+
+
 .PHONY: update-external-dependencies
 update-external-dependencies: ## Updates external dependencies like images, test vectors or proto files
 	go run ./scripts/cmd/... updatedeps

cmd/jsonschema.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"github.com/0xPolygonHermez/zkevm-node/config"
+	"github.com/urfave/cli/v2"
+)
+
+func genJSONSchema(cli *cli.Context) error {
+	file_config := cli.String(config.FlagDocumentationFileType)
+	output := cli.String(config.FlagOutputFile)
+	switch file_config {
+	case NODE_CONFIGFILE:
+		{
+			generator := config.NewNodeConfigJsonSchemaGenerater()
+			return generator.GenerateJsonSchemaAndWriteToFile(cli, output)
+		}
+	case NETWORK_CONFIGFILE:
+		{
+			generator := config.NewNetworkConfigJsonSchemaGenerater()
+			return generator.GenerateJsonSchemaAndWriteToFile(cli, output)
+		}
+	default:
+		panic("Not supported this config file: " + file_config)
+	}
+}

cmd/main.go

@@ -30,6 +30,13 @@ const (
 	SEQUENCE_SENDER = "sequence-sender"
 )
 
+const (
+	// NODE_CONFIGFILE name to identify the node config-file
+	NODE_CONFIGFILE = "node"
+	// NETWORK_CONFIGFILE name to identify the netowk_custom (genesis) config-file
+	NETWORK_CONFIGFILE = "custom_network"
+)
+
 var (
 	configFileFlag = cli.StringFlag{
 		Name:     config.FlagCfg,
@@ -75,6 +82,16 @@ var (
 		Usage:    "Blocks the migrations in stateDB to not run them",
 		Required: false,
 	}
+	outputFileFlag = cli.StringFlag{
+		Name:     config.FlagOutputFile,
+		Usage:    "Indicate the output file",
+		Required: true,
+	}
+	documentationFileTypeFlag = cli.StringFlag{
+		Name:     config.FlagDocumentationFileType,
+		Usage:    fmt.Sprintf("Indicate the type of file to generate json-schema: %v,%v ", NODE_CONFIGFILE, NETWORK_CONFIGFILE),
+		Required: true,
+	}
 )
 
 func main() {
@@ -149,6 +166,12 @@ func main() {
 			Action:  dumpState,
 			Flags:   dumpStateFlags,
 		},
+		{
+			Name:   "generate-json-schema",
+			Usage:  "Generate the json-schema for the configuration file, and store it on docs/schema.json",
+			Action: genJSONSchema,
+			Flags:  []cli.Flag{&outputFileFlag, &documentationFileTypeFlag},
+		},
 		{
 			Name:    "snapshot",
 			Aliases: []string{"snap"},

cmd/run.go

@@ -341,14 +341,14 @@ func runJSONRPCServer(c config.Config, etherman *etherman.Client, chainID uint64
 	if _, ok := apis[jsonrpc.APINet]; ok {
 		services = append(services, jsonrpc.Service{
 			Name:    jsonrpc.APINet,
-			Service: jsonrpc.NewNetEndpoints(chainID),
+			Service: jsonrpc.NewNetEndpoints(c.RPC, chainID),
 		})
 	}
 
 	if _, ok := apis[jsonrpc.APIZKEVM]; ok {
 		services = append(services, jsonrpc.Service{
 			Name:    jsonrpc.APIZKEVM,
-			Service: jsonrpc.NewZKEVMEndpoints(st),
+			Service: jsonrpc.NewZKEVMEndpoints(c.RPC, st),
 		})
 	}
 
@@ -362,7 +362,7 @@ func runJSONRPCServer(c config.Config, etherman *etherman.Client, chainID uint64
 	if _, ok := apis[jsonrpc.APIDebug]; ok {
 		services = append(services, jsonrpc.Service{
 			Name:    jsonrpc.APIDebug,
-			Service: jsonrpc.NewDebugEndpoints(st),
+			Service: jsonrpc.NewDebugEndpoints(c.RPC, st),
 		})
 	}
 

config/config.go

@@ -48,32 +48,74 @@ const (
 	FlagPassword = "password"
 	// FlagMigrations is the flag for migrations.
 	FlagMigrations = "migrations"
+	// FlagOutputFile is the flag for the output file
+	FlagOutputFile = "output"
 	// FlagMaxAmount is the flag to avoid to use the flag FlagAmount
 	FlagMaxAmount = "max-amount"
+	// FlagDocumentationFileType is the flag for the choose which file generate json-schema
+	FlagDocumentationFileType = "config-file"
 )
 
-// Config represents the configuration of the entire Hermez Node
+/*
+Config represents the configuration of the entire Hermez Node
+The file is [TOML format]
+You could find some examples:
+  - `config/environments/local/local.node.config.toml`: running a permisionless node
+  - `config/environments/mainnet/public.node.config.toml`
+  - `config/environments/public/public.node.config.toml`
+  - `test/config/test.node.config.toml`: configuration for a trusted node used in CI
+
+[TOML format]: https://en.wikipedia.org/wiki/TOML
+*/
 type Config struct {
-	IsTrustedSequencer     bool   `mapstructure:"IsTrustedSequencer"`
+	// This define is a trusted node (`true`) or a permission less (`false`). If you don't known
+	// set to `false`
+	IsTrustedSequencer bool `mapstructure:"IsTrustedSequencer"`
+	// Last batch number before  a forkid change (fork upgrade). That implies that
+	// greater batch numbers are going to be trusted but no virtualized neither verified.
+	// So after the batch number `ForkUpgradeBatchNumber` is virtualized and verified you could update
+	// the system (SC,...) to new forkId and remove this value to allow the system to keep
+	// Virtualizing and verifying the new batchs.
+	// Check issue [#2236](https://github.com/0xPolygonHermez/zkevm-node/issues/2236) to known more
+	// This value overwrite `SequenceSender.ForkUpgradeBatchNumber`
 	ForkUpgradeBatchNumber uint64 `mapstructure:"ForkUpgradeBatchNumber"`
-	ForkUpgradeNewForkId   uint64 `mapstructure:"ForkUpgradeNewForkId"`
-	Log                    log.Config
-	Etherman               etherman.Config
-	EthTxManager           ethtxmanager.Config
-	Pool                   pool.Config
-	RPC                    jsonrpc.Config
-	Synchronizer           synchronizer.Config
-	Sequencer              sequencer.Config
-	SequenceSender         sequencesender.Config
-	Aggregator             aggregator.Config
-	NetworkConfig          NetworkConfig
-	L2GasPriceSuggester    gasprice.Config
-	Executor               executor.Config
-	MTClient               merkletree.Config
-	StateDB                db.Config
-	Metrics                metrics.Config
-	EventLog               event.Config
-	HashDB                 db.Config
+	// Which is the new forkId
+	ForkUpgradeNewForkId uint64 `mapstructure:"ForkUpgradeNewForkId"`
+	// Configure Log level for all the services, allow also to store the logs in a file
+	Log log.Config
+	// Configuration of the etherman (client for access L1)
+	Etherman etherman.Config
+	// Configuration for ethereum transaction manager
+	EthTxManager ethtxmanager.Config
+	// Pool service configuration
+	Pool pool.Config
+	// Configuration for RPC service. THis one offers a extended Ethereum JSON-RPC API interface to interact with the node
+	RPC jsonrpc.Config
+	// Configuration of service `Syncrhonizer`. For this service is also really important the value of `IsTrustedSequencer`
+	// because depending of this values is going to ask to a trusted node for trusted transactions or not
+	Synchronizer synchronizer.Config
+	// Configuration of the sequencer service
+	Sequencer sequencer.Config
+	// Configuration of the sequence sender service
+	SequenceSender sequencesender.Config
+	// Configuration of the aggregator service
+	Aggregator aggregator.Config
+	// Configuration of the genesis of the network. This is used to known the initial state of the network
+	NetworkConfig NetworkConfig
+	// Configuration of the gas price suggester service
+	L2GasPriceSuggester gasprice.Config
+	// Configuration of the executor service
+	Executor executor.Config
+	// Configuration of the merkle tree client service. Not use in the node, only for testing
+	MTClient merkletree.Config
+	// Configuration of the state database connection
+	StateDB db.Config
+	// Configuration of the metrics service, basically is where is going to publish the metrics
+	Metrics metrics.Config
+	// Configuration of the event database connection
+	EventLog event.Config
+	// Configuration of the hash database connection
+	HashDB db.Config
 }
 
 // Default parses the default configuration values.

config/default.go

@@ -62,6 +62,7 @@ WriteTimeout = "60s"
 MaxRequestsPerIPAndSecond = 500
 SequencerNodeURI = ""
 EnableL2SuggestedGasPricePolling = true
+TraceBatchUseHTTPS = true
 	[RPC.WebSockets]
 		Enabled = true
 		Host = "0.0.0.0"