Merge branch 'development' of https://github.com/threefoldtech/tfgrid…

…-sdk-go into development_proxy_indexer

.github/workflows/grid-client-integration.yml

@@ -5,17 +5,17 @@ defaults:
     working-directory: grid-client
 on:
   schedule:
-    - cron: "0 3 * * *"
+    - cron: 0 3 * * *
   workflow_dispatch:
 
 jobs:
   test:
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       fail-fast: false
       matrix:
-        network: ["dev", "qa", "test", "main"]
+        network: [dev, qa, test, main]
 
     steps:
       - name: Set up Go
@@ -51,4 +51,4 @@ jobs:
         env:
           MNEMONICS: ${{ secrets.MNEMONICS }}
           NETWORK: ${{ matrix.network }}
-        run: go test -v ./integration_tests
+        run: go test -v ./integration_tests -timeout 1800s

.github/workflows/grid-proxy-integration.yml

@@ -5,7 +5,7 @@ on:
   pull_request:
     types: [opened, reopened, review_requested]
   schedule:
-    - cron: "25 5 * * *"
+    - cron: 25 5 * * *
   workflow_dispatch:
 
 jobs:

.github/workflows/lint.yml

@@ -18,7 +18,7 @@ jobs:
           go-version: 1.21
 
       - name: Run golangci-lint
-        uses: golangci/golangci-lint-action@v3
+        uses: golangci/golangci-lint-action@v4
         with:
           working-directory: ${{ matrix.dir }}
           args: --config=../.golangci.yml

.github/workflows/release.yml

@@ -3,7 +3,7 @@ name: Release
 on:
   push:
     tags:
-      - "v*"
+      - v*
 
 jobs:
   goreleaser:

activation-service/go.mod

@@ -8,8 +8,8 @@ require (
 	github.com/hashicorp/go-envparse v0.1.0
 	github.com/rs/zerolog v1.32.0
 	github.com/spf13/cobra v1.8.0
-	github.com/stretchr/testify v1.8.4
-	github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240116163757-68c63d80a9e0
+	github.com/stretchr/testify v1.9.0
+	github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240227171040-f2a20ee3e965
 	gopkg.in/validator.v2 v2.0.1
 )
 
@@ -43,9 +43,9 @@ require (
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/vedhavyas/go-subkey v1.0.3 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	golang.org/x/crypto v0.20.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

activation-service/go.sum

@@ -87,10 +87,10 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
-github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240116163757-68c63d80a9e0 h1:0ZMm/xYPgYv3vICNRw3vXTiJ/jVg8LkBLe8TPBEO0MQ=
-github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240116163757-68c63d80a9e0/go.mod h1:dtDKAPiUDxAwIkfHV7xcAFZcOm+xwNIuOI1MLFS+MeQ=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240227171040-f2a20ee3e965 h1:A4EF0webCGCQPym/uWrhhL6H3j/FKn60G99tR+Zl93o=
+github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240227171040-f2a20ee3e965/go.mod h1:dtDKAPiUDxAwIkfHV7xcAFZcOm+xwNIuOI1MLFS+MeQ=
 github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
 github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
@@ -101,20 +101,20 @@ github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPR
 github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg=
+golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
-golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
+golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=

docs/release.md

@@ -7,6 +7,7 @@
 - Update [zos](https://github.com/threefoldtech/zos) with the latest version of the rmb.
 - Create a pr against development branch to update zos with the latest commit (make sure it's merged).
 - Update the helm app version for the grid proxy as described [here](../grid-proxy/docs/release.md)
+- Update the helm app version for the relay cache warmer [here](../tools/relay-cache-warmer/chart/relay-cache-warmer/Chart.yaml)
 - Run `make release`
 
 After all the release workflows are finished you should create an issue on <https://github.com/threefoldtech/tf_operations> with type of `Update Request` to use the new images/binaries.

farmerbot/README.md

@@ -1,6 +1,6 @@
 # Farmerbot
 
-<a href='https://github.com/jpoles1/gopherbadger' target='_blank'>![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-78%25-brightgreen.svg?longCache=true&style=flat)</a>
+<a href='https://github.com/jpoles1/gopherbadger' target='_blank'>![gopherbadger-tag-do-not-edit](https://img.shields.io/badge/Go%20Coverage-75%25-brightgreen.svg?longCache=true&style=flat)</a>
 
 Farmerbot is a service that farmers can run allowing them to automatically manage power of the nodes of their farm.
 
@@ -30,7 +30,9 @@ Where:
 
 ```bash
 Flags:
--c, --config string     your config file that includes your farm, node and power configs. Available format is yml/yaml
+-c, --config string             your config file that includes your farm, node and power configs. Available format is yml/yaml
+    --continue-power-on-error   when set, the farmerbot will run even if there was an error powering on some of the nodes
+-h, --help                      help for run
 
 Global Flags:
 -d, --debug             by setting this flag the farmerbot will print debug logs too
@@ -62,6 +64,7 @@ mv farmerbot /usr/local/bin
 ```env
 NETWORK="the grid network to use (default is mainnet)"
 MNEMONIC_OR_SEED="your farm mnemonic or seed"
+KEY_TYPE="your key type [ed25519, sr25519], default is sr25519"
 ```
 
 2. Add your [configurations](#config)

farmerbot/cmd/root.go

@@ -31,6 +31,8 @@ var farmerBotCmd = &cobra.Command{
 func Execute() {
 	log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
 
+	farmerBotCmd.Root().CompletionOptions.DisableDefaultCmd = true
+
 	farmerBotCmd.AddCommand(versionCmd)
 	farmerBotCmd.AddCommand(runCmd)
 	farmerBotCmd.AddCommand(startCmd)
@@ -59,6 +61,7 @@ func init() {
 	farmerBotCmd.PersistentFlags().BoolP("debug", "d", false, "by setting this flag the farmerbot will print debug logs too")
 
 	runCmd.Flags().StringP("config", "c", "", "enter your config file that includes your farm, node and power configs. Allowed format is yml/yaml")
+	runCmd.Flags().BoolP("continue-power-on-error", "", false, "when set, the farmerbot will run even if there was an error powering on some of the nodes")
 
 	startCmd.Flags().Uint32("node", 0, "enter the node ID you want to use")
 

farmerbot/cmd/run.go

@@ -12,6 +12,10 @@ var runCmd = &cobra.Command{
 	Use:   "run",
 	Short: "Run farmerbot to manage your farm",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(cmd.Flags().Args()) != 0 {
+			return fmt.Errorf("'run' and %v cannot be used together, please use one command at a time", cmd.Flags().Args())
+		}
+
 		network, mnemonicOrSeed, keyType, err := getDefaultFlags(cmd)
 		if err != nil {
 			return err
@@ -22,6 +26,11 @@ var runCmd = &cobra.Command{
 			return fmt.Errorf("invalid config path '%s'", configPath)
 		}
 
+		continueOnPoweringOnErr, err := cmd.Flags().GetBool("continue-power-on-error")
+		if err != nil {
+			return fmt.Errorf("invalid `continue-power-on-error` flag %v", continueOnPoweringOnErr)
+		}
+
 		fileContent, err := parser.ReadFile(configPath)
 		if err != nil {
 			return err
@@ -32,6 +41,8 @@ var runCmd = &cobra.Command{
 			return err
 		}
 
+		config.ContinueOnPoweringOnErr = continueOnPoweringOnErr
+
 		farmerBot, err := internal.NewFarmerBot(cmd.Context(), config, network, mnemonicOrSeed, keyType)
 		if err != nil {
 			return err

farmerbot/cmd/start.go

@@ -13,6 +13,10 @@ var startCmd = &cobra.Command{
 	Use:   "start",
 	Short: "start a node in your farm",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(cmd.Flags().Args()) != 0 {
+			return fmt.Errorf("'start' and %v cannot be used together, please use one command at a time", cmd.Flags().Args())
+		}
+
 		network, mnemonicOrSeed, keyType, err := getDefaultFlags(cmd)
 		if err != nil {
 			return err

farmerbot/cmd/start_all.go

@@ -13,6 +13,10 @@ var startAllCmd = &cobra.Command{
 	Use:   "all",
 	Short: "start all nodes in your farm",
 	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(cmd.Flags().Args()) != 0 {
+			return fmt.Errorf("'all' and %v cannot be used together, please use one command at a time", cmd.Flags().Args())
+		}
+
 		network, mnemonicOrSeed, keyType, err := getDefaultFlags(cmd)
 		if err != nil {
 			return err

farmerbot/cmd/version.go

@@ -15,7 +15,12 @@ import (
 var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "get farmerbot latest version and commit",
-	Run: func(cmd *cobra.Command, args []string) {
+	RunE: func(cmd *cobra.Command, args []string) error {
+		if len(cmd.Flags().Args()) != 0 {
+			return fmt.Errorf("'run' and %v cannot be used together, please use one command at a time", cmd.Flags().Args())
+		}
+
 		fmt.Println(version.Version)
+		return nil
 	},
 }

farmerbot/go.mod

@@ -11,11 +11,11 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.32.0
 	github.com/spf13/cobra v1.8.0
-	github.com/stretchr/testify v1.8.4
-	github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240116163757-68c63d80a9e0
+	github.com/stretchr/testify v1.9.0
+	github.com/threefoldtech/tfchain/clients/tfchain-client-go v0.0.0-20240227171040-f2a20ee3e965
 	github.com/threefoldtech/tfgrid-sdk-go/grid-proxy v0.14.0
 	github.com/threefoldtech/tfgrid-sdk-go/rmb-sdk-go v0.11.10
-	github.com/threefoldtech/zos v0.5.6-0.20240201092442-d2ba5be539d2
+	github.com/threefoldtech/zos v0.5.6-0.20240226114056-364e04acbed3
 	github.com/vedhavyas/go-subkey v1.0.3
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d
 	gopkg.in/yaml.v3 v3.0.1
@@ -32,9 +32,12 @@ require (
 	github.com/decred/dcrd/crypto/blake256 v1.0.1 // indirect
 	github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect
 	github.com/ethereum/go-ethereum v1.11.6 // indirect
+	github.com/garyburd/redigo v1.6.2 // indirect
+	github.com/go-co-op/gocron v1.33.1 // indirect
 	github.com/go-ole/go-ole v1.2.6 // indirect
 	github.com/go-stack/stack v1.8.1 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
 	github.com/google/uuid v1.5.0 // indirect
 	github.com/gorilla/schema v1.2.1 // indirect
@@ -48,21 +51,25 @@ require (
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mimoo/StrobeGo v0.0.0-20220103164710-9a04d6ca976b // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 	github.com/pierrec/xxHash v0.1.5 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
+	github.com/robfig/cron/v3 v3.0.1 // indirect
 	github.com/rs/cors v1.10.1 // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/threefoldtech/zbus v1.0.1 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/vishvananda/netlink v1.2.1-beta.2 // indirect
 	github.com/vishvananda/netns v0.0.4 // indirect
+	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
-	golang.org/x/crypto v0.17.0 // indirect
-	golang.org/x/net v0.19.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	golang.org/x/crypto v0.20.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/sys v0.17.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )