Merge pull request #612 from SiaFoundation/nate/update-core

Update core and coreutils

.changeset/implement_rpcreplenish.md

@@ -0,0 +1,7 @@
+---
+default: minor
+---
+
+# Add support for RPCReplenish
+
+Adds support RPCReplenish, simplifying management for renters with a large number of accounts.  

cmd/hostd/run.go

@@ -18,6 +18,7 @@ import (
 	"go.sia.tech/coreutils"
 	"go.sia.tech/coreutils/chain"
 	rhp4 "go.sia.tech/coreutils/rhp/v4"
+	"go.sia.tech/coreutils/rhp/v4/siamux"
 	"go.sia.tech/coreutils/syncer"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/hostd/alerts"
@@ -436,7 +437,7 @@ func runRootCmd(ctx context.Context, cfg config.Config, walletKey types.PrivateK
 			}
 			log.Debug("started RHP4 listener", zap.String("address", l.Addr().String()))
 			stopListenerFuncs = append(stopListenerFuncs, l.Close)
-			go rhp.ServeRHP4SiaMux(l, rhp4, log.Named("rhp4"))
+			go siamux.Serve(l, rhp4, log.Named("rhp4"))
 		default:
 			return fmt.Errorf("unsupported protocol: %s", addr.Protocol)
 		}

go.mod

@@ -10,10 +10,9 @@ require (
 	github.com/hashicorp/golang-lru/v2 v2.0.7
 	github.com/mattn/go-sqlite3 v1.14.24
 	github.com/shopspring/decimal v1.4.0
-	go.sia.tech/core v0.10.1
-	go.sia.tech/coreutils v0.11.1
+	go.sia.tech/core v0.10.3-0.20250225045648-07b92f8cf455
+	go.sia.tech/coreutils v0.11.2-0.20250225051012-d01f7fa285c7
 	go.sia.tech/jape v0.12.1
-	go.sia.tech/mux v1.3.0
 	go.sia.tech/web/hostd v0.58.0
 	go.uber.org/goleak v1.3.0
 	go.uber.org/zap v1.27.0
@@ -35,14 +34,15 @@ require (
 	github.com/julienschmidt/httprouter v1.3.0 // indirect
 	github.com/onsi/ginkgo/v2 v2.12.0 // indirect
 	github.com/quic-go/qpack v0.5.1 // indirect
-	github.com/quic-go/quic-go v0.49.0 // indirect
+	github.com/quic-go/quic-go v0.50.0 // indirect
 	github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 // indirect
 	github.com/rogpeppe/go-internal v1.12.0 // indirect
 	go.etcd.io/bbolt v1.4.0 // indirect
+	go.sia.tech/mux v1.4.0 // indirect
 	go.sia.tech/web v0.0.0-20240610131903-5611d44a533e // indirect
 	go.uber.org/mock v0.5.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.33.0 // indirect
+	golang.org/x/crypto v0.34.0 // indirect
 	golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 // indirect
 	golang.org/x/mod v0.18.0 // indirect
 	golang.org/x/net v0.34.0 // indirect

go.sum

@@ -44,8 +44,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
 github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
-github.com/quic-go/quic-go v0.49.0 h1:w5iJHXwHxs1QxyBv1EHKuC50GX5to8mJAxvtnttJp94=
-github.com/quic-go/quic-go v0.49.0/go.mod h1:s2wDnmCdooUQBmQfpUSTCYBl1/D4FcqbULMMkASvR6s=
+github.com/quic-go/quic-go v0.50.0 h1:3H/ld1pa3CYhkcc20TPIyG1bNsdhn9qZBGN3b9/UyUo=
+github.com/quic-go/quic-go v0.50.0/go.mod h1:Vim6OmUvlYdwBhXP9ZVrtGmCMWa3wEqhq3NgYrI8b4E=
 github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66 h1:4WFk6u3sOT6pLa1kQ50ZVdm8BQFgJNA117cepZxtLIg=
 github.com/quic-go/webtransport-go v0.8.1-0.20241018022711-4ac2c9250e66/go.mod h1:Vp72IJajgeOL6ddqrAhmp7IM9zbTcgkQxD/YdxrVwMw=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
@@ -58,14 +58,14 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 go.etcd.io/bbolt v1.4.0 h1:TU77id3TnN/zKr7CO/uk+fBCwF2jGcMuw2B/FMAzYIk=
 go.etcd.io/bbolt v1.4.0/go.mod h1:AsD+OCi/qPN1giOX1aiLAha3o1U8rAz65bvN4j0sRuk=
-go.sia.tech/core v0.10.1 h1:96lmgO50oKPiQU46H14Ga+6NYo6IB++VQ4DI3QCc6/o=
-go.sia.tech/core v0.10.1/go.mod h1:FRg3rOIM8oSvf5wJoAJEgqqbTtKBDNeqL5/bH1lRuDk=
-go.sia.tech/coreutils v0.11.1 h1:rpR2a5oB/TRScPK9d0nBM5k2jL5/f0oy5ZgVzfyS4oo=
-go.sia.tech/coreutils v0.11.1/go.mod h1:vnY0haOx1InIQR0Pc5YAXDe4WnF6po8dv5bNP73CAnE=
+go.sia.tech/core v0.10.3-0.20250225045648-07b92f8cf455 h1:52vFjFvwiZACmrusVcsG8HHfskhwt7NYVRD/1XQzXQg=
+go.sia.tech/core v0.10.3-0.20250225045648-07b92f8cf455/go.mod h1:Sk8dcAAveVjQ2j6TLt1CVpMoQXqDfhFaTdt5eAplQTQ=
+go.sia.tech/coreutils v0.11.2-0.20250225051012-d01f7fa285c7 h1:z6eevJ2MNC0sYhWZwgLzdFdgbk/n4x3b0Py3tHZeOpw=
+go.sia.tech/coreutils v0.11.2-0.20250225051012-d01f7fa285c7/go.mod h1:kWJq7IrMCeBEQguZwTQ9L6Ml4X9bYbb4dqGOBR/N4c8=
 go.sia.tech/jape v0.12.1 h1:xr+o9V8FO8ScRqbSaqYf9bjj1UJ2eipZuNcI1nYousU=
 go.sia.tech/jape v0.12.1/go.mod h1:wU+h6Wh5olDjkPXjF0tbZ1GDgoZ6VTi4naFw91yyWC4=
-go.sia.tech/mux v1.3.0 h1:hgR34IEkqvfBKUJkAzGi31OADeW2y7D6Bmy/Jcbop9c=
-go.sia.tech/mux v1.3.0/go.mod h1:I46++RD4beqA3cW9Xm9SwXbezwPqLvHhVs9HLpDtt58=
+go.sia.tech/mux v1.4.0 h1:LgsLHtn7l+25MwrgaPaUCaS8f2W2/tfvHIdXps04sVo=
+go.sia.tech/mux v1.4.0/go.mod h1:iNFi9ifFb2XhuD+LF4t2HBb4Mvgq/zIPKqwXU/NlqHA=
 go.sia.tech/web v0.0.0-20240610131903-5611d44a533e h1:oKDz6rUExM4a4o6n/EXDppsEka2y/+/PgFOZmHWQRSI=
 go.sia.tech/web v0.0.0-20240610131903-5611d44a533e/go.mod h1:4nyDlycPKxTlCqvOeRO0wUfXxyzWCEE7+2BRrdNqvWk=
 go.sia.tech/web/hostd v0.58.0 h1:2rupaA+X1Qdu4XDlJscfmowQ9XncvC4yvErsXu/+i60=
@@ -78,8 +78,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
-golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
+golang.org/x/crypto v0.34.0 h1:+/C6tk6rf/+t5DhUketUbD1aNGqiSX3j15Z6xuIDlBA=
+golang.org/x/crypto v0.34.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM=
 golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0=

host/contracts/accounts.go

@@ -20,3 +20,10 @@ func (cm *Manager) CreditAccountsWithContract(deposits []proto4.AccountDeposit,
 func (cm *Manager) DebitAccount(account proto4.Account, usage proto4.Usage) error {
 	return cm.store.RHP4DebitAccount(account, usage)
 }
+
+// AccountBalances returns the balances of multiple accounts. Balances is returned
+// in the same order as the input accounts. If an account does not exist, the balance
+// at that index will be types.ZeroCurrency.
+func (cm *Manager) AccountBalances(accounts []proto4.Account) ([]types.Currency, error) {
+	return cm.store.RHP4AccountBalances(accounts)
+}

host/contracts/persist.go

@@ -58,6 +58,10 @@ type (
 
 		// RHP4AccountBalance returns the balance of an account.
 		RHP4AccountBalance(proto4.Account) (types.Currency, error)
+		// RHP4AccountBalances returns the balances of multiple accounts. Balances is returned
+		// in the same order as the input accounts. If an account does not exist, the balance
+		// at that index will be types.ZeroCurrency.
+		RHP4AccountBalances([]proto4.Account) ([]types.Currency, error)
 		// RHP4CreditAccounts atomically revises a contract and credits the accounts
 		RHP4CreditAccounts([]proto4.AccountDeposit, types.FileContractID, types.V2FileContract, proto4.Usage) (balances []types.Currency, err error)
 		// RHP4DebitAccount debits an account.

host/contracts/update.go

@@ -22,11 +22,6 @@ import (
 const chainIndexBuffer = 144
 
 type (
-	stateUpdater interface {
-		ForEachFileContractElement(func(types.FileContractElement, bool, *types.FileContractElement, bool, bool))
-		ForEachV2FileContractElement(func(types.V2FileContractElement, bool, *types.V2FileContractElement, types.V2FileContractResolutionType))
-	}
-
 	// LifecycleActions contains the actions that need to be taken to maintain
 	// the lifecycle of active contracts.
 	LifecycleActions struct {

host/settings/announce.go

@@ -9,7 +9,7 @@ import (
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
-	rhp4 "go.sia.tech/coreutils/rhp/v4"
+	"go.sia.tech/coreutils/rhp/v4/siamux"
 	"go.uber.org/zap"
 )
 
@@ -74,7 +74,7 @@ func (m *ConfigManager) Announce() error {
 			Attestations: []types.Attestation{
 				chain.V2HostAnnouncement{
 					{
-						Protocol: rhp4.ProtocolTCPSiaMux,
+						Protocol: siamux.Protocol,
 						Address:  m.rhp4NetAddress(),
 					},
 				}.ToAttestation(cs, m.hostKey),

host/settings/announce_test.go

@@ -6,7 +6,7 @@ import (
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
-	rhp4 "go.sia.tech/coreutils/rhp/v4"
+	"go.sia.tech/coreutils/rhp/v4/siamux"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/hostd/host/contracts"
 	"go.sia.tech/hostd/host/settings"
@@ -99,7 +99,7 @@ func TestAutoAnnounce(t *testing.T) {
 
 		netaddress := net.JoinHostPort(expectedHost, "9984")
 		h := types.NewHasher()
-		types.EncodeSlice(h.E, chain.V2HostAnnouncement{{Protocol: rhp4.ProtocolTCPSiaMux, Address: netaddress}})
+		types.EncodeSlice(h.E, chain.V2HostAnnouncement{{Protocol: siamux.Protocol, Address: netaddress}})
 		if err := h.E.Flush(); err != nil {
 			t.Fatal(err)
 		}
@@ -229,7 +229,7 @@ func TestAutoAnnounceV2(t *testing.T) {
 		}
 
 		h := types.NewHasher()
-		types.EncodeSlice(h.E, chain.V2HostAnnouncement{{Protocol: rhp4.ProtocolTCPSiaMux, Address: net.JoinHostPort(expectedHost, "9984")}})
+		types.EncodeSlice(h.E, chain.V2HostAnnouncement{{Protocol: siamux.Protocol, Address: net.JoinHostPort(expectedHost, "9984")}})
 		if err := h.E.Flush(); err != nil {
 			t.Fatal(err)
 		}

host/settings/update.go

@@ -5,7 +5,7 @@ import (
 
 	"go.sia.tech/core/types"
 	"go.sia.tech/coreutils/chain"
-	rhp4 "go.sia.tech/coreutils/rhp/v4"
+	"go.sia.tech/coreutils/rhp/v4/siamux"
 	"go.uber.org/zap"
 )
 
@@ -145,7 +145,7 @@ func (m *ConfigManager) ProcessActions(index types.ChainIndex) error {
 		h := types.NewHasher()
 		types.EncodeSlice(h.E, chain.V2HostAnnouncement{
 			{
-				Protocol: rhp4.ProtocolTCPSiaMux,
+				Protocol: siamux.Protocol,
 				Address:  m.rhp4NetAddress(),
 			},
 		})

internal/integration/rhp/v4/rhp4_test.go

@@ -3,6 +3,7 @@ package rhp_test
 import (
 	"bytes"
 	"context"
+	"math"
 	"net"
 	"path/filepath"
 	"reflect"
@@ -15,9 +16,9 @@ import (
 	proto4 "go.sia.tech/core/rhp/v4"
 	"go.sia.tech/core/types"
 	rhp4 "go.sia.tech/coreutils/rhp/v4"
+	"go.sia.tech/coreutils/rhp/v4/siamux"
 	"go.sia.tech/coreutils/wallet"
 	"go.sia.tech/hostd/internal/testutil"
-	"go.sia.tech/hostd/rhp"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zaptest"
 	"lukechampine.com/frand"
@@ -56,9 +57,9 @@ func testRenterHostPair(tb testing.TB, hostKey types.PrivateKey, hn *testutil.Ho
 		tb.Fatal(err)
 	}
 	tb.Cleanup(func() { l.Close() })
-	go rhp.ServeRHP4SiaMux(l, rs, log.Named("siamux"))
+	go siamux.Serve(l, rs, log.Named("siamux"))
 
-	transport, err := rhp4.DialSiaMux(context.Background(), l.Addr().String(), hostKey.PublicKey())
+	transport, err := siamux.Dial(context.Background(), l.Addr().String(), hostKey.PublicKey())
 	if err != nil {
 		tb.Fatal(err)
 	}
@@ -498,6 +499,131 @@ func TestRPCRenew(t *testing.T) {
 	})
 }
 
+func TestReplenishAccounts(t *testing.T) {
+	log := zaptest.NewLogger(t)
+	n, genesis := testutil.V2Network()
+	hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey()
+
+	hn := testutil.NewHostNode(t, hostKey, n, genesis, log)
+	cm := hn.Chain
+	w := hn.Wallet
+
+	testutil.MineAndSync(t, hn, w.Address(), int(n.MaturityDelay+20))
+
+	transport := testRenterHostPair(t, hostKey, hn, log.Named("renterhost"))
+
+	settings, err := rhp4.RPCSettings(context.Background(), transport)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fundAndSign := &fundAndSign{w, renterKey}
+	renterAllowance, hostCollateral := types.Siacoins(100), types.Siacoins(200)
+	formResult, err := rhp4.RPCFormContract(context.Background(), transport, cm, fundAndSign, cm.TipState(), settings.Prices, hostKey.PublicKey(), settings.WalletAddress, proto4.RPCFormContractParams{
+		RenterPublicKey: renterKey.PublicKey(),
+		RenterAddress:   w.Address(),
+		Allowance:       renterAllowance,
+		Collateral:      hostCollateral,
+		ProofHeight:     cm.Tip().Height + 50,
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	revision := formResult.Contract
+
+	cs := cm.TipState()
+
+	var balances []rhp4.AccountBalance
+	// add some random unknown accounts
+	for i := 0; i < 5; i++ {
+		balances = append(balances, rhp4.AccountBalance{
+			Account: proto4.Account(frand.Entropy256()),
+			Balance: types.ZeroCurrency,
+		})
+	}
+
+	var deposits []proto4.AccountDeposit
+	for i := 0; i < 10; i++ {
+		// fund an account with random values below 1SC
+		account1 := frand.Entropy256()
+		deposits = append(deposits, proto4.AccountDeposit{
+			Account: account1,
+			Amount:  types.NewCurrency64(frand.Uint64n(math.MaxUint64)),
+		})
+
+		// fund an account with 1SC
+		account2 := frand.Entropy256()
+		deposits = append(deposits, proto4.AccountDeposit{
+			Account: account2,
+			Amount:  types.Siacoins(1),
+		})
+
+		// fund an account with > 1SC
+		account3 := frand.Entropy256()
+		deposits = append(deposits, proto4.AccountDeposit{
+			Account: account3,
+			Amount:  types.Siacoins(1 + uint32(frand.Uint64n(5))),
+		})
+	}
+
+	// fund the initial set of accounts
+	fundResult, err := rhp4.RPCFundAccounts(context.Background(), transport, cs, renterKey, revision, deposits)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(fundResult.Balances) != len(deposits) {
+		t.Fatalf("expected %v, got %v", len(deposits), len(balances))
+	}
+	for i, deposit := range deposits {
+		if fundResult.Balances[i].Account != deposit.Account {
+			t.Fatalf("expected %v, got %v", deposit.Account, fundResult.Balances[i].Account)
+		} else if !fundResult.Balances[i].Balance.Equals(deposit.Amount) {
+			t.Fatalf("expected %v, got %v", deposit.Amount, fundResult.Balances[i].Balance)
+		}
+	}
+	revision.Revision = fundResult.Revision
+	balances = append(balances, fundResult.Balances...)
+
+	replenishParams := rhp4.RPCReplenishAccountsParams{
+		Contract: revision,
+		Target:   types.Siacoins(1),
+	}
+	var expectedCost types.Currency
+	for _, balance := range balances {
+		replenishParams.Accounts = append(replenishParams.Accounts, balance.Account)
+		if balance.Balance.Cmp(replenishParams.Target) < 0 {
+			expectedCost = expectedCost.Add(replenishParams.Target.Sub(balance.Balance))
+		}
+	}
+
+	// replenish the accounts
+	replenishResult, err := rhp4.RPCReplenishAccounts(context.Background(), transport, replenishParams, cs, fundAndSign)
+	if err != nil {
+		t.Fatal(err)
+	} else if !replenishResult.Usage.AccountFunding.Equals(expectedCost) {
+		t.Fatalf("expected %v, got %v", expectedCost, replenishResult.Usage.AccountFunding)
+	}
+	revisionTransfer := revision.Revision.RenterOutput.Value.Sub(replenishResult.Revision.RenterOutput.Value)
+	if !revisionTransfer.Equals(replenishResult.Usage.AccountFunding) {
+		t.Fatalf("expected %v, got %v", replenishResult.Usage.AccountFunding, revisionTransfer)
+	}
+
+	for _, account := range balances {
+		balance, err := rhp4.RPCAccountBalance(context.Background(), transport, account.Account)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if account.Balance.Cmp(replenishParams.Target) < 0 {
+			if !balance.Equals(replenishParams.Target) {
+				t.Fatalf("expected %v, got %v", replenishParams.Target, balance)
+			}
+		} else if !balance.Equals(account.Balance) {
+			// balances that were already >= 1SC should not have changed
+			t.Fatalf("expected %v, got %v", account.Balance, balance)
+		}
+	}
+}
+
 func TestAccounts(t *testing.T) {
 	n, genesis := testutil.V2Network()
 	hostKey, renterKey := types.GeneratePrivateKey(), types.GeneratePrivateKey()

persist/sqlite/accounts.go

@@ -137,6 +137,30 @@ func (s *Store) RHP4CreditAccounts(deposits []proto4.AccountDeposit, contractID
 	return
 }
 
+// RHP4AccountBalances returns the balances of the accounts with the given IDs. The
+// balances are returned in the same order as the input accounts. If an account does
+// not exist, the balance at that index will be types.ZeroCurrency.
+func (s *Store) RHP4AccountBalances(accounts []proto4.Account) (balances []types.Currency, err error) {
+	err = s.transaction(func(tx *txn) error {
+		stmt, err := tx.Prepare(`SELECT balance FROM accounts WHERE account_id=$1`)
+		if err != nil {
+			return fmt.Errorf("failed to prepare get balance statement: %w", err)
+		}
+		defer stmt.Close()
+
+		for _, account := range accounts {
+			var balance types.Currency
+			err := stmt.QueryRow(encode(account)).Scan(decode(&balance))
+			if err != nil && !errors.Is(err, sql.ErrNoRows) { // missing accounts have a balance of 0
+				return fmt.Errorf("failed to get balance: %w", err)
+			}
+			balances = append(balances, balance)
+		}
+		return nil
+	})
+	return
+}
+
 // AccountBalance returns the balance of the account with the given ID.
 func (s *Store) AccountBalance(accountID rhp3.Account) (balance types.Currency, err error) {
 	err = s.transaction(func(tx *txn) error {