Merge remote-tracking branch 'origin/main' into aaron/autogc

go/go.mod

@@ -56,7 +56,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/creasty/defaults v1.6.0
 	github.com/dolthub/flatbuffers/v23 v23.3.3-dh.2
-	github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc
+	github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a
 	github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63
 	github.com/dolthub/swiss v0.1.0
 	github.com/esote/minmaxheap v1.0.0

go/go.sum

@@ -179,8 +179,8 @@ github.com/dolthub/fslock v0.0.3 h1:iLMpUIvJKMKm92+N1fmHVdxJP5NdyDK5bK7z7Ba2s2U=
 github.com/dolthub/fslock v0.0.3/go.mod h1:QWql+P17oAAMLnL4HGB5tiovtDuAjdDTPbuqx7bYfa0=
 github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90 h1:Sni8jrP0sy/w9ZYXoff4g/ixe+7bFCZlfCqXKJSU+zM=
 github.com/dolthub/go-icu-regex v0.0.0-20241215010122-db690dd53c90/go.mod h1:ylU4XjUpsMcvl/BKeRRMXSH7e7WBrPXdSLvnRJYrxEA=
-github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc h1:SdN7GRPtaqmLwfi6cVcyF4Oc8FbFUJ+mwsFRV++6iH4=
-github.com/dolthub/go-mysql-server v0.19.1-0.20250207201905-b3a4c87c4fdc/go.mod h1:QQxZvPHOtycbC2bVmqmT6/Fov2g1/T1Rtm76wLd/Y1E=
+github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a h1:3aeB6IsvQKOTxSRDeDuBjvOkHBolqYrhLUXa0UmgWjc=
+github.com/dolthub/go-mysql-server v0.19.1-0.20250214204118-1e0e5e8f244a/go.mod h1:QQxZvPHOtycbC2bVmqmT6/Fov2g1/T1Rtm76wLd/Y1E=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63 h1:OAsXLAPL4du6tfbBgK0xXHZkOlos63RdKYS3Sgw/dfI=
 github.com/dolthub/gozstd v0.0.0-20240423170813-23a2903bca63/go.mod h1:lV7lUeuDhH5thVGDCKXbatwKy2KW80L4rMT46n+Y2/Q=
 github.com/dolthub/ishell v0.0.0-20240701202509-2b217167d718 h1:lT7hE5k+0nkBdj/1UOSFwjWpNxf+LCApbRHgnCA17XE=

go/libraries/doltcore/remotesrv/grpc.go

@@ -295,7 +295,12 @@ func (rs *RemoteChunkStore) StreamDownloadLocations(stream remotesapi.ChunkStore
 			var ranges []*remotesapi.RangeChunk
 			for h, r := range hashToRange {
 				hCpy := h
-				ranges = append(ranges, &remotesapi.RangeChunk{Hash: hCpy[:], Offset: r.Offset, Length: r.Length})
+				ranges = append(ranges, &remotesapi.RangeChunk{
+					Hash:             hCpy[:],
+					Offset:           r.Offset,
+					Length:           r.Length,
+					DictionaryOffset: r.DictOffset,
+					DictionaryLength: r.DictLength})
 			}
 
 			url := rs.getDownloadUrl(md, prefix+"/"+loc)
@@ -606,7 +611,7 @@ func getTableFileInfo(
 	}
 	appendixTableFileInfo := make([]*remotesapi.TableFileInfo, 0)
 	for _, t := range tableList {
-		url := rs.getDownloadUrl(md, prefix+"/"+t.LocationPrefix()+t.FileID())
+		url := rs.getDownloadUrl(md, prefix+"/"+t.LocationPrefix()+t.FileID()+t.LocationSuffix())
 		url, err = rs.sealer.Seal(url)
 		if err != nil {
 			return nil, status.Error(codes.Internal, "failed to get seal download url for "+t.FileID())

go/libraries/doltcore/remotesrv/http.go

@@ -35,6 +35,7 @@ import (
 
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 	"github.com/dolthub/dolt/go/store/hash"
+	"github.com/dolthub/dolt/go/store/nbs"
 	"github.com/dolthub/dolt/go/store/types"
 )
 
@@ -94,12 +95,18 @@ func (fh filehandler) ServeHTTP(respWr http.ResponseWriter, req *http.Request) {
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
-		_, ok := hash.MaybeParse(path[i+1:])
+
+		fileName := path[i+1:]
+		if strings.HasSuffix(fileName, nbs.ArchiveFileSuffix) {
+			fileName = fileName[:len(fileName)-len(nbs.ArchiveFileSuffix)]
+		}
+		_, ok := hash.MaybeParse(fileName)
 		if !ok {
-			logger.WithField("last_path_component", path[i+1:]).Warn("bad request with unparseable last path component")
+			logger.WithField("last_path_component", fileName).Warn("bad request with unparseable last path component")
 			respWr.WriteHeader(http.StatusBadRequest)
 			return
 		}
+
 		abs, err := fh.fs.Abs(path)
 		if err != nil {
 			logger.WithError(err).Error("could not get absolute path")

go/libraries/doltcore/remotesrv/server.go

@@ -17,7 +17,6 @@ package remotesrv
 import (
 	"context"
 	"crypto/tls"
-	"errors"
 	"net"
 	"net/http"
 	"strings"
@@ -29,7 +28,6 @@ import (
 	"google.golang.org/grpc"
 
 	remotesapi "github.com/dolthub/dolt/go/gen/proto/dolt/services/remotesapi/v1alpha1"
-	"github.com/dolthub/dolt/go/libraries/doltcore/env"
 	"github.com/dolthub/dolt/go/libraries/utils/filesys"
 )
 
@@ -80,14 +78,6 @@ func NewServer(args ServerArgs) (*Server, error) {
 		args.Logger = logrus.NewEntry(logrus.StandardLogger())
 	}
 
-	storageMetadata, err := env.GetMultiEnvStorageMetadata(args.FS)
-	if err != nil {
-		return nil, err
-	}
-	if storageMetadata.ArchiveFilesPresent() {
-		return nil, errors.New("archive files present. Please run `dolt archive --revert` before running the server.")
-	}
-
 	s := new(Server)
 	s.stopChan = make(chan struct{})
 

go/libraries/doltcore/remotestorage/chunk_cache.go

@@ -21,21 +21,20 @@ import (
 
 // ChunkCache is an interface used for caching chunks
 type ChunkCache interface {
-	// Put puts a slice of chunks into the cache.
-	Put(c []nbs.CompressedChunk) bool
+	// Put puts a slice of chunks into the cache. Error returned if the cache capacity has been exceeded.
+	Put(c []nbs.ToChunker) error
 
 	// Get gets a map of hash to chunk for a set of hashes.  In the event that a chunk is not in the cache, chunks.Empty.
 	// is put in it's place
-	Get(h hash.HashSet) map[hash.Hash]nbs.CompressedChunk
+	Get(h hash.HashSet) map[hash.Hash]nbs.ToChunker
 
 	// Has takes a set of hashes and returns the set of hashes that the cache currently does not have in it.
 	Has(h hash.HashSet) (absent hash.HashSet)
 
-	// PutChunk puts a single chunk in the cache.  true returns in the event that the chunk was cached successfully
-	// and false is returned if that chunk is already is the cache.
-	PutChunk(chunk nbs.CompressedChunk) bool
+	// PutChunk puts a single chunk in the cache. Returns an error if the cache capacity has been exceeded.
+	PutChunk(chunk nbs.ToChunker) error
 
 	// GetAndClearChunksToFlush gets a map of hash to chunk which includes all the chunks that were put in the cache
 	// between the last time GetAndClearChunksToFlush was called and now.
-	GetAndClearChunksToFlush() map[hash.Hash]nbs.CompressedChunk
+	GetAndClearChunksToFlush() map[hash.Hash]nbs.ToChunker
 }