Skip to content

Commit

Permalink
Make no grouping the default
Browse files Browse the repository at this point in the history
Bats tests were hanging does to CGO pathological case
  • Loading branch information
macneale4 committed Jul 22, 2024
1 parent 0ea272d commit 8f0f55a
Show file tree
Hide file tree
Showing 3 changed files with 13 additions and 9 deletions.
8 changes: 4 additions & 4 deletions go/cmd/dolt/commands/admin/archive.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,11 @@ var docs = cli.CommandDocumentationContent{
table files into archives. Currently, for safety, table files are left in place.`,

Synopsis: []string{
`[--no-grouping]`,
`[--group-chunks]`,
},
}

const noGroupingFlag = "no-grouping"
const groupChunksFlag = "group-chunks"

// Description returns a description of the command
func (cmd ArchiveCmd) Description() string {
Expand All @@ -68,7 +68,7 @@ func (cmd ArchiveCmd) Docs() *cli.CommandDocumentation {

func (cmd ArchiveCmd) ArgParser() *argparser.ArgParser {
ap := argparser.NewArgParserWithMaxArgs(cmd.Name(), 0)
ap.SupportsFlag(noGroupingFlag, "", "Do not attempt to group chunks. Default dictionary will be used for all chunks")
ap.SupportsFlag(groupChunksFlag, "", "Attempt to group chunks. This will produce smaller archives, but can take much longer to build.")
/* TODO: Implement these flags
ap.SupportsFlag("purge", "", "remove table files after archiving")
ap.SupportsFlag("revert", "", "Return to unpurged table files, or rebuilt table files from archives")
Expand Down Expand Up @@ -104,7 +104,7 @@ func (cmd ArchiveCmd) Exec(ctx context.Context, commandStr string, args []string
})

groupings := nbs.NewChunkRelations()
if !apr.Contains(noGroupingFlag) {
if apr.Contains(groupChunksFlag) {
err = historicalFuzzyMatching(ctx, hs, &groupings, dEnv.DoltDB)
if err != nil {
cli.PrintErrln(err)
Expand Down
12 changes: 8 additions & 4 deletions go/store/nbs/archive_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -528,14 +528,17 @@ func (cg *chunkGroup) worstZScore() float64 {
// rebuild - recalculate the entire group's compression ratio. Dictionary and total compression ratio are updated as well.
// This method is called after a new chunk is added to the group. Ensures that stats about the group are up-to-date.
func (cg *chunkGroup) rebuild(ctx context.Context, chunkCache *SimpleChunkSourceCache, defaultDict *gozstd.CDict, stats *Stats) error {
chks := make([]*chunks.Chunk, len(cg.chks))
chks := make([]*chunks.Chunk, 0, len(cg.chks))

for i, cs := range cg.chks {
for _, cs := range cg.chks {
chk, err := chunkCache.get(ctx, cs.chunkId, stats)
if err != nil {
return err
}
chks[i] = chk
if chk == nil {
continue
}
chks = append(chks, chk)
}

totalBytes := 0
Expand Down Expand Up @@ -798,13 +801,14 @@ func newSimpleChunkSourceCache(cs chunkSource) (*SimpleChunkSourceCache, error)
}

// get a chunk from the cache. If the chunk is not in the cache, it will be fetched from the ChunkSource.
// If the ChunkSource doesn't have the chunk, return nil - this is a valid case.
func (csc *SimpleChunkSourceCache) get(ctx context.Context, h hash.Hash, stats *Stats) (*chunks.Chunk, error) {
if chk, ok := csc.cache.Get(h); ok {
return chk, nil
}

bytes, err := csc.cs.get(ctx, h, stats)
if err != nil {
if bytes == nil || err != nil {
return nil, err
}

Expand Down
2 changes: 1 addition & 1 deletion go/store/nbs/archive_writer.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func (aw *archiveWriter) writeByteSpan(b []byte) (uint32, error) {
}

if len(b) == 0 {
return 0, nil
return 0, fmt.Errorf("Rutime error: empty compressed byte span")
}

offset := aw.bytesWritten
Expand Down

0 comments on commit 8f0f55a

Please sign in to comment.