gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rework volume compaction (a.k.a vacuuming) logic to cleanly support new parameters. (#8337)

Browse Source 
We'll leverage on this to support a "ignore broken needles" option, necessary
to properly recover damaged volumes, as described in
https://github.com/seaweedfs/seaweedfs/issues/7442#issuecomment-3897784283 .
This commit is contained in:
Lisandro Pin
2026-02-16 11:15:14 +01:00
committed by GitHub
parent 703d5e27b3
commit 0721e3c1e9
5 changed files with 92 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/command/backup.go
View File

@@ -137,7 +137,7 @@ func backupFromLocation(volumeServer pb.ServerAddress, grpcDialOption grpc.DialO
// Handle compaction if needed
if v.SuperBlock.CompactionRevision < uint16(stats.CompactRevision) {
if err = v.Compact2(0, 0, nil); err != nil {
if err = v.CompactByIndex(nil); err != nil {
v.Close()
return fmt.Errorf("compacting volume: %w", err), false
}

29
weed/command/compact.go
View File

@@ -1,6 +1,8 @@
package command
import (
"strings"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
@@ -18,8 +20,9 @@ var cmdCompact = &Command{
The compacted .dat file is stored as .cpd file.
The compacted .idx file is stored as .cpx file.
For method=0, it compacts based on the .dat file, works if .idx file is corrupted.
For method=1, it compacts based on the .idx file, works if deletion happened but not written to .dat files.
Supports two compaction methods:
* data: compacts based on the .dat file, works if .idx file is corrupted.
* index: compacts based on the .idx file, works if deletion happened but not written to .dat files.
`,
}
@@ -28,7 +31,7 @@ var (
compactVolumePath = cmdCompact.Flag.String("dir", ".", "data directory to store files")
compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name")
compactVolumeId = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
compactMethod = cmdCompact.Flag.Int("method", 0, "option to choose which compact method. use 0 (default) or 1.")
compactMethod = cmdCompact.Flag.String("method", "data", "option to choose which compact method (data/index)")
compactVolumePreallocate = cmdCompact.Flag.Int64("preallocateMB", 0, "preallocate volume disk space")
)
@@ -38,21 +41,29 @@ func runCompact(cmd *Command, args []string) bool {
return false
}
preallocate := *compactVolumePreallocate * (1 << 20)
preallocateBytes := *compactVolumePreallocate * (1 << 20)
vid := needle.VolumeId(*compactVolumeId)
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocate, needle.GetCurrentVersion(), 0, 0)
v, err := storage.NewVolume(util.ResolvePath(*compactVolumePath), util.ResolvePath(*compactVolumePath), *compactVolumeCollection, vid, storage.NeedleMapInMemory, nil, nil, preallocateBytes, needle.GetCurrentVersion(), 0, 0)
if err != nil {
glog.Fatalf("Load Volume [ERROR] %s\n", err)
}
if *compactMethod == 0 {
if err = v.Compact(preallocate, 0); err != nil {
opts := &storage.CompactOptions{
PreallocateBytes: preallocateBytes,
MaxBytesPerSecond: 0, // unlimited
}
switch strings.ToLower(*compactMethod) {
case "data":
if err = v.CompactByVolumeData(opts); err != nil {
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
}
} else {
if err = v.Compact2(preallocate, 0, nil); err != nil {
case "index":
if err = v.CompactByIndex(opts); err != nil {
glog.Fatalf("Compact Volume [ERROR] %s\n", err)
}
default:
glog.Fatalf("unsupported compaction method %q", *compactMethod)
}
return true