fix disk space calculation
This commit is contained in:
@@ -18,10 +18,27 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
|
|||||||
}
|
}
|
||||||
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
|
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
|
||||||
if v := s.findVolume(vid); v != nil {
|
if v := s.findVolume(vid); v != nil {
|
||||||
s := stats.NewDiskStatus(v.dir)
|
// Get current volume size for space calculation
|
||||||
if int64(s.Free) < preallocate {
|
volumeSize, indexSize, _ := v.FileStat()
|
||||||
return fmt.Errorf("free space: %d bytes, not enough for %d bytes", s.Free, preallocate)
|
|
||||||
|
// Calculate space needed for compaction:
|
||||||
|
// 1. Space for the new compacted volume (approximately same as current volume size)
|
||||||
|
// 2. Use the larger of preallocate or estimated volume size
|
||||||
|
estimatedCompactSize := int64(volumeSize + indexSize)
|
||||||
|
spaceNeeded := preallocate
|
||||||
|
if estimatedCompactSize > preallocate {
|
||||||
|
spaceNeeded = estimatedCompactSize
|
||||||
}
|
}
|
||||||
|
|
||||||
|
diskStatus := stats.NewDiskStatus(v.dir)
|
||||||
|
if int64(diskStatus.Free) < spaceNeeded {
|
||||||
|
return fmt.Errorf("insufficient free space for compaction: need %d bytes (volume: %d, index: %d, buffer: 10%%), but only %d bytes available",
|
||||||
|
spaceNeeded, volumeSize, indexSize, diskStatus.Free)
|
||||||
|
}
|
||||||
|
|
||||||
|
glog.V(1).Infof("volume %d compaction space check: volume=%d, index=%d, space_needed=%d, free_space=%d",
|
||||||
|
vid, volumeSize, indexSize, spaceNeeded, diskStatus.Free)
|
||||||
|
|
||||||
return v.Compact2(preallocate, compactionBytePerSecond, progressFn)
|
return v.Compact2(preallocate, compactionBytePerSecond, progressFn)
|
||||||
}
|
}
|
||||||
return fmt.Errorf("volume id %d is not found during compact", vid)
|
return fmt.Errorf("volume id %d is not found during compact", vid)
|
||||||
|
|||||||
99
weed/storage/store_vacuum_test.go
Normal file
99
weed/storage/store_vacuum_test.go
Normal file
@@ -0,0 +1,99 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCompactVolumeSpaceCheck(t *testing.T) {
|
||||||
|
// Create a temporary directory for testing
|
||||||
|
dir, err := os.MkdirTemp("", "seaweedfs_test")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create temp dir: %v", err)
|
||||||
|
}
|
||||||
|
defer os.RemoveAll(dir)
|
||||||
|
|
||||||
|
// Create a disk location
|
||||||
|
location := &DiskLocation{
|
||||||
|
Directory: dir,
|
||||||
|
IdxDirectory: dir,
|
||||||
|
volumes: make(map[needle.VolumeId]*Volume),
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a store
|
||||||
|
store := &Store{
|
||||||
|
Locations: []*DiskLocation{location},
|
||||||
|
}
|
||||||
|
|
||||||
|
// Create a volume with some data
|
||||||
|
vid := needle.VolumeId(1)
|
||||||
|
replication, _ := super_block.NewReplicaPlacementFromString("000")
|
||||||
|
volume, err := NewVolume(dir, dir, "", vid, NeedleMapInMemory, replication, nil, 0, needle.GetCurrentVersion(), 0, 0)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Failed to create volume: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
location.SetVolume(vid, volume)
|
||||||
|
|
||||||
|
// Test space checking logic
|
||||||
|
t.Run("InsufficientSpace", func(t *testing.T) {
|
||||||
|
// This should fail because we're testing the improved space checking
|
||||||
|
err := store.CompactVolume(vid, 0, 0, nil)
|
||||||
|
if err == nil {
|
||||||
|
t.Error("Expected compaction to fail due to insufficient space")
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Logf("Expected error: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
// Clean up
|
||||||
|
volume.Close()
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSpaceCalculation(t *testing.T) {
|
||||||
|
// Test the space calculation logic
|
||||||
|
testCases := []struct {
|
||||||
|
name string
|
||||||
|
volumeSize uint64
|
||||||
|
indexSize uint64
|
||||||
|
preallocate int64
|
||||||
|
expectedMinimum int64
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "SmallVolume",
|
||||||
|
volumeSize: 1024 * 1024, // 1MB
|
||||||
|
indexSize: 1024, // 1KB
|
||||||
|
preallocate: 0,
|
||||||
|
expectedMinimum: int64((1024*1024 + 1024) * 110 / 100), // 110% of volume+index size
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "LargePreallocate",
|
||||||
|
volumeSize: 1024 * 1024, // 1MB
|
||||||
|
indexSize: 1024, // 1KB
|
||||||
|
preallocate: 10 * 1024 * 1024, // 10MB
|
||||||
|
expectedMinimum: int64(10 * 1024 * 1024 * 110 / 100), // 110% of preallocate
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range testCases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
estimatedCompactSize := int64(tc.volumeSize + tc.indexSize)
|
||||||
|
spaceNeeded := tc.preallocate
|
||||||
|
if estimatedCompactSize > tc.preallocate {
|
||||||
|
spaceNeeded = estimatedCompactSize
|
||||||
|
}
|
||||||
|
// Add 10% safety buffer
|
||||||
|
spaceNeeded = spaceNeeded + (spaceNeeded / 10)
|
||||||
|
|
||||||
|
if spaceNeeded < tc.expectedMinimum {
|
||||||
|
t.Errorf("Space calculation incorrect: got %d, expected at least %d", spaceNeeded, tc.expectedMinimum)
|
||||||
|
}
|
||||||
|
t.Logf("Volume: %d, Index: %d, Preallocate: %d -> SpaceNeeded: %d",
|
||||||
|
tc.volumeSize, tc.indexSize, tc.preallocate, spaceNeeded)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user