gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

volume vacuum: avoid timeout with streaming progress report

Browse Source 
fix https://github.com/chrislusf/seaweedfs/issues/2396
This commit is contained in:
Chris Lu
2021-10-24 01:55:34 -07:00
parent 07dd4873db
commit 3be3c17f59
9 changed files with 979 additions and 899 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
weed/storage/store_vacuum.go
View File

@@ -15,13 +15,13 @@ func (s *Store) CheckCompactVolume(volumeId needle.VolumeId) (float64, error) {
}
return 0, fmt.Errorf("volume id %d is not found during check compact", volumeId)
}
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64) error {
func (s *Store) CompactVolume(vid needle.VolumeId, preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
if v := s.findVolume(vid); v != nil {
s := stats.NewDiskStatus(v.dir)
if int64(s.Free) < preallocate {
return fmt.Errorf("free space: %d bytes, not enough for %d bytes", s.Free, preallocate)
}
return v.Compact2(preallocate, compactionBytePerSecond)
return v.Compact2(preallocate, compactionBytePerSecond, progressFn)
}
return fmt.Errorf("volume id %d is not found during compact", vid)
}

14
weed/storage/volume_vacuum.go
View File

@@ -17,6 +17,8 @@ import (
"github.com/chrislusf/seaweedfs/weed/util"
)
type ProgressFunc func(processed int64) bool
func (v *Volume) garbageLevel() float64 {
if v.ContentSize() == 0 {
return 0
@@ -62,7 +64,7 @@ func (v *Volume) Compact(preallocate int64, compactionBytePerSecond int64) error
}
// compact a volume based on deletions in .idx files
func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64) error {
func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64, progressFn ProgressFunc) error {
if v.MemoryMapMaxSizeMb != 0 { //it makes no sense to compact in memory
return nil
@@ -83,7 +85,7 @@ func (v *Volume) Compact2(preallocate int64, compactionBytePerSecond int64) erro
if err := v.nm.Sync(); err != nil {
glog.V(0).Infof("compact2 fail to sync volume idx %d: %v", v.Id, err)
}
return copyDataBasedOnIndexFile(v.FileName(".dat"), v.FileName(".idx"), v.FileName(".cpd"), v.FileName(".cpx"), v.SuperBlock, v.Version(), preallocate, compactionBytePerSecond)
return copyDataBasedOnIndexFile(v.FileName(".dat"), v.FileName(".idx"), v.FileName(".cpd"), v.FileName(".cpx"), v.SuperBlock, v.Version(), preallocate, compactionBytePerSecond, progressFn)
}
func (v *Volume) CommitCompact() error {
@@ -382,7 +384,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca
return
}
func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName string, sb super_block.SuperBlock, version needle.Version, preallocate int64, compactionBytePerSecond int64) (err error) {
func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName string, sb super_block.SuperBlock, version needle.Version, preallocate, compactionBytePerSecond int64, progressFn ProgressFunc) (err error) {
var (
srcDatBackend, dstDatBackend backend.BackendStorageFile
dataFile *os.File
@@ -421,6 +423,12 @@ func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName str
return nil
}
if progressFn != nil {
if !progressFn(offset.ToActualOffset()) {
return fmt.Errorf("interrupted")
}
}
n := new(needle.Needle)
err := n.ReadData(srcDatBackend, offset.ToActualOffset(), size, version)
if err != nil {

2
weed/storage/volume_vacuum_test.go
View File

@@ -83,7 +83,7 @@ func TestCompaction(t *testing.T) {
}
startTime := time.Now()
v.Compact2(0, 0)
v.Compact2(0, 0, nil)
speed := float64(v.ContentSize()) / time.Now().Sub(startTime).Seconds()
t.Logf("compaction speed: %.2f bytes/s", speed)