fix(ec.decode): purge EC shards when volume is empty (#7749)

* fix(ec.decode): purge EC shards when volume is empty

When an EC volume has no live entries (all deleted), ec.decode should not generate an empty normal volume. Instead, treat decode as a no-op and allow shard purge to proceed cleanly.\n\nFixes: #7748

* chore: address PR review comments

* test: cover live EC index + avoid magic string

* chore: harden empty-EC handling

- Make shard cleanup best-effort (collect errors)\n- Remove unreachable EOF handling in HasLiveNeedles\n- Add empty ecx test case\n- Share no-live-entries substring between server/client\n

* perf: parallelize EC shard unmount/delete across locations

* refactor: combine unmount+delete into single goroutine per location

* refactor: use errors.Join for multi-error aggregation

* refactor: use existing ErrorWaitGroup for parallel execution

* fix: capture loop variables + clarify SuperBlockSize safety
This commit is contained in:
Chris Lu
2025-12-14 17:06:13 -08:00
committed by GitHub
parent 8bdc4390a0
commit 7ed7578424
4 changed files with 164 additions and 18 deletions

View File

@@ -14,6 +14,23 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util"
)
// EcNoLiveEntriesSubstring is used for server/client coordination when ec.decode determines that
// decoding should be a no-op (all entries are deleted).
const EcNoLiveEntriesSubstring = "has no live entries"
// HasLiveNeedles returns whether the EC index (.ecx) contains at least one live (non-deleted) entry.
// This is used by ec.decode to avoid generating an empty normal volume when all entries were deleted.
func HasLiveNeedles(indexBaseFileName string) (hasLive bool, err error) {
err = iterateEcxFile(indexBaseFileName, func(_ types.NeedleId, _ types.Offset, size types.Size) error {
if !size.IsDeleted() {
hasLive = true
return io.EOF // stop early
}
return nil
})
return
}
// write .idx file from .ecx and .ecj files
func WriteIdxFileFromEcIndex(baseFileName string) (err error) {
@@ -52,6 +69,11 @@ func FindDatFileSize(dataBaseFileName, indexBaseFileName string) (datSize int64,
return 0, fmt.Errorf("read ec volume %s version: %v", dataBaseFileName, err)
}
// Safety: ensure datSize is at least SuperBlockSize. While the caller typically
// checks HasLiveNeedles first, this protects against direct calls to FindDatFileSize
// when all needles are deleted (see issue #7748).
datSize = int64(super_block.SuperBlockSize)
err = iterateEcxFile(indexBaseFileName, func(key types.NeedleId, offset types.Offset, size types.Size) error {
if size.IsDeleted() {

View File

@@ -0,0 +1,81 @@
package erasure_coding_test
import (
"os"
"path/filepath"
"testing"
erasure_coding "github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
)
func TestHasLiveNeedles_AllDeletedIsFalse(t *testing.T) {
dir := t.TempDir()
collection := "foo"
base := filepath.Join(dir, collection+"_1")
// Build an ecx file with only deleted entries.
// ecx file entries are the same format as .idx entries.
ecx := makeNeedleMapEntry(types.NeedleId(1), types.Offset{}, types.TombstoneFileSize)
if err := os.WriteFile(base+".ecx", ecx, 0644); err != nil {
t.Fatalf("write ecx: %v", err)
}
hasLive, err := erasure_coding.HasLiveNeedles(base)
if err != nil {
t.Fatalf("HasLiveNeedles: %v", err)
}
if hasLive {
t.Fatalf("expected no live entries")
}
}
func TestHasLiveNeedles_WithLiveEntryIsTrue(t *testing.T) {
dir := t.TempDir()
collection := "foo"
base := filepath.Join(dir, collection+"_1")
// Build an ecx file containing at least one live entry.
// ecx file entries are the same format as .idx entries.
live := makeNeedleMapEntry(types.NeedleId(1), types.Offset{}, types.Size(1))
if err := os.WriteFile(base+".ecx", live, 0644); err != nil {
t.Fatalf("write ecx: %v", err)
}
hasLive, err := erasure_coding.HasLiveNeedles(base)
if err != nil {
t.Fatalf("HasLiveNeedles: %v", err)
}
if !hasLive {
t.Fatalf("expected live entries")
}
}
func TestHasLiveNeedles_EmptyFileIsFalse(t *testing.T) {
dir := t.TempDir()
base := filepath.Join(dir, "foo_1")
// Create an empty ecx file.
if err := os.WriteFile(base+".ecx", []byte{}, 0644); err != nil {
t.Fatalf("write ecx: %v", err)
}
hasLive, err := erasure_coding.HasLiveNeedles(base)
if err != nil {
t.Fatalf("HasLiveNeedles: %v", err)
}
if hasLive {
t.Fatalf("expected no live entries for empty file")
}
}
func makeNeedleMapEntry(key types.NeedleId, offset types.Offset, size types.Size) []byte {
b := make([]byte, types.NeedleIdSize+types.OffsetSize+types.SizeSize)
types.NeedleIdToBytes(b[0:types.NeedleIdSize], key)
types.OffsetToBytes(b[types.NeedleIdSize:types.NeedleIdSize+types.OffsetSize], offset)
types.SizeToBytes(b[types.NeedleIdSize+types.OffsetSize:types.NeedleIdSize+types.OffsetSize+types.SizeSize], size)
return b
}