fix(ec): gather shards from all disk locations before rebuild (#8633)

* fix(ec): gather shards from all disk locations before rebuild (#8631)

Fix "too few shards given" error during ec.rebuild on multi-disk volume
servers. The root cause has two parts:

1. VolumeEcShardsRebuild only looked at a single disk location for shard
   files. On multi-disk servers, the existing local shards could be on one
   disk while copied shards were placed on another, causing the rebuild to
   see fewer shards than actually available.

2. VolumeEcShardsCopy had a DiskId condition (req.DiskId == 0 &&
   len(vs.store.Locations) > 0) that was always true, making the
   FindFreeLocation fallback dead code. This meant copies always went to
   Locations[0] regardless of where existing shards were.

Changes:
- VolumeEcShardsRebuild now finds the location with the most shards,
  then gathers shard files from other locations via hard links (or
  symlinks for cross-device) before rebuilding. Gathered files are
  cleaned up after rebuild.
- VolumeEcShardsCopy now only uses Locations[DiskId] when DiskId > 0
  (explicitly set). Otherwise, it prefers the location that already has
  the EC volume, falling back to HDD then any free location.
- generateMissingEcFiles now logs shard counts and provides a clear
  error message when not enough shards are found, instead of passing
  through to the opaque reedsolomon "too few shards given" error.

* fix(ec): update test to match skip behavior for unrepairable volumes

The test expected an error for volumes with insufficient shards, but
commit 5acb4578a changed unrepairable volumes to be skipped with a log
message instead of returning an error. Update the test to verify the
skip behavior and log output.

* fix(ec): address PR review comments

- Add comment clarifying DiskId=0 means "not specified" (protobuf default),
  callers must use DiskId >= 1 to target a specific disk.
- Log warnings on cleanup failures for gathered shard links.

* fix(ec): read shard files from other disks directly instead of linking

Replace the hard link / symlink gathering approach with passing
additional search directories into RebuildEcFiles. The rebuild
function now opens shard files directly from whichever disk they
live on, avoiding filesystem link operations and cleanup.

RebuildEcFiles and RebuildEcFilesWithContext gain a variadic
additionalDirs parameter (backward compatible with existing callers).

* fix(ec): clarify DiskId selection semantics in VolumeEcShardsCopy comment

* fix(ec): avoid empty files on failed rebuild; don't skip ecx-only locations

- generateMissingEcFiles: two-pass approach — first discover present/missing
  shards and check reconstructability, only then create output files. This
  avoids leaving behind empty truncated shard files when there are too few
  shards to rebuild.

- VolumeEcShardsRebuild: compute hasEcx before skipping zero-shard locations.
  A location with an .ecx file but no shard files (all shards on other disks)
  is now a valid rebuild candidate instead of being silently skipped.

* fix(ec): select ecx-only location as rebuildLocation when none chosen yet

When rebuildLocation is nil and a location has hasEcx=true but
existingShardCount=0 (all shards on other disks), the condition
0 > 0 was false so it was never promoted to rebuildLocation.
Add rebuildLocation == nil to the predicate so the first location
with an .ecx file is always selected as a candidate.

weed/storage/erasure_coding/ec_encoder.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 
 	"github.com/klauspost/reedsolomon"
 
@@ -67,7 +68,10 @@ func WriteEcFilesWithContext(baseFileName string, ctx *ECContext) error {
 	return generateEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx)
 }
 
-func RebuildEcFiles(baseFileName string) ([]uint32, error) {
+// RebuildEcFiles rebuilds missing EC shard files.
+// additionalDirs are extra directories to search for existing shard files,
+// which handles multi-disk servers where shards may be spread across disks.
+func RebuildEcFiles(baseFileName string, additionalDirs ...string) ([]uint32, error) {
 	// Attempt to load EC config from .vif file to preserve original configuration
 	var ctx *ECContext
 	if volumeInfo, _, found, _ := volume_info.MaybeLoadVolumeInfo(baseFileName + ".vif"); found && volumeInfo.EcShardConfig != nil {
@@ -90,12 +94,13 @@ func RebuildEcFiles(baseFileName string) ([]uint32, error) {
 		ctx = NewDefaultECContext("", 0)
 	}
 
-	return RebuildEcFilesWithContext(baseFileName, ctx)
+	return RebuildEcFilesWithContext(baseFileName, ctx, additionalDirs...)
 }
 
-// RebuildEcFilesWithContext rebuilds missing EC files using the provided context
-func RebuildEcFilesWithContext(baseFileName string, ctx *ECContext) ([]uint32, error) {
-	return generateMissingEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx)
+// RebuildEcFilesWithContext rebuilds missing EC files using the provided context.
+// additionalDirs are extra directories to search for existing shard files.
+func RebuildEcFilesWithContext(baseFileName string, ctx *ECContext, additionalDirs ...string) ([]uint32, error) {
+	return generateMissingEcFiles(baseFileName, 256*1024, ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, ctx, additionalDirs)
 }
 
 func ToExt(ecIndex int) string {
@@ -122,30 +127,71 @@ func generateEcFiles(baseFileName string, bufferSize int, largeBlockSize int64,
 	return nil
 }
 
-func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext) (generatedShardIds []uint32, err error) {
+// findShardFile looks for a shard file at baseFileName+ext, then in additionalDirs.
+func findShardFile(baseFileName string, ext string, additionalDirs []string) string {
+	primary := baseFileName + ext
+	if util.FileExists(primary) {
+		return primary
+	}
+	baseName := filepath.Base(baseFileName)
+	for _, dir := range additionalDirs {
+		candidate := filepath.Join(dir, baseName+ext)
+		if util.FileExists(candidate) {
+			return candidate
+		}
+	}
+	return ""
+}
 
+func generateMissingEcFiles(baseFileName string, bufferSize int, largeBlockSize int64, smallBlockSize int64, ctx *ECContext, additionalDirs []string) (generatedShardIds []uint32, err error) {
+
+	// Pass 1: discover which shards exist and which are missing,
+	// opening input files but NOT creating output files yet.
 	shardHasData := make([]bool, ctx.Total())
+	shardPaths := make([]string, ctx.Total()) // non-empty for present shards
 	inputFiles := make([]*os.File, ctx.Total())
-	outputFiles := make([]*os.File, ctx.Total())
+	presentCount := 0
 	for shardId := 0; shardId < ctx.Total(); shardId++ {
-		shardFileName := baseFileName + ctx.ToExt(shardId)
-		if util.FileExists(shardFileName) {
+		ext := ctx.ToExt(shardId)
+		shardPath := findShardFile(baseFileName, ext, additionalDirs)
+		if shardPath != "" {
 			shardHasData[shardId] = true
-			inputFiles[shardId], err = os.OpenFile(shardFileName, os.O_RDONLY, 0)
+			shardPaths[shardId] = shardPath
+			inputFiles[shardId], err = os.OpenFile(shardPath, os.O_RDONLY, 0)
 			if err != nil {
 				return nil, err
 			}
 			defer inputFiles[shardId].Close()
+			presentCount++
 		} else {
-			outputFiles[shardId], err = os.OpenFile(shardFileName, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
-			if err != nil {
-				return nil, err
-			}
-			defer outputFiles[shardId].Close()
 			generatedShardIds = append(generatedShardIds, uint32(shardId))
 		}
 	}
 
+	// Pre-check: bail out before creating any output files.
+	if presentCount < ctx.DataShards {
+		return nil, fmt.Errorf("not enough shards to rebuild %s: found %d shards, need at least %d (data shards), missing shards: %v",
+			baseFileName, presentCount, ctx.DataShards, generatedShardIds)
+	}
+
+	glog.V(0).Infof("rebuilding %s: %d shards present, %d missing %v, config %s",
+		baseFileName, presentCount, len(generatedShardIds), generatedShardIds, ctx.String())
+
+	// Pass 2: create output files for missing shards now that we know
+	// reconstruction is possible.
+	outputFiles := make([]*os.File, ctx.Total())
+	for shardId := 0; shardId < ctx.Total(); shardId++ {
+		if shardHasData[shardId] {
+			continue
+		}
+		outputFileName := baseFileName + ctx.ToExt(shardId)
+		outputFiles[shardId], err = os.OpenFile(outputFileName, os.O_TRUNC|os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			return nil, err
+		}
+		defer outputFiles[shardId].Close()
+	}
+
 	err = rebuildEcFiles(shardHasData, inputFiles, outputFiles, ctx)
 	if err != nil {
 		return nil, fmt.Errorf("rebuildEcFiles: %w", err)