Admin: misc improvements on admin server and workers. EC now works. (#7055)

* initial design

* added simulation as tests

* reorganized the codebase to move the simulation framework and tests into their own dedicated package

* integration test. ec worker task

* remove "enhanced" reference

* start master, volume servers, filer

Current Status
✅ Master: Healthy and running (port 9333)
✅ Filer: Healthy and running (port 8888)
✅ Volume Servers: All 6 servers running (ports 8080-8085)
🔄 Admin/Workers: Will start when dependencies are ready

* generate write load

* tasks are assigned

* admin start wtih grpc port. worker has its own working directory

* Update .gitignore

* working worker and admin. Task detection is not working yet.

* compiles, detection uses volumeSizeLimitMB from master

* compiles

* worker retries connecting to admin

* build and restart

* rendering pending tasks

* skip task ID column

* sticky worker id

* test canScheduleTaskNow

* worker reconnect to admin

* clean up logs

* worker register itself first

* worker can run ec work and report status

but:
1. one volume should not be repeatedly worked on.
2. ec shards needs to be distributed and source data should be deleted.

* move ec task logic

* listing ec shards

* local copy, ec. Need to distribute.

* ec is mostly working now

* distribution of ec shards needs improvement
* need configuration to enable ec

* show ec volumes

* interval field UI component

* rename

* integration test with vauuming

* garbage percentage threshold

* fix warning

* display ec shard sizes

* fix ec volumes list

* Update ui.go

* show default values

* ensure correct default value

* MaintenanceConfig use ConfigField

* use schema defined defaults

* config

* reduce duplication

* refactor to use BaseUIProvider

* each task register its schema

* checkECEncodingCandidate use ecDetector

* use vacuumDetector

* use volumeSizeLimitMB

* remove

remove

* remove unused

* refactor

* use new framework

* remove v2 reference

* refactor

* left menu can scroll now

* The maintenance manager was not being initialized when no data directory was configured for persistent storage.

* saving config

* Update task_config_schema_templ.go

* enable/disable tasks

* protobuf encoded task configurations

* fix system settings

* use ui component

* remove logs

* interface{} Reduction

* reduce interface{}

* reduce interface{}

* avoid from/to map

* reduce interface{}

* refactor

* keep it DRY

* added logging

* debug messages

* debug level

* debug

* show the log caller line

* use configured task policy

* log level

* handle admin heartbeat response

* Update worker.go

* fix EC rack and dc count

* Report task status to admin server

* fix task logging, simplify interface checking, use erasure_coding constants

* factor in empty volume server during task planning

* volume.list adds disk id

* track disk id also

* fix locking scheduled and manual scanning

* add active topology

* simplify task detector

* ec task completed, but shards are not showing up

* implement ec in ec_typed.go

* adjust log level

* dedup

* implementing ec copying shards and only ecx files

* use disk id when distributing ec shards

🎯 Planning: ActiveTopology creates DestinationPlan with specific TargetDisk
📦 Task Creation: maintenance_integration.go creates ECDestination with DiskId
🚀 Task Execution: EC task passes DiskId in VolumeEcShardsCopyRequest
💾 Volume Server: Receives disk_id and stores shards on specific disk (vs.store.Locations[req.DiskId])
📂 File System: EC shards and metadata land in the exact disk directory planned

* Delete original volume from all locations

* clean up existing shard locations

* local encoding and distributing

* Update docker/admin_integration/EC-TESTING-README.md

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* check volume id range

* simplify

* fix tests

* fix types

* clean up logs and tests

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

weed/server/volume_grpc_copy.go

@@ -402,3 +402,120 @@ func (vs *VolumeServer) CopyFile(req *volume_server_pb.CopyFileRequest, stream v
 
 	return nil
 }
+
+// ReceiveFile receives a file stream from client and writes it to storage
+func (vs *VolumeServer) ReceiveFile(stream volume_server_pb.VolumeServer_ReceiveFileServer) error {
+	var fileInfo *volume_server_pb.ReceiveFileInfo
+	var targetFile *os.File
+	var filePath string
+	var bytesWritten uint64
+
+	defer func() {
+		if targetFile != nil {
+			targetFile.Close()
+		}
+	}()
+
+	for {
+		req, err := stream.Recv()
+		if err == io.EOF {
+			// Stream completed successfully
+			if targetFile != nil {
+				targetFile.Sync()
+				glog.V(1).Infof("Successfully received file %s (%d bytes)", filePath, bytesWritten)
+			}
+			return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{
+				BytesWritten: bytesWritten,
+			})
+		}
+		if err != nil {
+			// Clean up on error
+			if targetFile != nil {
+				targetFile.Close()
+				os.Remove(filePath)
+			}
+			glog.Errorf("Failed to receive stream: %v", err)
+			return fmt.Errorf("failed to receive stream: %v", err)
+		}
+
+		switch data := req.Data.(type) {
+		case *volume_server_pb.ReceiveFileRequest_Info:
+			// First message contains file info
+			fileInfo = data.Info
+			glog.V(1).Infof("ReceiveFile: volume %d, ext %s, collection %s, shard %d, size %d",
+				fileInfo.VolumeId, fileInfo.Ext, fileInfo.Collection, fileInfo.ShardId, fileInfo.FileSize)
+
+			// Create file path based on file info
+			if fileInfo.IsEcVolume {
+				// Find storage location for EC shard
+				var targetLocation *storage.DiskLocation
+				for _, location := range vs.store.Locations {
+					if location.DiskType == types.HardDriveType {
+						targetLocation = location
+						break
+					}
+				}
+				if targetLocation == nil && len(vs.store.Locations) > 0 {
+					targetLocation = vs.store.Locations[0] // Fall back to first available location
+				}
+				if targetLocation == nil {
+					glog.Errorf("ReceiveFile: no storage location available")
+					return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{
+						Error: "no storage location available",
+					})
+				}
+
+				// Create EC shard file path
+				baseFileName := erasure_coding.EcShardBaseFileName(fileInfo.Collection, int(fileInfo.VolumeId))
+				filePath = util.Join(targetLocation.Directory, baseFileName+fileInfo.Ext)
+			} else {
+				// Regular volume file
+				v := vs.store.GetVolume(needle.VolumeId(fileInfo.VolumeId))
+				if v == nil {
+					glog.Errorf("ReceiveFile: volume %d not found", fileInfo.VolumeId)
+					return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{
+						Error: fmt.Sprintf("volume %d not found", fileInfo.VolumeId),
+					})
+				}
+				filePath = v.FileName(fileInfo.Ext)
+			}
+
+			// Create target file
+			targetFile, err = os.Create(filePath)
+			if err != nil {
+				glog.Errorf("ReceiveFile: failed to create file %s: %v", filePath, err)
+				return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{
+					Error: fmt.Sprintf("failed to create file: %v", err),
+				})
+			}
+			glog.V(1).Infof("ReceiveFile: created target file %s", filePath)
+
+		case *volume_server_pb.ReceiveFileRequest_FileContent:
+			// Subsequent messages contain file content
+			if targetFile == nil {
+				glog.Errorf("ReceiveFile: file info must be sent first")
+				return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{
+					Error: "file info must be sent first",
+				})
+			}
+
+			n, err := targetFile.Write(data.FileContent)
+			if err != nil {
+				targetFile.Close()
+				os.Remove(filePath)
+				glog.Errorf("ReceiveFile: failed to write to file %s: %v", filePath, err)
+				return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{
+					Error: fmt.Sprintf("failed to write file: %v", err),
+				})
+			}
+			bytesWritten += uint64(n)
+			glog.V(2).Infof("ReceiveFile: wrote %d bytes to %s (total: %d)", n, filePath, bytesWritten)
+
+		default:
+			glog.Errorf("ReceiveFile: unknown message type")
+			return stream.SendAndClose(&volume_server_pb.ReceiveFileResponse{
+				Error: "unknown message type",
+			})
+		}
+	}
+}

weed/server/volume_grpc_erasure_coding.go

@@ -141,20 +141,31 @@ func (vs *VolumeServer) VolumeEcShardsCopy(ctx context.Context, req *volume_serv
 	glog.V(0).Infof("VolumeEcShardsCopy: %v", req)
 
 	var location *storage.DiskLocation
-	if req.CopyEcxFile {
-		location = vs.store.FindFreeLocation(func(location *storage.DiskLocation) bool {
-			return location.DiskType == types.HardDriveType
-		})
+
+	// Use disk_id if provided (disk-aware storage)
+	if req.DiskId > 0 || (req.DiskId == 0 && len(vs.store.Locations) > 0) {
+		// Validate disk ID is within bounds
+		if int(req.DiskId) >= len(vs.store.Locations) {
+			return nil, fmt.Errorf("invalid disk_id %d: only have %d disks", req.DiskId, len(vs.store.Locations))
+		}
+
+		// Use the specific disk location
+		location = vs.store.Locations[req.DiskId]
+		glog.V(1).Infof("Using disk %d for EC shard copy: %s", req.DiskId, location.Directory)
 	} else {
-		location = vs.store.FindFreeLocation(func(location *storage.DiskLocation) bool {
-			//(location.FindEcVolume) This method is error, will cause location is nil, redundant judgment
-			// _, found := location.FindEcVolume(needle.VolumeId(req.VolumeId))
-			// return found
-			return true
-		})
-	}
-	if location == nil {
-		return nil, fmt.Errorf("no space left")
+		// Fallback to old behavior for backward compatibility
+		if req.CopyEcxFile {
+			location = vs.store.FindFreeLocation(func(location *storage.DiskLocation) bool {
+				return location.DiskType == types.HardDriveType
+			})
+		} else {
+			location = vs.store.FindFreeLocation(func(location *storage.DiskLocation) bool {
+				return true
+			})
+		}
+		if location == nil {
+			return nil, fmt.Errorf("no space left")
+		}
 	}
 
 	dataBaseFileName := storage.VolumeFileName(location.Directory, req.Collection, int(req.VolumeId))
@@ -467,3 +478,30 @@ func (vs *VolumeServer) VolumeEcShardsToVolume(ctx context.Context, req *volume_
 
 	return &volume_server_pb.VolumeEcShardsToVolumeResponse{}, nil
 }
+
+func (vs *VolumeServer) VolumeEcShardsInfo(ctx context.Context, req *volume_server_pb.VolumeEcShardsInfoRequest) (*volume_server_pb.VolumeEcShardsInfoResponse, error) {
+	glog.V(0).Infof("VolumeEcShardsInfo: volume %d", req.VolumeId)
+
+	var ecShardInfos []*volume_server_pb.EcShardInfo
+
+	// Find the EC volume
+	for _, location := range vs.store.Locations {
+		if v, found := location.FindEcVolume(needle.VolumeId(req.VolumeId)); found {
+			// Get shard details from the EC volume
+			shardDetails := v.ShardDetails()
+			for _, shardDetail := range shardDetails {
+				ecShardInfo := &volume_server_pb.EcShardInfo{
+					ShardId:    uint32(shardDetail.ShardId),
+					Size:       shardDetail.Size,
+					Collection: v.Collection,
+				}
+				ecShardInfos = append(ecShardInfos, ecShardInfo)
+			}
+			break
+		}
+	}
+
+	return &volume_server_pb.VolumeEcShardsInfoResponse{
+		EcShardInfos: ecShardInfos,
+	}, nil
+}

weed/server/volume_server_handlers_admin.go

@@ -1,11 +1,12 @@
 package weed_server
 
 import (
-	"github.com/seaweedfs/seaweedfs/weed/topology"
-	"github.com/seaweedfs/seaweedfs/weed/util/version"
 	"net/http"
 	"path/filepath"
 
+	"github.com/seaweedfs/seaweedfs/weed/topology"
+	"github.com/seaweedfs/seaweedfs/weed/util/version"
+
 	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 )

weed/server/volume_server_ui/volume.html

@@ -175,8 +175,8 @@
             <tr>
                 <th>Id</th>
                 <th>Collection</th>
-                <th>Shard Size</th>
-                <th>Shards</th>
+                <th>Total Size</th>
+                <th>Shard Details</th>
                 <th>CreatedAt</th>
             </tr>
             </thead>
@@ -185,8 +185,14 @@
             <tr>
                 <td><code>{{ .VolumeId }}</code></td>
                 <td>{{ .Collection }}</td>
-                <td>{{ bytesToHumanReadable .ShardSize }}</td>
-                <td>{{ .ShardIdList }}</td>
+                <td>{{ bytesToHumanReadable .Size }}</td>
+                <td>
+                    {{ range .ShardDetails }}
+                        <span class="label label-info" style="margin-right: 5px;">
+                            {{ .ShardId }}: {{ bytesToHumanReadable .Size }}
+                        </span>
+                    {{ end }}
+                </td>
                 <td>{{ .CreatedAt.Format "2006-01-02 15:04" }}</td>
             </tr>
             {{ end }}