Admin: misc improvements on admin server and workers. EC now works. (#7055)

* initial design * added simulation as tests * reorganized the codebase to move the simulation framework and tests into their own dedicated package * integration test. ec worker task * remove "enhanced" reference * start master, volume servers, filer Current Status ✅ Master: Healthy and running (port 9333) ✅ Filer: Healthy and running (port 8888) ✅ Volume Servers: All 6 servers running (ports 8080-8085) 🔄 Admin/Workers: Will start when dependencies are ready * generate write load * tasks are assigned * admin start wtih grpc port. worker has its own working directory * Update .gitignore * working worker and admin. Task detection is not working yet. * compiles, detection uses volumeSizeLimitMB from master * compiles * worker retries connecting to admin * build and restart * rendering pending tasks * skip task ID column * sticky worker id * test canScheduleTaskNow * worker reconnect to admin * clean up logs * worker register itself first * worker can run ec work and report status but: 1. one volume should not be repeatedly worked on. 2. ec shards needs to be distributed and source data should be deleted. * move ec task logic * listing ec shards * local copy, ec. Need to distribute. * ec is mostly working now * distribution of ec shards needs improvement * need configuration to enable ec * show ec volumes * interval field UI component * rename * integration test with vauuming * garbage percentage threshold * fix warning * display ec shard sizes * fix ec volumes list * Update ui.go * show default values * ensure correct default value * MaintenanceConfig use ConfigField * use schema defined defaults * config * reduce duplication * refactor to use BaseUIProvider * each task register its schema * checkECEncodingCandidate use ecDetector * use vacuumDetector * use volumeSizeLimitMB * remove remove * remove unused * refactor * use new framework * remove v2 reference * refactor * left menu can scroll now * The maintenance manager was not being initialized when no data directory was configured for persistent storage. * saving config * Update task_config_schema_templ.go * enable/disable tasks * protobuf encoded task configurations * fix system settings * use ui component * remove logs * interface{} Reduction * reduce interface{} * reduce interface{} * avoid from/to map * reduce interface{} * refactor * keep it DRY * added logging * debug messages * debug level * debug * show the log caller line * use configured task policy * log level * handle admin heartbeat response * Update worker.go * fix EC rack and dc count * Report task status to admin server * fix task logging, simplify interface checking, use erasure_coding constants * factor in empty volume server during task planning * volume.list adds disk id * track disk id also * fix locking scheduled and manual scanning * add active topology * simplify task detector * ec task completed, but shards are not showing up * implement ec in ec_typed.go * adjust log level * dedup * implementing ec copying shards and only ecx files * use disk id when distributing ec shards 🎯 Planning: ActiveTopology creates DestinationPlan with specific TargetDisk 📦 Task Creation: maintenance_integration.go creates ECDestination with DiskId 🚀 Task Execution: EC task passes DiskId in VolumeEcShardsCopyRequest 💾 Volume Server: Receives disk_id and stores shards on specific disk (vs.store.Locations[req.DiskId]) 📂 File System: EC shards and metadata land in the exact disk directory planned * Delete original volume from all locations * clean up existing shard locations * local encoding and distributing * Update docker/admin_integration/EC-TESTING-README.md Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> * check volume id range * simplify * fix tests * fix types * clean up logs and tests --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
2025-07-30 12:38:03 -07:00
parent 64198dad83
commit 891a2fb6eb
130 changed files with 27737 additions and 4429 deletions
--- a/weed/admin/topology/active_topology_test.go
+++ b/weed/admin/topology/active_topology_test.go
@@ -0,0 +1,654 @@
+package topology
+
+import (
+	"testing"
+	"time"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// TestActiveTopologyBasicOperations tests basic topology management
+func TestActiveTopologyBasicOperations(t *testing.T) {
+	topology := NewActiveTopology(10)
+	assert.NotNil(t, topology)
+	assert.Equal(t, 10, topology.recentTaskWindowSeconds)
+
+	// Test empty topology
+	assert.Equal(t, 0, len(topology.nodes))
+	assert.Equal(t, 0, len(topology.disks))
+	assert.Equal(t, 0, len(topology.pendingTasks))
+}
+
+// TestActiveTopologyUpdate tests topology updates from master
+func TestActiveTopologyUpdate(t *testing.T) {
+	topology := NewActiveTopology(10)
+
+	// Create sample topology info
+	topologyInfo := createSampleTopology()
+
+	err := topology.UpdateTopology(topologyInfo)
+	require.NoError(t, err)
+
+	// Verify topology structure
+	assert.Equal(t, 2, len(topology.nodes)) // 2 nodes
+	assert.Equal(t, 4, len(topology.disks)) // 4 disks total (2 per node)
+
+	// Verify node structure
+	node1, exists := topology.nodes["10.0.0.1:8080"]
+	require.True(t, exists)
+	assert.Equal(t, "dc1", node1.dataCenter)
+	assert.Equal(t, "rack1", node1.rack)
+	assert.Equal(t, 2, len(node1.disks))
+
+	// Verify disk structure
+	disk1, exists := topology.disks["10.0.0.1:8080:0"]
+	require.True(t, exists)
+	assert.Equal(t, uint32(0), disk1.DiskID)
+	assert.Equal(t, "hdd", disk1.DiskType)
+	assert.Equal(t, "dc1", disk1.DataCenter)
+}
+
+// TestTaskLifecycle tests the complete task lifecycle
+func TestTaskLifecycle(t *testing.T) {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+
+	taskID := "balance-001"
+
+	// 1. Add pending task
+	topology.AddPendingTask(taskID, TaskTypeBalance, 1001,
+		"10.0.0.1:8080", 0, "10.0.0.2:8080", 1)
+
+	// Verify pending state
+	assert.Equal(t, 1, len(topology.pendingTasks))
+	assert.Equal(t, 0, len(topology.assignedTasks))
+	assert.Equal(t, 0, len(topology.recentTasks))
+
+	task := topology.pendingTasks[taskID]
+	assert.Equal(t, TaskStatusPending, task.Status)
+	assert.Equal(t, uint32(1001), task.VolumeID)
+
+	// Verify task assigned to disks
+	sourceDisk := topology.disks["10.0.0.1:8080:0"]
+	targetDisk := topology.disks["10.0.0.2:8080:1"]
+	assert.Equal(t, 1, len(sourceDisk.pendingTasks))
+	assert.Equal(t, 1, len(targetDisk.pendingTasks))
+
+	// 2. Assign task
+	err := topology.AssignTask(taskID)
+	require.NoError(t, err)
+
+	// Verify assigned state
+	assert.Equal(t, 0, len(topology.pendingTasks))
+	assert.Equal(t, 1, len(topology.assignedTasks))
+	assert.Equal(t, 0, len(topology.recentTasks))
+
+	task = topology.assignedTasks[taskID]
+	assert.Equal(t, TaskStatusInProgress, task.Status)
+
+	// Verify task moved to assigned on disks
+	assert.Equal(t, 0, len(sourceDisk.pendingTasks))
+	assert.Equal(t, 1, len(sourceDisk.assignedTasks))
+	assert.Equal(t, 0, len(targetDisk.pendingTasks))
+	assert.Equal(t, 1, len(targetDisk.assignedTasks))
+
+	// 3. Complete task
+	err = topology.CompleteTask(taskID)
+	require.NoError(t, err)
+
+	// Verify completed state
+	assert.Equal(t, 0, len(topology.pendingTasks))
+	assert.Equal(t, 0, len(topology.assignedTasks))
+	assert.Equal(t, 1, len(topology.recentTasks))
+
+	task = topology.recentTasks[taskID]
+	assert.Equal(t, TaskStatusCompleted, task.Status)
+	assert.False(t, task.CompletedAt.IsZero())
+}
+
+// TestTaskDetectionScenarios tests various task detection scenarios
+func TestTaskDetectionScenarios(t *testing.T) {
+	tests := []struct {
+		name          string
+		scenario      func() *ActiveTopology
+		expectedTasks map[string]bool // taskType -> shouldDetect
+	}{
+		{
+			name: "Empty cluster - no tasks needed",
+			scenario: func() *ActiveTopology {
+				topology := NewActiveTopology(10)
+				topology.UpdateTopology(createEmptyTopology())
+				return topology
+			},
+			expectedTasks: map[string]bool{
+				"balance": false,
+				"vacuum":  false,
+				"ec":      false,
+			},
+		},
+		{
+			name: "Unbalanced cluster - balance task needed",
+			scenario: func() *ActiveTopology {
+				topology := NewActiveTopology(10)
+				topology.UpdateTopology(createUnbalancedTopology())
+				return topology
+			},
+			expectedTasks: map[string]bool{
+				"balance": true,
+				"vacuum":  false,
+				"ec":      false,
+			},
+		},
+		{
+			name: "High garbage ratio - vacuum task needed",
+			scenario: func() *ActiveTopology {
+				topology := NewActiveTopology(10)
+				topology.UpdateTopology(createHighGarbageTopology())
+				return topology
+			},
+			expectedTasks: map[string]bool{
+				"balance": false,
+				"vacuum":  true,
+				"ec":      false,
+			},
+		},
+		{
+			name: "Large volumes - EC task needed",
+			scenario: func() *ActiveTopology {
+				topology := NewActiveTopology(10)
+				topology.UpdateTopology(createLargeVolumeTopology())
+				return topology
+			},
+			expectedTasks: map[string]bool{
+				"balance": false,
+				"vacuum":  false,
+				"ec":      true,
+			},
+		},
+		{
+			name: "Recent tasks - no immediate re-detection",
+			scenario: func() *ActiveTopology {
+				topology := NewActiveTopology(10)
+				topology.UpdateTopology(createUnbalancedTopology())
+				// Add recent balance task
+				topology.recentTasks["recent-balance"] = &taskState{
+					VolumeID:    1001,
+					TaskType:    TaskTypeBalance,
+					Status:      TaskStatusCompleted,
+					CompletedAt: time.Now().Add(-5 * time.Second), // 5 seconds ago
+				}
+				return topology
+			},
+			expectedTasks: map[string]bool{
+				"balance": false, // Should not detect due to recent task
+				"vacuum":  false,
+				"ec":      false,
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			topology := tt.scenario()
+
+			// Test balance task detection
+			shouldDetectBalance := tt.expectedTasks["balance"]
+			actualDetectBalance := !topology.HasRecentTaskForVolume(1001, TaskTypeBalance)
+			if shouldDetectBalance {
+				assert.True(t, actualDetectBalance, "Should detect balance task")
+			} else {
+				// Note: In real implementation, task detection would be more sophisticated
+				// This is a simplified test of the recent task prevention mechanism
+			}
+
+			// Test that recent tasks prevent re-detection
+			if len(topology.recentTasks) > 0 {
+				for _, task := range topology.recentTasks {
+					hasRecent := topology.HasRecentTaskForVolume(task.VolumeID, task.TaskType)
+					assert.True(t, hasRecent, "Should find recent task for volume %d", task.VolumeID)
+				}
+			}
+		})
+	}
+}
+
+// TestTargetSelectionScenarios tests target selection for different task types
+func TestTargetSelectionScenarios(t *testing.T) {
+	tests := []struct {
+		name               string
+		topology           *ActiveTopology
+		taskType           TaskType
+		excludeNode        string
+		expectedTargets    int
+		expectedBestTarget string
+	}{
+		{
+			name:            "Balance task - find least loaded disk",
+			topology:        createTopologyWithLoad(),
+			taskType:        TaskTypeBalance,
+			excludeNode:     "10.0.0.1:8080", // Exclude source node
+			expectedTargets: 2,               // 2 disks on other node
+		},
+		{
+			name:            "EC task - find multiple available disks",
+			topology:        createTopologyForEC(),
+			taskType:        TaskTypeErasureCoding,
+			excludeNode:     "", // Don't exclude any nodes
+			expectedTargets: 4,  // All 4 disks available
+		},
+		{
+			name:            "Vacuum task - avoid conflicting disks",
+			topology:        createTopologyWithConflicts(),
+			taskType:        TaskTypeVacuum,
+			excludeNode:     "",
+			expectedTargets: 1, // Only 1 disk without conflicts (conflicts exclude more disks)
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			availableDisks := tt.topology.GetAvailableDisks(tt.taskType, tt.excludeNode)
+			assert.Equal(t, tt.expectedTargets, len(availableDisks),
+				"Expected %d available disks, got %d", tt.expectedTargets, len(availableDisks))
+
+			// Verify disks are actually available
+			for _, disk := range availableDisks {
+				assert.NotEqual(t, tt.excludeNode, disk.NodeID,
+					"Available disk should not be on excluded node")
+
+				load := tt.topology.GetDiskLoad(disk.NodeID, disk.DiskID)
+				assert.Less(t, load, 2, "Disk load should be less than 2")
+			}
+		})
+	}
+}
+
+// TestDiskLoadCalculation tests disk load calculation
+func TestDiskLoadCalculation(t *testing.T) {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+
+	// Initially no load
+	load := topology.GetDiskLoad("10.0.0.1:8080", 0)
+	assert.Equal(t, 0, load)
+
+	// Add pending task
+	topology.AddPendingTask("task1", TaskTypeBalance, 1001,
+		"10.0.0.1:8080", 0, "10.0.0.2:8080", 1)
+
+	// Check load increased
+	load = topology.GetDiskLoad("10.0.0.1:8080", 0)
+	assert.Equal(t, 1, load)
+
+	// Add another task to same disk
+	topology.AddPendingTask("task2", TaskTypeVacuum, 1002,
+		"10.0.0.1:8080", 0, "", 0)
+
+	load = topology.GetDiskLoad("10.0.0.1:8080", 0)
+	assert.Equal(t, 2, load)
+
+	// Move one task to assigned
+	topology.AssignTask("task1")
+
+	// Load should still be 2 (1 pending + 1 assigned)
+	load = topology.GetDiskLoad("10.0.0.1:8080", 0)
+	assert.Equal(t, 2, load)
+
+	// Complete one task
+	topology.CompleteTask("task1")
+
+	// Load should decrease to 1
+	load = topology.GetDiskLoad("10.0.0.1:8080", 0)
+	assert.Equal(t, 1, load)
+}
+
+// TestTaskConflictDetection tests task conflict detection
+func TestTaskConflictDetection(t *testing.T) {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+
+	// Add a balance task
+	topology.AddPendingTask("balance1", TaskTypeBalance, 1001,
+		"10.0.0.1:8080", 0, "10.0.0.2:8080", 1)
+	topology.AssignTask("balance1")
+
+	// Try to get available disks for vacuum (conflicts with balance)
+	availableDisks := topology.GetAvailableDisks(TaskTypeVacuum, "")
+
+	// Source disk should not be available due to conflict
+	sourceDiskAvailable := false
+	for _, disk := range availableDisks {
+		if disk.NodeID == "10.0.0.1:8080" && disk.DiskID == 0 {
+			sourceDiskAvailable = true
+			break
+		}
+	}
+	assert.False(t, sourceDiskAvailable, "Source disk should not be available due to task conflict")
+}
+
+// TestPublicInterfaces tests the public interface methods
+func TestPublicInterfaces(t *testing.T) {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+
+	// Test GetAllNodes
+	nodes := topology.GetAllNodes()
+	assert.Equal(t, 2, len(nodes))
+	assert.Contains(t, nodes, "10.0.0.1:8080")
+	assert.Contains(t, nodes, "10.0.0.2:8080")
+
+	// Test GetNodeDisks
+	disks := topology.GetNodeDisks("10.0.0.1:8080")
+	assert.Equal(t, 2, len(disks))
+
+	// Test with non-existent node
+	disks = topology.GetNodeDisks("non-existent")
+	assert.Nil(t, disks)
+}
+
+// Helper functions to create test topologies
+
+func createSampleTopology() *master_pb.TopologyInfo {
+	return &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "10.0.0.1:8080",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {DiskId: 0, VolumeCount: 10, MaxVolumeCount: 100},
+									"ssd": {DiskId: 1, VolumeCount: 5, MaxVolumeCount: 50},
+								},
+							},
+							{
+								Id: "10.0.0.2:8080",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {DiskId: 0, VolumeCount: 8, MaxVolumeCount: 100},
+									"ssd": {DiskId: 1, VolumeCount: 3, MaxVolumeCount: 50},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func createEmptyTopology() *master_pb.TopologyInfo {
+	return &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "10.0.0.1:8080",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {DiskId: 0, VolumeCount: 0, MaxVolumeCount: 100},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func createUnbalancedTopology() *master_pb.TopologyInfo {
+	return &master_pb.TopologyInfo{
+		DataCenterInfos: []*master_pb.DataCenterInfo{
+			{
+				Id: "dc1",
+				RackInfos: []*master_pb.RackInfo{
+					{
+						Id: "rack1",
+						DataNodeInfos: []*master_pb.DataNodeInfo{
+							{
+								Id: "10.0.0.1:8080",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {DiskId: 0, VolumeCount: 90, MaxVolumeCount: 100}, // Very loaded
+								},
+							},
+							{
+								Id: "10.0.0.2:8080",
+								DiskInfos: map[string]*master_pb.DiskInfo{
+									"hdd": {DiskId: 0, VolumeCount: 10, MaxVolumeCount: 100}, // Lightly loaded
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func createHighGarbageTopology() *master_pb.TopologyInfo {
+	// In a real implementation, this would include volume-level garbage metrics
+	return createSampleTopology()
+}
+
+func createLargeVolumeTopology() *master_pb.TopologyInfo {
+	// In a real implementation, this would include volume-level size metrics
+	return createSampleTopology()
+}
+
+func createTopologyWithLoad() *ActiveTopology {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+
+	// Add some existing tasks to create load
+	topology.AddPendingTask("existing1", TaskTypeVacuum, 2001,
+		"10.0.0.1:8080", 0, "", 0)
+	topology.AssignTask("existing1")
+
+	return topology
+}
+
+func createTopologyForEC() *ActiveTopology {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+	return topology
+}
+
+func createTopologyWithConflicts() *ActiveTopology {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+
+	// Add conflicting tasks
+	topology.AddPendingTask("balance1", TaskTypeBalance, 3001,
+		"10.0.0.1:8080", 0, "10.0.0.2:8080", 0)
+	topology.AssignTask("balance1")
+
+	topology.AddPendingTask("ec1", TaskTypeErasureCoding, 3002,
+		"10.0.0.1:8080", 1, "", 0)
+	topology.AssignTask("ec1")
+
+	return topology
+}
+
+// TestDestinationPlanning tests destination planning functionality
+func TestDestinationPlanning(t *testing.T) {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+
+	// Test balance destination planning
+	t.Run("Balance destination planning", func(t *testing.T) {
+		plan, err := topology.PlanBalanceDestination(1001, "10.0.0.1:8080", "rack1", "dc1", 1024*1024) // 1MB
+		require.NoError(t, err)
+		require.NotNil(t, plan)
+
+		// Should not target the source node
+		assert.NotEqual(t, "10.0.0.1:8080", plan.TargetNode)
+		assert.Equal(t, "10.0.0.2:8080", plan.TargetNode)
+		assert.NotEmpty(t, plan.TargetRack)
+		assert.NotEmpty(t, plan.TargetDC)
+		assert.Greater(t, plan.PlacementScore, 0.0)
+	})
+
+	// Test EC destination planning
+	t.Run("EC destination planning", func(t *testing.T) {
+		multiPlan, err := topology.PlanECDestinations(1002, "10.0.0.1:8080", "rack1", "dc1", 3) // Ask for 3 shards - source node can be included
+		require.NoError(t, err)
+		require.NotNil(t, multiPlan)
+		assert.Greater(t, len(multiPlan.Plans), 0)
+		assert.LessOrEqual(t, len(multiPlan.Plans), 3) // Should get at most 3 shards
+		assert.Equal(t, len(multiPlan.Plans), multiPlan.TotalShards)
+
+		// Check that all plans have valid target nodes
+		for _, plan := range multiPlan.Plans {
+			assert.NotEmpty(t, plan.TargetNode)
+			assert.NotEmpty(t, plan.TargetRack)
+			assert.NotEmpty(t, plan.TargetDC)
+			assert.GreaterOrEqual(t, plan.PlacementScore, 0.0)
+		}
+
+		// Check diversity metrics
+		assert.GreaterOrEqual(t, multiPlan.SuccessfulRack, 1)
+		assert.GreaterOrEqual(t, multiPlan.SuccessfulDCs, 1)
+	})
+
+	// Test destination planning with load
+	t.Run("Destination planning considers load", func(t *testing.T) {
+		// Add load to one disk
+		topology.AddPendingTask("task1", TaskTypeBalance, 2001,
+			"10.0.0.2:8080", 0, "", 0)
+
+		plan, err := topology.PlanBalanceDestination(1003, "10.0.0.1:8080", "rack1", "dc1", 1024*1024)
+		require.NoError(t, err)
+		require.NotNil(t, plan)
+
+		// Should prefer less loaded disk (disk 1 over disk 0 on node2)
+		assert.Equal(t, "10.0.0.2:8080", plan.TargetNode)
+		assert.Equal(t, uint32(1), plan.TargetDisk) // Should prefer SSD (disk 1) which has no load
+	})
+
+	// Test insufficient destinations
+	t.Run("Handle insufficient destinations", func(t *testing.T) {
+		// Try to plan for more EC shards than available disks
+		multiPlan, err := topology.PlanECDestinations(1004, "10.0.0.1:8080", "rack1", "dc1", 100)
+
+		// Should get an error for insufficient disks
+		assert.Error(t, err)
+		assert.Nil(t, multiPlan)
+	})
+}
+
+// TestDestinationPlanningWithActiveTopology tests the integration between task detection and destination planning
+func TestDestinationPlanningWithActiveTopology(t *testing.T) {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createUnbalancedTopology())
+
+	// Test that tasks are created with destinations
+	t.Run("Balance task with destination", func(t *testing.T) {
+		// Simulate what the balance detector would create
+		sourceNode := "10.0.0.1:8080" // Overloaded node
+		volumeID := uint32(1001)
+
+		plan, err := topology.PlanBalanceDestination(volumeID, sourceNode, "rack1", "dc1", 1024*1024)
+		require.NoError(t, err)
+		require.NotNil(t, plan)
+
+		// Verify the destination is different from source
+		assert.NotEqual(t, sourceNode, plan.TargetNode)
+		assert.Equal(t, "10.0.0.2:8080", plan.TargetNode) // Should be the lightly loaded node
+
+		// Verify placement quality
+		assert.Greater(t, plan.PlacementScore, 0.0)
+		assert.LessOrEqual(t, plan.PlacementScore, 1.0)
+	})
+
+	// Test task state integration
+	t.Run("Task state affects future planning", func(t *testing.T) {
+		volumeID := uint32(1002)
+		sourceNode := "10.0.0.1:8080"
+		targetNode := "10.0.0.2:8080"
+
+		// Plan first destination
+		plan1, err := topology.PlanBalanceDestination(volumeID, sourceNode, "rack1", "dc1", 1024*1024)
+		require.NoError(t, err)
+		require.NotNil(t, plan1)
+
+		// Add a pending task to the target
+		topology.AddPendingTask("task1", TaskTypeBalance, volumeID, sourceNode, 0, targetNode, 0)
+
+		// Plan another destination - should consider the pending task load
+		plan2, err := topology.PlanBalanceDestination(1003, sourceNode, "rack1", "dc1", 1024*1024)
+		require.NoError(t, err)
+		require.NotNil(t, plan2)
+
+		// The placement score should reflect the increased load
+		// (This test might need adjustment based on the actual scoring algorithm)
+		glog.V(1).Infof("Plan1 score: %.3f, Plan2 score: %.3f", plan1.PlacementScore, plan2.PlacementScore)
+	})
+}
+
+// TestECDestinationPlanningDetailed tests the EC destination planning with multiple shards
+func TestECDestinationPlanningDetailed(t *testing.T) {
+	topology := NewActiveTopology(10)
+	topology.UpdateTopology(createSampleTopology())
+
+	t.Run("EC multiple destinations", func(t *testing.T) {
+		// Plan for 3 EC shards (now including source node, we have 4 disks total)
+		multiPlan, err := topology.PlanECDestinations(1005, "10.0.0.1:8080", "rack1", "dc1", 3)
+		require.NoError(t, err)
+		require.NotNil(t, multiPlan)
+
+		// Should get 3 destinations (can include source node's disks)
+		assert.Equal(t, 3, len(multiPlan.Plans))
+		assert.Equal(t, 3, multiPlan.TotalShards)
+
+		// Count node distribution - source node can now be included
+		nodeCount := make(map[string]int)
+		for _, plan := range multiPlan.Plans {
+			nodeCount[plan.TargetNode]++
+		}
+
+		// Should distribute across available nodes (both nodes can be used)
+		assert.GreaterOrEqual(t, len(nodeCount), 1, "Should use at least 1 node")
+		assert.LessOrEqual(t, len(nodeCount), 2, "Should use at most 2 nodes")
+		glog.V(1).Infof("EC destinations node distribution: %v", nodeCount)
+
+		glog.V(1).Infof("EC destinations: %d plans across %d racks, %d DCs",
+			multiPlan.TotalShards, multiPlan.SuccessfulRack, multiPlan.SuccessfulDCs)
+	})
+
+	t.Run("EC destination planning with task conflicts", func(t *testing.T) {
+		// Create a fresh topology for this test to avoid conflicts from previous test
+		freshTopology := NewActiveTopology(10)
+		freshTopology.UpdateTopology(createSampleTopology())
+
+		// Add tasks to create conflicts on some disks
+		freshTopology.AddPendingTask("conflict1", TaskTypeVacuum, 2001, "10.0.0.2:8080", 0, "", 0)
+		freshTopology.AddPendingTask("conflict2", TaskTypeBalance, 2002, "10.0.0.1:8080", 0, "", 0)
+		freshTopology.AssignTask("conflict1")
+		freshTopology.AssignTask("conflict2")
+
+		// Plan EC destinations - should still succeed using available disks
+		multiPlan, err := freshTopology.PlanECDestinations(1006, "10.0.0.1:8080", "rack1", "dc1", 2)
+		require.NoError(t, err)
+		require.NotNil(t, multiPlan)
+
+		// Should get destinations (using disks that don't have conflicts)
+		assert.GreaterOrEqual(t, len(multiPlan.Plans), 1)
+		assert.LessOrEqual(t, len(multiPlan.Plans), 2)
+
+		// Available disks should be: node1/disk1 and node2/disk1 (since disk0 on both nodes have conflicts)
+		for _, plan := range multiPlan.Plans {
+			assert.Equal(t, uint32(1), plan.TargetDisk, "Should prefer disk 1 which has no conflicts")
+		}
+
+		glog.V(1).Infof("EC destination planning with conflicts: found %d destinations", len(multiPlan.Plans))
+	})
+}