Add support for TLS in gRPC communication between worker and volume server (#8370)

* Add support for TLS in gRPC communication between worker and volume server

* address comments

* worker: capture shared grpc.DialOption in BalanceTask registration closure

* worker: capture shared grpc.DialOption in ErasureCodingTask registration closure

* worker: capture shared grpc.DialOption in VacuumTask registration closure

* worker: use grpc.worker security configuration section for tasks

* plugin/worker: fix compilation errors by passing grpc.DialOption to task constructors

* plugin/worker: prevent double-counting in EC skip counters

---------

Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/worker/tasks/erasure_coding/ec_task.go

@@ -26,11 +26,12 @@ import (
 // ErasureCodingTask implements the Task interface
 type ErasureCodingTask struct {
 	*base.BaseTask
-	server     string
-	volumeID   uint32
-	collection string
-	workDir    string
-	progress   float64
+	server         string
+	volumeID       uint32
+	collection     string
+	workDir        string
+	progress       float64
+	grpcDialOption grpc.DialOption
 
 	// EC parameters
 	dataShards      int32
@@ -41,14 +42,15 @@ type ErasureCodingTask struct {
 }
 
 // NewErasureCodingTask creates a new unified EC task instance
-func NewErasureCodingTask(id string, server string, volumeID uint32, collection string) *ErasureCodingTask {
+func NewErasureCodingTask(id string, server string, volumeID uint32, collection string, grpcDialOption grpc.DialOption) *ErasureCodingTask {
 	return &ErasureCodingTask{
-		BaseTask:     base.NewBaseTask(id, types.TaskTypeErasureCoding),
-		server:       server,
-		volumeID:     volumeID,
-		collection:   collection,
-		dataShards:   erasure_coding.DataShardsCount,   // Default values
-		parityShards: erasure_coding.ParityShardsCount, // Default values
+		BaseTask:       base.NewBaseTask(id, types.TaskTypeErasureCoding),
+		server:         server,
+		volumeID:       volumeID,
+		collection:     collection,
+		dataShards:     erasure_coding.DataShardsCount,   // Default values
+		parityShards:   erasure_coding.ParityShardsCount, // Default values
+		grpcDialOption: grpcDialOption,
 	}
 }
 
@@ -243,7 +245,7 @@ func (t *ErasureCodingTask) GetProgress() float64 {
 
 // markVolumeReadonly marks the volume as readonly on the source server
 func (t *ErasureCodingTask) markVolumeReadonly() error {
-	return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
+	return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
 		func(client volume_server_pb.VolumeServerClient) error {
 			_, err := client.VolumeMarkReadonly(context.Background(), &volume_server_pb.VolumeMarkReadonlyRequest{
 				VolumeId: t.volumeID,
@@ -301,7 +303,7 @@ func (t *ErasureCodingTask) copyVolumeFilesToWorker(workDir string) (map[string]
 
 // copyFileFromSource copies a file from source server to local path using gRPC streaming
 func (t *ErasureCodingTask) copyFileFromSource(ext, localPath string) error {
-	return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), grpc.WithInsecure(),
+	return operation.WithVolumeServerClient(false, pb.ServerAddress(t.server), t.grpcDialOption,
 		func(client volume_server_pb.VolumeServerClient) error {
 			stream, err := client.CopyFile(context.Background(), &volume_server_pb.CopyFileRequest{
 				VolumeId:   t.volumeID,
@@ -533,7 +535,7 @@ func (t *ErasureCodingTask) distributeEcShards(shardFiles map[string]string) err
 
 // sendShardFileToDestination sends a single shard file to a destination server using ReceiveFile API
 func (t *ErasureCodingTask) sendShardFileToDestination(destServer, filePath, shardType string) error {
-	return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), grpc.WithInsecure(),
+	return operation.WithVolumeServerClient(false, pb.ServerAddress(destServer), t.grpcDialOption,
 		func(client volume_server_pb.VolumeServerClient) error {
 			// Open the local shard file
 			file, err := os.Open(filePath)
@@ -665,7 +667,7 @@ func (t *ErasureCodingTask) mountEcShards() error {
 			continue
 		}
 
-		err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), grpc.WithInsecure(),
+		err := operation.WithVolumeServerClient(false, pb.ServerAddress(destNode), t.grpcDialOption,
 			func(client volume_server_pb.VolumeServerClient) error {
 				_, mountErr := client.VolumeEcShardsMount(context.Background(), &volume_server_pb.VolumeEcShardsMountRequest{
 					VolumeId:   t.volumeID,
@@ -722,7 +724,7 @@ func (t *ErasureCodingTask) deleteOriginalVolume() error {
 			"volume_id":      t.volumeID,
 		}).Info("Deleting volume from replica server")
 
-		err := operation.WithVolumeServerClient(false, pb.ServerAddress(replicaServer), grpc.WithInsecure(),
+		err := operation.WithVolumeServerClient(false, pb.ServerAddress(replicaServer), t.grpcDialOption,
 			func(client volume_server_pb.VolumeServerClient) error {
 				_, err := client.VolumeDelete(context.Background(), &volume_server_pb.VolumeDeleteRequest{
 					VolumeId:  t.volumeID,

weed/worker/tasks/erasure_coding/register.go

@@ -6,6 +6,8 @@ import (
 
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/worker_pb"
+	"github.com/seaweedfs/seaweedfs/weed/security"
+	"github.com/seaweedfs/seaweedfs/weed/util"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks"
 	"github.com/seaweedfs/seaweedfs/weed/worker/tasks/base"
 	"github.com/seaweedfs/seaweedfs/weed/worker/types"
@@ -27,6 +29,9 @@ func RegisterErasureCodingTask() {
 	// Create configuration instance
 	config := NewDefaultConfig()
 
+	// Create shared gRPC dial option using TLS configuration
+	dialOpt := security.LoadClientTLS(util.GetViper(), "grpc.worker")
+
 	// Create complete task definition
 	taskDef := &base.TaskDefinition{
 		Type:         types.TaskTypeErasureCoding,
@@ -50,6 +55,7 @@ func RegisterErasureCodingTask() {
 				params.Sources[0].Node, // Use first source node
 				params.VolumeId,
 				params.Collection,
+				dialOpt,
 			), nil
 		},
 		DetectionFunc:  Detection,