Client disconnects create context cancelled errors, 500x errors and Filer lookup failures (#8845)

* Update stream.go

Client disconnects create context cancelled errors and Filer lookup failures

* s3api: handle canceled stream requests cleanly

* s3api: address canceled streaming review feedback

---------

Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/filer/stream.go

@@ -141,12 +141,26 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
 		var urlStrings []string
 		var err error
 		for _, backoff := range getLookupFileIdBackoffSchedule {
+			if err := ctx.Err(); err != nil {
+				return nil, err
+			}
 			urlStrings, err = masterClient.GetLookupFileIdFunction()(ctx, chunkView.FileId)
 			if err == nil && len(urlStrings) > 0 {
 				break
 			}
+			if err := ctx.Err(); err != nil {
+				return nil, err
+			}
 			glog.V(4).InfofCtx(ctx, "waiting for chunk: %s", chunkView.FileId)
-			time.Sleep(backoff)
+			timer := time.NewTimer(backoff)
+			select {
+			case <-ctx.Done():
+				if !timer.Stop() {
+					<-timer.C
+				}
+				return nil, ctx.Err()
+			case <-timer.C:
+			}
 		}
 		if err != nil {
 			glog.V(1).InfofCtx(ctx, "operation LookupFileId %s failed, err: %v", chunkView.FileId, err)
@@ -179,6 +193,10 @@ func PrepareStreamContentWithThrottler(ctx context.Context, masterClient wdclien
 			jwt := jwtFunc(chunkView.FileId)
 			written, err := retriedStreamFetchChunkData(ctx, writer, urlStrings, jwt, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.OffsetInChunk, int(chunkView.ViewSize))
 
+			if err != nil && ctx.Err() != nil {
+				return ctx.Err()
+			}
+
 			// If read failed, try to invalidate cache and re-lookup
 			if err != nil && written == 0 {
 				if invalidator, ok := masterClient.(CacheInvalidator); ok {

weed/filer/stream_failover_test.go

@@ -1,8 +1,11 @@
 package filer
 
 import (
+	"bytes"
 	"context"
+	"errors"
 	"testing"
+	"time"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/wdclient"
@@ -173,3 +176,106 @@ func TestRetryLogicSkipsSameUrls(t *testing.T) {
 		t.Error("Expected different URLs to not be equal")
 	}
 }
+
+func TestCanceledStreamSkipsCacheInvalidation(t *testing.T) {
+	ctx, cancel := context.WithCancel(context.Background())
+	fileId := "3,canceled"
+
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fid string) ([]string, error) {
+			return []string{"http://server:8080"}, nil
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: fileId,
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	streamFn, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if err != nil {
+		t.Fatalf("PrepareStreamContentWithThrottler failed: %v", err)
+	}
+
+	cancel()
+
+	err = streamFn(&bytes.Buffer{})
+	if err != context.Canceled {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if len(mock.invalidatedFileIds) != 0 {
+		t.Fatalf("expected no cache invalidation on cancellation, got %v", mock.invalidatedFileIds)
+	}
+}
+
+func TestPrepareStreamContentSkipsLookupWhenContextAlreadyCanceled(t *testing.T) {
+	oldSchedule := getLookupFileIdBackoffSchedule
+	getLookupFileIdBackoffSchedule = []time.Duration{time.Millisecond}
+	t.Cleanup(func() {
+		getLookupFileIdBackoffSchedule = oldSchedule
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	lookupCalls := 0
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
+			lookupCalls++
+			return nil, errors.New("lookup should not run")
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "3,precanceled",
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	_, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if lookupCalls != 0 {
+		t.Fatalf("expected no lookup calls after cancellation, got %d", lookupCalls)
+	}
+}
+
+func TestPrepareStreamContentStopsLookupRetriesAfterContextCancellation(t *testing.T) {
+	oldSchedule := getLookupFileIdBackoffSchedule
+	getLookupFileIdBackoffSchedule = []time.Duration{time.Millisecond, time.Millisecond, time.Millisecond}
+	t.Cleanup(func() {
+		getLookupFileIdBackoffSchedule = oldSchedule
+	})
+
+	ctx, cancel := context.WithCancel(context.Background())
+	lookupCalls := 0
+	mock := &mockMasterClient{
+		lookupFunc: func(ctx context.Context, fileId string) ([]string, error) {
+			lookupCalls++
+			cancel()
+			return nil, context.Canceled
+		},
+	}
+
+	chunks := []*filer_pb.FileChunk{
+		{
+			FileId: "3,cancel-during-lookup",
+			Offset: 0,
+			Size:   10,
+		},
+	}
+
+	_, err := PrepareStreamContentWithThrottler(ctx, mock, noJwtFunc, chunks, 0, 10, 0)
+	if !errors.Is(err, context.Canceled) {
+		t.Fatalf("expected context.Canceled, got %v", err)
+	}
+	if lookupCalls != 1 {
+		t.Fatalf("expected lookup retries to stop after cancellation, got %d calls", lookupCalls)
+	}
+}