Fix chown Input/output error on large file sets (#7996)

* Fix chown Input/output error on large file sets (Fixes #7911)

Implemented retry logic for MySQL/MariaDB backend to handle transient errors like deadlocks and timeouts.

* Fix syntax error: missing closing brace

* Refactor: Use %w for error wrapping and errors.As for extraction

* Fix: Disable retry logic inside transactions

weed/util/retry.go

@@ -58,13 +58,13 @@ func MultiRetry(name string, errList []string, job func() error) (err error) {
 }
 
 // RetryUntil retries until the job returns no error or onErrFn returns false
-func RetryUntil(name string, job func() error, onErrFn func(err error) (shouldContinue bool)) {
+func RetryUntil(name string, job func() error, onErrFn func(err error) (shouldContinue bool)) error {
 	waitTime := time.Second
 	for {
 		err := job()
 		if err == nil {
 			waitTime = time.Second
-			break
+			return nil
 		}
 		if onErrFn(err) {
 			if strings.Contains(err.Error(), "transport") || strings.Contains(err.Error(), "ResourceExhausted") || strings.Contains(err.Error(), "Unavailable") {
@@ -76,7 +76,7 @@ func RetryUntil(name string, job func() error, onErrFn func(err error) (shouldCo
 			}
 			continue
 		} else {
-			break
+			return err
 		}
 	}
 }

weed/util/retry_test.go

@@ -0,0 +1,65 @@
+package util
+
+import (
+	"errors"
+	"testing"
+)
+
+func TestRetryUntil(t *testing.T) {
+	// Test case 1: Function succeeds immediately
+	t.Run("SucceedsImmediately", func(t *testing.T) {
+		callCount := 0
+		err := RetryUntil("test", func() error {
+			callCount++
+			return nil
+		}, func(err error) bool {
+			return false
+		})
+
+		if err != nil {
+			t.Errorf("Expected no error, got %v", err)
+		}
+		if callCount != 1 {
+			t.Errorf("Expected 1 call, got %d", callCount)
+		}
+	})
+
+	// Test case 2: Function fails with retryable error, then succeeds
+	t.Run("SucceedsAfterRetry", func(t *testing.T) {
+		callCount := 0
+		err := RetryUntil("test", func() error {
+			callCount++
+			if callCount < 3 {
+				return errors.New("retryable error")
+			}
+			return nil
+		}, func(err error) bool {
+			return err.Error() == "retryable error"
+		})
+
+		if err != nil {
+			t.Errorf("Expected no error, got %v", err)
+		}
+		if callCount != 3 {
+			t.Errorf("Expected 3 calls, got %d", callCount)
+		}
+	})
+
+	// Test case 3: Function fails with non-retryable error
+	t.Run("FailsNonRetryable", func(t *testing.T) {
+		callCount := 0
+		err := RetryUntil("test", func() error {
+			callCount++
+			return errors.New("fatal error")
+		}, func(err error) bool {
+			return err.Error() == "retryable error"
+		})
+
+		if err == nil || err.Error() != "fatal error" {
+			t.Errorf("Expected 'fatal error', got %v", err)
+		}
+		if callCount != 1 {
+			t.Errorf("Expected 1 call, got %d", callCount)
+		}
+	})
+}