Filer: Fixed critical bugs in the Azure SDK migration (PR #7310) (#7401)

* Fixed critical bugs in the Azure SDK migration (PR #7310)

fix https://github.com/seaweedfs/seaweedfs/issues/5044

* purge emojis

* conditional delete

* Update azure_sink_test.go

* refactoring

* refactor

* add context to each call

* refactor

* address comments

* refactor

* defer

* DeleteSnapshots

The conditional delete in handleExistingBlob was missing DeleteSnapshots, which would cause the delete operation to fail on Azure storage accounts that have blob snapshots enabled.

* ensure the expected size

* adjust comment
This commit is contained in:
Chris Lu
2025-10-28 22:16:21 -07:00
committed by GitHub
parent 85bd593936
commit ec4f7cf33c
4 changed files with 267 additions and 51 deletions

View File

@@ -28,8 +28,35 @@ import (
const (
defaultBlockSize = 4 * 1024 * 1024
defaultConcurrency = 16
// DefaultAzureOpTimeout is the timeout for individual Azure blob operations.
// This should be larger than the maximum time the Azure SDK client will spend
// retrying. With MaxRetries=3 (4 total attempts) and TryTimeout=10s, the maximum
// time is roughly 4*10s + delays(~7s) = 47s. We use 60s to provide a reasonable
// buffer while still failing faster than indefinite hangs.
DefaultAzureOpTimeout = 60 * time.Second
)
// DefaultAzBlobClientOptions returns the default Azure blob client options
// with consistent retry configuration across the application.
// This centralizes the retry policy to ensure uniform behavior between
// remote storage and replication sink implementations.
//
// Related: Use DefaultAzureOpTimeout for context.WithTimeout when calling Azure operations
// to ensure the timeout accommodates all retry attempts configured here.
func DefaultAzBlobClientOptions() *azblob.ClientOptions {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Retry: policy.RetryOptions{
MaxRetries: 3, // Reasonable retry count - aggressive retries mask configuration errors
TryTimeout: 10 * time.Second, // Reduced from 1 minute to fail faster on auth issues
RetryDelay: 1 * time.Second,
MaxRetryDelay: 10 * time.Second,
},
},
}
}
// invalidMetadataChars matches any character that is not valid in Azure metadata keys.
// Azure metadata keys must be valid C# identifiers: letters, digits, and underscores only.
var invalidMetadataChars = regexp.MustCompile(`[^a-zA-Z0-9_]`)
@@ -86,16 +113,7 @@ func (s azureRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storag
}
serviceURL := fmt.Sprintf("https://%s.blob.core.windows.net/", accountName)
azClient, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
Retry: policy.RetryOptions{
MaxRetries: 10, // Increased from default 3 to maintain resiliency similar to old SDK's 20
TryTimeout: time.Minute,
RetryDelay: 2 * time.Second,
MaxRetryDelay: time.Minute,
},
},
})
azClient, err := azblob.NewClientWithSharedKeyCredential(serviceURL, credential, DefaultAzBlobClientOptions())
if err != nil {
return nil, fmt.Errorf("failed to create Azure client: %w", err)
}

View File

@@ -336,7 +336,10 @@ func TestAzureRemoteStorageMaker(t *testing.T) {
t.Error("Expected HasBucket() to return true")
}
// Test with missing credentials
// Test with missing credentials - unset env vars (auto-restored by t.Setenv)
t.Setenv("AZURE_STORAGE_ACCOUNT", "")
t.Setenv("AZURE_STORAGE_ACCESS_KEY", "")
conf := &remote_pb.RemoteConf{
Name: "test",
}