gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add statfile remote storage (#8443)

Browse Source 
* feat: add statfile; add error for remote storage misses

* feat: statfile implementations for storage providers

* test: add unit tests for StatFile method across providers

Add comprehensive unit tests for the StatFile implementation covering:
- S3: interface compliance and error constant accessibility
- Azure: interface compliance, error constants, and field population
- GCS: interface compliance, error constants, error detection, and field population

Also fix variable shadowing issue in S3 and Azure StatFile implementations where
named return parameters were being shadowed by local variable declarations.

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: address StatFile review feedback

- Use errors.New for ErrRemoteObjectNotFound sentinel
- Fix S3 HeadObject 404 detection to use awserr.Error code check
- Remove hollow field-population tests that tested nothing
- Remove redundant stdlib error detection tests
- Trim verbose doc comment on ErrRemoteObjectNotFound

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: address second round of StatFile review feedback

- Rename interface assertion tests to TestXxxRemoteStorageClientImplementsInterface
- Delegate readFileRemoteEntry to StatFile in all three providers
- Revert S3 404 detection to RequestFailure.StatusCode() check
- Fix double-slash in GCS error message format string
- Add storage type prefix to S3 error message for consistency

Co-authored-by: Cursor <cursoragent@cursor.com>

* fix: comments

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Peter Dodd
2026-02-25 18:24:06 +00:00
committed by GitHub
parent b565a0cc86
commit 0910252e31
7 changed files with 126 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
weed/remote_storage/azure/azure_storage_client.go
View File

@@ -41,9 +41,6 @@ const (
// with consistent retry configuration across the application.
// This centralizes the retry policy to ensure uniform behavior between
// remote storage and replication sink implementations.
//
// Related: Use DefaultAzureOpTimeout for context.WithTimeout when calling Azure operations
// to ensure the timeout accommodates all retry attempts configured here.
func DefaultAzBlobClientOptions() *azblob.ClientOptions {
return &azblob.ClientOptions{
ClientOptions: azcore.ClientOptions{
@@ -130,6 +127,32 @@ type azureRemoteStorageClient struct {
var _ = remote_storage.RemoteStorageClient(&azureRemoteStorageClient{})
func (az *azureRemoteStorageClient) StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
key := loc.Path[1:]
ctx, cancel := context.WithTimeout(context.Background(), DefaultAzureOpTimeout)
defer cancel()
resp, err := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlobClient(key).GetProperties(ctx, nil)
if err != nil {
if bloberror.HasCode(err, bloberror.BlobNotFound) {
return nil, remote_storage.ErrRemoteObjectNotFound
}
return nil, fmt.Errorf("stat azure %s%s: %w", loc.Bucket, loc.Path, err)
}
remoteEntry = &filer_pb.RemoteEntry{
StorageName: az.conf.Name,
}
if resp.ContentLength != nil {
remoteEntry.RemoteSize = *resp.ContentLength
}
if resp.LastModified != nil {
remoteEntry.RemoteMtime = resp.LastModified.Unix()
}
if resp.ETag != nil {
remoteEntry.RemoteETag = string(*resp.ETag)
}
return remoteEntry, nil
}
func (az *azureRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation, visitFn remote_storage.VisitFunc) (err error) {
pathKey := loc.Path[1:]
@@ -241,29 +264,7 @@ func (az *azureRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocati
}
func (az *azureRemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
key := loc.Path[1:]
blobClient := az.client.ServiceClient().NewContainerClient(loc.Bucket).NewBlockBlobClient(key)
props, err := blobClient.GetProperties(context.Background(), nil)
if err != nil {
return nil, err
}
remoteEntry := &filer_pb.RemoteEntry{
StorageName: az.conf.Name,
}
if props.LastModified != nil {
remoteEntry.RemoteMtime = props.LastModified.Unix()
}
if props.ContentLength != nil {
remoteEntry.RemoteSize = *props.ContentLength
}
if props.ETag != nil {
remoteEntry.RemoteETag = string(*props.ETag)
}
return remoteEntry, nil
return az.StatFile(loc)
}
func toMetadata(attributes map[string][]byte) map[string]*string {