feat: add statfile remote storage (#8443)
* feat: add statfile; add error for remote storage misses * feat: statfile implementations for storage providers * test: add unit tests for StatFile method across providers Add comprehensive unit tests for the StatFile implementation covering: - S3: interface compliance and error constant accessibility - Azure: interface compliance, error constants, and field population - GCS: interface compliance, error constants, error detection, and field population Also fix variable shadowing issue in S3 and Azure StatFile implementations where named return parameters were being shadowed by local variable declarations. Co-authored-by: Cursor <cursoragent@cursor.com> * fix: address StatFile review feedback - Use errors.New for ErrRemoteObjectNotFound sentinel - Fix S3 HeadObject 404 detection to use awserr.Error code check - Remove hollow field-population tests that tested nothing - Remove redundant stdlib error detection tests - Trim verbose doc comment on ErrRemoteObjectNotFound Co-authored-by: Cursor <cursoragent@cursor.com> * fix: address second round of StatFile review feedback - Rename interface assertion tests to TestXxxRemoteStorageClientImplementsInterface - Delegate readFileRemoteEntry to StatFile in all three providers - Revert S3 404 detection to RequestFailure.StatusCode() check - Fix double-slash in GCS error message format string - Add storage type prefix to S3 error message for consistency Co-authored-by: Cursor <cursoragent@cursor.com> * fix: comments --------- Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -2,11 +2,13 @@ package gcs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"cloud.google.com/go/storage"
|
||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||
@@ -126,6 +128,28 @@ func (gcs *gcsRemoteStorageClient) Traverse(loc *remote_pb.RemoteStorageLocation
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
const defaultGCSOpTimeout = 30 * time.Second
|
||||
|
||||
func (gcs *gcsRemoteStorageClient) StatFile(loc *remote_pb.RemoteStorageLocation) (remoteEntry *filer_pb.RemoteEntry, err error) {
|
||||
key := loc.Path[1:]
|
||||
ctx, cancel := context.WithTimeout(context.Background(), defaultGCSOpTimeout)
|
||||
defer cancel()
|
||||
attr, err := gcs.client.Bucket(loc.Bucket).Object(key).Attrs(ctx)
|
||||
if err != nil {
|
||||
if errors.Is(err, storage.ErrObjectNotExist) {
|
||||
return nil, remote_storage.ErrRemoteObjectNotFound
|
||||
}
|
||||
return nil, fmt.Errorf("stat gcs %s%s: %w", loc.Bucket, loc.Path, err)
|
||||
}
|
||||
return &filer_pb.RemoteEntry{
|
||||
StorageName: gcs.conf.Name,
|
||||
RemoteMtime: attr.Updated.Unix(),
|
||||
RemoteSize: attr.Size,
|
||||
RemoteETag: attr.Etag,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (gcs *gcsRemoteStorageClient) ReadFile(loc *remote_pb.RemoteStorageLocation, offset int64, size int64) (data []byte, err error) {
|
||||
|
||||
key := loc.Path[1:]
|
||||
@@ -170,20 +194,7 @@ func (gcs *gcsRemoteStorageClient) WriteFile(loc *remote_pb.RemoteStorageLocatio
|
||||
}
|
||||
|
||||
func (gcs *gcsRemoteStorageClient) readFileRemoteEntry(loc *remote_pb.RemoteStorageLocation) (*filer_pb.RemoteEntry, error) {
|
||||
key := loc.Path[1:]
|
||||
attr, err := gcs.client.Bucket(loc.Bucket).Object(key).Attrs(context.Background())
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &filer_pb.RemoteEntry{
|
||||
RemoteMtime: attr.Updated.Unix(),
|
||||
RemoteSize: attr.Size,
|
||||
RemoteETag: attr.Etag,
|
||||
StorageName: gcs.conf.Name,
|
||||
}, nil
|
||||
|
||||
return gcs.StatFile(loc)
|
||||
}
|
||||
|
||||
func toMetadata(attributes map[string][]byte) map[string]string {
|
||||
|
||||
17
weed/remote_storage/gcs/gcs_storage_client_test.go
Normal file
17
weed/remote_storage/gcs/gcs_storage_client_test.go
Normal file
@@ -0,0 +1,17 @@
|
||||
package gcs
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGCSRemoteStorageClientImplementsInterface(t *testing.T) {
|
||||
var _ remote_storage.RemoteStorageClient = (*gcsRemoteStorageClient)(nil)
|
||||
}
|
||||
|
||||
func TestGCSErrRemoteObjectNotFoundIsAccessible(t *testing.T) {
|
||||
require.Error(t, remote_storage.ErrRemoteObjectNotFound)
|
||||
require.Equal(t, "remote object not found", remote_storage.ErrRemoteObjectNotFound.Error())
|
||||
}
|
||||
Reference in New Issue
Block a user