16f2269a33
* Add remote storage index for lazy metadata pull Introduces remoteStorageIndex, which maintains a map of filer directory to remote storage client/location, refreshed periodically from the filer's mount mappings. Provides lazyFetchFromRemote, ensureRemoteEntryInFiler, and isRemoteBacked on S3ApiServer as integration points for handler-level work in a follow-up PR. Nothing is wired into the server yet. Made-with: Cursor * Add unit tests for remote storage index and wire field into S3ApiServer Adds tests covering isEmpty, findForPath (including longest-prefix resolution), and isRemoteBacked. Also removes a stray PR review annotation from the index file and adds the remoteStorageIdx field to S3ApiServer so the package compiles ahead of the wiring PR. Made-with: Cursor * Address review comments on remote storage index - Use filer_pb.CreateEntry helper so resp.Error is checked, not just the RPC error - Extract keepPrev closure to remove duplicated error-handling in refresh loop - Add comment explaining availability-over-consistency trade-off on filer save failure Made-with: Cursor * Move lazy metadata pull from S3 API to filer - Add maybeLazyFetchFromRemote in filer: on FindEntry miss, stat remote and CreateEntry when path is under a remote mount - Use singleflight for dedup; context guard prevents CreateEntry recursion - Availability-over-consistency: return in-memory entry if CreateEntry fails - Add longest-prefix test for nested mounts in remote_storage_test.go - Remove remoteStorageIndex, lazyFetchFromRemote, ensureRemoteEntryInFiler, doLazyFetch from s3api; filer now owns metadata operations - Add filer_lazy_remote_test.go with tests for hit, miss, not-found, CreateEntry failure, longest-prefix, and FindEntry integration Made-with: Cursor * Address review: fix context guard test, add FindMountDirectory comment, remove dead code Made-with: Cursor * Nitpicks: restore prev maker in registerStubMaker, instance-scope lazyFetchGroup, nil-check remoteEntry Made-with: Cursor * Fix remotePath when mountDir is root: ensure relPath has leading slash Made-with: Cursor * filer: decouple lazy-fetch persistence from caller context Use context.Background() inside the singleflight closure for CreateEntry so persistence is not cancelled when the winning request's context is cancelled. Fixes CreateEntry failing for all waiters when the first caller times out. Made-with: Cursor * filer: remove redundant Mode bitwise OR with zero Made-with: Cursor * filer: use bounded context for lazy-fetch persistence Replace context.Background() with context.WithTimeout(30s) and defer cancel() to prevent indefinite blocking and release resources. Made-with: Cursor * filer: use checked type assertion for singleflight result Made-with: Cursor * filer: rename persist context vars to avoid shadowing function parameter Made-with: Cursor
71 lines
2.1 KiB
Go
71 lines
2.1 KiB
Go
package filer
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
|
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
"github.com/stretchr/testify/assert"
|
|
)
|
|
|
|
func TestFilerRemoteStorage_FindRemoteStorageClient(t *testing.T) {
|
|
conf := &remote_pb.RemoteConf{
|
|
Name: "s7",
|
|
Type: "s3",
|
|
}
|
|
rs := NewFilerRemoteStorage()
|
|
rs.storageNameToConf[conf.Name] = conf
|
|
|
|
rs.mapDirectoryToRemoteStorage("/a/b/c", &remote_pb.RemoteStorageLocation{
|
|
Name: "s7",
|
|
Bucket: "some",
|
|
Path: "/dir",
|
|
})
|
|
|
|
_, _, found := rs.FindRemoteStorageClient("/a/b/c/d/e/f")
|
|
assert.Equal(t, true, found, "find storage client")
|
|
|
|
_, _, found2 := rs.FindRemoteStorageClient("/a/b")
|
|
assert.Equal(t, false, found2, "should not find storage client")
|
|
|
|
_, _, found3 := rs.FindRemoteStorageClient("/a/b/c")
|
|
assert.Equal(t, false, found3, "should not find storage client")
|
|
|
|
_, _, found4 := rs.FindRemoteStorageClient("/a/b/cc")
|
|
assert.Equal(t, false, found4, "should not find storage client")
|
|
}
|
|
|
|
func TestFilerRemoteStorage_FindMountDirectory_LongestPrefixWins(t *testing.T) {
|
|
conf := &remote_pb.RemoteConf{Name: "store", Type: "s3"}
|
|
rs := NewFilerRemoteStorage()
|
|
rs.storageNameToConf[conf.Name] = conf
|
|
|
|
rs.mapDirectoryToRemoteStorage("/buckets/mybucket", &remote_pb.RemoteStorageLocation{
|
|
Name: "store",
|
|
Bucket: "bucket-root",
|
|
Path: "/",
|
|
})
|
|
rs.mapDirectoryToRemoteStorage("/buckets/mybucket/prefix", &remote_pb.RemoteStorageLocation{
|
|
Name: "store",
|
|
Bucket: "bucket-prefix",
|
|
Path: "/",
|
|
})
|
|
|
|
tests := []struct {
|
|
path string
|
|
wantMount string
|
|
wantBucket string
|
|
}{
|
|
{"/buckets/mybucket/file.txt", "/buckets/mybucket", "bucket-root"},
|
|
{"/buckets/mybucket/prefix/file.txt", "/buckets/mybucket/prefix", "bucket-prefix"},
|
|
{"/buckets/mybucket/prefix/sub/file.txt", "/buckets/mybucket/prefix", "bucket-prefix"},
|
|
}
|
|
for _, tt := range tests {
|
|
mountDir, loc := rs.FindMountDirectory(util.FullPath(tt.path))
|
|
assert.Equal(t, util.FullPath(tt.wantMount), mountDir, "mount dir for %s", tt.path)
|
|
if assert.NotNil(t, loc, "location for %s", tt.path) {
|
|
assert.Equal(t, tt.wantBucket, loc.Bucket, "bucket for %s", tt.path)
|
|
}
|
|
}
|
|
}
|