gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

mount: add option to show system entries (#8829)

Browse Source 
* mount: add option to show system entries

* address gemini code review's suggested changes

* rename flag from -showSystemEntries to -includeSystemEntries

* meta_cache: purge hidden system entries on filer events

---------

Co-authored-by: Chris Lu <chris.lu@gmail.com>
This commit is contained in:
Simon Bråten
2026-03-29 22:33:17 +02:00
committed by GitHub
parent 46567ac06c
commit 479e72b5ab
11 changed files with 203 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/command/mount.go
View File

@@ -31,6 +31,7 @@ type MountOptions struct {
uidMap *string
gidMap *string
readOnly *bool
includeSystemEntries *bool
debug *bool
debugPort *int
localSocket *string
@@ -99,6 +100,7 @@ func init() {
mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
mountOptions.includeSystemEntries = cmdMount.Flag.Bool("includeSystemEntries", false, "show filer system entries (e.g. /topics, /etc) in directory listings")
mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging")
mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-<mount_dir_hash>.sock")

1
weed/command/mount_std.go
View File

@@ -340,6 +340,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
VolumeServerAccess: *mountOptions.volumeServerAccess,
Cipher: cipher,
UidGidMapper: uidGidMapper,
IncludeSystemEntries: *option.includeSystemEntries,
DisableXAttr: *option.disableXAttr,
IsMacOs: runtime.GOOS == "darwin",
MetadataFlushSeconds: *option.metadataFlushSeconds,

78
weed/mount/meta_cache/meta_cache.go
View File

@@ -27,18 +27,19 @@ type MetaCache struct {
localStore filer.VirtualFilerStore
leveldbStore *leveldb.LevelDBStore // direct reference for batch operations
sync.RWMutex
uidGidMapper *UidGidMapper
markCachedFn func(fullpath util.FullPath)
isCachedFn func(fullpath util.FullPath) bool
invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry)
onDirectoryUpdate func(dir util.FullPath)
visitGroup singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
applyCh chan metadataApplyRequest
applyDone chan struct{}
applyStateMu sync.Mutex
applyClosed bool
buildingDirs map[util.FullPath]*directoryBuildState
dedupRing dedupRingBuffer
uidGidMapper *UidGidMapper
markCachedFn func(fullpath util.FullPath)
isCachedFn func(fullpath util.FullPath) bool
invalidateFunc func(fullpath util.FullPath, entry *filer_pb.Entry)
onDirectoryUpdate func(dir util.FullPath)
visitGroup singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
applyCh chan metadataApplyRequest
applyDone chan struct{}
applyStateMu sync.Mutex
applyClosed bool
buildingDirs map[util.FullPath]*directoryBuildState
dedupRing dedupRingBuffer
includeSystemEntries bool
}
var errMetaCacheClosed = errors.New("metadata cache is shut down")
@@ -84,17 +85,18 @@ type metadataApplyRequest struct {
done chan error
}
func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath,
func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath, includeSystemEntries bool,
markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry), onDirectoryUpdate func(dir util.FullPath)) *MetaCache {
leveldbStore, virtualStore := openMetaStore(dbFolder)
mc := &MetaCache{
root: root,
localStore: virtualStore,
leveldbStore: leveldbStore,
markCachedFn: markCachedFn,
isCachedFn: isCachedFn,
uidGidMapper: uidGidMapper,
onDirectoryUpdate: onDirectoryUpdate,
root: root,
localStore: virtualStore,
leveldbStore: leveldbStore,
markCachedFn: markCachedFn,
isCachedFn: isCachedFn,
uidGidMapper: uidGidMapper,
onDirectoryUpdate: onDirectoryUpdate,
includeSystemEntries: includeSystemEntries,
invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) {
invalidateFunc(fullpath, entry)
},
@@ -182,6 +184,29 @@ func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPa
return nil
}
func (mc *MetaCache) shouldHideEntry(fullpath util.FullPath) bool {
if mc.includeSystemEntries {
return false
}
dir, name := fullpath.DirAndName()
return IsHiddenSystemEntry(dir, name)
}
func (mc *MetaCache) purgeEntryLocked(ctx context.Context, fullpath util.FullPath, isDirectory bool) error {
if fullpath == "" {
return nil
}
if err := mc.localStore.DeleteEntry(ctx, fullpath); err != nil {
return err
}
if isDirectory {
if err := mc.localStore.DeleteFolderChildren(ctx, fullpath); err != nil {
return err
}
}
return nil
}
func (mc *MetaCache) ApplyMetadataResponse(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error {
if resp == nil || resp.EventNotification == nil {
return nil
@@ -520,7 +545,9 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file
}
var oldPath util.FullPath
var newPath util.FullPath
var newEntry *filer.Entry
hideNewPath := false
if message.OldEntry != nil {
oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name)
}
@@ -530,11 +557,20 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file
if message.NewParentPath != "" {
dir = message.NewParentPath
}
newEntry = filer.FromPbEntry(dir, message.NewEntry)
newPath = util.NewFullPath(dir, message.NewEntry.Name)
hideNewPath = mc.shouldHideEntry(newPath)
if !hideNewPath {
newEntry = filer.FromPbEntry(dir, message.NewEntry)
}
}
mc.Lock()
err := mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, allowUncachedInsert)
if err == nil && hideNewPath {
if purgeErr := mc.purgeEntryLocked(ctx, newPath, message.NewEntry.IsDirectory); purgeErr != nil {
err = purgeErr
}
}
// When a directory is deleted or moved, remove its cached descendants
// so stale children cannot be served from the local cache.
if err == nil && oldPath != "" && message.OldEntry != nil && message.OldEntry.IsDirectory {

110
weed/mount/meta_cache/meta_cache_apply_test.go
View File

@@ -2,6 +2,7 @@ package meta_cache
import (
"context"
"os"
"path/filepath"
"sync"
"testing"
@@ -296,6 +297,114 @@ func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) {
}
}
func TestApplyMetadataResponseSkipsHiddenSystemEntryWhenDisabled(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
})
defer mc.Shutdown()
createResp := &filer_pb.SubscribeMetadataResponse{
Directory: "/",
EventNotification: &filer_pb.EventNotification{
NewEntry: &filer_pb.Entry{
Name: "topics",
Attributes: &filer_pb.FuseAttributes{
Crtime: 1,
Mtime: 1,
FileMode: uint32(os.ModeDir | 0o755),
},
IsDirectory: true,
},
},
}
if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
t.Fatalf("apply create: %v", err)
}
entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics"))
if err != filer_pb.ErrNotFound {
t.Fatalf("find hidden entry error = %v, want %v", err, filer_pb.ErrNotFound)
}
if entry != nil {
t.Fatalf("hidden entry still cached: %+v", entry)
}
}
func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) {
mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
"/": true,
"/src": true,
})
defer mc.Shutdown()
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/topics",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: os.ModeDir | 0o755,
},
}); err != nil {
t.Fatalf("insert stale hidden dir: %v", err)
}
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/topics/leaked.txt",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: 0o644,
FileSize: 7,
},
}); err != nil {
t.Fatalf("insert leaked hidden child: %v", err)
}
if err := mc.InsertEntry(context.Background(), &filer.Entry{
FullPath: "/src/visible",
Attr: filer.Attr{
Crtime: time.Unix(1, 0),
Mtime: time.Unix(1, 0),
Mode: os.ModeDir | 0o755,
},
}); err != nil {
t.Fatalf("insert source dir: %v", err)
}
renameResp := &filer_pb.SubscribeMetadataResponse{
Directory: "/src",
EventNotification: &filer_pb.EventNotification{
OldEntry: &filer_pb.Entry{
Name: "visible",
IsDirectory: true,
},
NewEntry: &filer_pb.Entry{
Name: "topics",
Attributes: &filer_pb.FuseAttributes{
Crtime: 2,
Mtime: 2,
FileMode: uint32(os.ModeDir | 0o755),
},
IsDirectory: true,
},
NewParentPath: "/",
},
}
if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
t.Fatalf("apply rename: %v", err)
}
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil {
t.Fatalf("source dir after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
}
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil {
t.Fatalf("hidden destination after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
}
if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil {
t.Fatalf("hidden child after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
}
}
func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) {
t.Helper()
@@ -312,6 +421,7 @@ func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache,
filepath.Join(t.TempDir(), "meta"),
mapper,
util.FullPath("/"),
false,
func(path util.FullPath) {
cachedMu.Lock()
defer cachedMu.Unlock()

2
weed/mount/meta_cache/meta_cache_init.go
View File

@@ -107,7 +107,7 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl
var err error
snapshotTsNs, err = filer_pb.ReadDirAllEntriesWithSnapshot(ctx, client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
entry := filer.FromPbEntry(string(path), pbEntry)
if IsHiddenSystemEntry(string(path), entry.Name()) {
if !mc.includeSystemEntries && IsHiddenSystemEntry(string(path), entry.Name()) {
return nil
}

2
weed/mount/weedfs.go
View File

@@ -65,6 +65,7 @@ type Option struct {
VolumeServerAccess string // how to access volume servers
Cipher bool // whether encrypt data on volume server
UidGidMapper *meta_cache.UidGidMapper
IncludeSystemEntries bool
// Periodic metadata flush interval in seconds (0 to disable)
// This protects chunks from being purged by volume.fsck for long-running writes
@@ -203,6 +204,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDirForRead(), "meta"), option.UidGidMapper,
util.FullPath(option.FilerMountRootPath),
option.IncludeSystemEntries,
func(path util.FullPath) {
wfs.inodeToPath.MarkChildrenCached(path)
}, func(path util.FullPath) bool {

8
weed/mount/weedfs_dir_read.go
View File

@@ -302,7 +302,7 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
if input.Offset >= dh.entryStreamOffset {
if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
skipCount := uint32(input.Offset-dh.entryStreamOffset) + batchSize
entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs)
entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
if err != nil {
glog.Errorf("list filer directory: %v", err)
return fuse.EIO
@@ -331,7 +331,7 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
}
}
entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs)
entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
if err != nil {
glog.Errorf("list filer directory: %v", err)
return fuse.EIO
@@ -360,13 +360,13 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
return fuse.OK
}
func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64) ([]*filer.Entry, int64, error) {
func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64, includeSystemEntries bool) ([]*filer.Entry, int64, error) {
entries := make([]*filer.Entry, 0, limit)
var actualSnapshotTsNs int64
err := client.WithFilerClient(false, func(sc filer_pb.SeaweedFilerClient) error {
var innerErr error
actualSnapshotTsNs, innerErr = filer_pb.DoSeaweedListWithSnapshot(ctx, sc, dirPath, "", func(entry *filer_pb.Entry, isLast bool) error {
if meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) {
if !includeSystemEntries && meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) {
return nil
}
if uidGidMapper != nil && entry.Attributes != nil {

24
weed/mount/weedfs_dir_read_test.go
View File

@@ -84,7 +84,7 @@ func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T)
},
}
entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0)
entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0, false)
if err != nil {
t.Fatalf("loadDirectoryEntriesDirect: %v", err)
}
@@ -98,3 +98,25 @@ func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T)
t.Fatalf("mapped uid/gid = %d/%d, want 10/20", entries[0].Attr.Uid, entries[0].Attr.Gid)
}
}
func TestLoadDirectoryEntriesDirectShowsSystemEntriesWhenEnabled(t *testing.T) {
client := &directoryFilerAccessor{
client: &directoryListClient{
responses: []*filer_pb.ListEntriesResponse{
{Entry: &filer_pb.Entry{Name: "topics"}},
{Entry: &filer_pb.Entry{Name: "visible"}},
},
},
}
entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, nil, util.FullPath("/"), "", false, 10, 0, true)
if err != nil {
t.Fatalf("loadDirectoryEntriesDirect: %v", err)
}
if got := len(entries); got != 2 {
t.Fatalf("entry count = %d, want 2", got)
}
if entries[0].Name() != "topics" || entries[1].Name() != "visible" {
t.Fatalf("entry names = %q, %q, want topics, visible", entries[0].Name(), entries[1].Name())
}
}

1
weed/mount/weedfs_file_copy_range_test.go
View File

@@ -338,6 +338,7 @@ func newCopyRangeTestWFSWithMetaCache(t *testing.T) *WFS {
filepath.Join(t.TempDir(), "meta"),
uidGidMapper,
root,
false,
func(path util.FullPath) {
wfs.inodeToPath.MarkChildrenCached(path)
},

1
weed/mount/weedfs_file_mkrm_test.go
View File

@@ -120,6 +120,7 @@ func newCreateTestWFS(t *testing.T) (*WFS, *createEntryTestServer) {
filepath.Join(t.TempDir(), "meta"),
uidGidMapper,
root,
false,
func(path util.FullPath) {
wfs.inodeToPath.MarkChildrenCached(path)
},

1
weed/mount/weedfs_rename_test.go
View File

@@ -23,6 +23,7 @@ func TestHandleRenameResponseLeavesUncachedTargetOutOfCache(t *testing.T) {
filepath.Join(t.TempDir(), "meta"),
uidGidMapper,
root,
false,
func(path util.FullPath) {
inodeToPath.MarkChildrenCached(path)
},