mount: add option to show system entries (#8829)

* mount: add option to show system entries

* address gemini code review's suggested changes

* rename flag from -showSystemEntries to -includeSystemEntries

* meta_cache: purge hidden system entries on filer events

---------

Co-authored-by: Chris Lu <chris.lu@gmail.com>

weed/command/mount.go

@@ -31,6 +31,7 @@ type MountOptions struct {
 	uidMap             *string
 	gidMap             *string
 	readOnly           *bool
+	includeSystemEntries *bool
 	debug              *bool
 	debugPort          *int
 	localSocket        *string
@@ -99,6 +100,7 @@ func init() {
 	mountOptions.uidMap = cmdMount.Flag.String("map.uid", "", "map local uid to uid on filer, comma-separated <local_uid>:<filer_uid>")
 	mountOptions.gidMap = cmdMount.Flag.String("map.gid", "", "map local gid to gid on filer, comma-separated <local_gid>:<filer_gid>")
 	mountOptions.readOnly = cmdMount.Flag.Bool("readOnly", false, "read only")
+	mountOptions.includeSystemEntries = cmdMount.Flag.Bool("includeSystemEntries", false, "show filer system entries (e.g. /topics, /etc) in directory listings")
 	mountOptions.debug = cmdMount.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
 	mountOptions.debugPort = cmdMount.Flag.Int("debug.port", 6061, "http port for debugging")
 	mountOptions.localSocket = cmdMount.Flag.String("localSocket", "", "default to /tmp/seaweedfs-mount-<mount_dir_hash>.sock")

weed/command/mount_std.go

@@ -340,6 +340,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		VolumeServerAccess:          *mountOptions.volumeServerAccess,
 		Cipher:                      cipher,
 		UidGidMapper:                uidGidMapper,
+		IncludeSystemEntries:        *option.includeSystemEntries,
 		DisableXAttr:                *option.disableXAttr,
 		IsMacOs:                     runtime.GOOS == "darwin",
 		MetadataFlushSeconds:        *option.metadataFlushSeconds,

weed/mount/meta_cache/meta_cache.go

@@ -27,18 +27,19 @@ type MetaCache struct {
 	localStore   filer.VirtualFilerStore
 	leveldbStore *leveldb.LevelDBStore // direct reference for batch operations
 	sync.RWMutex
-	uidGidMapper      *UidGidMapper
+	uidGidMapper         *UidGidMapper
-	markCachedFn      func(fullpath util.FullPath)
+	markCachedFn         func(fullpath util.FullPath)
-	isCachedFn        func(fullpath util.FullPath) bool
+	isCachedFn           func(fullpath util.FullPath) bool
-	invalidateFunc    func(fullpath util.FullPath, entry *filer_pb.Entry)
+	invalidateFunc       func(fullpath util.FullPath, entry *filer_pb.Entry)
-	onDirectoryUpdate func(dir util.FullPath)
+	onDirectoryUpdate    func(dir util.FullPath)
-	visitGroup        singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
+	visitGroup           singleflight.Group // deduplicates concurrent EnsureVisited calls for the same path
-	applyCh           chan metadataApplyRequest
+	applyCh              chan metadataApplyRequest
-	applyDone         chan struct{}
+	applyDone            chan struct{}
-	applyStateMu      sync.Mutex
+	applyStateMu         sync.Mutex
-	applyClosed       bool
+	applyClosed          bool
-	buildingDirs      map[util.FullPath]*directoryBuildState
+	buildingDirs         map[util.FullPath]*directoryBuildState
-	dedupRing         dedupRingBuffer
+	dedupRing            dedupRingBuffer
+	includeSystemEntries bool
 }
 
 var errMetaCacheClosed = errors.New("metadata cache is shut down")
@@ -84,17 +85,18 @@ type metadataApplyRequest struct {
 	done         chan error
 }
 
-func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath,
+func NewMetaCache(dbFolder string, uidGidMapper *UidGidMapper, root util.FullPath, includeSystemEntries bool,
 	markCachedFn func(path util.FullPath), isCachedFn func(path util.FullPath) bool, invalidateFunc func(util.FullPath, *filer_pb.Entry), onDirectoryUpdate func(dir util.FullPath)) *MetaCache {
 	leveldbStore, virtualStore := openMetaStore(dbFolder)
 	mc := &MetaCache{
-		root:              root,
+		root:                 root,
-		localStore:        virtualStore,
+		localStore:           virtualStore,
-		leveldbStore:      leveldbStore,
+		leveldbStore:         leveldbStore,
-		markCachedFn:      markCachedFn,
+		markCachedFn:         markCachedFn,
-		isCachedFn:        isCachedFn,
+		isCachedFn:           isCachedFn,
-		uidGidMapper:      uidGidMapper,
+		uidGidMapper:         uidGidMapper,
-		onDirectoryUpdate: onDirectoryUpdate,
+		onDirectoryUpdate:    onDirectoryUpdate,
+		includeSystemEntries: includeSystemEntries,
 		invalidateFunc: func(fullpath util.FullPath, entry *filer_pb.Entry) {
 			invalidateFunc(fullpath, entry)
 		},
@@ -182,6 +184,29 @@ func (mc *MetaCache) atomicUpdateEntryFromFilerLocked(ctx context.Context, oldPa
 	return nil
 }
 
+func (mc *MetaCache) shouldHideEntry(fullpath util.FullPath) bool {
+	if mc.includeSystemEntries {
+		return false
+	}
+	dir, name := fullpath.DirAndName()
+	return IsHiddenSystemEntry(dir, name)
+}
+
+func (mc *MetaCache) purgeEntryLocked(ctx context.Context, fullpath util.FullPath, isDirectory bool) error {
+	if fullpath == "" {
+		return nil
+	}
+	if err := mc.localStore.DeleteEntry(ctx, fullpath); err != nil {
+		return err
+	}
+	if isDirectory {
+		if err := mc.localStore.DeleteFolderChildren(ctx, fullpath); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (mc *MetaCache) ApplyMetadataResponse(ctx context.Context, resp *filer_pb.SubscribeMetadataResponse, options MetadataResponseApplyOptions) error {
 	if resp == nil || resp.EventNotification == nil {
 		return nil
@@ -520,7 +545,9 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file
 	}
 
 	var oldPath util.FullPath
+	var newPath util.FullPath
 	var newEntry *filer.Entry
+	hideNewPath := false
 	if message.OldEntry != nil {
 		oldPath = util.NewFullPath(resp.Directory, message.OldEntry.Name)
 	}
@@ -530,11 +557,20 @@ func (mc *MetaCache) applyMetadataResponseLocked(ctx context.Context, resp *file
 		if message.NewParentPath != "" {
 			dir = message.NewParentPath
 		}
-		newEntry = filer.FromPbEntry(dir, message.NewEntry)
+		newPath = util.NewFullPath(dir, message.NewEntry.Name)
+		hideNewPath = mc.shouldHideEntry(newPath)
+		if !hideNewPath {
+			newEntry = filer.FromPbEntry(dir, message.NewEntry)
+		}
 	}
 
 	mc.Lock()
 	err := mc.atomicUpdateEntryFromFilerLocked(ctx, oldPath, newEntry, allowUncachedInsert)
+	if err == nil && hideNewPath {
+		if purgeErr := mc.purgeEntryLocked(ctx, newPath, message.NewEntry.IsDirectory); purgeErr != nil {
+			err = purgeErr
+		}
+	}
 	// When a directory is deleted or moved, remove its cached descendants
 	// so stale children cannot be served from the local cache.
 	if err == nil && oldPath != "" && message.OldEntry != nil && message.OldEntry.IsDirectory {

weed/mount/meta_cache/meta_cache_apply_test.go

@@ -2,6 +2,7 @@ package meta_cache
 
 import (
 	"context"
+	"os"
 	"path/filepath"
 	"sync"
 	"testing"
@@ -296,6 +297,114 @@ func TestApplyMetadataResponseDeduplicatesRepeatedFilerEvent(t *testing.T) {
 	}
 }
 
+func TestApplyMetadataResponseSkipsHiddenSystemEntryWhenDisabled(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/": true,
+	})
+	defer mc.Shutdown()
+
+	createResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/",
+		EventNotification: &filer_pb.EventNotification{
+			NewEntry: &filer_pb.Entry{
+				Name: "topics",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   1,
+					Mtime:    1,
+					FileMode: uint32(os.ModeDir | 0o755),
+				},
+				IsDirectory: true,
+			},
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), createResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply create: %v", err)
+	}
+
+	entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics"))
+	if err != filer_pb.ErrNotFound {
+		t.Fatalf("find hidden entry error = %v, want %v", err, filer_pb.ErrNotFound)
+	}
+	if entry != nil {
+		t.Fatalf("hidden entry still cached: %+v", entry)
+	}
+}
+
+func TestApplyMetadataResponsePurgesHiddenDestinationPath(t *testing.T) {
+	mc, _, _, _ := newTestMetaCache(t, map[util.FullPath]bool{
+		"/":    true,
+		"/src": true,
+	})
+	defer mc.Shutdown()
+
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/topics",
+		Attr: filer.Attr{
+			Crtime: time.Unix(1, 0),
+			Mtime:  time.Unix(1, 0),
+			Mode:   os.ModeDir | 0o755,
+		},
+	}); err != nil {
+		t.Fatalf("insert stale hidden dir: %v", err)
+	}
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/topics/leaked.txt",
+		Attr: filer.Attr{
+			Crtime:   time.Unix(1, 0),
+			Mtime:    time.Unix(1, 0),
+			Mode:     0o644,
+			FileSize: 7,
+		},
+	}); err != nil {
+		t.Fatalf("insert leaked hidden child: %v", err)
+	}
+	if err := mc.InsertEntry(context.Background(), &filer.Entry{
+		FullPath: "/src/visible",
+		Attr: filer.Attr{
+			Crtime: time.Unix(1, 0),
+			Mtime:  time.Unix(1, 0),
+			Mode:   os.ModeDir | 0o755,
+		},
+	}); err != nil {
+		t.Fatalf("insert source dir: %v", err)
+	}
+
+	renameResp := &filer_pb.SubscribeMetadataResponse{
+		Directory: "/src",
+		EventNotification: &filer_pb.EventNotification{
+			OldEntry: &filer_pb.Entry{
+				Name:        "visible",
+				IsDirectory: true,
+			},
+			NewEntry: &filer_pb.Entry{
+				Name: "topics",
+				Attributes: &filer_pb.FuseAttributes{
+					Crtime:   2,
+					Mtime:    2,
+					FileMode: uint32(os.ModeDir | 0o755),
+				},
+				IsDirectory: true,
+			},
+			NewParentPath: "/",
+		},
+	}
+
+	if err := mc.ApplyMetadataResponse(context.Background(), renameResp, SubscriberMetadataResponseApplyOptions); err != nil {
+		t.Fatalf("apply rename: %v", err)
+	}
+
+	if entry, err := mc.FindEntry(context.Background(), util.FullPath("/src/visible")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("source dir after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+	if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("hidden destination after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+	if entry, err := mc.FindEntry(context.Background(), util.FullPath("/topics/leaked.txt")); err != filer_pb.ErrNotFound || entry != nil {
+		t.Fatalf("hidden child after rename = %+v, %v; want nil, %v", entry, err, filer_pb.ErrNotFound)
+	}
+}
+
 func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache, map[util.FullPath]bool, *recordedPaths, *recordedPaths) {
 	t.Helper()
 
@@ -312,6 +421,7 @@ func newTestMetaCache(t *testing.T, cached map[util.FullPath]bool) (*MetaCache,
 		filepath.Join(t.TempDir(), "meta"),
 		mapper,
 		util.FullPath("/"),
+		false,
 		func(path util.FullPath) {
 			cachedMu.Lock()
 			defer cachedMu.Unlock()

weed/mount/meta_cache/meta_cache_init.go

@@ -107,7 +107,7 @@ func doEnsureVisited(ctx context.Context, mc *MetaCache, client filer_pb.FilerCl
 			var err error
 			snapshotTsNs, err = filer_pb.ReadDirAllEntriesWithSnapshot(ctx, client, path, "", func(pbEntry *filer_pb.Entry, isLast bool) error {
 				entry := filer.FromPbEntry(string(path), pbEntry)
-				if IsHiddenSystemEntry(string(path), entry.Name()) {
+				if !mc.includeSystemEntries && IsHiddenSystemEntry(string(path), entry.Name()) {
 					return nil
 				}
 

weed/mount/weedfs.go

@@ -65,6 +65,7 @@ type Option struct {
 	VolumeServerAccess string // how to access volume servers
 	Cipher             bool   // whether encrypt data on volume server
 	UidGidMapper       *meta_cache.UidGidMapper
+	IncludeSystemEntries bool
 
 	// Periodic metadata flush interval in seconds (0 to disable)
 	// This protects chunks from being purged by volume.fsck for long-running writes
@@ -203,6 +204,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 
 	wfs.metaCache = meta_cache.NewMetaCache(path.Join(option.getUniqueCacheDirForRead(), "meta"), option.UidGidMapper,
 		util.FullPath(option.FilerMountRootPath),
+		option.IncludeSystemEntries,
 		func(path util.FullPath) {
 			wfs.inodeToPath.MarkChildrenCached(path)
 		}, func(path util.FullPath) bool {

weed/mount/weedfs_dir_read.go

@@ -302,7 +302,7 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
 	if input.Offset >= dh.entryStreamOffset {
 		if len(dh.entryStream) == 0 && input.Offset > dh.entryStreamOffset {
 			skipCount := uint32(input.Offset-dh.entryStreamOffset) + batchSize
-			entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs)
+			entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, "", false, skipCount, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
 			if err != nil {
 				glog.Errorf("list filer directory: %v", err)
 				return fuse.EIO
@@ -331,7 +331,7 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
 			}
 		}
 
-		entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs)
+		entries, snapshotTs, err := loadDirectoryEntriesDirect(context.Background(), wfs, wfs.option.UidGidMapper, dirPath, lastEntryName, false, batchSize, dh.snapshotTsNs, wfs.option.IncludeSystemEntries)
 		if err != nil {
 			glog.Errorf("list filer directory: %v", err)
 			return fuse.EIO
@@ -360,13 +360,13 @@ func (wfs *WFS) readDirectoryDirect(input *fuse.ReadIn, out *fuse.DirEntryList,
 	return fuse.OK
 }
 
-func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64) ([]*filer.Entry, int64, error) {
+func loadDirectoryEntriesDirect(ctx context.Context, client filer_pb.FilerClient, uidGidMapper *meta_cache.UidGidMapper, dirPath util.FullPath, startFileName string, includeStart bool, limit uint32, snapshotTsNs int64, includeSystemEntries bool) ([]*filer.Entry, int64, error) {
 	entries := make([]*filer.Entry, 0, limit)
 	var actualSnapshotTsNs int64
 	err := client.WithFilerClient(false, func(sc filer_pb.SeaweedFilerClient) error {
 		var innerErr error
 		actualSnapshotTsNs, innerErr = filer_pb.DoSeaweedListWithSnapshot(ctx, sc, dirPath, "", func(entry *filer_pb.Entry, isLast bool) error {
-			if meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) {
+			if !includeSystemEntries && meta_cache.IsHiddenSystemEntry(string(dirPath), entry.Name) {
 				return nil
 			}
 			if uidGidMapper != nil && entry.Attributes != nil {

weed/mount/weedfs_dir_read_test.go

@@ -84,7 +84,7 @@ func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T)
 		},
 	}
 
-	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0)
+	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, mapper, util.FullPath("/"), "", false, 10, 0, false)
 	if err != nil {
 		t.Fatalf("loadDirectoryEntriesDirect: %v", err)
 	}
@@ -98,3 +98,25 @@ func TestLoadDirectoryEntriesDirectFiltersHiddenEntriesAndMapsIds(t *testing.T)
 		t.Fatalf("mapped uid/gid = %d/%d, want 10/20", entries[0].Attr.Uid, entries[0].Attr.Gid)
 	}
 }
+
+func TestLoadDirectoryEntriesDirectShowsSystemEntriesWhenEnabled(t *testing.T) {
+	client := &directoryFilerAccessor{
+		client: &directoryListClient{
+			responses: []*filer_pb.ListEntriesResponse{
+				{Entry: &filer_pb.Entry{Name: "topics"}},
+				{Entry: &filer_pb.Entry{Name: "visible"}},
+			},
+		},
+	}
+
+	entries, _, err := loadDirectoryEntriesDirect(context.Background(), client, nil, util.FullPath("/"), "", false, 10, 0, true)
+	if err != nil {
+		t.Fatalf("loadDirectoryEntriesDirect: %v", err)
+	}
+	if got := len(entries); got != 2 {
+		t.Fatalf("entry count = %d, want 2", got)
+	}
+	if entries[0].Name() != "topics" || entries[1].Name() != "visible" {
+		t.Fatalf("entry names = %q, %q, want topics, visible", entries[0].Name(), entries[1].Name())
+	}
+}

weed/mount/weedfs_file_copy_range_test.go

@@ -338,6 +338,7 @@ func newCopyRangeTestWFSWithMetaCache(t *testing.T) *WFS {
 		filepath.Join(t.TempDir(), "meta"),
 		uidGidMapper,
 		root,
+		false,
 		func(path util.FullPath) {
 			wfs.inodeToPath.MarkChildrenCached(path)
 		},

weed/mount/weedfs_file_mkrm_test.go

@@ -120,6 +120,7 @@ func newCreateTestWFS(t *testing.T) (*WFS, *createEntryTestServer) {
 		filepath.Join(t.TempDir(), "meta"),
 		uidGidMapper,
 		root,
+		false,
 		func(path util.FullPath) {
 			wfs.inodeToPath.MarkChildrenCached(path)
 		},

weed/mount/weedfs_rename_test.go

@@ -23,6 +23,7 @@ func TestHandleRenameResponseLeavesUncachedTargetOutOfCache(t *testing.T) {
 		filepath.Join(t.TempDir(), "meta"),
 		uidGidMapper,
 		root,
+		false,
 		func(path util.FullPath) {
 			inodeToPath.MarkChildrenCached(path)
 		},