mount: refresh and evict hot dir cache (#8174)

* mount: refresh and evict hot dir cache

* mount: guard dir update window and extend TTL

* mount: reuse timestamp for cache mark

* Apply suggestion from @gemini-code-assist[bot]

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* mount: make dir cache tuning configurable

* mount: dedupe dir update notices

* mount: restore invalidate-all cache helper

* mount: keep hot dir tuning constants

* mount: centralize cache state reset

* mount: mark refresh completion time

* mount: allow disabling idle eviction

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

weed/mount/weedfs.go

@@ -75,6 +75,9 @@ type Option struct {
 	RdmaMaxConcurrent int
 	RdmaTimeoutMs     int
 
+	// Directory cache refresh/eviction controls
+	DirIdleEvictSec int
+
 	uniqueCacheDirForRead  string
 	uniqueCacheDirForWrite string
 }
@@ -102,8 +105,19 @@ type WFS struct {
 	rdmaClient           *RDMAMountClient
 	FilerConf            *filer.FilerConf
 	filerClient          *wdclient.FilerClient // Cached volume location client
+	refreshMu            sync.Mutex
+	refreshingDirs       map[util.FullPath]struct{}
+	dirHotWindow         time.Duration
+	dirHotThreshold      int
+	dirIdleEvict         time.Duration
 }
 
+const (
+	defaultDirHotWindow    = 2 * time.Second
+	defaultDirHotThreshold = 64
+	defaultDirIdleEvict    = 10 * time.Minute
+)
+
 func NewSeaweedFileSystem(option *Option) *WFS {
 	// Only create FilerClient for direct volume access modes
 	// When VolumeServerAccess == "filerProxy", all reads go through filer, so no volume lookup needed
@@ -127,15 +141,28 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 		)
 	}
 
+	dirHotWindow := defaultDirHotWindow
+	dirHotThreshold := defaultDirHotThreshold
+	dirIdleEvict := defaultDirIdleEvict
+	if option.DirIdleEvictSec != 0 {
+		dirIdleEvict = time.Duration(option.DirIdleEvictSec) * time.Second
+	} else {
+		dirIdleEvict = 0
+	}
+
 	wfs := &WFS{
-		RawFileSystem: fuse.NewDefaultRawFileSystem(),
-		option:        option,
-		signature:     util.RandomInt32(),
-		inodeToPath:   NewInodeToPath(util.FullPath(option.FilerMountRootPath), option.CacheMetaTTlSec),
-		fhMap:         NewFileHandleToInode(),
-		dhMap:         NewDirectoryHandleToInode(),
-		filerClient:   filerClient, // nil for proxy mode, initialized for direct access
-		fhLockTable:   util.NewLockTable[FileHandleId](),
+		RawFileSystem:   fuse.NewDefaultRawFileSystem(),
+		option:          option,
+		signature:       util.RandomInt32(),
+		inodeToPath:     NewInodeToPath(util.FullPath(option.FilerMountRootPath), option.CacheMetaTTlSec),
+		fhMap:           NewFileHandleToInode(),
+		dhMap:           NewDirectoryHandleToInode(),
+		filerClient:     filerClient, // nil for proxy mode, initialized for direct access
+		fhLockTable:     util.NewLockTable[FileHandleId](),
+		refreshingDirs:  make(map[util.FullPath]struct{}),
+		dirHotWindow:    dirHotWindow,
+		dirHotThreshold: dirHotThreshold,
+		dirIdleEvict:    dirIdleEvict,
 	}
 
 	wfs.option.filerIndex = int32(rand.IntN(len(option.FilerAddresses)))
@@ -171,6 +198,10 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 					}
 				}
 			}
+		}, func(dirPath util.FullPath) {
+			if wfs.inodeToPath.RecordDirectoryUpdate(dirPath, time.Now(), wfs.dirHotWindow, wfs.dirHotThreshold) {
+				wfs.maybeRefreshDirectory(dirPath)
+			}
 		})
 	grace.OnInterrupt(func() {
 		wfs.metaCache.Shutdown()
@@ -224,6 +255,7 @@ func (wfs *WFS) StartBackgroundTasks() error {
 	}, follower)
 	go wfs.loopCheckQuota()
 	go wfs.loopFlushDirtyMetadata()
+	go wfs.loopEvictIdleDirCache()
 
 	return nil
 }
@@ -339,6 +371,49 @@ func (wfs *WFS) ClearCacheDir() {
 	os.RemoveAll(wfs.option.getUniqueCacheDirForRead())
 }
 
+func (wfs *WFS) maybeRefreshDirectory(dirPath util.FullPath) {
+	if !wfs.inodeToPath.NeedsRefresh(dirPath) {
+		return
+	}
+	wfs.refreshMu.Lock()
+	if _, exists := wfs.refreshingDirs[dirPath]; exists {
+		wfs.refreshMu.Unlock()
+		return
+	}
+	wfs.refreshingDirs[dirPath] = struct{}{}
+	wfs.refreshMu.Unlock()
+
+	go func() {
+		defer func() {
+			wfs.refreshMu.Lock()
+			delete(wfs.refreshingDirs, dirPath)
+			wfs.refreshMu.Unlock()
+		}()
+		wfs.inodeToPath.InvalidateChildrenCache(dirPath)
+		if err := meta_cache.EnsureVisited(wfs.metaCache, wfs, dirPath); err != nil {
+			glog.Warningf("refresh dir cache %s: %v", dirPath, err)
+			return
+		}
+		wfs.inodeToPath.MarkDirectoryRefreshed(dirPath, time.Now())
+	}()
+}
+
+func (wfs *WFS) loopEvictIdleDirCache() {
+	if wfs.dirIdleEvict <= 0 {
+		return
+	}
+	ticker := time.NewTicker(wfs.dirIdleEvict / 2)
+	defer ticker.Stop()
+	for range ticker.C {
+		dirs := wfs.inodeToPath.CollectEvictableDirs(time.Now(), wfs.dirIdleEvict)
+		for _, dir := range dirs {
+			if err := wfs.metaCache.DeleteFolderChildren(context.Background(), dir); err != nil {
+				glog.V(2).Infof("evict dir cache %s: %v", dir, err)
+			}
+		}
+	}
+}
+
 func (option *Option) setupUniqueCacheDirectory() {
 	cacheUniqueId := util.Md5String([]byte(option.MountDirectory + string(option.FilerAddresses[0]) + option.FilerMountRootPath + version.Version()))[0:8]
 	option.uniqueCacheDirForRead = path.Join(option.CacheDirForRead, cacheUniqueId)