fix: keep metadata subscriptions progressing (#8730) (#8746)

* fix: keep metadata subscriptions progressing (#8730)

* test: cancel slow metadata writers with parent context

* filer: ignore missing persisted log chunks

weed/filer/filer.go

@@ -81,7 +81,7 @@ func NewFiler(masters pb.ServerDiscovery, grpcDialOption grpc.DialOption, filerH
 		f.UniqueFilerId = -f.UniqueFilerId
 	}
 
-	f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, nil, notifyFn)
+	f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, f.readPersistedLogBufferPosition, notifyFn)
 	f.metaLogCollection = collection
 	f.metaLogReplication = replication
 

weed/filer/filer_notify.go

@@ -2,8 +2,10 @@ package filer
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"io"
+	nethttp "net/http"
 	"regexp"
 	"strings"
 	"time"
@@ -16,6 +18,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/notification"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) {
@@ -174,6 +177,7 @@ func (f *Filer) logFlushFunc(logBuffer *log_buffer.LogBuffer, startTime, stopTim
 var (
 	volumeNotFoundPattern = regexp.MustCompile(`volume \d+? not found`)
 	chunkNotFoundPattern  = regexp.MustCompile(`(urls not found|File Not Found)`)
+	httpNotFoundPattern   = regexp.MustCompile(`404 Not Found: not found`)
 )
 
 // isChunkNotFoundError checks if the error indicates that a volume or chunk
@@ -183,8 +187,13 @@ func isChunkNotFoundError(err error) bool {
 	if err == nil {
 		return false
 	}
+	if errors.Is(err, util_http.ErrNotFound) || errors.Is(err, nethttp.ErrMissingFile) {
+		return true
+	}
 	errMsg := err.Error()
-	return volumeNotFoundPattern.MatchString(errMsg) || chunkNotFoundPattern.MatchString(errMsg)
+	return volumeNotFoundPattern.MatchString(errMsg) ||
+		chunkNotFoundPattern.MatchString(errMsg) ||
+		httpNotFoundPattern.MatchString(errMsg)
 }
 
 func (f *Filer) ReadPersistedLogBuffer(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastTsNs int64, isDone bool, err error) {
@@ -220,3 +229,17 @@ func (f *Filer) ReadPersistedLogBuffer(startPosition log_buffer.MessagePosition,
 
 	return
 }
+
+func (f *Filer) readPersistedLogBufferPosition(startPosition log_buffer.MessagePosition, stopTsNs int64, eachLogEntryFn log_buffer.EachLogEntryFuncType) (lastReadPosition log_buffer.MessagePosition, isDone bool, err error) {
+	lastReadPosition = startPosition
+
+	lastTsNs, isDone, err := f.ReadPersistedLogBuffer(startPosition, stopTsNs, eachLogEntryFn)
+	if err != nil {
+		return lastReadPosition, isDone, err
+	}
+	if lastTsNs != 0 {
+		lastReadPosition = log_buffer.NewMessagePosition(lastTsNs, 1)
+	}
+
+	return lastReadPosition, isDone, nil
+}

weed/filer/meta_aggregator.go

@@ -74,6 +74,18 @@ func (ma *MetaAggregator) OnPeerUpdate(update *master_pb.ClusterNodeUpdate, star
 	}
 }
 
+func (ma *MetaAggregator) HasRemotePeers() bool {
+	ma.peerChansLock.Lock()
+	defer ma.peerChansLock.Unlock()
+
+	for address := range ma.peerChans {
+		if address != ma.self {
+			return true
+		}
+	}
+	return false
+}
+
 func (ma *MetaAggregator) loopSubscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress, startFrom time.Time, stopChan chan struct{}) {
 	lastTsNs := startFrom.UnixNano()
 	for {