ensure metadata follow a specific folder

fix https://github.com/seaweedfs/seaweedfs/issues/5774
This commit is contained in:
chrislu
2024-07-12 11:17:28 -07:00
parent ec9e7493b3
commit 3a82f5ffad
7 changed files with 40 additions and 7 deletions

View File

@@ -10,6 +10,7 @@ import (
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/insecure"
"strconv" "strconv"
"strings"
"time" "time"
) )
@@ -79,12 +80,17 @@ func startGenerateMetadata() {
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) { func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
prefix := *dir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "tail", ClientName: "tail",
ClientId: 0, ClientId: 0,
ClientEpoch: 0, ClientEpoch: 0,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: *dir, PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: 0, StartTsNs: 0,

View File

@@ -9,6 +9,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
"regexp" "regexp"
"strings"
"time" "time"
) )
@@ -148,12 +149,17 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
}() }()
} }
prefix := sourcePath
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "backup_" + dataSink.GetName(), ClientName: "backup_" + dataSink.GetName(),
ClientId: clientId, ClientId: clientId,
ClientEpoch: clientEpoch, ClientEpoch: clientEpoch,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: sourcePath, PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: startFrom.UnixNano(), StartTsNs: startFrom.UnixNano(),

View File

@@ -8,6 +8,7 @@ import (
"github.com/spf13/viper" "github.com/spf13/viper"
"google.golang.org/grpc" "google.golang.org/grpc"
"reflect" "reflect"
"strings"
"time" "time"
"github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
@@ -197,12 +198,16 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
metaBackup.clientEpoch++ metaBackup.clientEpoch++
prefix := *metaBackup.filerDirectory
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "meta_backup", ClientName: "meta_backup",
ClientId: metaBackup.clientId, ClientId: metaBackup.clientId,
ClientEpoch: metaBackup.clientEpoch, ClientEpoch: metaBackup.clientEpoch,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: *metaBackup.filerDirectory, PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: startTime.UnixNano(), StartTsNs: startTime.UnixNano(),

View File

@@ -55,7 +55,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
ClientId: option.clientId, ClientId: option.clientId,
ClientEpoch: option.clientEpoch, ClientEpoch: option.clientEpoch,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: option.bucketsDir, PathPrefix: option.bucketsDir + "/",
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote}, AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: lastOffsetTs.UnixNano(), StartTsNs: lastOffsetTs.UnixNano(),

View File

@@ -64,12 +64,17 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
option.clientEpoch++ option.clientEpoch++
prefix := mountedDir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "filer.remote.sync", ClientName: "filer.remote.sync",
ClientId: option.clientId, ClientId: option.clientId,
ClientEpoch: option.clientEpoch, ClientEpoch: option.clientEpoch,
SelfSignature: 0, SelfSignature: 0,
PathPrefix: mountedDir, PathPrefix: prefix,
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote}, AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: lastOffsetTs.UnixNano(), StartTsNs: lastOffsetTs.UnixNano(),

View File

@@ -296,12 +296,17 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs) return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
}) })
prefix := sourcePath
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: clientName, ClientName: clientName,
ClientId: clientId, ClientId: clientId,
ClientEpoch: clientEpoch, ClientEpoch: clientEpoch,
SelfSignature: targetFilerSignature, SelfSignature: targetFilerSignature,
PathPrefix: sourcePath, PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: sourceFilerOffsetTsNs, StartTsNs: sourceFilerOffsetTsNs,

View File

@@ -7,6 +7,7 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"strings"
) )
func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error { func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error {
@@ -57,12 +58,17 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
} }
prefix := dir
if !strings.HasSuffix(prefix, "/") {
prefix = prefix + "/"
}
metadataFollowOption := &pb.MetadataFollowOption{ metadataFollowOption := &pb.MetadataFollowOption{
ClientName: "mount", ClientName: "mount",
ClientId: selfSignature, ClientId: selfSignature,
ClientEpoch: 1, ClientEpoch: 1,
SelfSignature: selfSignature, SelfSignature: selfSignature,
PathPrefix: dir, PathPrefix: prefix,
AdditionalPathPrefixes: nil, AdditionalPathPrefixes: nil,
DirectoriesToWatch: nil, DirectoriesToWatch: nil,
StartTsNs: lastTsNs, StartTsNs: lastTsNs,