ensure metadata follow a specific folder
fix https://github.com/seaweedfs/seaweedfs/issues/5774
This commit is contained in:
@@ -10,6 +10,7 @@ import (
|
|||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"google.golang.org/grpc/credentials/insecure"
|
"google.golang.org/grpc/credentials/insecure"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -79,12 +80,17 @@ func startGenerateMetadata() {
|
|||||||
|
|
||||||
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
|
func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
|
||||||
|
|
||||||
|
prefix := *dir
|
||||||
|
if !strings.HasSuffix(prefix, "/") {
|
||||||
|
prefix = prefix + "/"
|
||||||
|
}
|
||||||
|
|
||||||
metadataFollowOption := &pb.MetadataFollowOption{
|
metadataFollowOption := &pb.MetadataFollowOption{
|
||||||
ClientName: "tail",
|
ClientName: "tail",
|
||||||
ClientId: 0,
|
ClientId: 0,
|
||||||
ClientEpoch: 0,
|
ClientEpoch: 0,
|
||||||
SelfSignature: 0,
|
SelfSignature: 0,
|
||||||
PathPrefix: *dir,
|
PathPrefix: prefix,
|
||||||
AdditionalPathPrefixes: nil,
|
AdditionalPathPrefixes: nil,
|
||||||
DirectoriesToWatch: nil,
|
DirectoriesToWatch: nil,
|
||||||
StartTsNs: 0,
|
StartTsNs: 0,
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -148,12 +149,17 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
prefix := sourcePath
|
||||||
|
if !strings.HasSuffix(prefix, "/") {
|
||||||
|
prefix = prefix + "/"
|
||||||
|
}
|
||||||
|
|
||||||
metadataFollowOption := &pb.MetadataFollowOption{
|
metadataFollowOption := &pb.MetadataFollowOption{
|
||||||
ClientName: "backup_" + dataSink.GetName(),
|
ClientName: "backup_" + dataSink.GetName(),
|
||||||
ClientId: clientId,
|
ClientId: clientId,
|
||||||
ClientEpoch: clientEpoch,
|
ClientEpoch: clientEpoch,
|
||||||
SelfSignature: 0,
|
SelfSignature: 0,
|
||||||
PathPrefix: sourcePath,
|
PathPrefix: prefix,
|
||||||
AdditionalPathPrefixes: nil,
|
AdditionalPathPrefixes: nil,
|
||||||
DirectoriesToWatch: nil,
|
DirectoriesToWatch: nil,
|
||||||
StartTsNs: startFrom.UnixNano(),
|
StartTsNs: startFrom.UnixNano(),
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import (
|
|||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||||
@@ -197,12 +198,16 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
|
|||||||
|
|
||||||
metaBackup.clientEpoch++
|
metaBackup.clientEpoch++
|
||||||
|
|
||||||
|
prefix := *metaBackup.filerDirectory
|
||||||
|
if !strings.HasSuffix(prefix, "/") {
|
||||||
|
prefix = prefix + "/"
|
||||||
|
}
|
||||||
metadataFollowOption := &pb.MetadataFollowOption{
|
metadataFollowOption := &pb.MetadataFollowOption{
|
||||||
ClientName: "meta_backup",
|
ClientName: "meta_backup",
|
||||||
ClientId: metaBackup.clientId,
|
ClientId: metaBackup.clientId,
|
||||||
ClientEpoch: metaBackup.clientEpoch,
|
ClientEpoch: metaBackup.clientEpoch,
|
||||||
SelfSignature: 0,
|
SelfSignature: 0,
|
||||||
PathPrefix: *metaBackup.filerDirectory,
|
PathPrefix: prefix,
|
||||||
AdditionalPathPrefixes: nil,
|
AdditionalPathPrefixes: nil,
|
||||||
DirectoriesToWatch: nil,
|
DirectoriesToWatch: nil,
|
||||||
StartTsNs: startTime.UnixNano(),
|
StartTsNs: startTime.UnixNano(),
|
||||||
|
|||||||
@@ -55,7 +55,7 @@ func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSo
|
|||||||
ClientId: option.clientId,
|
ClientId: option.clientId,
|
||||||
ClientEpoch: option.clientEpoch,
|
ClientEpoch: option.clientEpoch,
|
||||||
SelfSignature: 0,
|
SelfSignature: 0,
|
||||||
PathPrefix: option.bucketsDir,
|
PathPrefix: option.bucketsDir + "/",
|
||||||
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
||||||
DirectoriesToWatch: nil,
|
DirectoriesToWatch: nil,
|
||||||
StartTsNs: lastOffsetTs.UnixNano(),
|
StartTsNs: lastOffsetTs.UnixNano(),
|
||||||
|
|||||||
@@ -64,12 +64,17 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
|
|||||||
|
|
||||||
option.clientEpoch++
|
option.clientEpoch++
|
||||||
|
|
||||||
|
prefix := mountedDir
|
||||||
|
if !strings.HasSuffix(prefix, "/") {
|
||||||
|
prefix = prefix + "/"
|
||||||
|
}
|
||||||
|
|
||||||
metadataFollowOption := &pb.MetadataFollowOption{
|
metadataFollowOption := &pb.MetadataFollowOption{
|
||||||
ClientName: "filer.remote.sync",
|
ClientName: "filer.remote.sync",
|
||||||
ClientId: option.clientId,
|
ClientId: option.clientId,
|
||||||
ClientEpoch: option.clientEpoch,
|
ClientEpoch: option.clientEpoch,
|
||||||
SelfSignature: 0,
|
SelfSignature: 0,
|
||||||
PathPrefix: mountedDir,
|
PathPrefix: prefix,
|
||||||
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
AdditionalPathPrefixes: []string{filer.DirectoryEtcRemote},
|
||||||
DirectoriesToWatch: nil,
|
DirectoriesToWatch: nil,
|
||||||
StartTsNs: lastOffsetTs.UnixNano(),
|
StartTsNs: lastOffsetTs.UnixNano(),
|
||||||
|
|||||||
@@ -296,12 +296,17 @@ func doSubscribeFilerMetaChanges(clientId int32, clientEpoch int32, grpcDialOpti
|
|||||||
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
|
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, offsetTsNs)
|
||||||
})
|
})
|
||||||
|
|
||||||
|
prefix := sourcePath
|
||||||
|
if !strings.HasSuffix(prefix, "/") {
|
||||||
|
prefix = prefix + "/"
|
||||||
|
}
|
||||||
|
|
||||||
metadataFollowOption := &pb.MetadataFollowOption{
|
metadataFollowOption := &pb.MetadataFollowOption{
|
||||||
ClientName: clientName,
|
ClientName: clientName,
|
||||||
ClientId: clientId,
|
ClientId: clientId,
|
||||||
ClientEpoch: clientEpoch,
|
ClientEpoch: clientEpoch,
|
||||||
SelfSignature: targetFilerSignature,
|
SelfSignature: targetFilerSignature,
|
||||||
PathPrefix: sourcePath,
|
PathPrefix: prefix,
|
||||||
AdditionalPathPrefixes: nil,
|
AdditionalPathPrefixes: nil,
|
||||||
DirectoriesToWatch: nil,
|
DirectoriesToWatch: nil,
|
||||||
StartTsNs: sourceFilerOffsetTsNs,
|
StartTsNs: sourceFilerOffsetTsNs,
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error {
|
func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.FilerClient, dir string, lastTsNs int64) error {
|
||||||
@@ -57,12 +58,17 @@ func SubscribeMetaEvents(mc *MetaCache, selfSignature int32, client filer_pb.Fil
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
prefix := dir
|
||||||
|
if !strings.HasSuffix(prefix, "/") {
|
||||||
|
prefix = prefix + "/"
|
||||||
|
}
|
||||||
|
|
||||||
metadataFollowOption := &pb.MetadataFollowOption{
|
metadataFollowOption := &pb.MetadataFollowOption{
|
||||||
ClientName: "mount",
|
ClientName: "mount",
|
||||||
ClientId: selfSignature,
|
ClientId: selfSignature,
|
||||||
ClientEpoch: 1,
|
ClientEpoch: 1,
|
||||||
SelfSignature: selfSignature,
|
SelfSignature: selfSignature,
|
||||||
PathPrefix: dir,
|
PathPrefix: prefix,
|
||||||
AdditionalPathPrefixes: nil,
|
AdditionalPathPrefixes: nil,
|
||||||
DirectoriesToWatch: nil,
|
DirectoriesToWatch: nil,
|
||||||
StartTsNs: lastTsNs,
|
StartTsNs: lastTsNs,
|
||||||
|
|||||||
Reference in New Issue
Block a user