gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add s3 replication sink

Browse Source
This commit is contained in:
Chris Lu
2018-10-03 23:36:52 -07:00
parent 56a5d5af8d
commit e8ef501f02
8 changed files with 369 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
weed/replication/replicator.go
View File

@@ -6,8 +6,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication/sink"
"github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/glog"
)
@@ -17,18 +16,15 @@ type Replicator struct {
source *source.FilerSource
}
func NewReplicator(sourceConfig, sinkConfig util.Configuration) *Replicator {
sink := &filersink.FilerSink{}
sink.Initialize(sinkConfig)
func NewReplicator(sourceConfig util.Configuration, dataSink sink.ReplicationSink) *Replicator {
source := &source.FilerSource{}
source.Initialize(sourceConfig)
sink.SetSourceFiler(source)
dataSink.SetSourceFiler(source)
return &Replicator{
sink: sink,
sink: dataSink,
source: source,
}
}
@@ -39,22 +35,19 @@ func (r *Replicator) Replicate(key string, message *filer_pb.EventNotification)
}
key = filepath.Join(r.sink.GetSinkToDirectory(), key[len(r.source.Dir):])
if message.OldEntry != nil && message.NewEntry == nil {
return r.sink.DeleteEntry(key, message.OldEntry, message.DeleteChunks)
return r.sink.DeleteEntry(key, message.OldEntry.IsDirectory, message.DeleteChunks)
}
if message.OldEntry == nil && message.NewEntry != nil {
return r.sink.CreateEntry(key, message.NewEntry)
}
if existingEntry, err := r.sink.LookupEntry(key); err == nil {
if message.OldEntry == nil && message.NewEntry == nil {
glog.V(0).Infof("message %+v existingEntry: %+v", message, existingEntry)
return r.sink.DeleteEntry(key, existingEntry, true)
}
return r.sink.UpdateEntry(key, message.OldEntry, message.NewEntry, existingEntry, message.DeleteChunks)
if message.OldEntry == nil && message.NewEntry == nil {
glog.V(0).Infof("weird message %+v", message)
return nil
}
glog.V(0).Infof("key:%s, message %+v", key, message)
if message.OldEntry == nil && message.NewEntry == nil {
return nil
foundExisting, err := r.sink.UpdateEntry(key, message.OldEntry, message.NewEntry, message.DeleteChunks)
if foundExisting {
return err
}
return r.sink.CreateEntry(key, message.NewEntry)