gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

cloud drive: add support for Wasabi

Browse Source 
* disable md5, sha256 checking to avoid reading one chunk twice
* single threaded upload to avoid chunk swapping (to be enhanced later)
This commit is contained in:
Chris Lu
2021-08-25 17:34:29 -07:00
parent 9bcf94b2b1
commit c08ac536ed
14 changed files with 298 additions and 155 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/remote_storage/s3/aliyun.go
View File

@@ -29,6 +29,7 @@ func (s AliyunRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storag
Endpoint: aws.String(conf.AliyunEndpoint),
Region: aws.String(conf.AliyunRegion),
S3ForcePathStyle: aws.Bool(false),
S3DisableContentMD5Validation: aws.Bool(true),
}
if accessKey != "" && secretKey != "" {
config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
@@ -38,6 +39,7 @@ func (s AliyunRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storag
if err != nil {
return nil, fmt.Errorf("create aliyun session: %v", err)
}
sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
client.conn = s3.New(sess)
return client, nil
}

2
weed/remote_storage/s3/backblaze.go
View File

@@ -24,6 +24,7 @@ func (s BackBlazeRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_sto
Endpoint: aws.String(conf.BackblazeEndpoint),
Region: aws.String("us-west-002"),
S3ForcePathStyle: aws.Bool(true),
S3DisableContentMD5Validation: aws.Bool(true),
}
if conf.BackblazeKeyId != "" && conf.BackblazeApplicationKey != "" {
config.Credentials = credentials.NewStaticCredentials(conf.BackblazeKeyId, conf.BackblazeApplicationKey, "")
@@ -33,6 +34,7 @@ func (s BackBlazeRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_sto
if err != nil {
return nil, fmt.Errorf("create backblaze session: %v", err)
}
sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
client.conn = s3.New(sess)
return client, nil
}

2
weed/remote_storage/s3/baidu.go
View File

@@ -29,6 +29,7 @@ func (s BaiduRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage
Endpoint: aws.String(conf.BaiduEndpoint),
Region: aws.String(conf.BaiduRegion),
S3ForcePathStyle: aws.Bool(true),
S3DisableContentMD5Validation: aws.Bool(true),
}
if accessKey != "" && secretKey != "" {
config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
@@ -38,6 +39,7 @@ func (s BaiduRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage
if err != nil {
return nil, fmt.Errorf("create baidu session: %v", err)
}
sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
client.conn = s3.New(sess)
return client, nil
}

12
weed/remote_storage/s3/s3_storage_client.go
View File

@@ -27,9 +27,10 @@ func (s s3RemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage.Re
conf: conf,
}
config := &aws.Config{
Region: aws.String(conf.S3Region),
Endpoint: aws.String(conf.S3Endpoint),
S3ForcePathStyle: aws.Bool(conf.S3ForcePathStyle),
Region: aws.String(conf.S3Region),
Endpoint: aws.String(conf.S3Endpoint),
S3ForcePathStyle: aws.Bool(conf.S3ForcePathStyle),
S3DisableContentMD5Validation: aws.Bool(true),
}
if conf.S3AccessKey != "" && conf.S3SecretKey != "" {
config.Credentials = credentials.NewStaticCredentials(conf.S3AccessKey, conf.S3SecretKey, "")
@@ -39,6 +40,7 @@ func (s s3RemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage.Re
if err != nil {
return nil, fmt.Errorf("create aws session: %v", err)
}
sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
client.conn = s3.New(sess)
return client, nil
}
@@ -129,7 +131,7 @@ func (s *s3RemoteStorageClient) WriteFile(loc *filer_pb.RemoteStorageLocation, e
// Create an uploader with the session and custom options
uploader := s3manager.NewUploaderWithClient(s.conn, func(u *s3manager.Uploader) {
u.PartSize = partSize
u.Concurrency = 5
u.Concurrency = 1
})
// process tagging
@@ -152,7 +154,7 @@ func (s *s3RemoteStorageClient) WriteFile(loc *filer_pb.RemoteStorageLocation, e
//in case it fails to upload
if err != nil {
return nil, fmt.Errorf("upload to s3 %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
return nil, fmt.Errorf("upload to %s/%s%s: %v", loc.Name, loc.Bucket, loc.Path, err)
}
// read back the remote entry

3
weed/remote_storage/s3/tencent.go
View File

@@ -27,7 +27,9 @@ func (s TencentRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_stora
config := &aws.Config{
Endpoint: aws.String(conf.TencentEndpoint),
Region: aws.String("us-west-2"),
S3ForcePathStyle: aws.Bool(true),
S3DisableContentMD5Validation: aws.Bool(true),
}
if accessKey != "" && secretKey != "" {
config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
@@ -37,6 +39,7 @@ func (s TencentRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_stora
if err != nil {
return nil, fmt.Errorf("create tencent session: %v", err)
}
sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
client.conn = s3.New(sess)
return client, nil
}

57
weed/remote_storage/s3/wasabi.go Normal file
View File

@@ -0,0 +1,57 @@
package s3
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/remote_storage"
"github.com/chrislusf/seaweedfs/weed/util"
)
func init() {
remote_storage.RemoteStorageClientMakers["wasabi"] = new(WasabiRemoteStorageMaker)
}
type WasabiRemoteStorageMaker struct{}
func (s WasabiRemoteStorageMaker) Make(conf *filer_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &s3RemoteStorageClient{
conf: conf,
}
accessKey := util.Nvl(conf.WasabiAccessKey)
secretKey := util.Nvl(conf.WasabiSecretKey)
config := &aws.Config{
Endpoint: aws.String(conf.WasabiEndpoint),
Region: aws.String(conf.WasabiRegion),
S3ForcePathStyle: aws.Bool(true),
S3DisableContentMD5Validation: aws.Bool(true),
}
if accessKey != "" && secretKey != "" {
config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
}
sess, err := session.NewSession(config)
if err != nil {
return nil, fmt.Errorf("create wasabi session: %v", err)
}
sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
client.conn = s3.New(sess)
return client, nil
}
var skipSha256PayloadSigning = func(r *request.Request) {
// see https://github.com/ceph/ceph/pull/15965/files
if r.ClientInfo.ServiceID != "S3" {
return
}
if r.Operation.Name == "PutObject" || r.Operation.Name == "UploadPart" {
if len(r.HTTPRequest.Header.Get("X-Amz-Content-Sha256")) == 0 {
r.HTTPRequest.Header.Set("X-Amz-Content-Sha256", "UNSIGNED-PAYLOAD")
}
}
}