gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

tiered storage: can copy to s3, read from s3

Browse Source 
master not aware tiered volume yet, file assigning is not working yet
This commit is contained in:
Chris Lu
2019-12-02 15:08:28 -08:00
parent 228231f3d7
commit ec8de250e2
13 changed files with 689 additions and 286 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
weed/storage/backend/backend.go
View File

@@ -8,6 +8,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/spf13/viper"
)
@@ -23,7 +24,8 @@ type BackendStorageFile interface {
type BackendStorage interface {
ToProperties() map[string]string
NewStorageFile(key string) BackendStorageFile
NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) BackendStorageFile
CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error)
}
type StringProperties interface {
@@ -46,13 +48,13 @@ func LoadConfiguration(config *viper.Viper) {
backendSub := config.Sub(StorageBackendPrefix)
for backendTypeName, _ := range config.GetStringMap(StorageBackendPrefix) {
for backendTypeName := range config.GetStringMap(StorageBackendPrefix) {
backendStorageFactory, found := BackendStorageFactories[StorageType(backendTypeName)]
if !found {
glog.Fatalf("backend storage type %s not found", backendTypeName)
}
backendTypeSub := backendSub.Sub(backendTypeName)
for backendStorageId, _ := range backendSub.GetStringMap(backendTypeName) {
for backendStorageId := range backendSub.GetStringMap(backendTypeName) {
if !backendTypeSub.GetBool(backendStorageId + ".enabled") {
continue
}
@@ -105,12 +107,10 @@ func (p *Properties) GetString(key string) string {
func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
for sName, s := range BackendStorages {
parts := strings.Split(sName, ".")
if len(parts) != 2 {
sType, sId := BackendNameToTypeId(sName)
if sType == "" {
continue
}
sType, sId := parts[0], parts[1]
backends = append(backends, &master_pb.StorageBackend{
Type: sType,
Id: sId,
@@ -119,3 +119,16 @@ func ToPbStorageBackends() (backends []*master_pb.StorageBackend) {
}
return
}
func BackendNameToTypeId(backendName string) (backendType, backendId string) {
parts := strings.Split(backendName, ".")
if len(parts) == 1 {
return backendName, "default"
}
if len(parts) != 2 {
return
}
backendType, backendId = parts[0], parts[1]
return
}

56
weed/storage/backend/s3_backend/s3_backend.go
View File

@@ -2,6 +2,7 @@ package s3_backend
import (
"fmt"
"io"
"os"
"strings"
"time"
@@ -9,7 +10,9 @@ import (
"github.com/aws/aws-sdk-go/service/s3"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/backend"
"github.com/google/uuid"
)
func init() {
@@ -57,7 +60,7 @@ func (s *S3BackendStorage) ToProperties() map[string]string {
return m
}
func (s *S3BackendStorage) NewStorageFile(key string) backend.BackendStorageFile {
func (s *S3BackendStorage) NewStorageFile(key string, tierInfo *volume_server_pb.VolumeTierInfo) backend.BackendStorageFile {
if strings.HasPrefix(key, "/") {
key = key[1:]
}
@@ -65,18 +68,35 @@ func (s *S3BackendStorage) NewStorageFile(key string) backend.BackendStorageFile
f := &S3BackendStorageFile{
backendStorage: s,
key: key,
tierInfo: tierInfo,
}
return f
}
func (s *S3BackendStorage) CopyFile(f *os.File, fn func(progressed int64, percentage float32) error) (key string, size int64, err error) {
randomUuid, _ := uuid.NewRandom()
key = randomUuid.String()
glog.V(1).Infof("copying dat file of", f.Name(), "to remote s3", s.id, "as", key)
size, err = uploadToS3(s.conn, f.Name(), s.bucket, key, fn)
return
}
type S3BackendStorageFile struct {
backendStorage *S3BackendStorage
key string
tierInfo *volume_server_pb.VolumeTierInfo
}
func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n int, err error) {
bytesRange := fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))-1)
// glog.V(0).Infof("read %s %s", s3backendStorageFile.key, bytesRange)
getObjectOutput, getObjectErr := s3backendStorageFile.backendStorage.conn.GetObject(&s3.GetObjectInput{
Bucket: &s3backendStorageFile.backendStorage.bucket,
Key: &s3backendStorageFile.key,
@@ -84,13 +104,26 @@ func (s3backendStorageFile S3BackendStorageFile) ReadAt(p []byte, off int64) (n
})
if getObjectErr != nil {
return 0, fmt.Errorf("bucket %s GetObject %s: %v",
s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr)
return 0, fmt.Errorf("bucket %s GetObject %s: %v", s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, getObjectErr)
}
defer getObjectOutput.Body.Close()
return getObjectOutput.Body.Read(p)
glog.V(4).Infof("read %s %s", s3backendStorageFile.key, bytesRange)
glog.V(4).Infof("content range: %s, contentLength: %d", *getObjectOutput.ContentRange, *getObjectOutput.ContentLength)
for {
if n, err = getObjectOutput.Body.Read(p); err == nil && n < len(p) {
p = p[n:]
} else {
break
}
}
if err == io.EOF {
err = nil
}
return
}
func (s3backendStorageFile S3BackendStorageFile) WriteAt(p []byte, off int64) (n int, err error) {
@@ -107,18 +140,15 @@ func (s3backendStorageFile S3BackendStorageFile) Close() error {
func (s3backendStorageFile S3BackendStorageFile) GetStat() (datSize int64, modTime time.Time, err error) {
headObjectOutput, headObjectErr := s3backendStorageFile.backendStorage.conn.HeadObject(&s3.HeadObjectInput{
Bucket: &s3backendStorageFile.backendStorage.bucket,
Key: &s3backendStorageFile.key,
})
files := s3backendStorageFile.tierInfo.GetFiles()
if headObjectErr != nil {
return 0, time.Now(), fmt.Errorf("bucket %s HeadObject %s: %v",
s3backendStorageFile.backendStorage.bucket, s3backendStorageFile.key, headObjectErr)
if len(files)==0 {
err = fmt.Errorf("remote file info not found")
return
}
datSize = int64(*headObjectOutput.ContentLength)
modTime = *headObjectOutput.LastModified
datSize = int64(files[0].FileSize)
modTime = time.Unix(int64(files[0].ModifiedTime),0)
return
}

66
weed/storage/backend/s3_backend/s3_upload.go
View File

@@ -3,27 +3,30 @@ package s3_backend
import (
"fmt"
"os"
"sync/atomic"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3/s3iface"
"github.com/aws/aws-sdk-go/service/s3/s3manager"
"github.com/chrislusf/seaweedfs/weed/glog"
)
func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string) error {
func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey string,
fn func(progressed int64, percentage float32)error) (fileSize int64, err error) {
//open the file
f, err := os.Open(filename)
if err != nil {
return fmt.Errorf("failed to open file %q, %v", filename, err)
return 0, fmt.Errorf("failed to open file %q, %v", filename, err)
}
defer f.Close()
info, err := f.Stat()
if err != nil {
return fmt.Errorf("failed to stat file %q, %v", filename, err)
return 0, fmt.Errorf("failed to stat file %q, %v", filename, err)
}
fileSize := info.Size()
fileSize = info.Size()
partSize := int64(64 * 1024 * 1024) // The minimum/default allowed part size is 5MB
for partSize*1000 < fileSize {
@@ -33,14 +36,22 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey
// Create an uploader with the session and custom options
uploader := s3manager.NewUploaderWithClient(sess, func(u *s3manager.Uploader) {
u.PartSize = partSize
u.Concurrency = 15 // default is 15
u.Concurrency = 5
})
fileReader := &s3UploadProgressedReader{
fp: f,
size:fileSize,
read:-fileSize,
fn:fn,
}
// Upload the file to S3.
result, err := uploader.Upload(&s3manager.UploadInput{
var result *s3manager.UploadOutput
result, err = uploader.Upload(&s3manager.UploadInput{
Bucket: aws.String(destBucket),
Key: aws.String(destKey),
Body: f,
Body: fileReader,
ACL: aws.String("private"),
ServerSideEncryption: aws.String("AES256"),
StorageClass: aws.String("STANDARD_IA"),
@@ -48,9 +59,44 @@ func uploadToS3(sess s3iface.S3API, filename string, destBucket string, destKey
//in case it fails to upload
if err != nil {
return fmt.Errorf("failed to upload file, %v", err)
return 0, fmt.Errorf("failed to upload file %s: %v", filename, err)
}
fmt.Printf("file %s uploaded to %s\n", filename, result.Location)
glog.V(1).Infof("file %s uploaded to %s\n", filename, result.Location)
return nil
return
}
// adapted from https://github.com/aws/aws-sdk-go/pull/1868
type s3UploadProgressedReader struct {
fp *os.File
size int64
read int64
fn func(progressed int64, percentage float32)error
}
func (r *s3UploadProgressedReader) Read(p []byte) (int, error) {
return r.fp.Read(p)
}
func (r *s3UploadProgressedReader) ReadAt(p []byte, off int64) (int, error) {
n, err := r.fp.ReadAt(p, off)
if err != nil {
return n, err
}
// Got the length have read( or means has uploaded), and you can construct your message
atomic.AddInt64(&r.read, int64(n))
if r.fn != nil {
read := r.read
if err := r.fn(read, float32(read*100)/float32(r.size)); err != nil {
return n, err
}
}
return n, err
}
func (r *s3UploadProgressedReader) Seek(offset int64, whence int) (int64, error) {
return r.fp.Seek(offset, whence)
}