gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use const multipart uploads folder

Browse Source 
avoid error bucket NotEmpty if multipart uploads folder exist
This commit is contained in:
Konstantin Lebedev
2022-06-29 16:21:16 +05:00
parent c6e6e303db
commit bcbdc4cb37
8 changed files with 16 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
weed/command/filer_remote_gateway_buckets.go
View File

@@ -9,6 +9,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
"github.com/chrislusf/seaweedfs/weed/remote_storage"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
"math"
@@ -181,7 +182,7 @@ func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *sour
if message.NewParentPath == option.bucketsDir {
return handleCreateBucket(message.NewEntry)
}
if strings.HasPrefix(message.NewParentPath, option.bucketsDir) && strings.Contains(message.NewParentPath, "/.uploads/") {
if strings.HasPrefix(message.NewParentPath, option.bucketsDir) && strings.Contains(message.NewParentPath, "/"+s3_constants.MultipartUploadsFolder+"/") {
return nil
}
if !filer.HasData(message.NewEntry) {

3
weed/replication/sink/localsink/local_sink.go
View File

@@ -7,6 +7,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/replication/repl_util"
"github.com/chrislusf/seaweedfs/weed/replication/sink"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/util"
"os"
"path/filepath"
@@ -32,7 +33,7 @@ func (localsink *LocalSink) GetName() string {
}
func (localsink *LocalSink) isMultiPartEntry(key string) bool {
return strings.HasSuffix(key, ".part") && strings.Contains(key, "/.uploads/")
return strings.HasSuffix(key, ".part") && strings.Contains(key, "/"+s3_constants.MultipartUploadsFolder+"/")
}
func (localsink *LocalSink) initialize(dir string, isIncremental bool) error {

1
weed/s3api/s3_constants/s3_actions.go
View File

@@ -8,4 +8,5 @@ const (
ACTION_LIST = "List"
SeaweedStorageDestinationHeader = "x-seaweedfs-destination"
MultipartUploadsFolder = ".uploads"
)

8
weed/s3api/s3api_bucket_handlers.go
View File

@@ -150,12 +150,14 @@ func (s3a *S3ApiServer) DeleteBucketHandler(w http.ResponseWriter, r *http.Reque
err := s3a.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
if !s3a.option.AllowDeleteBucketNotEmpty {
entries, _, err := s3a.list(s3a.option.BucketsPath+"/"+bucket, "", "", false, 1)
entries, _, err := s3a.list(s3a.option.BucketsPath+"/"+bucket, "", "", false, 2)
if err != nil {
return fmt.Errorf("failed to list bucket %s: %v", bucket, err)
}
if len(entries) > 0 {
return errors.New(s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code)
for _, entry := range entries {
if entry.Name != s3_constants.MultipartUploadsFolder {
return errors.New(s3err.GetAPIError(s3err.ErrBucketNotEmpty).Code)
}
}
}

2
weed/s3api/s3api_object_multipart_handlers.go
View File

@@ -268,7 +268,7 @@ func (s3a *S3ApiServer) PutObjectPartHandler(w http.ResponseWriter, r *http.Requ
}
func (s3a *S3ApiServer) genUploadsFolder(bucket string) string {
return fmt.Sprintf("%s/%s/.uploads", s3a.option.BucketsPath, bucket)
return fmt.Sprintf("%s/%s/%s", s3a.option.BucketsPath, bucket, s3_constants.MultipartUploadsFolder)
}
// Generate uploadID hash string from object

2
weed/s3api/s3api_objects_list_handlers.go
View File

@@ -309,7 +309,7 @@ func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, d
nextMarker = dir + "/" + entry.Name
if entry.IsDirectory {
// println("ListEntries", dir, "dir:", entry.Name)
if entry.Name == ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
if entry.Name == s3_constants.MultipartUploadsFolder { // FIXME no need to apply to all directories. this extra also affects maxKeys
continue
}
if delimiter == "" {

3
weed/shell/command_s3_clean_uploads.go
View File

@@ -3,6 +3,7 @@ package shell
import (
"flag"
"fmt"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"io"
@@ -68,7 +69,7 @@ func (c *commandS3CleanUploads) Do(args []string, commandEnv *CommandEnv, writer
}
func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io.Writer, filerBucketsPath string, bucket string, timeAgo time.Duration, signingKey string) error {
uploadsDir := filerBucketsPath + "/" + bucket + "/.uploads"
uploadsDir := filerBucketsPath + "/" + bucket + "/" + s3_constants.MultipartUploadsFolder
var staleUploads []string
now := time.Now()
err := filer_pb.List(commandEnv, uploadsDir, "", func(entry *filer_pb.Entry, isLast bool) error {