cloud drive: add createBucket() deleteBucket()

weed/remote_storage/azure/azure_storage_client.go

@@ -217,15 +217,36 @@ func (az *azureRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocat
 }
 
 func (az *azureRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
-	resp, err := az.ListBuckets()
-	if err != nil {
-		return nil, fmt.Errorf("list buckets: %v", err)
-	}
-	for _, b := range resp {
-		buckets = append(buckets, &remote_storage.Bucket{
-			Name:      b.Name,
-			CreatedAt: b.CreatedAt,
-		})
+	ctx := context.Background()
+	for containerMarker := (azblob.Marker{}); containerMarker.NotDone(); {
+		listContainer, err := az.serviceURL.ListContainersSegment(ctx, containerMarker, azblob.ListContainersSegmentOptions{})
+		if err == nil {
+			for _, v := range listContainer.ContainerItems {
+				buckets = append(buckets, &remote_storage.Bucket{
+					Name:      v.Name,
+					CreatedAt: v.Properties.LastModified,
+				})
+			}
+		} else {
+			return buckets, err
+		}
+		containerMarker = listContainer.NextMarker
+	}
+	return
+}
+
+func (az *azureRemoteStorageClient) CreateBucket(name string) (err error) {
+	containerURL := az.serviceURL.NewContainerURL(name)
+	if _, err = containerURL.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessNone); err != nil {
+		return fmt.Errorf("create bucket %s: %v", name, err)
+	}
+	return
+}
+
+func (az *azureRemoteStorageClient) DeleteBucket(name string) (err error) {
+	containerURL := az.serviceURL.NewContainerURL(name)
+	if _, err = containerURL.Delete(context.Background(), azblob.ContainerAccessConditions{}); err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
 	}
 	return
 }

weed/remote_storage/gcs/gcs_storage_client.go

@@ -4,6 +4,7 @@ import (
 	"cloud.google.com/go/storage"
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/remote_storage"
@@ -41,6 +42,15 @@ func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.
 		}
 	}
 
+	projectID := conf.GcsProjectId
+	if projectID == "" {
+		found := false
+		projectID, found = os.LookupEnv("GOOGLE_CLOUD_PROJECT")
+		if !found {
+			glog.Warningf("need to specific GOOGLE_CLOUD_PROJECT env variable")
+		}
+	}
+
 	googleApplicationCredentials = util.ResolvePath(googleApplicationCredentials)
 
 	c, err := storage.NewClient(context.Background(), option.WithCredentialsFile(googleApplicationCredentials))
@@ -49,12 +59,14 @@ func (s gcsRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.
 	}
 
 	client.client = c
+	client.projectID = projectID
 	return client, nil
 }
 
 type gcsRemoteStorageClient struct {
-	conf   *remote_pb.RemoteConf
-	client *storage.Client
+	conf      *remote_pb.RemoteConf
+	client    *storage.Client
+	projectID string
 }
 
 var _ = remote_storage.RemoteStorageClient(&gcsRemoteStorageClient{})
@@ -186,5 +198,41 @@ func (gcs *gcsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocati
 }
 
 func (gcs *gcsRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
+	if gcs.projectID == "" {
+		return nil, fmt.Errorf("gcs project id or GOOGLE_CLOUD_PROJECT env variable not set")
+	}
+	iter := gcs.client.Buckets(context.Background(), gcs.projectID)
+	for {
+		b, err := iter.Next()
+		if err == iterator.Done {
+			break
+		}
+		if err != nil {
+			return
+		}
+		buckets = append(buckets, &remote_storage.Bucket{
+			Name:      b.Name,
+			CreatedAt: b.Created,
+		})
+	}
+	return
+}
+
+func (gcs *gcsRemoteStorageClient) CreateBucket(name string) (err error) {
+	if gcs.projectID == "" {
+		return fmt.Errorf("gcs project id or GOOGLE_CLOUD_PROJECT env variable not set")
+	}
+	err = gcs.client.Bucket(name).Create(context.Background(), gcs.projectID, &storage.BucketAttrs{})
+	if err != nil {
+		return fmt.Errorf("create bucket %s: %v", name, err)
+	}
+	return
+}
+
+func (gcs *gcsRemoteStorageClient) DeleteBucket(name string) (err error) {
+	err = gcs.client.Bucket(name).Delete(context.Background())
+	if err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
+	}
 	return
 }

weed/remote_storage/hdfs/hdfs_storage_client.go

@@ -181,3 +181,11 @@ func (c *hdfsRemoteStorageClient) DeleteFile(loc *remote_pb.RemoteStorageLocatio
 func (c *hdfsRemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket, err error) {
 	return
 }
+
+func (c *hdfsRemoteStorageClient) CreateBucket(name string) (err error) {
+	return
+}
+
+func (c *hdfsRemoteStorageClient) DeleteBucket(name string) (err error) {
+	return
+}

weed/remote_storage/remote_storage.go

@@ -80,6 +80,8 @@ type RemoteStorageClient interface {
 	UpdateFileMetadata(loc *remote_pb.RemoteStorageLocation, oldEntry *filer_pb.Entry, newEntry *filer_pb.Entry) (err error)
 	DeleteFile(loc *remote_pb.RemoteStorageLocation) (err error)
 	ListBuckets() ([]*Bucket, error)
+	CreateBucket(name string) (err error)
+	DeleteBucket(name string) (err error)
 }
 
 type RemoteStorageClientMaker interface {

weed/remote_storage/s3/s3_storage_client.go

@@ -248,3 +248,31 @@ func (s *s3RemoteStorageClient) ListBuckets() (buckets []*remote_storage.Bucket,
 	}
 	return
 }
+
+func (s *s3RemoteStorageClient) CreateBucket(name string) (err error) {
+	_, err = s.conn.CreateBucket(&s3.CreateBucketInput{
+		ACL:                        nil,
+		Bucket:                     aws.String(name),
+		CreateBucketConfiguration:  nil,
+		GrantFullControl:           nil,
+		GrantRead:                  nil,
+		GrantReadACP:               nil,
+		GrantWrite:                 nil,
+		GrantWriteACP:              nil,
+		ObjectLockEnabledForBucket: nil,
+	})
+	if err != nil {
+		return fmt.Errorf("create bucket %s: %v", name, err)
+	}
+	return
+}
+
+func (s *s3RemoteStorageClient) DeleteBucket(name string) (err error) {
+	_, err = s.conn.DeleteBucket(&s3.DeleteBucketInput{
+		Bucket:                     aws.String(name),
+	})
+	if err != nil {
+		return fmt.Errorf("delete bucket %s: %v", name, err)
+	}
+	return
+}