"golang.org/x/exp/slices" => "slices" and go fmt
This commit is contained in:
@@ -11,7 +11,7 @@ import (
|
|||||||
|
|
||||||
hashicorpRaft "github.com/hashicorp/raft"
|
hashicorpRaft "github.com/hashicorp/raft"
|
||||||
|
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
"github.com/seaweedfs/raft/protobuf"
|
"github.com/seaweedfs/raft/protobuf"
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ package filer
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"log"
|
"log"
|
||||||
|
"slices"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package filer
|
|||||||
import (
|
import (
|
||||||
"container/list"
|
"container/list"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval]) {
|
func readResolvedChunks(chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles *IntervalList[*VisibleInterval]) {
|
||||||
|
|||||||
@@ -123,7 +123,7 @@ func (f *Filer) ReadPersistedLogBuffer(startPosition log_buffer.MessagePosition,
|
|||||||
err = fmt.Errorf("reading from persisted logs: %v", visitErr)
|
err = fmt.Errorf("reading from persisted logs: %v", visitErr)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var logEntry *filer_pb.LogEntry
|
var logEntry *filer_pb.LogEntry
|
||||||
for {
|
for {
|
||||||
logEntry, visitErr = visitor.GetNext()
|
logEntry, visitErr = visitor.GetNext()
|
||||||
if visitErr != nil {
|
if visitErr != nil {
|
||||||
|
|||||||
@@ -37,10 +37,9 @@ func Replay(filerStore FilerStore, resp *filer_pb.SubscribeMetadataResponse) err
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
// ParallelProcessDirectoryStructure processes each entry in parallel, and also ensure parent directories are processed first.
|
// ParallelProcessDirectoryStructure processes each entry in parallel, and also ensure parent directories are processed first.
|
||||||
// This also assumes the parent directories are in the entryChan already.
|
// This also assumes the parent directories are in the entryChan already.
|
||||||
func ParallelProcessDirectoryStructure(entryChan chan *Entry, concurrency int, eachEntryFn func(entry *Entry)(error)) (firstErr error) {
|
func ParallelProcessDirectoryStructure(entryChan chan *Entry, concurrency int, eachEntryFn func(entry *Entry) error) (firstErr error) {
|
||||||
|
|
||||||
executors := util.NewLimitedConcurrentExecutor(concurrency)
|
executors := util.NewLimitedConcurrentExecutor(concurrency)
|
||||||
|
|
||||||
|
|||||||
@@ -3,7 +3,7 @@ package redis
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
package redis3
|
package redis3
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
"github.com/go-redsync/redsync/v4"
|
"github.com/go-redsync/redsync/v4"
|
||||||
"github.com/go-redsync/redsync/v4/redis/goredis/v9"
|
"github.com/go-redsync/redsync/v4/redis/goredis/v9"
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -3,9 +3,9 @@ package redis3
|
|||||||
import (
|
import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
"github.com/go-redsync/redsync/v4"
|
"github.com/go-redsync/redsync/v4"
|
||||||
"github.com/go-redsync/redsync/v4/redis/goredis/v9"
|
"github.com/go-redsync/redsync/v4/redis/goredis/v9"
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
package redis3
|
package redis3
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/redis/go-redis/v9"
|
|
||||||
"github.com/go-redsync/redsync/v4"
|
"github.com/go-redsync/redsync/v4"
|
||||||
"github.com/go-redsync/redsync/v4/redis/goredis/v9"
|
"github.com/go-redsync/redsync/v4/redis/goredis/v9"
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/filer"
|
"github.com/seaweedfs/seaweedfs/weed/filer"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -9,14 +9,14 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
|
||||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||||
)
|
)
|
||||||
|
|
||||||
var getLookupFileIdBackoffSchedule = []time.Duration{
|
var getLookupFileIdBackoffSchedule = []time.Duration{
|
||||||
|
|||||||
@@ -5,11 +5,11 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
|
"github.com/seaweedfs/seaweedfs/weed/mq/client/pub_client"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||||
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||||
"log"
|
"log"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
|||||||
@@ -7,11 +7,11 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
|
"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
"google.golang.org/grpc/credentials/insecure"
|
"google.golang.org/grpc/credentials/insecure"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
|
|||||||
@@ -50,7 +50,7 @@ func (p *TopicPublisher) FinishPublish() error {
|
|||||||
inputBuffer.Enqueue(&mq_pb.DataMessage{
|
inputBuffer.Enqueue(&mq_pb.DataMessage{
|
||||||
TsNs: time.Now().UnixNano(),
|
TsNs: time.Now().UnixNano(),
|
||||||
Ctrl: &mq_pb.ControlMessage{
|
Ctrl: &mq_pb.ControlMessage{
|
||||||
IsClose: true,
|
IsClose: true,
|
||||||
PublisherName: p.config.PublisherName,
|
PublisherName: p.config.PublisherName,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -95,8 +95,8 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
|
|||||||
followerCount: 1,
|
followerCount: 1,
|
||||||
assignments: []*mq_pb.BrokerPartitionAssignment{
|
assignments: []*mq_pb.BrokerPartitionAssignment{
|
||||||
{
|
{
|
||||||
LeaderBroker: "",
|
LeaderBroker: "",
|
||||||
Partition: &mq_pb.Partition{},
|
Partition: &mq_pb.Partition{},
|
||||||
FollowerBroker: "localhost:2",
|
FollowerBroker: "localhost:2",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -110,8 +110,8 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
|
|||||||
followerCount: 1,
|
followerCount: 1,
|
||||||
assignments: []*mq_pb.BrokerPartitionAssignment{
|
assignments: []*mq_pb.BrokerPartitionAssignment{
|
||||||
{
|
{
|
||||||
LeaderBroker: "localhost:1",
|
LeaderBroker: "localhost:1",
|
||||||
Partition: &mq_pb.Partition{},
|
Partition: &mq_pb.Partition{},
|
||||||
FollowerBroker: "",
|
FollowerBroker: "",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -125,8 +125,8 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
|
|||||||
followerCount: 1,
|
followerCount: 1,
|
||||||
assignments: []*mq_pb.BrokerPartitionAssignment{
|
assignments: []*mq_pb.BrokerPartitionAssignment{
|
||||||
{
|
{
|
||||||
LeaderBroker: "localhost:1",
|
LeaderBroker: "localhost:1",
|
||||||
Partition: &mq_pb.Partition{},
|
Partition: &mq_pb.Partition{},
|
||||||
FollowerBroker: "localhost:200",
|
FollowerBroker: "localhost:200",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -140,8 +140,8 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
|
|||||||
followerCount: 1,
|
followerCount: 1,
|
||||||
assignments: []*mq_pb.BrokerPartitionAssignment{
|
assignments: []*mq_pb.BrokerPartitionAssignment{
|
||||||
{
|
{
|
||||||
LeaderBroker: "localhost:100",
|
LeaderBroker: "localhost:100",
|
||||||
Partition: &mq_pb.Partition{},
|
Partition: &mq_pb.Partition{},
|
||||||
FollowerBroker: "localhost:200",
|
FollowerBroker: "localhost:200",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -155,8 +155,8 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
|
|||||||
followerCount: 3,
|
followerCount: 3,
|
||||||
assignments: []*mq_pb.BrokerPartitionAssignment{
|
assignments: []*mq_pb.BrokerPartitionAssignment{
|
||||||
{
|
{
|
||||||
LeaderBroker: "localhost:1",
|
LeaderBroker: "localhost:1",
|
||||||
Partition: &mq_pb.Partition{},
|
Partition: &mq_pb.Partition{},
|
||||||
FollowerBroker: "localhost:2",
|
FollowerBroker: "localhost:2",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -184,8 +184,8 @@ func TestEnsureAssignmentsToActiveBrokersX(t *testing.T) {
|
|||||||
followerCount: 3,
|
followerCount: 3,
|
||||||
assignments: []*mq_pb.BrokerPartitionAssignment{
|
assignments: []*mq_pb.BrokerPartitionAssignment{
|
||||||
{
|
{
|
||||||
LeaderBroker: "localhost:1",
|
LeaderBroker: "localhost:1",
|
||||||
Partition: &mq_pb.Partition{},
|
Partition: &mq_pb.Partition{},
|
||||||
FollowerBroker: "localhost:2",
|
FollowerBroker: "localhost:2",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -16,8 +16,8 @@ type ConsumerGroupInstance struct {
|
|||||||
|
|
||||||
func NewConsumerGroupInstance(instanceId string, maxPartitionCount int32) *ConsumerGroupInstance {
|
func NewConsumerGroupInstance(instanceId string, maxPartitionCount int32) *ConsumerGroupInstance {
|
||||||
return &ConsumerGroupInstance{
|
return &ConsumerGroupInstance{
|
||||||
InstanceId: ConsumerGroupInstanceId(instanceId),
|
InstanceId: ConsumerGroupInstanceId(instanceId),
|
||||||
ResponseChan: make(chan *mq_pb.SubscriberToSubCoordinatorResponse, 1),
|
ResponseChan: make(chan *mq_pb.SubscriberToSubCoordinatorResponse, 1),
|
||||||
MaxPartitionCount: maxPartitionCount,
|
MaxPartitionCount: maxPartitionCount,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -84,9 +84,9 @@ type TimestampStatus struct {
|
|||||||
|
|
||||||
// RingBuffer represents a circular buffer to hold timestamps.
|
// RingBuffer represents a circular buffer to hold timestamps.
|
||||||
type RingBuffer struct {
|
type RingBuffer struct {
|
||||||
buffer []*TimestampStatus
|
buffer []*TimestampStatus
|
||||||
head int
|
head int
|
||||||
size int
|
size int
|
||||||
maxTimestamp int64
|
maxTimestamp int64
|
||||||
maxAllAckedTs int64
|
maxAllAckedTs int64
|
||||||
}
|
}
|
||||||
@@ -111,7 +111,7 @@ func (rb *RingBuffer) EnflightTimestamp(timestamp int64) {
|
|||||||
if rb.size < len(rb.buffer) {
|
if rb.size < len(rb.buffer) {
|
||||||
rb.size++
|
rb.size++
|
||||||
} else {
|
} else {
|
||||||
newBuf := newBuffer(2*len(rb.buffer))
|
newBuf := newBuffer(2 * len(rb.buffer))
|
||||||
for i := 0; i < rb.size; i++ {
|
for i := 0; i < rb.size; i++ {
|
||||||
newBuf[i] = rb.buffer[(rb.head+len(rb.buffer)-rb.size+i)%len(rb.buffer)]
|
newBuf[i] = rb.buffer[(rb.head+len(rb.buffer)-rb.size+i)%len(rb.buffer)]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import "sync"
|
|||||||
|
|
||||||
type LocalTopic struct {
|
type LocalTopic struct {
|
||||||
Topic
|
Topic
|
||||||
Partitions []*LocalPartition
|
Partitions []*LocalPartition
|
||||||
partitionLock sync.RWMutex
|
partitionLock sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,11 +9,11 @@ import (
|
|||||||
"io"
|
"io"
|
||||||
"mime"
|
"mime"
|
||||||
"mime/multipart"
|
"mime/multipart"
|
||||||
"sync"
|
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/textproto"
|
"net/textproto"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"strings"
|
"strings"
|
||||||
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
@@ -66,9 +66,9 @@ func (uploadResult *UploadResult) ToPbFileChunk(fileId string, offset int64, tsN
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "")
|
fileNameEscaper = strings.NewReplacer(`\`, `\\`, `"`, `\"`, "\n", "")
|
||||||
uploader *Uploader
|
uploader *Uploader
|
||||||
uploaderErr error
|
uploaderErr error
|
||||||
once sync.Once
|
once sync.Once
|
||||||
)
|
)
|
||||||
|
|
||||||
// HTTPClient interface for testing
|
// HTTPClient interface for testing
|
||||||
@@ -82,7 +82,7 @@ type Uploader struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func NewUploader() (*Uploader, error) {
|
func NewUploader() (*Uploader, error) {
|
||||||
once.Do(func () {
|
once.Do(func() {
|
||||||
// With Dial context
|
// With Dial context
|
||||||
var httpClient *util_http_client.HTTPClient
|
var httpClient *util_http_client.HTTPClient
|
||||||
httpClient, uploaderErr = util_http.NewGlobalHttpClient(util_http_client.AddDialContext)
|
httpClient, uploaderErr = util_http.NewGlobalHttpClient(util_http_client.AddDialContext)
|
||||||
@@ -96,7 +96,7 @@ func NewUploader() (*Uploader, error) {
|
|||||||
return uploader, uploaderErr
|
return uploader, uploaderErr
|
||||||
}
|
}
|
||||||
|
|
||||||
func newUploader(httpClient HTTPClient) (*Uploader) {
|
func newUploader(httpClient HTTPClient) *Uploader {
|
||||||
return &Uploader{
|
return &Uploader{
|
||||||
httpClient: httpClient,
|
httpClient: httpClient,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -69,7 +69,7 @@ func processOneDirectory(filerClient FilerClient, parentPath util.FullPath, queu
|
|||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry)error) (err error) {
|
func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64, fn func(parentPath util.FullPath, entry *Entry) error) (err error) {
|
||||||
glog.V(0).Infof("TraverseBfsMetadata %v if before %v", dir, time.Unix(0, olderThanTsNs))
|
glog.V(0).Infof("TraverseBfsMetadata %v if before %v", dir, time.Unix(0, olderThanTsNs))
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
defer cancel()
|
defer cancel()
|
||||||
@@ -87,7 +87,7 @@ func StreamBfs(client SeaweedFilerClient, dir util.FullPath, olderThanTsNs int64
|
|||||||
}
|
}
|
||||||
return fmt.Errorf("traverse bfs metadata: %v", err)
|
return fmt.Errorf("traverse bfs metadata: %v", err)
|
||||||
}
|
}
|
||||||
if err := fn(util.FullPath(resp.Directory), resp.Entry); err != nil {
|
if err := fn(util.FullPath(resp.Directory), resp.Entry); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -112,7 +112,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry, signatures []
|
|||||||
appendBlobURL := g.containerURL.NewAppendBlobURL(key)
|
appendBlobURL := g.containerURL.NewAppendBlobURL(key)
|
||||||
|
|
||||||
accessCondition := azblob.BlobAccessConditions{}
|
accessCondition := azblob.BlobAccessConditions{}
|
||||||
if entry.Attributes!=nil && entry.Attributes.Mtime>0 {
|
if entry.Attributes != nil && entry.Attributes.Mtime > 0 {
|
||||||
accessCondition.ModifiedAccessConditions.IfUnmodifiedSince = time.Unix(entry.Attributes.Mtime, 0)
|
accessCondition.ModifiedAccessConditions.IfUnmodifiedSince = time.Unix(entry.Attributes.Mtime, 0)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,9 +7,9 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/stats"
|
"github.com/seaweedfs/seaweedfs/weed/stats"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"math"
|
"math"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"sort"
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|||||||
@@ -9,9 +9,9 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
|
||||||
|
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strings"
|
"strings"
|
||||||
util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type AccountManager interface {
|
type AccountManager interface {
|
||||||
|
|||||||
@@ -3,8 +3,10 @@ package s3api
|
|||||||
import (
|
import (
|
||||||
"encoding/xml"
|
"encoding/xml"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht
|
|||||||
PairMap: pu.PairMap,
|
PairMap: pu.PairMap,
|
||||||
Jwt: auth,
|
Jwt: auth,
|
||||||
}
|
}
|
||||||
|
|
||||||
uploader, uploaderErr := operation.NewUploader()
|
uploader, uploaderErr := operation.NewUploader()
|
||||||
if uploaderErr != nil {
|
if uploaderErr != nil {
|
||||||
return nil, fmt.Errorf("uploader initialization error: %v", uploaderErr)
|
return nil, fmt.Errorf("uploader initialization error: %v", uploaderErr)
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||||
|
|||||||
@@ -18,8 +18,8 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
|
"slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
type DataCenterId string
|
type DataCenterId string
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
"github.com/seaweedfs/seaweedfs/weed/wdclient"
|
||||||
"golang.org/x/exp/maps"
|
"golang.org/x/exp/maps"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
||||||
|
|||||||
@@ -20,7 +20,7 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
"go.uber.org/atomic"
|
"go.uber.org/atomic"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||||
|
|||||||
@@ -17,8 +17,8 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/server/constants"
|
"github.com/seaweedfs/seaweedfs/weed/server/constants"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
|
"slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
|||||||
@@ -13,10 +13,8 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
|
||||||
|
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"google.golang.org/grpc"
|
"google.golang.org/grpc"
|
||||||
|
"slices"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/operation"
|
"github.com/seaweedfs/seaweedfs/weed/operation"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
@@ -346,7 +344,7 @@ func (c *commandVolumeFixReplication) fixOneUnderReplicatedVolume(commandEnv *Co
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if resp.ProcessedBytes > 0 {
|
if resp.ProcessedBytes > 0 {
|
||||||
fmt.Fprintf(writer, "volume %d processed %s bytes\n", replica.info.Id, util.BytesToHumanReadable(uint64(resp.ProcessedBytes)))
|
fmt.Fprintf(writer, "volume %d processed %d bytes\n", replica.info.Id, resp.ProcessedBytes)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -7,8 +7,8 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
|||||||
@@ -8,12 +8,12 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
"github.com/seaweedfs/seaweedfs/weed/util/grace"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"io"
|
"io"
|
||||||
"math/rand"
|
"math/rand"
|
||||||
"os"
|
"os"
|
||||||
"path"
|
"path"
|
||||||
"regexp"
|
"regexp"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/peterh/liner"
|
"github.com/peterh/liner"
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
|
||||||
|
|||||||
@@ -5,7 +5,7 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
type Interval struct {
|
type Interval struct {
|
||||||
BlockIndex int // the index of the block in either the large blocks or the small blocks
|
BlockIndex int // the index of the block in either the large blocks or the small blocks
|
||||||
InnerBlockOffset int64
|
InnerBlockOffset int64
|
||||||
Size types.Size
|
Size types.Size
|
||||||
IsLargeBlock bool // whether the block is a large block or a small block
|
IsLargeBlock bool // whether the block is a large block or a small block
|
||||||
@@ -50,7 +50,7 @@ func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, of
|
|||||||
|
|
||||||
func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
|
func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
|
||||||
largeRowSize := largeBlockLength * DataShardsCount
|
largeRowSize := largeBlockLength * DataShardsCount
|
||||||
nLargeBlockRows = (shardDatSize-1)/ largeBlockLength
|
nLargeBlockRows = (shardDatSize - 1) / largeBlockLength
|
||||||
|
|
||||||
// if offset is within the large block area
|
// if offset is within the large block area
|
||||||
if offset < nLargeBlockRows*largeRowSize {
|
if offset < nLargeBlockRows*largeRowSize {
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
|
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package topology
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/util"
|
"github.com/seaweedfs/seaweedfs/weed/util"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ package topology
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
|
||||||
"golang.org/x/exp/slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -5,8 +5,8 @@ import (
|
|||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage"
|
"github.com/seaweedfs/seaweedfs/weed/storage"
|
||||||
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
"github.com/seaweedfs/seaweedfs/weed/storage/types"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"path"
|
"path"
|
||||||
|
"slices"
|
||||||
)
|
)
|
||||||
|
|
||||||
type OnDiskCacheLayer struct {
|
type OnDiskCacheLayer struct {
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import (
|
|||||||
|
|
||||||
var (
|
var (
|
||||||
ConfigurationFileDirectory DirectoryValueType
|
ConfigurationFileDirectory DirectoryValueType
|
||||||
loadSecurityConfigOnce sync.Once
|
loadSecurityConfigOnce sync.Once
|
||||||
)
|
)
|
||||||
|
|
||||||
type DirectoryValueType string
|
type DirectoryValueType string
|
||||||
@@ -32,7 +32,7 @@ type Configuration interface {
|
|||||||
SetDefault(key string, value interface{})
|
SetDefault(key string, value interface{})
|
||||||
}
|
}
|
||||||
|
|
||||||
func LoadSecurityConfiguration(){
|
func LoadSecurityConfiguration() {
|
||||||
loadSecurityConfigOnce.Do(func() {
|
loadSecurityConfigOnce.Do(func() {
|
||||||
LoadConfiguration("security", false)
|
LoadConfiguration("security", false)
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -4,7 +4,7 @@ import (
|
|||||||
"sync"
|
"sync"
|
||||||
)
|
)
|
||||||
|
|
||||||
type node[T any]struct {
|
type node[T any] struct {
|
||||||
data T
|
data T
|
||||||
next *node[T]
|
next *node[T]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,8 +2,8 @@ package skiplist
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/seaweedfs/seaweedfs/weed/glog"
|
"github.com/seaweedfs/seaweedfs/weed/glog"
|
||||||
"golang.org/x/exp/slices"
|
|
||||||
"google.golang.org/protobuf/proto"
|
"google.golang.org/protobuf/proto"
|
||||||
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user