gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' into gentle_vacuum

Browse Source 
# Conflicts:
#	weed/pb/messaging_pb/messaging.pb.go
#	weed/pb/messaging_pb/messaging_grpc.pb.go
#	weed/pb/s3_pb/s3.pb.go
#	weed/pb/volume_server_pb/volume_server.pb.go
#	weed/server/volume_grpc_vacuum.go
This commit is contained in:
Konstantin Lebedev
2022-08-01 14:45:22 +05:00
parent c0d92f61a1 036566629a
commit 3c75479e2b
633 changed files with 6720 additions and 7638 deletions
Download Patch File Download Diff File Expand all files Collapse all files

296
weed/cluster/cluster.go
View File

@@ -1,8 +1,8 @@
package cluster
import (
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"math"
"sync"
"time"
@@ -15,88 +15,156 @@ const (
BrokerType = "broker"
)
type FilerGroup string
type Filers struct {
filers map[pb.ServerAddress]*ClusterNode
leaders *Leaders
}
type FilerGroupName string
type DataCenter string
type Rack string
type Leaders struct {
leaders [3]pb.ServerAddress
}
type ClusterNode struct {
Address pb.ServerAddress
Version string
counter int
CreatedTs time.Time
Address pb.ServerAddress
Version string
counter int
CreatedTs time.Time
DataCenter DataCenter
Rack Rack
}
type GroupMembers struct {
members map[pb.ServerAddress]*ClusterNode
leaders *Leaders
}
type ClusterNodeGroups struct {
groupMembers map[FilerGroupName]*GroupMembers
sync.RWMutex
}
type Cluster struct {
filerGroups *ClusterNodeGroups
brokerGroups *ClusterNodeGroups
}
type Cluster struct {
filerGroup2filers map[FilerGroup]*Filers
filersLock sync.RWMutex
brokers map[pb.ServerAddress]*ClusterNode
brokersLock sync.RWMutex
func newClusterNodeGroups() *ClusterNodeGroups {
return &ClusterNodeGroups{
groupMembers: map[FilerGroupName]*GroupMembers{},
}
}
func (g *ClusterNodeGroups) getGroupMembers(filerGroup FilerGroupName, createIfNotFound bool) *GroupMembers {
members, found := g.groupMembers[filerGroup]
if !found && createIfNotFound {
members = &GroupMembers{
members: make(map[pb.ServerAddress]*ClusterNode),
leaders: &Leaders{},
}
g.groupMembers[filerGroup] = members
}
return members
}
func (m *GroupMembers) addMember(dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) *ClusterNode {
if existingNode, found := m.members[address]; found {
existingNode.counter++
return nil
}
t := &ClusterNode{
Address: address,
Version: version,
counter: 1,
CreatedTs: time.Now(),
DataCenter: dataCenter,
Rack: rack,
}
m.members[address] = t
return t
}
func (m *GroupMembers) removeMember(address pb.ServerAddress) bool {
if existingNode, found := m.members[address]; !found {
return false
} else {
existingNode.counter--
if existingNode.counter <= 0 {
delete(m.members, address)
return true
}
}
return false
}
func (g *ClusterNodeGroups) AddClusterNode(filerGroup FilerGroupName, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, true)
if t := m.addMember(dataCenter, rack, address, version); t != nil {
return ensureGroupLeaders(m, true, filerGroup, nodeType, address)
}
return nil
}
func (g *ClusterNodeGroups) RemoveClusterNode(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
if m.removeMember(address) {
return ensureGroupLeaders(m, false, filerGroup, nodeType, address)
}
return nil
}
func (g *ClusterNodeGroups) ListClusterNode(filerGroup FilerGroupName) (nodes []*ClusterNode) {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
for _, node := range m.members {
nodes = append(nodes, node)
}
return
}
func (g *ClusterNodeGroups) IsOneLeader(filerGroup FilerGroupName, address pb.ServerAddress) bool {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return false
}
return m.leaders.isOneLeader(address)
}
func (g *ClusterNodeGroups) ListClusterNodeLeaders(filerGroup FilerGroupName) (nodes []pb.ServerAddress) {
g.Lock()
defer g.Unlock()
m := g.getGroupMembers(filerGroup, false)
if m == nil {
return nil
}
return m.leaders.GetLeaders()
}
func NewCluster() *Cluster {
return &Cluster{
filerGroup2filers: make(map[FilerGroup]*Filers),
brokers: make(map[pb.ServerAddress]*ClusterNode),
filerGroups: newClusterNodeGroups(),
brokerGroups: newClusterNodeGroups(),
}
}
func (cluster *Cluster) getFilers(filerGroup FilerGroup, createIfNotFound bool) *Filers {
filers, found := cluster.filerGroup2filers[filerGroup]
if !found && createIfNotFound {
filers = &Filers{
filers: make(map[pb.ServerAddress]*ClusterNode),
leaders: &Leaders{},
}
cluster.filerGroup2filers[filerGroup] = filers
}
return filers
}
func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroup(ns)
func (cluster *Cluster) getGroupMembers(filerGroup FilerGroupName, nodeType string, createIfNotFound bool) *GroupMembers {
switch nodeType {
case FilerType:
cluster.filersLock.Lock()
defer cluster.filersLock.Unlock()
filers := cluster.getFilers(filerGroup, true)
if existingNode, found := filers.filers[address]; found {
existingNode.counter++
return nil
}
filers.filers[address] = &ClusterNode{
Address: address,
Version: version,
counter: 1,
CreatedTs: time.Now(),
}
return cluster.ensureFilerLeaders(filers, true, filerGroup, nodeType, address)
return cluster.filerGroups.getGroupMembers(filerGroup, createIfNotFound)
case BrokerType:
cluster.brokersLock.Lock()
defer cluster.brokersLock.Unlock()
if existingNode, found := cluster.brokers[address]; found {
existingNode.counter++
return nil
}
cluster.brokers[address] = &ClusterNode{
Address: address,
Version: version,
counter: 1,
CreatedTs: time.Now(),
}
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: true,
},
},
}
return cluster.brokerGroups.getGroupMembers(filerGroup, createIfNotFound)
}
return nil
}
func (cluster *Cluster) AddClusterNode(ns, nodeType string, dataCenter DataCenter, rack Rack, address pb.ServerAddress, version string) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroupName(ns)
switch nodeType {
case FilerType:
return cluster.filerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
case BrokerType:
return cluster.brokerGroups.AddClusterNode(filerGroup, nodeType, dataCenter, rack, address, version)
case MasterType:
return []*master_pb.KeepConnectedResponse{
{
@@ -112,44 +180,12 @@ func (cluster *Cluster) AddClusterNode(ns, nodeType string, address pb.ServerAdd
}
func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb.ServerAddress) []*master_pb.KeepConnectedResponse {
filerGroup := FilerGroup(ns)
filerGroup := FilerGroupName(ns)
switch nodeType {
case FilerType:
cluster.filersLock.Lock()
defer cluster.filersLock.Unlock()
filers := cluster.getFilers(filerGroup, false)
if filers == nil {
return nil
}
if existingNode, found := filers.filers[address]; !found {
return nil
} else {
existingNode.counter--
if existingNode.counter <= 0 {
delete(filers.filers, address)
return cluster.ensureFilerLeaders(filers, false, filerGroup, nodeType, address)
}
}
return cluster.filerGroups.RemoveClusterNode(filerGroup, nodeType, address)
case BrokerType:
cluster.brokersLock.Lock()
defer cluster.brokersLock.Unlock()
if existingNode, found := cluster.brokers[address]; !found {
return nil
} else {
existingNode.counter--
if existingNode.counter <= 0 {
delete(cluster.brokers, address)
return []*master_pb.KeepConnectedResponse{
{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
NodeType: nodeType,
Address: string(address),
IsAdd: false,
},
},
}
}
}
return cluster.brokerGroups.RemoveClusterNode(filerGroup, nodeType, address)
case MasterType:
return []*master_pb.KeepConnectedResponse{
{
@@ -164,40 +200,42 @@ func (cluster *Cluster) RemoveClusterNode(ns string, nodeType string, address pb
return nil
}
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroup, nodeType string) (nodes []*ClusterNode) {
func (cluster *Cluster) ListClusterNode(filerGroup FilerGroupName, nodeType string) (nodes []*ClusterNode) {
switch nodeType {
case FilerType:
cluster.filersLock.RLock()
defer cluster.filersLock.RUnlock()
filers := cluster.getFilers(filerGroup, false)
if filers == nil {
return
}
for _, node := range filers.filers {
nodes = append(nodes, node)
}
return cluster.filerGroups.ListClusterNode(filerGroup)
case BrokerType:
cluster.brokersLock.RLock()
defer cluster.brokersLock.RUnlock()
for _, node := range cluster.brokers {
nodes = append(nodes, node)
}
return cluster.brokerGroups.ListClusterNode(filerGroup)
case MasterType:
}
return
}
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroup, address pb.ServerAddress) bool {
filers := cluster.getFilers(filerGroup, false)
if filers == nil {
return false
func (cluster *Cluster) ListClusterNodeLeaders(filerGroup FilerGroupName, nodeType string) (nodes []pb.ServerAddress) {
switch nodeType {
case FilerType:
return cluster.filerGroups.ListClusterNodeLeaders(filerGroup)
case BrokerType:
return cluster.brokerGroups.ListClusterNodeLeaders(filerGroup)
case MasterType:
}
return filers.leaders.isOneLeader(address)
return
}
func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGroup FilerGroup, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
func (cluster *Cluster) IsOneLeader(filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) bool {
switch nodeType {
case FilerType:
return cluster.filerGroups.IsOneLeader(filerGroup, address)
case BrokerType:
return cluster.brokerGroups.IsOneLeader(filerGroup, address)
case MasterType:
}
return false
}
func ensureGroupLeaders(m *GroupMembers, isAdd bool, filerGroup FilerGroupName, nodeType string, address pb.ServerAddress) (result []*master_pb.KeepConnectedResponse) {
if isAdd {
if filers.leaders.addLeaderIfVacant(address) {
if m.leaders.addLeaderIfVacant(address) {
// has added the address as one leader
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
@@ -220,7 +258,7 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
})
}
} else {
if filers.leaders.removeLeaderIfExists(address) {
if m.leaders.removeLeaderIfExists(address) {
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{
@@ -236,8 +274,8 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
var shortestDuration int64 = math.MaxInt64
now := time.Now()
var candidateAddress pb.ServerAddress
for _, node := range filers.filers {
if filers.leaders.isOneLeader(node.Address) {
for _, node := range m.members {
if m.leaders.isOneLeader(node.Address) {
continue
}
duration := now.Sub(node.CreatedTs).Nanoseconds()
@@ -247,7 +285,7 @@ func (cluster *Cluster) ensureFilerLeaders(filers *Filers, isAdd bool, filerGrou
}
}
if candidateAddress != "" {
filers.leaders.addLeaderIfVacant(candidateAddress)
m.leaders.addLeaderIfVacant(candidateAddress)
// added a new leader
result = append(result, &master_pb.KeepConnectedResponse{
ClusterNodeUpdate: &master_pb.ClusterNodeUpdate{

24
weed/cluster/cluster_test.go
View File

@@ -1,7 +1,7 @@
package cluster
import (
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/stretchr/testify/assert"
"strconv"
"sync"
@@ -11,29 +11,29 @@ import (
func TestClusterAddRemoveNodes(t *testing.T) {
c := NewCluster()
c.AddClusterNode("", "filer", pb.ServerAddress("111:1"), "23.45")
c.AddClusterNode("", "filer", pb.ServerAddress("111:2"), "23.45")
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:1"), "23.45")
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:2"), "23.45")
assert.Equal(t, []pb.ServerAddress{
pb.ServerAddress("111:1"),
pb.ServerAddress("111:2"),
}, c.getFilers("", false).leaders.GetLeaders())
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
c.AddClusterNode("", "filer", pb.ServerAddress("111:3"), "23.45")
c.AddClusterNode("", "filer", pb.ServerAddress("111:4"), "23.45")
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:3"), "23.45")
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:4"), "23.45")
assert.Equal(t, []pb.ServerAddress{
pb.ServerAddress("111:1"),
pb.ServerAddress("111:2"),
pb.ServerAddress("111:3"),
}, c.getFilers("", false).leaders.GetLeaders())
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
c.AddClusterNode("", "filer", pb.ServerAddress("111:5"), "23.45")
c.AddClusterNode("", "filer", pb.ServerAddress("111:6"), "23.45")
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:5"), "23.45")
c.AddClusterNode("", "filer", "", "", pb.ServerAddress("111:6"), "23.45")
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:4"))
assert.Equal(t, []pb.ServerAddress{
pb.ServerAddress("111:1"),
pb.ServerAddress("111:2"),
pb.ServerAddress("111:3"),
}, c.getFilers("", false).leaders.GetLeaders())
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
// remove oldest
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
@@ -41,7 +41,7 @@ func TestClusterAddRemoveNodes(t *testing.T) {
pb.ServerAddress("111:6"),
pb.ServerAddress("111:2"),
pb.ServerAddress("111:3"),
}, c.getFilers("", false).leaders.GetLeaders())
}, c.getGroupMembers("", "filer", true).leaders.GetLeaders())
// remove oldest
c.RemoveClusterNode("", "filer", pb.ServerAddress("111:1"))
@@ -56,7 +56,7 @@ func TestConcurrentAddRemoveNodes(t *testing.T) {
go func(i int) {
defer wg.Done()
address := strconv.Itoa(i)
c.AddClusterNode("", "filer", pb.ServerAddress(address), "23.45")
c.AddClusterNode("", "filer", "", "", pb.ServerAddress(address), "23.45")
}(i)
}
wg.Wait()

34
weed/cluster/master_client.go Normal file
View File

@@ -0,0 +1,34 @@
package cluster
import (
"context"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"google.golang.org/grpc"
)
func ListExistingPeerUpdates(master pb.ServerAddress, grpcDialOption grpc.DialOption, filerGroup string, clientType string) (existingNodes []*master_pb.ClusterNodeUpdate) {
if grpcErr := pb.WithMasterClient(false, master, grpcDialOption, func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
ClientType: clientType,
FilerGroup: filerGroup,
})
glog.V(0).Infof("the cluster has %d %s\n", len(resp.ClusterNodes), clientType)
for _, node := range resp.ClusterNodes {
existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
NodeType: FilerType,
Address: node.Address,
IsLeader: node.IsLeader,
IsAdd: true,
CreatedAtNs: node.CreatedAtNs,
})
}
return err
}); grpcErr != nil {
glog.V(0).Infof("connect to %s: %v", master, grpcErr)
}
return
}

2
weed/command/autocomplete.go
View File

@@ -2,9 +2,9 @@ package command
import (
"fmt"
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
"github.com/posener/complete"
completeinstall "github.com/posener/complete/cmd/install"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
"runtime"
)

14
weed/command/backup.go
View File

@@ -2,15 +2,15 @@ package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/storage"
)
var (

14
weed/command/benchmark.go
View File

@@ -3,7 +3,7 @@ package command
import (
"bufio"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"io"
"math"
"math/rand"
@@ -16,11 +16,11 @@ import (
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
)
type BenchmarkOptions struct {
@@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
defer pprof.StopCPUProfile()
}
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "", "client", "", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
go b.masterClient.KeepConnectedToMaster()
b.masterClient.WaitUntilConnected()

6
weed/command/command.go
View File

@@ -5,7 +5,7 @@ import (
"os"
"strings"
flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
flag "github.com/seaweedfs/seaweedfs/weed/util/fla9"
)
var Commands = []*Command{
@@ -28,12 +28,12 @@ var Commands = []*Command{
cmdFilerSynchronize,
cmdFix,
cmdFuse,
cmdIam,
cmdMaster,
cmdMasterFollower,
cmdMount,
cmdMqBroker,
cmdS3,
cmdIam,
cmdMsgBroker,
cmdScaffold,
cmdServer,
cmdShell,

8
weed/command/compact.go
View File

@@ -1,10 +1,10 @@
package command
import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {

8
weed/command/download.go
View File

@@ -10,10 +10,10 @@ import (
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

14
weed/command/export.go
View File

@@ -13,13 +13,13 @@ import (
"text/template"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (

16
weed/command/filer.go
View File

@@ -12,14 +12,14 @@ import (
"google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

10
weed/command/filer_backup.go
View File

@@ -2,11 +2,11 @@ package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"strings"
"time"

12
weed/command/filer_cat.go
View File

@@ -3,17 +3,17 @@ package command
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
"net/url"
"os"
"strings"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

18
weed/command/filer_copy.go
View File

@@ -14,15 +14,15 @@ import (
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
)
var (

12
weed/command/filer_meta_backup.go
View File

@@ -3,17 +3,17 @@ package command
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/spf13/viper"
"google.golang.org/grpc"
"reflect"
"time"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

8
weed/command/filer_meta_tail.go
View File

@@ -2,16 +2,16 @@ package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/golang/protobuf/jsonpb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"os"
"path/filepath"
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {

4
weed/command/filer_meta_tail_elastic.go
View File

@@ -5,10 +5,10 @@ package command
import (
"context"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
jsoniter "github.com/json-iterator/go"
elastic "github.com/olivere/elastic/v7"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"strings"
)

2
weed/command/filer_meta_tail_non_elastic.go
View File

@@ -4,7 +4,7 @@
package command
import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func sendToElasticSearchFunc(servers string, esIndex string) (func(resp *filer_pb.SubscribeMetadataResponse) error, error) {

14
weed/command/filer_remote_gateway.go
View File

@@ -3,13 +3,13 @@ package command
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"os"
"time"

18
weed/command/filer_remote_gateway_buckets.go
View File

@@ -2,16 +2,16 @@ package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
"github.com/chrislusf/seaweedfs/weed/remote_storage"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
"github.com/seaweedfs/seaweedfs/weed/util"
"math"
"math/rand"
"path/filepath"

12
weed/command/filer_remote_sync.go
View File

@@ -2,12 +2,12 @@ package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
"time"
)

16
weed/command/filer_remote_sync_dir.go
View File

@@ -7,15 +7,15 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
"github.com/chrislusf/seaweedfs/weed/remote_storage"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/remote_pb"
"github.com/seaweedfs/seaweedfs/weed/remote_storage"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc"
)

10
weed/command/filer_replication.go
View File

@@ -4,11 +4,11 @@ import (
"context"
"strings"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/replication"
"github.com/chrislusf/seaweedfs/weed/replication/sink"
"github.com/chrislusf/seaweedfs/weed/replication/sub"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/replication"
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/seaweedfs/seaweedfs/weed/replication/sub"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {

22
weed/command/filer_sync.go
View File

@@ -4,17 +4,17 @@ import (
"context"
"errors"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/replication"
"github.com/chrislusf/seaweedfs/weed/replication/sink"
"github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security"
statsCollect "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/replication"
"github.com/seaweedfs/seaweedfs/weed/replication/sink"
"github.com/seaweedfs/seaweedfs/weed/replication/sink/filersink"
"github.com/seaweedfs/seaweedfs/weed/replication/source"
"github.com/seaweedfs/seaweedfs/weed/security"
statsCollect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"google.golang.org/grpc"
"os"
"strings"

14
weed/command/fix.go
View File

@@ -8,13 +8,13 @@ import (
"strconv"
"strings"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
"github.com/chrislusf/seaweedfs/weed/storage/super_block"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/needle_map"
"github.com/seaweedfs/seaweedfs/weed/storage/super_block"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {

12
weed/command/iam.go
View File

@@ -5,13 +5,13 @@ import (
"fmt"
"net/http"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/iamapi"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/iamapi"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
"time"
)

56
weed/command/imports.go
View File

@@ -3,34 +3,34 @@ package command
import (
_ "net/http/pprof"
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/azure"
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/gcs"
_ "github.com/chrislusf/seaweedfs/weed/remote_storage/s3"
_ "github.com/seaweedfs/seaweedfs/weed/remote_storage/azure"
_ "github.com/seaweedfs/seaweedfs/weed/remote_storage/gcs"
_ "github.com/seaweedfs/seaweedfs/weed/remote_storage/s3"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/azuresink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/b2sink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/gcssink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/localsink"
_ "github.com/chrislusf/seaweedfs/weed/replication/sink/s3sink"
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/azuresink"
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/b2sink"
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/filersink"
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/gcssink"
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/localsink"
_ "github.com/seaweedfs/seaweedfs/weed/replication/sink/s3sink"
_ "github.com/chrislusf/seaweedfs/weed/filer/arangodb"
_ "github.com/chrislusf/seaweedfs/weed/filer/cassandra"
_ "github.com/chrislusf/seaweedfs/weed/filer/elastic/v7"
_ "github.com/chrislusf/seaweedfs/weed/filer/etcd"
_ "github.com/chrislusf/seaweedfs/weed/filer/hbase"
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb"
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb2"
_ "github.com/chrislusf/seaweedfs/weed/filer/leveldb3"
_ "github.com/chrislusf/seaweedfs/weed/filer/mongodb"
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql"
_ "github.com/chrislusf/seaweedfs/weed/filer/mysql2"
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres"
_ "github.com/chrislusf/seaweedfs/weed/filer/postgres2"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
_ "github.com/chrislusf/seaweedfs/weed/filer/redis3"
_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
_ "github.com/chrislusf/seaweedfs/weed/filer/tikv"
_ "github.com/chrislusf/seaweedfs/weed/filer/ydb"
_ "github.com/seaweedfs/seaweedfs/weed/filer/arangodb"
_ "github.com/seaweedfs/seaweedfs/weed/filer/cassandra"
_ "github.com/seaweedfs/seaweedfs/weed/filer/elastic/v7"
_ "github.com/seaweedfs/seaweedfs/weed/filer/etcd"
_ "github.com/seaweedfs/seaweedfs/weed/filer/hbase"
_ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb"
_ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb2"
_ "github.com/seaweedfs/seaweedfs/weed/filer/leveldb3"
_ "github.com/seaweedfs/seaweedfs/weed/filer/mongodb"
_ "github.com/seaweedfs/seaweedfs/weed/filer/mysql"
_ "github.com/seaweedfs/seaweedfs/weed/filer/mysql2"
_ "github.com/seaweedfs/seaweedfs/weed/filer/postgres"
_ "github.com/seaweedfs/seaweedfs/weed/filer/postgres2"
_ "github.com/seaweedfs/seaweedfs/weed/filer/redis"
_ "github.com/seaweedfs/seaweedfs/weed/filer/redis2"
_ "github.com/seaweedfs/seaweedfs/weed/filer/redis3"
_ "github.com/seaweedfs/seaweedfs/weed/filer/sqlite"
_ "github.com/seaweedfs/seaweedfs/weed/filer/tikv"
_ "github.com/seaweedfs/seaweedfs/weed/filer/ydb"
)

25
weed/command/master.go
View File

@@ -2,28 +2,30 @@ package command
import (
"fmt"
"golang.org/x/exp/slices"
"net/http"
"os"
"path"
"strings"
"time"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
"golang.org/x/exp/slices"
"github.com/gorilla/mux"
"github.com/seaweedfs/raft/protobuf"
"github.com/spf13/viper"
"google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/util/grace"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/security"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/storage/backend"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/storage/backend"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (
@@ -174,11 +176,12 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
} else {
raftServer, err = weed_server.NewRaftServer(raftServerOption)
if raftServer == nil {
glog.Fatalf("please verify %s is writable, see https://github.com/chrislusf/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
glog.Fatalf("please verify %s is writable, see https://github.com/seaweedfs/seaweedfs/issues/717: %s", *masterOption.metaFolder, err)
}
}
ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
if *m.raftHashicorp {
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
}

12
weed/command/master_follower.go
View File

@@ -7,13 +7,13 @@ import (
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/security"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/gorilla/mux"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc/reflection"
)

22
weed/command/mount_std.go
View File

@@ -6,16 +6,16 @@ package command
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/mount"
"github.com/chrislusf/seaweedfs/weed/mount/meta_cache"
"github.com/chrislusf/seaweedfs/weed/mount/unmount"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/mount_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mount"
"github.com/seaweedfs/seaweedfs/weed/mount/meta_cache"
"github.com/seaweedfs/seaweedfs/weed/mount/unmount"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mount_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"google.golang.org/grpc/reflection"
"net"
"net/http"
@@ -26,8 +26,8 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
)
func runMount(cmd *Command, args []string) bool {

94
weed/command/mq_broker.go Normal file
View File

@@ -0,0 +1,94 @@
package command
import (
"google.golang.org/grpc/reflection"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mq/broker"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (
mqBrokerStandaloneOptions MessageQueueBrokerOptions
)
type MessageQueueBrokerOptions struct {
masters map[string]pb.ServerAddress
mastersString *string
filerGroup *string
ip *string
port *int
dataCenter *string
rack *string
cpuprofile *string
memprofile *string
}
func init() {
cmdMqBroker.Run = runMqBroker // break init cycle
mqBrokerStandaloneOptions.mastersString = cmdMqBroker.Flag.String("master", "localhost:9333", "comma-separated master servers")
mqBrokerStandaloneOptions.filerGroup = cmdMqBroker.Flag.String("filerGroup", "", "share metadata with other filers in the same filerGroup")
mqBrokerStandaloneOptions.ip = cmdMqBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
mqBrokerStandaloneOptions.port = cmdMqBroker.Flag.Int("port", 17777, "broker gRPC listen port")
mqBrokerStandaloneOptions.dataCenter = cmdMqBroker.Flag.String("dataCenter", "", "prefer to read and write to volumes in this data center")
mqBrokerStandaloneOptions.rack = cmdMqBroker.Flag.String("rack", "", "prefer to write to volumes in this rack")
mqBrokerStandaloneOptions.cpuprofile = cmdMqBroker.Flag.String("cpuprofile", "", "cpu profile output file")
mqBrokerStandaloneOptions.memprofile = cmdMqBroker.Flag.String("memprofile", "", "memory profile output file")
}
var cmdMqBroker = &Command{
UsageLine: "mq.broker [-port=17777] [-master=<ip:port>]",
Short: "<WIP> start a message queue broker",
Long: `start a message queue broker
The broker can accept gRPC calls to write or read messages. The messages are stored via filer.
The brokers are stateless. To scale up, just add more brokers.
`,
}
func runMqBroker(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
mqBrokerStandaloneOptions.masters = pb.ServerAddresses(*mqBrokerStandaloneOptions.mastersString).ToAddressMap()
return mqBrokerStandaloneOptions.startQueueServer()
}
func (mqBrokerOpt *MessageQueueBrokerOptions) startQueueServer() bool {
grace.SetupProfiling(*mqBrokerStandaloneOptions.cpuprofile, *mqBrokerStandaloneOptions.memprofile)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker")
qs, err := broker.NewMessageBroker(&broker.MessageQueueBrokerOption{
Masters: mqBrokerOpt.masters,
FilerGroup: *mqBrokerOpt.filerGroup,
DataCenter: *mqBrokerOpt.dataCenter,
Rack: *mqBrokerOpt.rack,
DefaultReplication: "",
MaxMB: 0,
Ip: *mqBrokerOpt.ip,
Port: *mqBrokerOpt.port,
}, grpcDialOption)
// start grpc listener
grpcL, _, err := util.NewIpAndLocalListeners("", *mqBrokerOpt.port, 0)
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", *mqBrokerOpt.port, err)
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
mq_pb.RegisterSeaweedMessagingServer(grpcS, qs)
reflection.Register(grpcS)
grpcS.Serve(grpcL)
return true
}

109
weed/command/msg_broker.go
View File

@@ -1,109 +0,0 @@
package command
import (
"context"
"fmt"
"time"
"google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/messaging/broker"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
)
var (
messageBrokerStandaloneOptions MessageBrokerOptions
)
type MessageBrokerOptions struct {
filer *string
ip *string
port *int
cpuprofile *string
memprofile *string
}
func init() {
cmdMsgBroker.Run = runMsgBroker // break init cycle
messageBrokerStandaloneOptions.filer = cmdMsgBroker.Flag.String("filer", "localhost:8888", "filer server address")
messageBrokerStandaloneOptions.ip = cmdMsgBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address")
messageBrokerStandaloneOptions.port = cmdMsgBroker.Flag.Int("port", 17777, "broker gRPC listen port")
messageBrokerStandaloneOptions.cpuprofile = cmdMsgBroker.Flag.String("cpuprofile", "", "cpu profile output file")
messageBrokerStandaloneOptions.memprofile = cmdMsgBroker.Flag.String("memprofile", "", "memory profile output file")
}
var cmdMsgBroker = &Command{
UsageLine: "msgBroker [-port=17777] [-filer=<ip:port>]",
Short: "start a message queue broker",
Long: `start a message queue broker
The broker can accept gRPC calls to write or read messages. The messages are stored via filer.
The brokers are stateless. To scale up, just add more brokers.
`,
}
func runMsgBroker(cmd *Command, args []string) bool {
util.LoadConfiguration("security", false)
return messageBrokerStandaloneOptions.startQueueServer()
}
func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool {
grace.SetupProfiling(*messageBrokerStandaloneOptions.cpuprofile, *messageBrokerStandaloneOptions.memprofile)
filerAddress := pb.ServerAddress(*msgBrokerOpt.filer)
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker")
cipher := false
for {
err := pb.WithGrpcFilerClient(false, filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
if err != nil {
return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
}
cipher = resp.Cipher
return nil
})
if err != nil {
glog.V(0).Infof("wait to connect to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress())
time.Sleep(time.Second)
} else {
glog.V(0).Infof("connected to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress())
break
}
}
qs, err := broker.NewMessageBroker(&broker.MessageBrokerOption{
Filers: []pb.ServerAddress{filerAddress},
DefaultReplication: "",
MaxMB: 0,
Ip: *msgBrokerOpt.ip,
Port: *msgBrokerOpt.port,
Cipher: cipher,
}, grpcDialOption)
// start grpc listener
grpcL, _, err := util.NewIpAndLocalListeners("", *msgBrokerOpt.port, 0)
if err != nil {
glog.Fatalf("failed to listen on grpc port %d: %v", *msgBrokerOpt.port, err)
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
messaging_pb.RegisterSeaweedMessagingServer(grpcS, qs)
reflection.Register(grpcS)
grpcS.Serve(grpcL)
return true
}

18
weed/command/s3.go
View File

@@ -3,22 +3,22 @@ package command
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"github.com/seaweedfs/seaweedfs/weed/s3api/s3err"
"google.golang.org/grpc/reflection"
"net/http"
"time"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/s3_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/gorilla/mux"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/s3api"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/s3api"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

4
weed/command/scaffold.go
View File

@@ -2,10 +2,10 @@ package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util"
"path/filepath"
"github.com/chrislusf/seaweedfs/weed/command/scaffold"
"github.com/seaweedfs/seaweedfs/weed/command/scaffold"
)
func init() {

42
weed/command/server.go
View File

@@ -7,12 +7,12 @@ import (
"strings"
"time"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
)
type ServerOptions struct {
@@ -24,13 +24,13 @@ type ServerOptions struct {
}
var (
serverOptions ServerOptions
masterOptions MasterOptions
filerOptions FilerOptions
s3Options S3Options
iamOptions IamOptions
webdavOptions WebDavOption
msgBrokerOptions MessageBrokerOptions
serverOptions ServerOptions
masterOptions MasterOptions
filerOptions FilerOptions
s3Options S3Options
iamOptions IamOptions
webdavOptions WebDavOption
mqBrokerOptions MessageQueueBrokerOptions
)
func init() {
@@ -74,7 +74,7 @@ var (
isStartingS3 = cmdServer.Flag.Bool("s3", false, "whether to start S3 gateway")
isStartingIam = cmdServer.Flag.Bool("iam", false, "whether to start IAM service")
isStartingWebDav = cmdServer.Flag.Bool("webdav", false, "whether to start WebDAV gateway")
isStartingMsgBroker = cmdServer.Flag.Bool("msgBroker", false, "whether to start message broker")
isStartingMqBroker = cmdServer.Flag.Bool("mq.broker", false, "whether to start message queue broker")
serverWhiteList []string
@@ -155,7 +155,7 @@ func init() {
webdavOptions.cacheDir = cmdServer.Flag.String("webdav.cacheDir", os.TempDir(), "local cache directory for file chunks")
webdavOptions.cacheSizeMB = cmdServer.Flag.Int64("webdav.cacheCapacityMB", 0, "local cache capacity in MB")
msgBrokerOptions.port = cmdServer.Flag.Int("msgBroker.port", 17777, "broker gRPC listen port")
mqBrokerOptions.port = cmdServer.Flag.Int("mq.broker.port", 17777, "message queue broker gRPC listen port")
}
@@ -179,7 +179,7 @@ func runServer(cmd *Command, args []string) bool {
if *isStartingWebDav {
*isStartingFiler = true
}
if *isStartingMsgBroker {
if *isStartingMqBroker {
*isStartingFiler = true
}
@@ -208,7 +208,9 @@ func runServer(cmd *Command, args []string) bool {
serverOptions.v.idleConnectionTimeout = serverTimeout
serverOptions.v.dataCenter = serverDataCenter
serverOptions.v.rack = serverRack
msgBrokerOptions.ip = serverIp
mqBrokerOptions.ip = serverIp
mqBrokerOptions.masters = filerOptions.masters
mqBrokerOptions.filerGroup = filerOptions.filerGroup
// serverOptions.v.pulseSeconds = pulseSeconds
// masterOptions.pulseSeconds = pulseSeconds
@@ -217,6 +219,8 @@ func runServer(cmd *Command, args []string) bool {
filerOptions.dataCenter = serverDataCenter
filerOptions.rack = serverRack
mqBrokerOptions.dataCenter = serverDataCenter
mqBrokerOptions.rack = serverRack
filerOptions.disableHttp = serverDisableHttp
masterOptions.disableHttp = serverDisableHttp
@@ -224,7 +228,7 @@ func runServer(cmd *Command, args []string) bool {
s3Options.filer = &filerAddress
iamOptions.filer = &filerAddress
webdavOptions.filer = &filerAddress
msgBrokerOptions.filer = &filerAddress
mqBrokerOptions.filerGroup = filerOptions.filerGroup
go stats_collect.StartMetricsServer(*serverMetricsHttpPort)
@@ -276,10 +280,10 @@ func runServer(cmd *Command, args []string) bool {
}()
}
if *isStartingMsgBroker {
if *isStartingMqBroker {
go func() {
time.Sleep(2 * time.Second)
msgBrokerOptions.startQueueServer()
mqBrokerOptions.startQueueServer()
}()
}

8
weed/command/shell.go
View File

@@ -2,11 +2,11 @@ package command
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/shell"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/shell"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

10
weed/command/update.go
View File

@@ -19,8 +19,8 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/util"
"golang.org/x/net/context/ctxhttp"
)
@@ -76,8 +76,8 @@ func init() {
var cmdUpdate = &Command{
UsageLine: "update [-dir=/path/to/dir] [-name=name] [-version=x.xx]",
Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs",
Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`,
Short: "get latest or specific version from https://github.com/seaweedfs/seaweedfs",
Long: `get latest or specific version from https://github.com/seaweedfs/seaweedfs`,
}
func runUpdate(cmd *Command, args []string) bool {
@@ -118,7 +118,7 @@ func runUpdate(cmd *Command, args []string) bool {
func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) {
currentVersion := util.VERSION_NUMBER
rel, err := GitHubLatestRelease(ctx, ver, "chrislusf", "seaweedfs")
rel, err := GitHubLatestRelease(ctx, ver, "seaweedfs", "seaweedfs")
if err != nil {
return "", err
}

10
weed/command/upload.go
View File

@@ -4,15 +4,15 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"google.golang.org/grpc"
"os"
"path/filepath"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

2
weed/command/version.go
View File

@@ -4,7 +4,7 @@ import (
"fmt"
"runtime"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var cmdVersion = &Command{

22
weed/command/volume.go
View File

@@ -10,25 +10,25 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"github.com/spf13/viper"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"github.com/seaweedfs/seaweedfs/weed/util/grace"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/util/httpdown"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/util/httpdown"
"google.golang.org/grpc/reflection"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
weed_server "github.com/seaweedfs/seaweedfs/weed/server"
stats_collect "github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

2
weed/command/volume_test.go
View File

@@ -5,7 +5,7 @@ import (
"testing"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func TestXYZ(t *testing.T) {

12
weed/command/webdav.go
View File

@@ -9,12 +9,12 @@ import (
"strconv"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/security"
"github.com/seaweedfs/seaweedfs/weed/server"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (

8
weed/filer/abstract_sql/abstract_sql_store.go
View File

@@ -4,10 +4,10 @@ import (
"context"
"database/sql"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"strings"
"sync"
)

6
weed/filer/abstract_sql/abstract_sql_store_kv.go
View File

@@ -5,9 +5,9 @@ import (
"database/sql"
"encoding/base64"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/util"
"strings"
)

8
weed/filer/arangodb/arangodb_store.go
View File

@@ -11,10 +11,10 @@ import (
"github.com/arangodb/go-driver"
"github.com/arangodb/go-driver/http"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {

4
weed/filer/arangodb/arangodb_store_bucket.go
View File

@@ -5,9 +5,9 @@ import (
"github.com/arangodb/go-driver"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
var _ filer.BucketAware = (*ArangodbStore)(nil)

4
weed/filer/arangodb/arangodb_store_kv.go
View File

@@ -5,8 +5,8 @@ import (
"fmt"
"github.com/arangodb/go-driver"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func (store *ArangodbStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {

2
weed/filer/arangodb/helpers.go
View File

@@ -9,7 +9,7 @@ import (
"strings"
"github.com/arangodb/go-driver"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util"
)
//convert a string into arango-key safe hex bytes hash

8
weed/filer/cassandra/cassandra_store.go
View File

@@ -6,10 +6,10 @@ import (
"github.com/gocql/gocql"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {

2
weed/filer/cassandra/cassandra_store_kv.go
View File

@@ -4,8 +4,8 @@ import (
"context"
"encoding/base64"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/gocql/gocql"
"github.com/seaweedfs/seaweedfs/weed/filer"
)
func (store *CassandraStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {

4
weed/filer/configuration.go
View File

@@ -1,8 +1,8 @@
package filer
import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/util"
"os"
"reflect"
"strings"

8
weed/filer/elastic/v7/elastic_store.go
View File

@@ -9,12 +9,12 @@ import (
"math"
"strings"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
jsoniter "github.com/json-iterator/go"
elastic "github.com/olivere/elastic/v7"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/seaweedfs/seaweedfs/weed/util"
)
var (

4
weed/filer/elastic/v7/elastic_store_kv.go
View File

@@ -7,11 +7,11 @@ import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
jsoniter "github.com/json-iterator/go"
elastic "github.com/olivere/elastic/v7"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func (store *ElasticStore) KvDelete(ctx context.Context, key []byte) (err error) {

4
weed/filer/entry.go
View File

@@ -4,8 +4,8 @@ import (
"os"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
type Attr struct {

2
weed/filer/entry_codec.go
View File

@@ -8,7 +8,7 @@ import (
"github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {

8
weed/filer/etcd/etcd_store.go
View File

@@ -9,10 +9,10 @@ import (
"go.etcd.io/etcd/client/v3"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/seaweedfs/seaweedfs/weed/util"
)
const (

2
weed/filer/etcd/etcd_store_kv.go
View File

@@ -3,7 +3,7 @@ package etcd
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
)
func (store *EtcdStore) KvPut(ctx context.Context, key []byte, value []byte) (err error) {

2
weed/filer/etcd/etcd_store_test.go
View File

@@ -1,7 +1,7 @@
package etcd
import (
"github.com/chrislusf/seaweedfs/weed/filer/store_test"
"github.com/seaweedfs/seaweedfs/weed/filer/store_test"
"testing"
)

8
weed/filer/filechunk_manifest.go
View File

@@ -10,13 +10,13 @@ import (
"sync"
"time"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (

2
weed/filer/filechunk_manifest_test.go
View File

@@ -7,7 +7,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestDoMaybeManifestize(t *testing.T) {

11
weed/filer/filechunks.go
View File

@@ -3,12 +3,12 @@ package filer
import (
"bytes"
"fmt"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"golang.org/x/exp/slices"
"math"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
@@ -114,9 +114,12 @@ func DoMinusChunksBySourceFileId(as, bs []*filer_pb.FileChunk) (delta []*filer_p
fileIds := make(map[string]bool)
for _, interval := range bs {
fileIds[interval.GetFileIdString()] = true
fileIds[interval.GetSourceFileId()] = true
}
for _, chunk := range as {
if _, found := fileIds[chunk.GetSourceFileId()]; !found {
_, sourceFileIdFound := fileIds[chunk.GetSourceFileId()]
_, fileIdFound := fileIds[chunk.GetFileId()]
if !sourceFileIdFound && !fileIdFound {
delta = append(delta, chunk)
}
}

43
weed/filer/filechunks2_test.go
View File

@@ -1,13 +1,52 @@
package filer
import (
"github.com/stretchr/testify/assert"
"golang.org/x/exp/slices"
"log"
"testing"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestDoMinusChunks(t *testing.T) {
// https://github.com/seaweedfs/seaweedfs/issues/3328
// clusterA and clusterB using filer.sync to sync file: hello.txt
// clusterA append a new line and then clusterB also append a new line
// clusterA append a new line again
chunksInA := []*filer_pb.FileChunk{
{Offset: 0, Size: 3, FileId: "11", Mtime: 100},
{Offset: 3, Size: 3, FileId: "22", SourceFileId: "2", Mtime: 200},
{Offset: 6, Size: 3, FileId: "33", Mtime: 300},
}
chunksInB := []*filer_pb.FileChunk{
{Offset: 0, Size: 3, FileId: "1", SourceFileId: "11", Mtime: 100},
{Offset: 3, Size: 3, FileId: "2", Mtime: 200},
{Offset: 6, Size: 3, FileId: "3", SourceFileId: "33", Mtime: 300},
}
// clusterB using command "echo 'content' > hello.txt" to overwrite file
// clusterA will receive two evenNotification, need to empty the whole file content first and add new content
// the first one is oldEntry is chunksInB and newEntry is empty fileChunks
firstOldEntry := chunksInB
var firstNewEntry []*filer_pb.FileChunk
// clusterA received the first one event, gonna empty the whole chunk, according the code in filer_sink 194
// we can get the deleted chunks and newChunks
firstDeletedChunks := DoMinusChunks(firstOldEntry, firstNewEntry)
log.Println("first deleted chunks:", firstDeletedChunks)
//firstNewEntry := DoMinusChunks(firstNewEntry, firstOldEntry)
// clusterA need to delete all chunks in firstDeletedChunks
emptiedChunksInA := DoMinusChunksBySourceFileId(chunksInA, firstDeletedChunks)
// chunksInA supposed to be empty by minus the deletedChunks but it just delete the chunk which sync from clusterB
log.Println("clusterA synced empty chunks event result:", emptiedChunksInA)
// clusterB emptied it's chunks and clusterA must sync the change and empty chunks too
assert.Equalf(t, firstNewEntry, emptiedChunksInA, "empty")
}
func TestCompactFileChunksRealCase(t *testing.T) {
chunks := []*filer_pb.FileChunk{

2
weed/filer/filechunks_read.go
View File

@@ -1,7 +1,7 @@
package filer
import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"golang.org/x/exp/slices"
)

2
weed/filer/filechunks_read_test.go
View File

@@ -2,7 +2,7 @@ package filer
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"math/rand"
"testing"
)

2
weed/filer/filechunks_test.go
View File

@@ -10,7 +10,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func TestCompactFileChunks(t *testing.T) {

41
weed/filer/filer.go
View File

@@ -3,9 +3,9 @@ package filer
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/cluster"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"os"
"sort"
"strings"
@@ -13,11 +13,11 @@ import (
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
)
const (
@@ -52,7 +52,7 @@ type Filer struct {
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption, filerHost pb.ServerAddress,
filerGroup string, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
f := &Filer{
MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, masters),
MasterClient: wdclient.NewMasterClient(grpcDialOption, filerGroup, cluster.FilerType, filerHost, dataCenter, "", masters),
fileIdDeletionQueue: util.NewUnboundedQueue(),
GrpcDialOption: grpcDialOption,
FilerConf: NewFilerConf(),
@@ -105,28 +105,7 @@ func (f *Filer) AggregateFromPeers(self pb.ServerAddress, existingNodes []*maste
}
func (f *Filer) ListExistingPeerUpdates() (existingNodes []*master_pb.ClusterNodeUpdate) {
if grpcErr := pb.WithMasterClient(false, f.MasterClient.GetMaster(), f.GrpcDialOption, func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
ClientType: cluster.FilerType,
FilerGroup: f.MasterClient.FilerGroup,
})
glog.V(0).Infof("the cluster has %d filers\n", len(resp.ClusterNodes))
for _, node := range resp.ClusterNodes {
existingNodes = append(existingNodes, &master_pb.ClusterNodeUpdate{
NodeType: cluster.FilerType,
Address: node.Address,
IsLeader: node.IsLeader,
IsAdd: true,
CreatedAtNs: node.CreatedAtNs,
})
}
return err
}); grpcErr != nil {
glog.V(0).Infof("connect to %s: %v", f.MasterClient.GetMaster(), grpcErr)
}
return
return cluster.ListExistingPeerUpdates(f.GetMaster(), f.GrpcDialOption, f.MasterClient.FilerGroup, cluster.FilerType)
}
func (f *Filer) SetStore(store FilerStore) (isFresh bool) {

10
weed/filer/filer_conf.go
View File

@@ -4,15 +4,15 @@ import (
"bytes"
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
"google.golang.org/grpc"
"io"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/golang/protobuf/jsonpb"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/viant/ptrie"
)

2
weed/filer/filer_conf_test.go
View File

@@ -3,7 +3,7 @@ package filer
import (
"testing"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/stretchr/testify/assert"
)

14
weed/filer/filer_delete_entry.go
View File

@@ -3,10 +3,10 @@ package filer
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (
@@ -122,11 +122,7 @@ func (f *Filer) doDeleteEntryMetaAndData(ctx context.Context, entry *Entry, shou
glog.V(3).Infof("deleting entry %v, delete chunks: %v", entry.FullPath, shouldDeleteChunks)
if !entry.IsDirectory() && !shouldDeleteChunks {
if storeDeletionErr := f.Store.DeleteOneEntrySkipHardlink(ctx, entry.FullPath); storeDeletionErr != nil {
return fmt.Errorf("filer store delete skip hardlink: %v", storeDeletionErr)
}
} else if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
if storeDeletionErr := f.Store.DeleteOneEntry(ctx, entry); storeDeletionErr != nil {
return fmt.Errorf("filer store delete: %v", storeDeletionErr)
}
if !entry.IsDirectory() {

8
weed/filer/filer_deletion.go
View File

@@ -5,10 +5,10 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/wdclient"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
)
func LookupByMasterClientFn(masterClient *wdclient.MasterClient) func(vids []string) (map[string]*operation.LookupResult, error) {

2
weed/filer/filer_hardlink.go
View File

@@ -1,7 +1,7 @@
package filer
import (
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (

8
weed/filer/filer_notify.go
View File

@@ -10,10 +10,10 @@ import (
"github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/notification"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/notification"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool, signatures []int32) {

6
weed/filer/filer_notify_append.go
View File

@@ -6,9 +6,9 @@ import (
"os"
"time"
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/operation"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func (f *Filer) appendToFile(targetFile string, data []byte) error {

4
weed/filer/filer_notify_test.go
View File

@@ -4,8 +4,8 @@ import (
"testing"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/golang/protobuf/proto"
)

6
weed/filer/filer_on_meta_event.go
View File

@@ -2,9 +2,9 @@ package filer
import (
"bytes"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
// onMetadataChangeEvent is triggered after filer processed change events from local or remote filers

2
weed/filer/filer_rename.go
View File

@@ -2,7 +2,7 @@ package filer
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util"
"strings"
)

2
weed/filer/filer_search.go
View File

@@ -2,7 +2,7 @@ package filer
import (
"context"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util"
"math"
"path/filepath"
"strings"

2
weed/filer/filerstore.go
View File

@@ -3,7 +3,7 @@ package filer
import (
"context"
"errors"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
)

4
weed/filer/filerstore_hardlink.go
View File

@@ -4,8 +4,8 @@ import (
"bytes"
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
func (fsw *FilerStoreWrapper) handleUpdateToHardLinks(ctx context.Context, entry *Entry) error {

2
weed/filer/filerstore_translate_path.go
View File

@@ -2,7 +2,7 @@ package filer
import (
"context"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util"
"math"
"strings"
)

21
weed/filer/filerstore_wrapper.go
View File

@@ -2,16 +2,16 @@ package filer
import (
"context"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/viant/ptrie"
"io"
"math"
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/util"
)
var (
@@ -23,7 +23,6 @@ type VirtualFilerStore interface {
FilerStore
DeleteHardLink(ctx context.Context, hardLinkId HardLinkId) error
DeleteOneEntry(ctx context.Context, entry *Entry) error
DeleteOneEntrySkipHardlink(ctx context.Context, fullpath util.FullPath) error
AddPathSpecificStore(path string, storeId string, store FilerStore)
OnBucketCreation(bucket string)
OnBucketDeletion(bucket string)
@@ -217,18 +216,6 @@ func (fsw *FilerStoreWrapper) DeleteOneEntry(ctx context.Context, existingEntry
return actualStore.DeleteEntry(ctx, existingEntry.FullPath)
}
func (fsw *FilerStoreWrapper) DeleteOneEntrySkipHardlink(ctx context.Context, fullpath util.FullPath) (err error) {
actualStore := fsw.getActualStore(fullpath)
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "delete").Inc()
start := time.Now()
defer func() {
stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "delete").Observe(time.Since(start).Seconds())
}()
glog.V(4).Infof("DeleteOneEntrySkipHardlink %s", fullpath)
return actualStore.DeleteEntry(ctx, fullpath)
}
func (fsw *FilerStoreWrapper) DeleteFolderChildren(ctx context.Context, fp util.FullPath) (err error) {
actualStore := fsw.getActualStore(fp + "/")
stats.FilerStoreCounter.WithLabelValues(actualStore.GetName(), "deleteFolderChildren").Inc()

8
weed/filer/hbase/hbase_store.go
View File

@@ -4,10 +4,10 @@ import (
"bytes"
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/tsuna/gohbase"
"github.com/tsuna/gohbase/hrpc"
"io"

2
weed/filer/hbase/hbase_store_kv.go
View File

@@ -2,7 +2,7 @@ package hbase
import (
"context"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/tsuna/gohbase/hrpc"
"time"
)

8
weed/filer/leveldb/leveldb_store.go
View File

@@ -12,10 +12,10 @@ import (
"io"
"os"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/seaweedfs/seaweedfs/weed/util"
)
const (

2
weed/filer/leveldb/leveldb_store_kv.go
View File

@@ -3,7 +3,7 @@ package leveldb
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/syndtr/goleveldb/leveldb"
)

4
weed/filer/leveldb/leveldb_store_test.go
View File

@@ -7,8 +7,8 @@ import (
"testing"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func TestCreateAndFind(t *testing.T) {

8
weed/filer/leveldb2/leveldb2_store.go
View File

@@ -14,10 +14,10 @@ import (
"github.com/syndtr/goleveldb/leveldb/opt"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/seaweedfs/seaweedfs/weed/util"
)
func init() {

2
weed/filer/leveldb2/leveldb2_store_kv.go
View File

@@ -4,7 +4,7 @@ import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/syndtr/goleveldb/leveldb"
)

4
weed/filer/leveldb2/leveldb2_store_test.go
View File

@@ -4,8 +4,8 @@ import (
"context"
"testing"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func TestCreateAndFind(t *testing.T) {

8
weed/filer/leveldb3/leveldb3_store.go
View File

@@ -16,10 +16,10 @@ import (
"github.com/syndtr/goleveldb/leveldb/opt"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
weed_util "github.com/seaweedfs/seaweedfs/weed/util"
)
const (

2
weed/filer/leveldb3/leveldb3_store_bucket.go
View File

@@ -1,7 +1,7 @@
package leveldb
import (
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
"os"
)

2
weed/filer/leveldb3/leveldb3_store_kv.go
View File

@@ -4,7 +4,7 @@ import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/syndtr/goleveldb/leveldb"
)

4
weed/filer/leveldb3/leveldb3_store_test.go
View File

@@ -4,8 +4,8 @@ import (
"context"
"testing"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func TestCreateAndFind(t *testing.T) {

14
weed/filer/meta_aggregator.go
View File

@@ -3,9 +3,9 @@ package filer
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/cluster"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"io"
"strings"
"sync"
@@ -14,10 +14,10 @@ import (
"github.com/golang/protobuf/proto"
"google.golang.org/grpc"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util/log_buffer"
)
type MetaAggregator struct {

6
weed/filer/meta_replay.go
View File

@@ -3,9 +3,9 @@ package filer
import (
"context"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
)
func Replay(filerStore FilerStore, resp *filer_pb.SubscribeMetadataResponse) error {

8
weed/filer/mongodb/mongodb_store.go
View File

@@ -3,10 +3,10 @@ package mongodb
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"

4
weed/filer/mongodb/mongodb_store_kv.go
View File

@@ -3,8 +3,8 @@ package mongodb
import (
"context"
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"

2
weed/filer/mysql/mysql_sql_gen.go
View File

@@ -3,8 +3,8 @@ package mysql
import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
_ "github.com/go-sql-driver/mysql"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
)
type SqlGenMysql struct {

6
weed/filer/mysql/mysql_store.go
View File

@@ -5,11 +5,11 @@ import (
"fmt"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
"github.com/chrislusf/seaweedfs/weed/util"
_ "github.com/go-sql-driver/mysql"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (

8
weed/filer/mysql2/mysql2_store.go
View File

@@ -7,11 +7,11 @@ import (
"strings"
"time"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/filer/abstract_sql"
"github.com/chrislusf/seaweedfs/weed/filer/mysql"
"github.com/chrislusf/seaweedfs/weed/util"
_ "github.com/go-sql-driver/mysql"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/filer/abstract_sql"
"github.com/seaweedfs/seaweedfs/weed/filer/mysql"
"github.com/seaweedfs/seaweedfs/weed/util"
)
const (

Some files were not shown because too many files have changed in this diff Show More