gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

remove go.uber.org/atomic

Browse Source
This commit is contained in:
石昌林
2022-06-20 12:35:29 +08:00
parent 3dd60529c5
commit 9e036df356
7 changed files with 172 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
weed/s3api/s3_constants/s3_config.go
View File

@@ -8,8 +8,8 @@ var (
CircuitBreakerConfigDir = "/etc/s3"
CircuitBreakerConfigFile = "circuit_breaker.json"
AllowedActions = []string{ACTION_READ, ACTION_WRITE, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN}
LimitTypeCount = "count"
LimitTypeBytes = "bytes"
LimitTypeCount = "Count"
LimitTypeBytes = "MB"
Separator = ":"
)

17
weed/s3api/s3api_circuit_breaker.go
View File

@@ -10,21 +10,21 @@ import (
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"github.com/gorilla/mux"
"go.uber.org/atomic"
"net/http"
"sync"
"sync/atomic"
)
type CircuitBreaker struct {
sync.Mutex
Enabled bool
counters map[string]*atomic.Int64
counters map[string]*int64
limitations map[string]int64
}
func NewCircuitBreaker(option *S3ApiServerOption) *CircuitBreaker {
cb := &CircuitBreaker{
counters: make(map[string]*atomic.Int64),
counters: make(map[string]*int64),
limitations: make(map[string]int64),
}
@@ -156,21 +156,22 @@ func (cb *CircuitBreaker) loadCounterAndCompare(bucket, action, limitType string
cb.Lock()
counter, exists = cb.counters[key]
if !exists {
counter = atomic.NewInt64(0)
var newCounter int64
counter = &newCounter
cb.counters[key] = counter
}
cb.Unlock()
}
current := counter.Load()
current := atomic.LoadInt64(counter)
if current+inc > max {
e = errCode
return
} else {
counter.Add(inc)
current := atomic.AddInt64(counter, inc)
f = func() {
counter.Sub(inc)
atomic.AddInt64(counter, -inc)
}
current = counter.Load()
current = atomic.LoadInt64(counter)
if current > max {
e = errCode
return

12
weed/s3api/s3api_circuit_breaker_test.go
View File

@@ -4,9 +4,9 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"go.uber.org/atomic"
"net/http"
"sync"
"sync/atomic"
"testing"
)
@@ -32,7 +32,7 @@ var (
{action, s3_constants.LimitTypeBytes, 1024, 1024, 6, 200, 5},
{action, s3_constants.LimitTypeBytes, 1200, 1200, 6, 200, 6},
{action, s3_constants.LimitTypeBytes, 11990, 11990, 60, 200, 59},
{action, s3_constants.LimitTypeBytes, 11790, 11990, 60, 200, 58},
{action, s3_constants.LimitTypeBytes, 11790, 11990, 70, 200, 58},
}
)
@@ -56,7 +56,7 @@ func TestLimit(t *testing.T) {
},
}
circuitBreaker := &CircuitBreaker{
counters: make(map[string]*atomic.Int64),
counters: make(map[string]*int64),
limitations: make(map[string]int64),
}
err := circuitBreaker.loadCircuitBreakerConfig(circuitBreakerConfig)
@@ -72,7 +72,7 @@ func TestLimit(t *testing.T) {
}
func doLimit(circuitBreaker *CircuitBreaker, routineCount int, r *http.Request) int64 {
var successCounter atomic.Int64
var successCounter int64
resultCh := make(chan []func(), routineCount)
var wg sync.WaitGroup
for i := 0; i < routineCount; i++ {
@@ -81,7 +81,7 @@ func doLimit(circuitBreaker *CircuitBreaker, routineCount int, r *http.Request)
defer wg.Done()
rollbackFn, errCode := circuitBreaker.limit(r, bucket, action)
if errCode == s3err.ErrNone {
successCounter.Inc()
atomic.AddInt64(&successCounter, 1)
}
resultCh <- rollbackFn
}()
@@ -93,5 +93,5 @@ func doLimit(circuitBreaker *CircuitBreaker, routineCount int, r *http.Request)
fn()
}
}
return successCounter.Load()
return successCounter
}

15
weed/shell/command_s3_circuitbreaker.go
View File

@@ -4,7 +4,6 @@ import (
"bytes"
"flag"
"fmt"
"github.com/alecthomas/units"
"github.com/chrislusf/seaweedfs/weed/filer"
"github.com/chrislusf/seaweedfs/weed/pb/s3_pb"
"github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
@@ -61,8 +60,8 @@ func (c *commandS3CircuitBreaker) Do(args []string, commandEnv *CommandEnv, writ
global := s3CircuitBreakerCommand.Bool("global", false, "configure global circuit breaker")
actions := s3CircuitBreakerCommand.String("actions", "", "comma separated actions names: Read,Write,List,Tagging,Admin")
limitType := s3CircuitBreakerCommand.String("type", "", "count|bytes simultaneous requests count")
values := s3CircuitBreakerCommand.String("values", "", "comma separated max values,Maximum number of simultaneous requests content length, support byte unit: eg: 1k, 10m, 1g")
limitType := s3CircuitBreakerCommand.String("type", "", "'Count' or 'MB'; Count represents the number of simultaneous requests, and MB represents the content size of all simultaneous requests")
values := s3CircuitBreakerCommand.String("values", "", "comma separated values")
disabled := s3CircuitBreakerCommand.Bool("disable", false, "disable global or buckets circuit breaker")
deleted := s3CircuitBreakerCommand.Bool("delete", false, "delete circuit breaker config")
@@ -326,7 +325,7 @@ func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitTy
if len(elements) != 1 || len(elements) == 0 {
return nil, nil, nil, fmt.Errorf("values count of -actions and -values not equal")
}
v, err := units.ParseStrictBytes(elements[0])
v, err := parseMBToBytes(elements[0])
if err != nil {
return nil, nil, nil, fmt.Errorf("value of -max must be a legal number(s)")
}
@@ -335,7 +334,7 @@ func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitTy
}
} else {
for _, value := range elements {
v, err := units.ParseStrictBytes(value)
v, err := parseMBToBytes(value)
if err != nil {
return nil, nil, nil, fmt.Errorf("value of -max must be a legal number(s)")
}
@@ -351,3 +350,9 @@ func (c *commandS3CircuitBreaker) initActionsAndValues(buckets, actions, limitTy
}
return cmdBuckets, cmdActions, cmdValues, nil
}
func parseMBToBytes(valStr string) (int64, error) {
v, err := strconv.Atoi(valStr)
v *= 1024 * 1024
return int64(v), err
}

157
weed/shell/command_s3_circuitbreaker_test.go
View File

@@ -17,13 +17,13 @@ var (
TestCases = []*Case{
//add circuit breaker config for global
{
args: strings.Split("-global -type count -actions Read,Write -values 500,200", " "),
args: strings.Split("-global -type Count -actions Read,Write -values 500,200", " "),
result: `{
"global": {
"enabled": true,
"actions": {
"Read:count": "500",
"Write:count": "200"
"Read:Count": "500",
"Write:Count": "200"
}
}
}`,
@@ -35,8 +35,8 @@ var (
result: `{
"global": {
"actions": {
"Read:count": "500",
"Write:count": "200"
"Read:Count": "500",
"Write:Count": "200"
}
}
}`,
@@ -44,34 +44,34 @@ var (
//add circuit breaker config for buckets x,y,z
{
args: strings.Split("-buckets x,y,z -type count -actions Read,Write -values 200,100", " "),
args: strings.Split("-buckets x,y,z -type Count -actions Read,Write -values 200,100", " "),
result: `{
"global": {
"actions": {
"Read:count": "500",
"Write:count": "200"
"Read:Count": "500",
"Write:Count": "200"
}
},
"buckets": {
"x": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
"Read:Count": "200",
"Write:Count": "100"
}
},
"y": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
"Read:Count": "200",
"Write:Count": "100"
}
},
"z": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
"Read:Count": "200",
"Write:Count": "100"
}
}
}
@@ -84,29 +84,29 @@ var (
result: `{
"global": {
"actions": {
"Read:count": "500",
"Write:count": "200"
"Read:Count": "500",
"Write:Count": "200"
}
},
"buckets": {
"x": {
"actions": {
"Read:count": "200",
"Write:count": "100"
"Read:Count": "200",
"Write:Count": "100"
}
},
"y": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
"Read:Count": "200",
"Write:Count": "100"
}
},
"z": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
"Read:Count": "200",
"Write:Count": "100"
}
}
}
@@ -119,23 +119,124 @@ var (
result: `{
"global": {
"actions": {
"Read:count": "500",
"Write:count": "200"
"Read:Count": "500",
"Write:Count": "200"
}
},
"buckets": {
"y": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
"Read:Count": "200",
"Write:Count": "100"
}
},
"z": {
"enabled": true,
"actions": {
"Read:count": "200",
"Write:count": "100"
"Read:Count": "200",
"Write:Count": "100"
}
}
}
}`,
},
//configure the circuit breaker for the size of the uploaded file for bucket x,y
{
args: strings.Split("-buckets x,y -type MB -actions Write -values 1024", " "),
result: `{
"global": {
"actions": {
"Read:Count": "500",
"Write:Count": "200"
}
},
"buckets": {
"x": {
"enabled": true,
"actions": {
"Write:MB": "1073741824"
}
},
"y": {
"enabled": true,
"actions": {
"Read:Count": "200",
"Write:Count": "100",
"Write:MB": "1073741824"
}
},
"z": {
"enabled": true,
"actions": {
"Read:Count": "200",
"Write:Count": "100"
}
}
}
}`,
},
//delete the circuit breaker configuration for the size of the uploaded file of bucket x,y
{
args: strings.Split("-buckets x,y -type MB -actions Write -delete", " "),
result: `{
"global": {
"actions": {
"Read:Count": "500",
"Write:Count": "200"
}
},
"buckets": {
"x": {
"enabled": true
},
"y": {
"enabled": true,
"actions": {
"Read:Count": "200",
"Write:Count": "100"
}
},
"z": {
"enabled": true,
"actions": {
"Read:Count": "200",
"Write:Count": "100"
}
}
}
}`,
},
//enable global circuit breaker config (without -disable flag)
{
args: strings.Split("-global", " "),
result: `{
"global": {
"enabled": true,
"actions": {
"Read:Count": "500",
"Write:Count": "200"
}
},
"buckets": {
"x": {
"enabled": true
},
"y": {
"enabled": true,
"actions": {
"Read:Count": "200",
"Write:Count": "100"
}
},
"z": {
"enabled": true,
"actions": {
"Read:Count": "200",
"Write:Count": "100"
}
}
}