gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

filer: fix ttl parsing

Browse Source 
fix https://github.com/chrislusf/seaweedfs/issues/1225
This commit is contained in:
Chris Lu
2020-03-09 00:16:10 -07:00
parent c32f95c380
commit 8a899992f2
3 changed files with 26 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
weed/server/filer_server_handlers_write.go
View File

@@ -23,6 +23,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
) )
@@ -39,7 +40,7 @@ type FilerPostResult struct {
Url string `json:"url,omitempty"` Url string `json:"url,omitempty"`
} }
func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection string, dataCenter string) (fileId, urlLocation string, auth security.EncodedJwt, err error) { func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection, dataCenter, ttlString string) (fileId, urlLocation string, auth security.EncodedJwt, err error) {
stats.FilerRequestCounter.WithLabelValues("assign").Inc() stats.FilerRequestCounter.WithLabelValues("assign").Inc()
start := time.Now() start := time.Now()
@@ -49,7 +50,7 @@ func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request,
Count: 1, Count: 1,
Replication: replication, Replication: replication,
Collection: collection, Collection: collection,
Ttl: r.URL.Query().Get("ttl"), Ttl: ttlString,
DataCenter: dataCenter, DataCenter: dataCenter,
} }
var altRequest *operation.VolumeAssignRequest var altRequest *operation.VolumeAssignRequest
@@ -58,7 +59,7 @@ func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request,
Count: 1, Count: 1,
Replication: replication, Replication: replication,
Collection: collection, Collection: collection,
Ttl: r.URL.Query().Get("ttl"), Ttl: ttlString,
DataCenter: "", DataCenter: "",
} }
} }
@@ -86,13 +87,21 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
if dataCenter == "" { if dataCenter == "" {
dataCenter = fs.option.DataCenter dataCenter = fs.option.DataCenter
} }
ttlString := r.URL.Query().Get("ttl")
if autoChunked := fs.autoChunk(ctx, w, r, replication, collection, dataCenter); autoChunked { // read ttl in seconds
ttl, err := needle.ReadTTL(ttlString)
ttlSeconds := int32(0)
if err == nil {
ttlSeconds = int32(ttl.Minutes()) * 60
}
if autoChunked := fs.autoChunk(ctx, w, r, replication, collection, dataCenter, ttlSeconds, ttlString); autoChunked {
return return
} }
if fs.option.Cipher { if fs.option.Cipher {
reply, err := fs.encrypt(ctx, w, r, replication, collection, dataCenter) reply, err := fs.encrypt(ctx, w, r, replication, collection, dataCenter, ttlSeconds, ttlString)
if err != nil { if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err) writeJsonError(w, r, http.StatusInternalServerError, err)
} else if reply != nil { } else if reply != nil {
@@ -102,7 +111,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter) fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString)
if err != nil || fileId == "" || urlLocation == "" { if err != nil || fileId == "" || urlLocation == "" {
glog.V(0).Infof("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter) glog.V(0).Infof("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter)
@@ -118,7 +127,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
return return
} }
if err = fs.updateFilerStore(ctx, r, w, replication, collection, ret, fileId); err != nil { if err = fs.updateFilerStore(ctx, r, w, replication, collection, ret, fileId, ttlSeconds); err != nil {
return return
} }
@@ -136,7 +145,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
// update metadata in filer store // update metadata in filer store
func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w http.ResponseWriter, func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w http.ResponseWriter,
replication string, collection string, ret *operation.UploadResult, fileId string) (err error) { replication string, collection string, ret *operation.UploadResult, fileId string, ttlSeconds int32) (err error) {
stats.FilerRequestCounter.WithLabelValues("postStoreWrite").Inc() stats.FilerRequestCounter.WithLabelValues("postStoreWrite").Inc()
start := time.Now() start := time.Now()
@@ -175,7 +184,7 @@ func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w
Gid: OS_GID, Gid: OS_GID,
Replication: replication, Replication: replication,
Collection: collection, Collection: collection,
TtlSec: int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)), TtlSec: ttlSeconds,
Mime: ret.Mime, Mime: ret.Mime,
}, },
Chunks: []*filer_pb.FileChunk{{ Chunks: []*filer_pb.FileChunk{{

11
weed/server/filer_server_handlers_write_autochunk.go
View File

@@ -16,11 +16,10 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
) )
func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
replication string, collection string, dataCenter string) bool { replication string, collection string, dataCenter string, ttlSec int32, ttlString string) bool {
if r.Method != "POST" { if r.Method != "POST" {
glog.V(4).Infoln("AutoChunking not supported for method", r.Method) glog.V(4).Infoln("AutoChunking not supported for method", r.Method)
return false return false
@@ -56,7 +55,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
return false return false
} }
reply, err := fs.doAutoChunk(ctx, w, r, contentLength, chunkSize, replication, collection, dataCenter) reply, err := fs.doAutoChunk(ctx, w, r, contentLength, chunkSize, replication, collection, dataCenter, ttlSec, ttlString)
if err != nil { if err != nil {
writeJsonError(w, r, http.StatusInternalServerError, err) writeJsonError(w, r, http.StatusInternalServerError, err)
} else if reply != nil { } else if reply != nil {
@@ -66,7 +65,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
} }
func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request, func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
contentLength int64, chunkSize int32, replication string, collection string, dataCenter string) (filerResult *FilerPostResult, replyerr error) { contentLength int64, chunkSize int32, replication string, collection string, dataCenter string, ttlSec int32, ttlString string) (filerResult *FilerPostResult, replyerr error) {
stats.FilerRequestCounter.WithLabelValues("postAutoChunk").Inc() stats.FilerRequestCounter.WithLabelValues("postAutoChunk").Inc()
start := time.Now() start := time.Now()
@@ -100,7 +99,7 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r
limitedReader := io.LimitReader(part1, int64(chunkSize)) limitedReader := io.LimitReader(part1, int64(chunkSize))
// assign one file id for one chunk // assign one file id for one chunk
fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(w, r, replication, collection, dataCenter) fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString)
if assignErr != nil { if assignErr != nil {
return nil, assignErr return nil, assignErr
} }
@@ -158,7 +157,7 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r
Gid: OS_GID, Gid: OS_GID,
Replication: replication, Replication: replication,
Collection: collection, Collection: collection,
TtlSec: int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)), TtlSec: ttlSec,
Mime: contentType, Mime: contentType,
}, },
Chunks: fileChunks, Chunks: fileChunks,

7
weed/server/filer_server_handlers_write_cipher.go
View File

@@ -12,14 +12,13 @@ import (
"github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle" "github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/util"
) )
// handling single chunk POST or PUT upload // handling single chunk POST or PUT upload
func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request, func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request,
replication string, collection string, dataCenter string) (filerResult *FilerPostResult, err error) { replication string, collection string, dataCenter string, ttlSeconds int32, ttlString string) (filerResult *FilerPostResult, err error) {
fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter) fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString)
if err != nil || fileId == "" || urlLocation == "" { if err != nil || fileId == "" || urlLocation == "" {
return nil, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter) return nil, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter)
@@ -77,7 +76,7 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht
Gid: OS_GID, Gid: OS_GID,
Replication: replication, Replication: replication,
Collection: collection, Collection: collection,
TtlSec: int32(util.ParseInt(r.URL.Query().Get("ttl"), 0)), TtlSec: ttlSeconds,
Mime: pu.MimeType, Mime: pu.MimeType,
}, },
Chunks: fileChunks, Chunks: fileChunks,