gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

sampling whether the data can be gzipped

Browse Source
This commit is contained in:
Chris Lu
2020-03-26 23:50:48 -07:00
parent 5463560912
commit 2286eda575
1 changed files with 11 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
weed/operation/upload_content.go
View File

@@ -78,9 +78,13 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
contentIsGzipped := isInputGzipped contentIsGzipped := isInputGzipped
shouldGzipNow := false shouldGzipNow := false
if !isInputGzipped { if !isInputGzipped {
if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); mtype == "" || iAmSure && shouldBeZipped { if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
shouldGzipNow = true shouldGzipNow = true
contentIsGzipped = true contentIsGzipped = true
} else if len(data) > 128 {
var compressed []byte
compressed, err = util.GzipData(data[0:128])
shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%
} }
} }
@@ -90,7 +94,12 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
// this could be double copying // this could be double copying
clearDataLen = len(data) clearDataLen = len(data)
if shouldGzipNow { if shouldGzipNow {
data, err = util.GzipData(data) compressed, compressErr := util.GzipData(data)
// fmt.Printf("data is compressed from %d ==> %d\n", len(data), len(compressed))
if compressErr == nil {
data = compressed
contentIsGzipped = true
}
} else if isInputGzipped { } else if isInputGzipped {
// just to get the clear data length // just to get the clear data length
clearData, err := util.UnGzipData(data) clearData, err := util.UnGzipData(data)