Delete all chunks when delete a ChunkManifest

LoadChunkManifest can uncompress buffer
move compress.go from storage to operation because of import cycle
MakeFile add cross complete command

go/operation/chunked_file.go

@@ -30,10 +30,10 @@ type ChunkInfo struct {
 type ChunkList []*ChunkInfo
 
 type ChunkManifest struct {
-	Name      string    `json:"name,omitempty"`
-	Mime      string    `json:"mime,omitempty"`
-	Size      int64     `json:"size,omitempty"`
-	Chunks    ChunkList `json:"chunks,omitempty"`
+	Name   string    `json:"name,omitempty"`
+	Mime   string    `json:"mime,omitempty"`
+	Size   int64     `json:"size,omitempty"`
+	Chunks ChunkList `json:"chunks,omitempty"`
 }
 
 // seekable chunked file reader
@@ -50,7 +50,13 @@ func (s ChunkList) Len() int           { return len(s) }
 func (s ChunkList) Less(i, j int) bool { return s[i].Offset < s[j].Offset }
 func (s ChunkList) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
 
-func LoadChunkedManifest(buffer []byte) (*ChunkManifest, error) {
+func LoadChunkManifest(buffer []byte, isGzipped bool) (*ChunkManifest, error) {
+	if isGzipped {
+		var err error
+		if buffer, err = UnGzipData(buffer); err != nil {
+			return nil, err
+		}
+	}
 	cm := ChunkManifest{}
 	if e := json.Unmarshal(buffer, &cm); e != nil {
 		return nil, e

go/operation/compress.go

@@ -0,0 +1,59 @@
+package operation
+
+import (
+	"bytes"
+	"compress/flate"
+	"compress/gzip"
+	"io/ioutil"
+	"strings"
+
+	"github.com/chrislusf/seaweedfs/go/glog"
+)
+
+/*
+* Default more not to gzip since gzip can be done on client side.
+ */
+func IsGzippable(ext, mtype string) bool {
+	if strings.HasPrefix(mtype, "text/") {
+		return true
+	}
+	switch ext {
+	case ".zip", ".rar", ".gz", ".bz2", ".xz":
+		return false
+	case ".pdf", ".txt", ".html", ".htm", ".css", ".js", ".json":
+		return true
+	}
+	if strings.HasPrefix(mtype, "application/") {
+		if strings.HasSuffix(mtype, "xml") {
+			return true
+		}
+		if strings.HasSuffix(mtype, "script") {
+			return true
+		}
+	}
+	return false
+}
+
+func GzipData(input []byte) ([]byte, error) {
+	buf := new(bytes.Buffer)
+	w, _ := gzip.NewWriterLevel(buf, flate.BestCompression)
+	if _, err := w.Write(input); err != nil {
+		glog.V(2).Infoln("error compressing data:", err)
+		return nil, err
+	}
+	if err := w.Close(); err != nil {
+		glog.V(2).Infoln("error closing compressed data:", err)
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+func UnGzipData(input []byte) ([]byte, error) {
+	buf := bytes.NewBuffer(input)
+	r, _ := gzip.NewReader(buf)
+	defer r.Close()
+	output, err := ioutil.ReadAll(r)
+	if err != nil {
+		glog.V(2).Infoln("error uncompressing data:", err)
+	}
+	return output, err
+}