pooling []byte

reduce the number of requests to make([]byte)

go/storage/needle.go

@@ -13,6 +13,7 @@ import (
 	"github.com/chrislusf/seaweedfs/go/glog"
 	"github.com/chrislusf/seaweedfs/go/images"
 	"github.com/chrislusf/seaweedfs/go/operation"
+	"github.com/chrislusf/seaweedfs/go/util"
 )
 
 const (
@@ -22,6 +23,14 @@ const (
 	MaxPossibleVolumeSize = 4 * 1024 * 1024 * 1024 * 8
 )
 
+var (
+	BYTESPOOL *util.BytesPool
+)
+
+func init() {
+	BYTESPOOL = util.NewBytesPool()
+}
+
 /*
 * A Needle means a uploaded and stored file.
 * Needle file size is limited to 4GB for now.
@@ -43,6 +52,8 @@ type Needle struct {
 
 	Checksum CRC    `comment:"CRC32 to check integrity"`
 	Padding  []byte `comment:"Aligned to 8 bytes"`
+
+	rawBytes []byte // underlying supporing []byte, fetched and released into a pool
 }
 
 func (n *Needle) String() (str string) {

go/storage/needle_read_write.go

@@ -136,15 +136,33 @@ func (n *Needle) Append(w io.Writer, version Version) (size uint32, err error) {
 	return 0, fmt.Errorf("Unsupported Version! (%d)", version)
 }
 
-func ReadNeedleBlob(r *os.File, offset int64, size uint32) (bytes []byte, err error) {
+func ReleaseBytes(b []byte) {
+	// println("Releasing", len(b))
+	BYTESPOOL.Put(b)
+}
+
+func BorrwoBytes(size int) []byte {
+	ret := BYTESPOOL.Get(size)
+	// println("Reading", len(ret))
+	return ret
+}
+
+func ReadNeedleBlob(r *os.File, offset int64, size uint32) (dataSlice, rawBytes []byte, err error) {
 	padding := NeedlePaddingSize - ((NeedleHeaderSize + size + NeedleChecksumSize) % NeedlePaddingSize)
-	bytes = make([]byte, NeedleHeaderSize+size+NeedleChecksumSize+padding)
-	_, err = r.ReadAt(bytes, offset)
+	readSize := NeedleHeaderSize + size + NeedleChecksumSize + padding
+	rawBytes = BorrwoBytes(int(readSize))
+	dataSlice = rawBytes[0:int(readSize)]
+	_, err = r.ReadAt(dataSlice, offset)
 	return
 }
 
+func (n *Needle) ReleaseMemory() {
+	ReleaseBytes(n.rawBytes)
+}
+
 func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
-	bytes, err := ReadNeedleBlob(r, offset, size)
+	bytes, rawBytes, err := ReadNeedleBlob(r, offset, size)
+	n.rawBytes = rawBytes
 	if err != nil {
 		return err
 	}

go/storage/volume.go

@@ -159,6 +159,7 @@ func (v *Volume) isFileUnchanged(n *Needle) bool {
 	if ok && nv.Offset > 0 {
 		oldNeedle := new(Needle)
 		err := oldNeedle.ReadData(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
+		defer oldNeedle.ReleaseMemory()
 		if err != nil {
 			glog.V(0).Infof("Failed to check updated file %v", err)
 			return false
@@ -288,6 +289,7 @@ func (v *Volume) readNeedle(n *Needle) (int, error) {
 	}
 	err := n.ReadData(v.dataFile, int64(nv.Offset)*NeedlePaddingSize, nv.Size, v.Version())
 	if err != nil {
+		n.ReleaseMemory()
 		return 0, err
 	}
 	bytesRead := len(n.Data)
@@ -304,6 +306,7 @@ func (v *Volume) readNeedle(n *Needle) (int, error) {
 	if uint64(time.Now().Unix()) < n.LastModified+uint64(ttlMinutes*60) {
 		return bytesRead, nil
 	}
+	n.ReleaseMemory()
 	return -1, errors.New("Not Found")
 }