Merge branch 'master' into bptree

weed/util/compression.go

@@ -2,10 +2,7 @@ package util
 
 import (
 	"bytes"
-	"compress/flate"
-	"compress/gzip"
 	"fmt"
-	"io/ioutil"
 	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -42,17 +39,21 @@ func MaybeDecompressData(input []byte) []byte {
 }
 
 func GzipData(input []byte) ([]byte, error) {
-	buf := new(bytes.Buffer)
-	w, _ := gzip.NewWriterLevel(buf, flate.BestSpeed)
-	if _, err := w.Write(input); err != nil {
-		glog.V(2).Infof("error gzip data: %v", err)
+	w := new(bytes.Buffer)
+	_, err := GzipStream(w, bytes.NewReader(input))
+	if err != nil {
 		return nil, err
 	}
-	if err := w.Close(); err != nil {
-		glog.V(2).Infof("error closing gzipped data: %v", err)
+	return w.Bytes(), nil
+}
+
+func ungzipData(input []byte) ([]byte, error) {
+	w := new(bytes.Buffer)
+	_, err := GunzipStream(w, bytes.NewReader(input))
+	if err != nil {
 		return nil, err
 	}
-	return buf.Bytes(), nil
+	return w.Bytes(), nil
 }
 
 func DecompressData(input []byte) ([]byte, error) {
@@ -67,17 +68,6 @@ func DecompressData(input []byte) ([]byte, error) {
 	return input, UnsupportedCompression
 }
 
-func ungzipData(input []byte) ([]byte, error) {
-	buf := bytes.NewBuffer(input)
-	r, _ := gzip.NewReader(buf)
-	defer r.Close()
-	output, err := ioutil.ReadAll(r)
-	if err != nil {
-		glog.V(2).Infof("error ungzip data: %v", err)
-	}
-	return output, err
-}
-
 func IsGzippedContent(data []byte) bool {
 	if len(data) < 2 {
 		return false

weed/util/compression_stream.go

@@ -0,0 +1,53 @@
+package util
+
+import (
+	"compress/gzip"
+	"fmt"
+	"io"
+	"sync"
+)
+
+var (
+	gzipReaderPool = sync.Pool{
+		New: func() interface{} {
+			return new(gzip.Reader)
+			//return gzip.NewReader()
+		},
+	}
+
+	gzipWriterPool = sync.Pool{
+		New: func() interface{} {
+			w, _ := gzip.NewWriterLevel(nil, gzip.BestSpeed)
+			return w
+		},
+	}
+)
+
+func GzipStream(w io.Writer, r io.Reader) (int64, error) {
+	gw, ok := gzipWriterPool.Get().(*gzip.Writer)
+	if !ok {
+		return 0, fmt.Errorf("gzip: new writer error")
+	}
+	gw.Reset(w)
+	defer func() {
+		gw.Close()
+		gzipWriterPool.Put(gw)
+	}()
+	return io.Copy(gw, r)
+}
+
+func GunzipStream(w io.Writer, r io.Reader) (int64, error) {
+	gr, ok := gzipReaderPool.Get().(*gzip.Reader)
+	if !ok {
+		return 0, fmt.Errorf("gzip: new reader error")
+	}
+
+	if err := gr.Reset(r); err != nil {
+		return 0, err
+	}
+	defer func() {
+		gr.Close()
+		gzipReaderPool.Put(gr)
+	}()
+	return io.Copy(w, gr)
+}

weed/util/compression_test.go

@@ -1,21 +0,0 @@
-package util
-
-import (
-	"testing"
-
-	"golang.org/x/tools/godoc/util"
-)
-
-func TestIsGzippable(t *testing.T) {
-	buf := make([]byte, 1024)
-
-	isText := util.IsText(buf)
-
-	if isText {
-		t.Error("buf with zeros are not text")
-	}
-
-	compressed, _ := GzipData(buf)
-
-	t.Logf("compressed size %d\n", len(compressed))
-}

weed/util/constants.go

@@ -5,8 +5,9 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %.02f", sizeLimit, 2.63)
-	COMMIT  = ""
+	VERSION_NUMBER = fmt.Sprintf("%.02f", 2.68)
+	VERSION        = sizeLimit + " " + VERSION_NUMBER
+	COMMIT         = ""
 )
 
 func Version() string {

weed/util/constants_4bytes.go

@@ -1,3 +1,4 @@
+//go:build !5BytesOffset
 // +build !5BytesOffset
 
 package util

weed/util/constants_5bytes.go

@@ -1,3 +1,4 @@
+//go:build 5BytesOffset
 // +build 5BytesOffset
 
 package util

weed/util/file_util_non_posix.go

@@ -1,3 +1,4 @@
+//go:build linux || darwin || freebsd || netbsd || openbsd || plan9 || solaris || zos
 // +build linux darwin freebsd netbsd openbsd plan9 solaris zos
 
 package util

weed/util/file_util_posix.go

@@ -1,3 +1,4 @@
+//go:build windows
 // +build windows
 
 package util

weed/util/grace/signal_handling.go

@@ -1,3 +1,4 @@
+//go:build !plan9
 // +build !plan9
 
 package grace

weed/util/grace/signal_handling_notsupported.go

@@ -1,3 +1,4 @@
+//go:build plan9
 // +build plan9
 
 package grace

weed/util/network.go

@@ -2,6 +2,8 @@ package util
 
 import (
 	"net"
+	"strconv"
+	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 )
@@ -13,6 +15,18 @@ func DetectedHostAddress() string {
 		return ""
 	}
 
+	if v4Address := selectIpV4(netInterfaces, true); v4Address != "" {
+		return v4Address
+	}
+
+	if v6Address := selectIpV4(netInterfaces, false); v6Address != "" {
+		return v6Address
+	}
+
+	return "localhost"
+}
+
+func selectIpV4(netInterfaces []net.Interface, isIpV4 bool) string {
 	for _, netInterface := range netInterfaces {
 		if (netInterface.Flags & net.FlagUp) == 0 {
 			continue
@@ -24,12 +38,25 @@ func DetectedHostAddress() string {
 
 		for _, a := range addrs {
 			if ipNet, ok := a.(*net.IPNet); ok && !ipNet.IP.IsLoopback() {
-				if ipNet.IP.To4() != nil {
-					return ipNet.IP.String()
+				if isIpV4 {
+					if ipNet.IP.To4() != nil {
+						return ipNet.IP.String()
+					}
+				} else {
+					if ipNet.IP.To16() != nil {
+						return ipNet.IP.String()
+					}
 				}
 			}
 		}
 	}
-
-	return "localhost"
+	return ""
+}
+
+func JoinHostPort(host string, port int) string {
+	portStr := strconv.Itoa(port)
+	if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
+		return host + ":" + portStr
+	}
+	return net.JoinHostPort(host, portStr)
 }

weed/util/retry.go

@@ -37,6 +37,7 @@ func RetryForever(name string, job func() error, onErrFn func(err error) bool) {
 	for {
 		err := job()
 		if err == nil {
+			waitTime = time.Second
 			break
 		}
 		if onErrFn(err) {