Merge branch 'master' into messaging

weed/filer/filechunks.go

@@ -3,11 +3,9 @@ package filer
 import (
 	"bytes"
 	"fmt"
-	"math"
-	"sync"
-
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 	"golang.org/x/exp/slices"
+	"math"
 
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -54,11 +52,11 @@ func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
 	if len(chunks) == 1 {
 		return fmt.Sprintf("%x", util.Base64Md5ToBytes(chunks[0].ETag))
 	}
-	md5_digests := [][]byte{}
+	var md5Digests [][]byte
 	for _, c := range chunks {
-		md5_digests = append(md5_digests, util.Base64Md5ToBytes(c.ETag))
+		md5Digests = append(md5Digests, util.Base64Md5ToBytes(c.ETag))
 	}
-	return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5_digests, nil)), len(chunks))
+	return fmt.Sprintf("%x-%d", util.Md5(bytes.Join(md5Digests, nil)), len(chunks))
 }
 
 func CompactFileChunks(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
@@ -189,12 +187,6 @@ func logPrintf(name string, visibles []VisibleInterval) {
 	*/
 }
 
-var bufPool = sync.Pool{
-	New: func() interface{} {
-		return new(VisibleInterval)
-	},
-}
-
 func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (newVisibles []VisibleInterval) {
 
 	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, 0, chunk.Size, chunk.CipherKey, chunk.IsCompressed)

weed/filer/reader_at.go

@@ -164,6 +164,10 @@ func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {
 func (c *ChunkReadAt) readChunkSliceAt(buffer []byte, chunkView *ChunkView, nextChunkViews []*ChunkView, offset uint64) (n int, err error) {
 
 	if c.readerPattern.IsRandomMode() {
+		n, err := c.readerCache.chunkCache.ReadChunkAt(buffer, chunkView.FileId, offset)
+		if n > 0 {
+			return n, err
+		}
 		return fetchChunkRange(buffer, c.readerCache.lookupFileIdFn, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped, int64(offset))
 	}
 

weed/filer/reader_cache.go

@@ -19,6 +19,7 @@ type ReaderCache struct {
 
 type SingleChunkCacher struct {
 	sync.RWMutex
+	cond          *sync.Cond
 	parent        *ReaderCache
 	chunkFileId   string
 	data          []byte
@@ -140,6 +141,7 @@ func newSingleChunkCacher(parent *ReaderCache, fileId string, cipherKey []byte,
 		chunkSize:   chunkSize,
 		shouldCache: shouldCache,
 	}
+	t.cond = sync.NewCond(t)
 	return t
 }
 
@@ -168,6 +170,7 @@ func (s *SingleChunkCacher) startCaching() {
 	if s.shouldCache {
 		s.parent.chunkCache.SetChunk(s.chunkFileId, s.data)
 	}
+	s.cond.Broadcast()
 
 	return
 }
@@ -183,6 +186,10 @@ func (s *SingleChunkCacher) readChunkAt(buf []byte, offset int64) (int, error) {
 	s.RLock()
 	defer s.RUnlock()
 
+	for s.completedTime.IsZero() {
+		s.cond.Wait()
+	}
+
 	if s.err != nil {
 		return 0, s.err
 	}

weed/filer/tikv/tikv_store.go

@@ -15,6 +15,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/tikv/client-go/v2/config"
 	"github.com/tikv/client-go/v2/txnkv"
 )
 
@@ -38,21 +39,25 @@ func (store *TikvStore) GetName() string {
 }
 
 func (store *TikvStore) Initialize(config util.Configuration, prefix string) error {
-	pdAddrs := []string{}
-	pdAddrsStr := config.GetString(prefix + "pdaddrs")
-	for _, item := range strings.Split(pdAddrsStr, ",") {
-		pdAddrs = append(pdAddrs, strings.TrimSpace(item))
-	}
+	ca := config.GetString(prefix + "ca_path")
+	cert := config.GetString(prefix + "cert_path")
+	key := config.GetString(prefix + "key_path")
+	verify_cn := strings.Split(config.GetString(prefix+"verify_cn"), ",")
+	pdAddrs := strings.Split(config.GetString(prefix+"pdaddrs"), ",")
+
 	drc := config.GetInt(prefix + "deleterange_concurrency")
 	if drc <= 0 {
 		drc = 1
 	}
 	store.onePC = config.GetBool(prefix + "enable_1pc")
 	store.deleteRangeConcurrency = drc
-	return store.initialize(pdAddrs)
+	return store.initialize(ca, cert, key, verify_cn, pdAddrs)
 }
 
-func (store *TikvStore) initialize(pdAddrs []string) error {
+func (store *TikvStore) initialize(ca, cert, key string, verify_cn, pdAddrs []string) error {
+	config.UpdateGlobal(func(conf *config.Config) {
+		conf.Security = config.NewSecurity(ca, cert, key, verify_cn)
+	})
 	client, err := txnkv.NewClient(pdAddrs)
 	store.client = client
 	return err