gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

use one readerCache for the whole file

Browse Source
This commit is contained in:
chrislu
2023-01-16 22:43:02 -08:00
parent 11d055bfc1
commit bfe5d910c6
7 changed files with 24 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
weed/filer/filechunk_group.go
View File

@@ -12,13 +12,15 @@ type ChunkGroup struct {
chunkCache chunk_cache.ChunkCache
sections map[SectionIndex]*FileChunkSection
sectionsLock sync.RWMutex
readerCache *ReaderCache
}
func NewChunkGroup(lookupFn wdclient.LookupFileIdFunctionType, chunkCache chunk_cache.ChunkCache, chunks []*filer_pb.FileChunk) (*ChunkGroup, error) {
group := &ChunkGroup{
lookupFn: lookupFn,
chunkCache: chunkCache,
sections: make(map[SectionIndex]*FileChunkSection),
lookupFn: lookupFn,
chunkCache: chunkCache,
sections: make(map[SectionIndex]*FileChunkSection),
readerCache: NewReaderCache(32, chunkCache, lookupFn),
}
err := group.SetChunks(chunks)

2
weed/filer/filechunk_section.go
View File

@@ -74,7 +74,7 @@ func (section *FileChunkSection) setupForRead(group *ChunkGroup, fileSize int64)
}
if section.reader == nil {
section.reader = NewChunkReaderAtFromClient(group.lookupFn, section.chunkViews, group.chunkCache, min(int64(section.sectionIndex+1)*SectionSize, fileSize))
section.reader = NewChunkReaderAtFromClient(group.readerCache, section.chunkViews, min(int64(section.sectionIndex+1)*SectionSize, fileSize))
}
section.reader.fileSize = fileSize
}

5
weed/filer/reader_at.go
View File

@@ -10,7 +10,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/util"
"github.com/seaweedfs/seaweedfs/weed/util/chunk_cache"
"github.com/seaweedfs/seaweedfs/weed/wdclient"
)
@@ -88,12 +87,12 @@ func LookupFn(filerClient filer_pb.FilerClient) wdclient.LookupFileIdFunctionTyp
}
}
func NewChunkReaderAtFromClient(lookupFn wdclient.LookupFileIdFunctionType, chunkViews *IntervalList[*ChunkView], chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt {
func NewChunkReaderAtFromClient(readerCache *ReaderCache, chunkViews *IntervalList[*ChunkView], fileSize int64) *ChunkReadAt {
return &ChunkReadAt{
chunkViews: chunkViews,
fileSize: fileSize,
readerCache: newReaderCache(32, chunkCache, lookupFn),
readerCache: readerCache,
readerPattern: NewReaderPattern(),
}
}

10
weed/filer/reader_at_test.go
View File

@@ -68,7 +68,7 @@ func TestReaderAt(t *testing.T) {
readerAt := &ChunkReadAt{
chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
fileSize: 10,
readerCache: newReaderCache(3, &mockChunkCache{}, nil),
readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
readerPattern: NewReaderPattern(),
}
@@ -115,7 +115,7 @@ func TestReaderAt0(t *testing.T) {
readerAt := &ChunkReadAt{
chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
fileSize: 10,
readerCache: newReaderCache(3, &mockChunkCache{}, nil),
readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
readerPattern: NewReaderPattern(),
}
@@ -141,7 +141,7 @@ func TestReaderAt1(t *testing.T) {
readerAt := &ChunkReadAt{
chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
fileSize: 20,
readerCache: newReaderCache(3, &mockChunkCache{}, nil),
readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
readerPattern: NewReaderPattern(),
}
@@ -174,7 +174,7 @@ func TestReaderAtGappedChunksDoNotLeak(t *testing.T) {
readerAt := &ChunkReadAt{
chunkViews: ViewFromVisibleIntervals(visibles, 0, math.MaxInt64),
fileSize: 9,
readerCache: newReaderCache(3, &mockChunkCache{}, nil),
readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
readerPattern: NewReaderPattern(),
}
@@ -186,7 +186,7 @@ func TestReaderAtSparseFileDoesNotLeak(t *testing.T) {
readerAt := &ChunkReadAt{
chunkViews: ViewFromVisibleIntervals(NewIntervalList[*VisibleInterval](), 0, math.MaxInt64),
fileSize: 3,
readerCache: newReaderCache(3, &mockChunkCache{}, nil),
readerCache: NewReaderCache(3, &mockChunkCache{}, nil),
readerPattern: NewReaderPattern(),
}

2
weed/filer/reader_cache.go
View File

@@ -34,7 +34,7 @@ type SingleChunkCacher struct {
completedTimeNew int64
}
func newReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType) *ReaderCache {
func NewReaderCache(limit int, chunkCache chunk_cache.ChunkCache, lookupFileIdFn wdclient.LookupFileIdFunctionType) *ReaderCache {
return &ReaderCache{
limit: limit,
chunkCache: chunkCache,