gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

merge chunks during upload (#4130)

Browse Source 
* merge chunks during upload

* fix test
This commit is contained in:
Chris Lu
2023-01-12 23:07:36 -08:00
committed by GitHub
parent 2452f3b2f7
commit 1cd2e64aac
3 changed files with 31 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
weed/mount/page_writer/page_chunk_mem.go
View File

@@ -105,4 +105,18 @@ func (mc *MemChunk) SaveContent(saveFn SaveToStorageFunc) {
saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+t.StartOffset, t.Size(), t.TsNs, func() {
})
}
for t := mc.usage.head.next; t != mc.usage.tail; t = t.next {
startOffset := t.StartOffset
stopOffset := t.stopOffset
tsNs := t.TsNs
for t != mc.usage.tail && t.next.StartOffset == stopOffset {
stopOffset = t.next.stopOffset
t = t.next
tsNs = max(tsNs, t.TsNs)
}
reader := util.NewBytesReader(mc.buf[startOffset:stopOffset])
saveFn(reader, int64(mc.logicChunkIndex)*mc.chunkSize+startOffset, stopOffset-startOffset, tsNs, func() {
})
}
}

15
weed/mount/page_writer/page_chunk_swapfile.go
View File

@@ -176,11 +176,20 @@ func (sc *SwapFileChunk) SaveContent(saveFn SaveToStorageFunc) {
}
// println(sc.logicChunkIndex, "|", "save")
for t := sc.usage.head.next; t != sc.usage.tail; t = t.next {
data := mem.Allocate(int(t.Size()))
n, _ := sc.swapfile.file.ReadAt(data, t.StartOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
startOffset := t.StartOffset
stopOffset := t.stopOffset
tsNs := t.TsNs
for t != sc.usage.tail && t.next.StartOffset == stopOffset {
stopOffset = t.next.stopOffset
t = t.next
tsNs = max(tsNs, t.TsNs)
}
data := mem.Allocate(int(stopOffset - startOffset))
n, _ := sc.swapfile.file.ReadAt(data, startOffset+int64(sc.actualChunkIndex)*sc.swapfile.chunkSize)
if n > 0 {
reader := util.NewBytesReader(data[:n])
saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+t.StartOffset, int64(n), t.TsNs, func() {
saveFn(reader, int64(sc.logicChunkIndex)*sc.swapfile.chunkSize+startOffset, int64(n), tsNs, func() {
})
}
mem.Free(data)