gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

each log function adds a "done" return parameter

Browse Source
This commit is contained in:
chrislu
2024-03-07 10:50:09 -08:00
parent 62397f2371
commit 34f2b600ac
6 changed files with 23 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
weed/util/log_buffer/log_buffer.go
View File

@@ -22,8 +22,8 @@ type dataToFlush struct {
data *bytes.Buffer
}
type EachLogEntryFuncType func(logEntry *filer_pb.LogEntry) error
type LogFlushFuncType func(startTime, stopTime time.Time, buf []byte)
type EachLogEntryFuncType func(logEntry *filer_pb.LogEntry) (isDone bool, err error)
type LogFlushFuncType func(logBuffer *LogBuffer, startTime, stopTime time.Time, buf []byte)
type LogReadFromDiskFuncType func(startPosition MessagePosition, stopTsNs int64, eachLogEntryFn EachLogEntryFuncType) (lastReadPosition MessagePosition, isDone bool, err error)
type LogBuffer struct {
@@ -146,7 +146,7 @@ func (logBuffer *LogBuffer) loopFlush() {
for d := range logBuffer.flushChan {
if d != nil {
// glog.V(4).Infof("%s flush [%v, %v] size %d", m.name, d.startTime, d.stopTime, len(d.data.Bytes()))
logBuffer.flushFn(d.startTime, d.stopTime, d.data.Bytes())
logBuffer.flushFn(logBuffer, d.startTime, d.stopTime, d.data.Bytes())
d.releaseMemory()
// local logbuffer is different from aggregate logbuffer here
logBuffer.lastFlushTime = d.stopTime

8
weed/util/log_buffer/log_read.go
View File

@@ -30,7 +30,7 @@ func NewMessagePosition(tsNs int64, batchIndex int64) MessagePosition {
}
func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition MessagePosition, stopTsNs int64,
waitForDataFn func() bool, eachLogDataFn func(logEntry *filer_pb.LogEntry) error) (lastReadPosition MessagePosition, isDone bool, err error) {
waitForDataFn func() bool, eachLogDataFn EachLogEntryFuncType) (lastReadPosition MessagePosition, isDone bool, err error) {
// loop through all messages
var bytesBuf *bytes.Buffer
var batchIndex int64
@@ -69,6 +69,7 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition
if waitForDataFn() {
continue
} else {
isDone = true
return
}
}
@@ -101,10 +102,13 @@ func (logBuffer *LogBuffer) LoopProcessLogData(readerName string, startPosition
}
lastReadPosition = NewMessagePosition(logEntry.TsNs, batchIndex)
if err = eachLogDataFn(logEntry); err != nil {
if isDone, err = eachLogDataFn(logEntry); err != nil {
glog.Errorf("LoopProcessLogData: %s process log entry %d %v: %v", readerName, batchSize+1, logEntry, err)
return
}
if isDone {
return
}
pos += 4 + int(size)
batchSize++