fix: skip exhausted blocks before creating an interval (#8180)
* fix: skip exhausted blocks before creating an interval * refactor: optimize interval creation and fix logic duplication * docs: add docstring for LocateData * refactor: extract moveToNextBlock helper to deduplicate logic * fix: use int64 for block index comparison to prevent overflow * test: add unit test for LocateData boundary crossing (issue #8179) * fix: skip exhausted blocks to prevent negative interval size and panics (issue #8179) * refactor: apply review suggestions for test maintainability and code style
This commit is contained in:
@@ -12,10 +12,23 @@ type Interval struct {
|
|||||||
LargeBlockRowsCount int
|
LargeBlockRowsCount int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LocateData finds the intervals of data within erasure coding blocks for a given offset and size.
|
||||||
func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) (intervals []Interval) {
|
func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64, size types.Size) (intervals []Interval) {
|
||||||
blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, shardDatSize, offset)
|
blockIndex, isLargeBlock, nLargeBlockRows, innerBlockOffset := locateOffset(largeBlockLength, smallBlockLength, shardDatSize, offset)
|
||||||
|
|
||||||
for size > 0 {
|
for size > 0 {
|
||||||
|
blockRemaining := largeBlockLength - innerBlockOffset
|
||||||
|
if !isLargeBlock {
|
||||||
|
blockRemaining = smallBlockLength - innerBlockOffset
|
||||||
|
}
|
||||||
|
|
||||||
|
if blockRemaining <= 0 {
|
||||||
|
// move to next block
|
||||||
|
blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
|
||||||
|
innerBlockOffset = 0
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
interval := Interval{
|
interval := Interval{
|
||||||
BlockIndex: blockIndex,
|
BlockIndex: blockIndex,
|
||||||
InnerBlockOffset: innerBlockOffset,
|
InnerBlockOffset: innerBlockOffset,
|
||||||
@@ -23,11 +36,6 @@ func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, of
|
|||||||
LargeBlockRowsCount: int(nLargeBlockRows),
|
LargeBlockRowsCount: int(nLargeBlockRows),
|
||||||
}
|
}
|
||||||
|
|
||||||
blockRemaining := largeBlockLength - innerBlockOffset
|
|
||||||
if !isLargeBlock {
|
|
||||||
blockRemaining = smallBlockLength - innerBlockOffset
|
|
||||||
}
|
|
||||||
|
|
||||||
if int64(size) <= blockRemaining {
|
if int64(size) <= blockRemaining {
|
||||||
interval.Size = size
|
interval.Size = size
|
||||||
intervals = append(intervals, interval)
|
intervals = append(intervals, interval)
|
||||||
@@ -37,17 +45,23 @@ func LocateData(largeBlockLength, smallBlockLength int64, shardDatSize int64, of
|
|||||||
intervals = append(intervals, interval)
|
intervals = append(intervals, interval)
|
||||||
|
|
||||||
size -= interval.Size
|
size -= interval.Size
|
||||||
blockIndex += 1
|
blockIndex, isLargeBlock = moveToNextBlock(blockIndex, isLargeBlock, nLargeBlockRows)
|
||||||
if isLargeBlock && blockIndex == interval.LargeBlockRowsCount*DataShardsCount {
|
|
||||||
isLargeBlock = false
|
|
||||||
blockIndex = 0
|
|
||||||
}
|
|
||||||
innerBlockOffset = 0
|
innerBlockOffset = 0
|
||||||
|
|
||||||
}
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func moveToNextBlock(blockIndex int, isLargeBlock bool, nLargeBlockRows int64) (int, bool) {
|
||||||
|
nextBlockIndex := blockIndex + 1
|
||||||
|
nextIsLargeBlock := isLargeBlock
|
||||||
|
if isLargeBlock && int64(nextBlockIndex) == nLargeBlockRows*DataShardsCount {
|
||||||
|
nextIsLargeBlock = false
|
||||||
|
nextBlockIndex = 0
|
||||||
|
}
|
||||||
|
return nextBlockIndex, nextIsLargeBlock
|
||||||
|
}
|
||||||
|
|
||||||
func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
|
func locateOffset(largeBlockLength, smallBlockLength int64, shardDatSize int64, offset int64) (blockIndex int, isLargeBlock bool, nLargeBlockRows int64, innerBlockOffset int64) {
|
||||||
largeRowSize := largeBlockLength * DataShardsCount
|
largeRowSize := largeBlockLength * DataShardsCount
|
||||||
nLargeBlockRows = (shardDatSize - 1) / largeBlockLength
|
nLargeBlockRows = (shardDatSize - 1) / largeBlockLength
|
||||||
|
|||||||
@@ -236,3 +236,20 @@ func TestLocateData3(t *testing.T) {
|
|||||||
{BlockIndex: 8876, InnerBlockOffset: 912752, Size: 112568, IsLargeBlock: false, LargeBlockRowsCount: 2},
|
{BlockIndex: 8876, InnerBlockOffset: 912752, Size: 112568, IsLargeBlock: false, LargeBlockRowsCount: 2},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLocateData_Issue8179(t *testing.T) {
|
||||||
|
large := int64(10000)
|
||||||
|
small := int64(100)
|
||||||
|
shardSize := int64(259092) // Resulting in nLargeBlockRows = 25 as seen in panic log
|
||||||
|
|
||||||
|
// Testing range through the large-to-small transition boundary
|
||||||
|
nLargeBlockRows := (shardSize - 1) / large
|
||||||
|
largeAreaSize := nLargeBlockRows * int64(DataShardsCount) * large
|
||||||
|
|
||||||
|
for offset := largeAreaSize - 500; offset < largeAreaSize+500; offset++ {
|
||||||
|
intervals := LocateData(large, small, shardSize, offset, 200)
|
||||||
|
for _, interval := range intervals {
|
||||||
|
assert.True(t, interval.Size > 0, "Interval size must be positive at offset %d, got %+v", offset, interval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user