gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add grpc ec shard read

Browse Source
This commit is contained in:
Chris Lu
2019-05-27 11:59:03 -07:00
parent a4f3d82c57
commit b4b407e403
12 changed files with 418 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
weed/storage/erasure_coding/ec_locate.go
View File

@@ -69,3 +69,15 @@ func locateOffsetWithinBlocks(blockLength int64, offset int64) (blockIndex int,
innerBlockOffset = offset % blockLength
return
}
func (interval Interval) ToShardIdAndOffset(largeBlockSize, smallBlockSize int64) (ShardId, int64) {
ecFileOffset := interval.InnerBlockOffset
rowIndex := interval.BlockIndex / DataShardsCount
if interval.IsLargeBlock {
ecFileOffset += int64(rowIndex) * largeBlockSize
} else {
ecFileOffset += int64(interval.LargeBlockRowsCount)*largeBlockSize + int64(rowIndex)*smallBlockSize
}
ecFileIndex := interval.BlockIndex % DataShardsCount
return ShardId(ecFileIndex), ecFileOffset
}

6
weed/storage/erasure_coding/ec_shard.go
View File

@@ -105,3 +105,9 @@ func (shard *EcVolumeShard) findNeedleFromEcx(needleId types.NeedleId) (offset t
err = fmt.Errorf("needle id %d not found", needleId)
return
}
func (shard *EcVolumeShard) ReadAt(buf []byte, offset int64) (int, error) {
return shard.ecdFile.ReadAt(buf, offset)
}

19
weed/storage/erasure_coding/ec_test.go
View File

@@ -105,10 +105,8 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin
intervals := LocateData(largeBlockSize, smallBlockSize, datSize, offset.ToAcutalOffset(), size)
nLargeBlockRows := int(datSize / (largeBlockSize * DataShardsCount))
for i, interval := range intervals {
if d, e := readOneInterval(interval, ecFiles, nLargeBlockRows); e != nil {
if d, e := readOneInterval(interval, ecFiles); e != nil {
return nil, e
} else {
if i == 0 {
@@ -122,21 +120,14 @@ func readEcFile(datSize int64, ecFiles []*os.File, offset types.Offset, size uin
return data, nil
}
func readOneInterval(interval Interval, ecFiles []*os.File, nLargeBlockRows int) (data []byte, err error) {
ecFileOffset := interval.InnerBlockOffset
rowIndex := interval.BlockIndex / DataShardsCount
if interval.IsLargeBlock {
ecFileOffset += int64(rowIndex) * largeBlockSize
} else {
ecFileOffset += int64(nLargeBlockRows)*largeBlockSize + int64(rowIndex)*smallBlockSize
}
func readOneInterval(interval Interval, ecFiles []*os.File) (data []byte, err error) {
ecFileIndex := interval.BlockIndex % DataShardsCount
ecFileOffset, ecFileIndex := interval.ToShardIdAndOffset(largeBlockSize, smallBlockSize)
data = make([]byte, interval.Size)
err = readFromFile(ecFiles[ecFileIndex], data, ecFileOffset)
{ // do some ec testing
ecData, err := readFromOtherEcFiles(ecFiles, ecFileIndex, ecFileOffset, interval.Size)
ecData, err := readFromOtherEcFiles(ecFiles, int(ecFileIndex), ecFileOffset, interval.Size)
if err != nil {
return nil, fmt.Errorf("ec reconstruct error: %v", err)
}
@@ -198,7 +189,7 @@ func TestLocateData(t *testing.T) {
if len(intervals) != 1 {
t.Errorf("unexpected interval size %d", len(intervals))
}
if !intervals[0].sameAs(Interval{0, 0, 1, false}) {
if !intervals[0].sameAs(Interval{0, 0, 1, false, 1}) {
t.Errorf("unexpected interval %+v", intervals[0])
}

13
weed/storage/erasure_coding/ec_volume.go
View File

@@ -6,6 +6,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
type EcVolumeShards []*EcVolumeShard
@@ -71,19 +72,17 @@ func (shards *EcVolumeShards) ToVolumeEcShardInformationMessage() (messages []*m
return
}
func (shards *EcVolumeShards) ReadEcShardNeedle(n *needle.Needle) (int, error) {
func (shards *EcVolumeShards) LocateEcShardNeedle(n *needle.Needle) (offset types.Offset, size uint32, intervals []Interval, err error) {
shard := (*shards)[0]
// find the needle from ecx file
offset, size, err := shard.findNeedleFromEcx(n.Id)
offset, size, err = shard.findNeedleFromEcx(n.Id)
if err != nil {
return 0, err
return types.Offset{}, 0, nil, err
}
// calculate the locations in the ec shards
intervals := LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size)
intervals = LocateData(ErasureCodingLargeBlockSize, ErasureCodingSmallBlockSize, shard.ecxFileSize, offset.ToAcutalOffset(), size)
// TODO read the intervals
return len(intervals), nil
return
}

16
weed/storage/needle/needle_read_write.go
View File

@@ -170,11 +170,8 @@ func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dat
return dataSlice, err
}
func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
bytes, err := ReadNeedleBlob(r, offset, size, version)
if err != nil {
return err
}
// ReadBytes hydrates the needle from the bytes buffer, with only n.Id is set.
func (n *Needle) ReadBytes(bytes []byte, offset int64, size uint32, version Version) (err error) {
n.ParseNeedleHeader(bytes)
if n.Size != size {
return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size)
@@ -203,6 +200,15 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version
return nil
}
// ReadData hydrates the needle from the file, with only n.Id is set.
func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
bytes, err := ReadNeedleBlob(r, offset, size, version)
if err != nil {
return err
}
return n.ReadBytes(bytes, offset, size, version)
}
func (n *Needle) ParseNeedleHeader(bytes []byte) {
n.Cookie = BytesToCookie(bytes[0:CookieSize])
n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize])

57
weed/storage/store_ec.go
View File

@@ -89,9 +89,62 @@ func (s *Store) HasEcShard(vid needle.VolumeId) (erasure_coding.EcVolumeShards,
func (s *Store) ReadEcShardNeedle(vid needle.VolumeId, n *needle.Needle) (int, error) {
for _, location := range s.Locations {
if ecShards, found := location.HasEcShard(vid); found {
return ecShards.ReadEcShardNeedle(n)
if localEcShards, found := location.HasEcShard(vid); found {
offset, size, intervals, err := localEcShards.LocateEcShardNeedle(n)
if err != nil {
return 0, err
}
bytes, err := s.ReadEcShardIntervals(vid, localEcShards, intervals)
if err != nil {
return 0, fmt.Errorf("ReadEcShardIntervals: %v", err)
}
version := needle.CurrentVersion
err = n.ReadBytes(bytes, offset.ToAcutalOffset(), size, version)
if err != nil {
return 0, fmt.Errorf("readbytes: %v", err)
}
return len(bytes), nil
}
}
return 0, fmt.Errorf("ec shard %d not found", vid)
}
func (s *Store) ReadEcShardIntervals(vid needle.VolumeId, localEcShards erasure_coding.EcVolumeShards, intervals []erasure_coding.Interval) (data []byte, err error) {
for i, interval := range intervals {
if d, e := s.readOneEcShardInterval(vid, localEcShards, interval); e != nil {
return nil, e
} else {
if i == 0 {
data = d
} else {
data = append(data, d...)
}
}
}
return
}
func (s *Store) readOneEcShardInterval(vid needle.VolumeId, localEcShards erasure_coding.EcVolumeShards, interval erasure_coding.Interval) (data []byte, err error) {
shardId, actualOffset := interval.ToShardIdAndOffset(erasure_coding.ErasureCodingLargeBlockSize, erasure_coding.ErasureCodingSmallBlockSize)
data = make([]byte, interval.Size)
if shard, found := localEcShards.FindEcVolumeShard(shardId); found {
if _, err = shard.ReadAt(data, actualOffset); err != nil {
return
}
} else {
s.readOneRemoteEcShardInterval(vid, shardId, data, actualOffset)
}
return
}
func (s *Store) readOneRemoteEcShardInterval(vid needle.VolumeId, shardId erasure_coding.ShardId, buf []byte, offset int64) (n int, err error) {
return
}