update ChunkedFile to seekable reader, so we can use io.* to read data
This commit is contained in:
tnextday
@@ -8,10 +8,18 @@ import (
|
||||
"net/http"
|
||||
"sort"
|
||||
|
||||
"sync"
|
||||
|
||||
"github.com/chrislusf/seaweedfs/go/glog"
|
||||
"github.com/chrislusf/seaweedfs/go/util"
|
||||
)
|
||||
|
||||
var ErrOutOfRange = errors.New("Out of Range")
|
||||
var (
|
||||
// when the remote server does not allow range requests (Accept-Ranges was not set)
|
||||
ErrRangeRequestsNotSupported = errors.New("Range requests are not supported by the remote server")
|
||||
// ErrInvalidRange is returned by Read when trying to read past the end of the file
|
||||
ErrInvalidRange = errors.New("Invalid range")
|
||||
)
|
||||
|
||||
type ChunkInfo struct {
|
||||
Fid string `json:"fid,omitempty"`
|
||||
@@ -21,41 +29,74 @@ type ChunkInfo struct {
|
||||
|
||||
type ChunkList []*ChunkInfo
|
||||
|
||||
type ChunkedFile struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Mime string `json:"mime,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
Chunks ChunkList `json:"chunks,omitempty"`
|
||||
type ChunkManifest struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
Mime string `json:"mime,omitempty"`
|
||||
Size int64 `json:"size,omitempty"`
|
||||
Chunks ChunkList `json:"chunks,omitempty"`
|
||||
}
|
||||
|
||||
master string `json:"-"`
|
||||
// seekable chunked file reader
|
||||
type ChunkedFileReader struct {
|
||||
Manifest *ChunkManifest
|
||||
Master string
|
||||
pos int64
|
||||
pr *io.PipeReader
|
||||
pw *io.PipeWriter
|
||||
mutex sync.Mutex
|
||||
}
|
||||
|
||||
func (s ChunkList) Len() int { return len(s) }
|
||||
func (s ChunkList) Less(i, j int) bool { return s[i].Offset < s[j].Offset }
|
||||
func (s ChunkList) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
|
||||
|
||||
func NewChunkedNeedle(buffer []byte, master string) (*ChunkedFile, error) {
|
||||
c := ChunkedFile{}
|
||||
|
||||
if e := json.Unmarshal(buffer, c); e != nil {
|
||||
func LoadChunkedManifest(buffer []byte) (*ChunkManifest, error) {
|
||||
cm := ChunkManifest{}
|
||||
if e := json.Unmarshal(buffer, cm); e != nil {
|
||||
return nil, e
|
||||
}
|
||||
sort.Sort(c.Chunks)
|
||||
c.master = master
|
||||
return &c, nil
|
||||
sort.Sort(cm.Chunks)
|
||||
return &cm, nil
|
||||
}
|
||||
|
||||
func (c *ChunkedFile) Marshal() ([]byte, error) {
|
||||
return json.Marshal(c)
|
||||
func (cm *ChunkManifest) GetData() ([]byte, error) {
|
||||
return json.Marshal(cm)
|
||||
}
|
||||
|
||||
func copyChunk(fileUrl string, w io.Writer, startOffset, size int64) (written int64, e error) {
|
||||
func (cm *ChunkManifest) DeleteChunks(master string) error {
|
||||
fileIds := make([]string, 0, len(cm.Chunks))
|
||||
for _, ci := range cm.Chunks {
|
||||
fileIds = append(fileIds, ci.Fid)
|
||||
}
|
||||
results, e := DeleteFiles(master, fileIds)
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
deleteError := 0
|
||||
for _, ret := range results.Results {
|
||||
if ret.Error != "" {
|
||||
deleteError++
|
||||
glog.V(0).Infoln("delete error:", ret.Error, ret.Fid)
|
||||
}
|
||||
}
|
||||
if deleteError > 0 {
|
||||
return errors.New("Not all chunks deleted.")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cm *ChunkManifest) StoredHelper() error {
|
||||
//TODO
|
||||
return nil
|
||||
}
|
||||
|
||||
func httpRangeDownload(fileUrl string, w io.Writer, offset int64) (written int64, e error) {
|
||||
req, err := http.NewRequest("GET", fileUrl, nil)
|
||||
if err != nil {
|
||||
return written, err
|
||||
}
|
||||
if startOffset > 0 {
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", startOffset))
|
||||
if offset > 0 {
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", offset))
|
||||
}
|
||||
|
||||
resp, err := util.Do(req)
|
||||
@@ -63,65 +104,117 @@ func copyChunk(fileUrl string, w io.Writer, startOffset, size int64) (written in
|
||||
return written, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if startOffset > 0 && resp.StatusCode != 206 {
|
||||
return written, fmt.Errorf("Cannot Read Needle Position: %d [%s]", startOffset, fileUrl)
|
||||
}
|
||||
|
||||
if size > 0 {
|
||||
return io.CopyN(w, resp.Body, size)
|
||||
} else {
|
||||
return io.Copy(w, resp.Body)
|
||||
switch resp.StatusCode {
|
||||
case http.StatusRequestedRangeNotSatisfiable:
|
||||
return written, ErrInvalidRange
|
||||
case http.StatusOK:
|
||||
if offset > 0 {
|
||||
return written, ErrRangeRequestsNotSupported
|
||||
}
|
||||
case http.StatusPartialContent:
|
||||
break
|
||||
default:
|
||||
return written, fmt.Errorf("Read Needle http error: [%d] %s", resp.StatusCode, fileUrl)
|
||||
|
||||
}
|
||||
return io.Copy(w, resp.Body)
|
||||
}
|
||||
|
||||
func (c *ChunkedFile) WriteBuffer(w io.Writer, offset, size int64) (written int64, e error) {
|
||||
if offset >= c.Size || offset+size > c.Size {
|
||||
return written, ErrOutOfRange
|
||||
func (cf *ChunkedFileReader) Seek(offset int64, whence int) (int64, error) {
|
||||
var err error
|
||||
switch whence {
|
||||
case 0:
|
||||
case 1:
|
||||
offset += cf.pos
|
||||
case 2:
|
||||
offset = cf.Manifest.Size - offset
|
||||
}
|
||||
if offset > cf.Manifest.Size {
|
||||
err = ErrInvalidRange
|
||||
}
|
||||
if cf.pos != offset {
|
||||
cf.Close()
|
||||
}
|
||||
cf.pos = offset
|
||||
return cf.pos, err
|
||||
}
|
||||
|
||||
func (cf *ChunkedFileReader) WriteTo(w io.Writer) (n int64, err error) {
|
||||
cm := cf.Manifest
|
||||
chunkIndex := -1
|
||||
chunkStartOffset := int64(0)
|
||||
for i, ci := range c.Chunks {
|
||||
if offset >= ci.Offset && offset < ci.Offset+ci.Size {
|
||||
for i, ci := range cm.Chunks {
|
||||
if cf.pos >= ci.Offset && cf.pos < ci.Offset+ci.Size {
|
||||
chunkIndex = i
|
||||
chunkStartOffset = offset - ci.Offset
|
||||
chunkStartOffset = cf.pos - ci.Offset
|
||||
break
|
||||
}
|
||||
}
|
||||
if chunkIndex < 0 {
|
||||
return written, ErrOutOfRange
|
||||
return n, ErrInvalidRange
|
||||
}
|
||||
//preload next chunk?
|
||||
for ; chunkIndex < c.Chunks.Len(); chunkIndex++ {
|
||||
ci := c.Chunks[chunkIndex]
|
||||
fileUrl, lookupError := LookupFileId(c.master, ci.Fid)
|
||||
for ; chunkIndex < cm.Chunks.Len(); chunkIndex++ {
|
||||
ci := cm.Chunks[chunkIndex]
|
||||
// if we need read date from local volume server first?
|
||||
fileUrl, lookupError := LookupFileId(cf.Master, ci.Fid)
|
||||
if lookupError != nil {
|
||||
return written, lookupError
|
||||
return n, lookupError
|
||||
}
|
||||
rsize := int64(0)
|
||||
if size > 0 {
|
||||
rsize = size - written
|
||||
}
|
||||
if n, e := copyChunk(fileUrl, w, chunkStartOffset, rsize); e != nil {
|
||||
return written, e
|
||||
if wn, e := httpRangeDownload(fileUrl, w, chunkStartOffset); e != nil {
|
||||
return n, e
|
||||
} else {
|
||||
written += n
|
||||
n += wn
|
||||
cf.pos += wn
|
||||
}
|
||||
|
||||
if size > 0 && written >= size {
|
||||
break
|
||||
}
|
||||
chunkStartOffset = 0
|
||||
}
|
||||
|
||||
return written, nil
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (c *ChunkedFile) DeleteHelper() error {
|
||||
//TODO Delete all chunks
|
||||
return nil
|
||||
func (cf *ChunkedFileReader) ReadAt(p []byte, off int64) (n int, err error) {
|
||||
cf.Seek(off, 0)
|
||||
return cf.Read(p)
|
||||
}
|
||||
|
||||
func (c *ChunkedFile) StoredHelper() error {
|
||||
//TODO
|
||||
return nil
|
||||
func (cf *ChunkedFileReader) Read(p []byte) (int, error) {
|
||||
return cf.getPipeReader().Read(p)
|
||||
}
|
||||
|
||||
func (cf *ChunkedFileReader) Close() (e error) {
|
||||
cf.mutex.Lock()
|
||||
defer cf.mutex.Unlock()
|
||||
return cf.closePipe()
|
||||
}
|
||||
|
||||
func (cf *ChunkedFileReader) closePipe() (e error) {
|
||||
if cf.pr != nil {
|
||||
if err := cf.pr.Close(); err != nil {
|
||||
e = err
|
||||
}
|
||||
}
|
||||
cf.pr = nil
|
||||
if cf.pw != nil {
|
||||
if err := cf.pw.Close(); err != nil {
|
||||
e = err
|
||||
}
|
||||
}
|
||||
cf.pw = nil
|
||||
return e
|
||||
}
|
||||
|
||||
func (cf *ChunkedFileReader) getPipeReader() io.Reader {
|
||||
cf.mutex.Lock()
|
||||
defer cf.mutex.Unlock()
|
||||
if cf.pr != nil && cf.pw != nil {
|
||||
return cf.pr
|
||||
}
|
||||
cf.closePipe()
|
||||
cf.pr, cf.pw = io.Pipe()
|
||||
go func(pw *io.PipeWriter) {
|
||||
_, e := cf.WriteTo(pw)
|
||||
pw.CloseWithError(e)
|
||||
}(cf.pw)
|
||||
return cf.pr
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user