fix(replication): resume partial chunk reads on EOF instead of re-downloading (#8607)
* fix(replication): resume partial chunk reads on EOF instead of re-downloading When replicating chunks and the source connection drops mid-transfer, accumulate the bytes already received and retry with a Range header to fetch only the remaining bytes. This avoids re-downloading potentially large chunks from scratch on each retry, reducing load on busy source servers and speeding up recovery. * test(replication): add tests for downloadWithRange including gzip partial reads Tests cover: - No offset (no Range header sent) - With offset (Range header verified) - Content-Disposition filename extraction - Partial read + resume: server drops connection mid-transfer, client resumes with Range from the offset of received bytes - Gzip partial read + resume: first response is gzip-encoded (Go auto- decompresses), connection drops, resume request gets decompressed data (Go doesn't add Accept-Encoding when Range is set, so the server decompresses), combined bytes match original * fix(replication): address PR review comments - Consolidate downloadWithRange into DownloadFile with optional offset parameter (variadic), eliminating code duplication (DRY) - Validate HTTP response status: require 206 + correct Content-Range when offset > 0, reject when server ignores Range header - Use if/else for fullData assignment for clarity - Add test for rejected Range (server returns 200 instead of 206) * refactor(replication): remove unused ReplicationSource interface The interface was never referenced and its signature didn't match the actual FilerSource.ReadPart method. --------- Co-authored-by: Copilot <copilot@github.com>
This commit is contained in:
@@ -201,7 +201,7 @@ func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) e
|
||||
return readFn(r.Body)
|
||||
}
|
||||
|
||||
func DownloadFile(fileUrl string, jwt string) (filename string, header http.Header, resp *http.Response, e error) {
|
||||
func DownloadFile(fileUrl string, jwt string, offset ...int64) (filename string, header http.Header, resp *http.Response, e error) {
|
||||
req, err := http.NewRequest(http.MethodGet, fileUrl, nil)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
@@ -209,10 +209,29 @@ func DownloadFile(fileUrl string, jwt string) (filename string, header http.Head
|
||||
|
||||
maybeAddAuth(req, jwt)
|
||||
|
||||
var rangeOffset int64
|
||||
if len(offset) > 0 {
|
||||
rangeOffset = offset[0]
|
||||
}
|
||||
if rangeOffset > 0 {
|
||||
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", rangeOffset))
|
||||
}
|
||||
|
||||
response, err := GetGlobalHttpClient().Do(req)
|
||||
if err != nil {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
|
||||
if rangeOffset > 0 {
|
||||
expected := fmt.Sprintf("bytes %d-", rangeOffset)
|
||||
if response.StatusCode != http.StatusPartialContent ||
|
||||
!strings.HasPrefix(response.Header.Get("Content-Range"), expected) {
|
||||
CloseResponse(response)
|
||||
return "", nil, nil, fmt.Errorf("range request %q to %s returned %s with Content-Range %q",
|
||||
req.Header.Get("Range"), fileUrl, response.Status, response.Header.Get("Content-Range"))
|
||||
}
|
||||
}
|
||||
|
||||
header = response.Header
|
||||
contentDisposition := response.Header["Content-Disposition"]
|
||||
if len(contentDisposition) > 0 {
|
||||
|
||||
Reference in New Issue
Block a user