fix(gcs): resolve credential conflict and improve backup logging (#7951)

* fix(gcs): resolve credential conflict and improve backup logging

- Workaround GCS SDK's "multiple credential options" error by manually constructing an authenticated HTTP client.
- Include source entry path in filer backup error logs for better visibility on missing volumes/404s.

* fix: address PR review feedback

- Add nil check for EventNotification in getSourceKey
- Avoid reassigning google_application_credentials parameter in gcs_sink.go

* fix(gcs): return errors instead of calling glog.Fatalf in initialize

Adheres to Go best practices and allows for more graceful failure handling by callers.

* read from bind ip
This commit is contained in:
Chris Lu
2026-01-03 14:41:25 -08:00
committed by GitHub
parent 23fc3f2621
commit 25975bacfb
3 changed files with 50 additions and 17 deletions

View File

@@ -148,13 +148,13 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
}
// ignore HTTP 404 from remote reads
if errors.Is(err, http.ErrNotFound) {
glog.V(0).Infof("got 404 error, ignore it: %s", err.Error())
glog.V(0).Infof("got 404 error for %s, ignore it: %s", getSourceKey(resp), err.Error())
return nil
}
// also ignore missing volume/lookup errors coming from LookupFileId or vid map
errStr := err.Error()
if strings.Contains(errStr, "LookupFileId") || (strings.Contains(errStr, "volume id") && strings.Contains(errStr, "not found")) {
glog.V(0).Infof("got missing-volume error, ignore it: %s", errStr)
glog.V(0).Infof("got missing-volume error for %s, ignore it: %s", getSourceKey(resp), errStr)
return nil
}
return err
@@ -206,3 +206,17 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
return pb.FollowMetadata(sourceFiler, grpcDialOption, metadataFollowOption, processEventFnWithOffset)
}
func getSourceKey(resp *filer_pb.SubscribeMetadataResponse) string {
if resp == nil || resp.EventNotification == nil {
return ""
}
message := resp.EventNotification
if message.NewEntry != nil {
return string(util.FullPath(message.NewParentPath).Child(message.NewEntry.Name))
}
if message.OldEntry != nil {
return string(util.FullPath(resp.Directory).Child(message.OldEntry.Name))
}
return ""
}