gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor filer_pb.Entry and filer.Entry to use GetChunks()

Browse Source 
for later locking on reading chunks
This commit is contained in:
chrislu
2022-11-15 06:33:36 -08:00
parent 371972a1c2
commit 70a4c98b00
56 changed files with 107 additions and 103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
weed/server/filer_grpc_server.go
View File

@@ -203,14 +203,14 @@ func (fs *FilerServer) cleanupChunks(fullpath string, existingEntry *filer.Entry
// remove old chunks if not included in the new ones
if existingEntry != nil {
garbage, err = filer.MinusChunks(fs.lookupFileId, existingEntry.Chunks, newEntry.Chunks)
garbage, err = filer.MinusChunks(fs.lookupFileId, existingEntry.GetChunks(), newEntry.GetChunks())
if err != nil {
return newEntry.Chunks, nil, fmt.Errorf("MinusChunks: %v", err)
return newEntry.GetChunks(), nil, fmt.Errorf("MinusChunks: %v", err)
}
}
// files with manifest chunks are usually large and append only, skip calculating covered chunks
manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(newEntry.Chunks)
manifestChunks, nonManifestChunks := filer.SeparateManifestChunks(newEntry.GetChunks())
chunks, coveredChunks := filer.CompactFileChunks(fs.lookupFileId, nonManifestChunks)
garbage = append(garbage, coveredChunks...)
@@ -256,7 +256,7 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo
},
}
} else {
offset = int64(filer.TotalSize(entry.Chunks))
offset = int64(filer.TotalSize(entry.GetChunks()))
}
for _, chunk := range req.Chunks {
@@ -264,13 +264,13 @@ func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendTo
offset += int64(chunk.Size)
}
entry.Chunks = append(entry.Chunks, req.Chunks...)
entry.Chunks = append(entry.GetChunks(), req.Chunks...)
so, err := fs.detectStorageOption(string(fullpath), "", "", entry.TtlSec, "", "", "", "")
if err != nil {
glog.Warningf("detectStorageOption: %v", err)
return &filer_pb.AppendToEntryResponse{}, err
}
entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), entry.Chunks)
entry.Chunks, err = filer.MaybeManifestize(fs.saveAsChunk(so), entry.GetChunks())
if err != nil {
// not good, but should be ok
glog.V(0).Infof("MaybeManifestize: %v", err)

2
weed/server/filer_grpc_server_remote.go
View File

@@ -169,7 +169,7 @@ func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req
return nil, fetchAndWriteErr
}
garbage := entry.Chunks
garbage := entry.GetChunks()
newEntry := entry.ShallowClone()
newEntry.Chunks = chunks

2
weed/server/filer_grpc_server_rename.go
View File

@@ -165,7 +165,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, stream filer_pb.Seawee
newEntry := &filer.Entry{
FullPath: newPath,
Attr: entry.Attr,
Chunks: entry.Chunks,
Chunks: entry.GetChunks(),
Extended: entry.Extended,
Content: entry.Content,
HardLinkCounter: entry.HardLinkCounter,

8
weed/server/filer_server_handlers_read.go
View File

@@ -135,7 +135,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
if query.Get("resolveManifest") == "true" {
if entry.Chunks, _, err = filer.ResolveChunkManifest(
fs.filer.MasterClient.GetLookupFileIdFunction(),
entry.Chunks, 0, math.MaxInt64); err != nil {
entry.GetChunks(), 0, math.MaxInt64); err != nil {
err = fmt.Errorf("failed to resolve chunk manifest, err: %s", err.Error())
writeJsonError(w, r, http.StatusInternalServerError, err)
}
@@ -212,7 +212,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
if shouldResize {
data := mem.Allocate(int(totalSize))
defer mem.Free(data)
err := filer.ReadAll(data, fs.filer.MasterClient, entry.Chunks)
err := filer.ReadAll(data, fs.filer.MasterClient, entry.GetChunks())
if err != nil {
glog.Errorf("failed to read %s: %v", path, err)
w.WriteHeader(http.StatusInternalServerError)
@@ -233,7 +233,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
}
return err
}
chunks := entry.Chunks
chunks := entry.GetChunks()
if entry.IsInRemoteOnly() {
dir, name := entry.FullPath.DirAndName()
if resp, err := fs.CacheRemoteObjectToLocalCluster(context.Background(), &filer_pb.CacheRemoteObjectToLocalClusterRequest{
@@ -244,7 +244,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request)
glog.Errorf("CacheRemoteObjectToLocalCluster %s: %v", entry.FullPath, err)
return fmt.Errorf("cache %s: %v", entry.FullPath, err)
} else {
chunks = resp.Entry.Chunks
chunks = resp.Entry.GetChunks()
}
}

2
weed/server/filer_server_handlers_write_autochunk.go
View File

@@ -181,7 +181,7 @@ func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileNa
}
entry.FileSize += uint64(chunkOffset)
}
newChunks = append(entry.Chunks, fileChunks...)
newChunks = append(entry.GetChunks(), fileChunks...)
// TODO
if len(entry.Content) > 0 {

2
weed/server/filer_server_handlers_write_cipher.go
View File

@@ -91,7 +91,7 @@ func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *ht
}
if dbErr := fs.filer.CreateEntry(ctx, entry, false, false, nil, false); dbErr != nil {
fs.filer.DeleteChunks(entry.Chunks)
fs.filer.DeleteChunks(entry.GetChunks())
err = dbErr
filerResult.Error = dbErr.Error()
return

6
weed/server/webdav_server.go
View File

@@ -438,13 +438,13 @@ func (f *WebDavFile) Write(buf []byte) (int, error) {
}
f.entry.Content = nil
f.entry.Chunks = append(f.entry.Chunks, chunk)
f.entry.Chunks = append(f.entry.GetChunks(), chunk)
return flushErr
}
f.bufWriter.CloseFunc = func() error {
manifestedChunks, manifestErr := filer.MaybeManifestize(f.saveDataAsChunk, f.entry.Chunks)
manifestedChunks, manifestErr := filer.MaybeManifestize(f.saveDataAsChunk, f.entry.GetChunks())
if manifestErr != nil {
// not good, but should be ok
glog.V(0).Infof("file %s close MaybeManifestize: %v", f.name, manifestErr)
@@ -514,7 +514,7 @@ func (f *WebDavFile) Read(p []byte) (readSize int, err error) {
return 0, io.EOF
}
if f.entryViewCache == nil {
f.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(f.fs), f.entry.Chunks, 0, fileSize)
f.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(filer.LookupFn(f.fs), f.entry.GetChunks(), 0, fileSize)
f.reader = nil
}
if f.reader == nil {