gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

filer: remove replication, collection, disk_type info from entry metadata

Browse Source 
these metadata can change and are not used
This commit is contained in:
chrislu
2022-06-06 00:39:35 -07:00
parent cbf46de5f4
commit 4fd5f96598
20 changed files with 537 additions and 725 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
weed/filer/entry.go
View File

@@ -15,10 +15,7 @@ type Attr struct {
Uid uint32 // owner uid
Gid uint32 // group gid
Mime string // mime type
Replication string // replication
Collection string // collection name
TtlSec int32 // ttl in seconds
DiskType string
UserName string
GroupNames []string
SymlinkTarget string

6
weed/filer/entry_codec.go
View File

@@ -39,10 +39,7 @@ func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
Uid: entry.Uid,
Gid: entry.Gid,
Mime: entry.Mime,
Collection: entry.Attr.Collection,
Replication: entry.Attr.Replication,
TtlSec: entry.Attr.TtlSec,
DiskType: entry.Attr.DiskType,
UserName: entry.Attr.UserName,
GroupName: entry.Attr.GroupNames,
SymlinkTarget: entry.Attr.SymlinkTarget,
@@ -67,10 +64,7 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
t.Uid = attr.Uid
t.Gid = attr.Gid
t.Mime = attr.Mime
t.Collection = attr.Collection
t.Replication = attr.Replication
t.TtlSec = attr.TtlSec
t.DiskType = attr.DiskType
t.UserName = attr.UserName
t.GroupNames = attr.GroupName
t.SymlinkTarget = attr.SymlinkTarget

20
weed/filer/filer.go
View File

@@ -37,8 +37,6 @@ type Filer struct {
fileIdDeletionQueue *util.UnboundedQueue
GrpcDialOption grpc.DialOption
DirBucketsPath string
FsyncBuckets []string
buckets *FilerBuckets
Cipher bool
LocalMetaLogBuffer *log_buffer.LogBuffer
metaLogCollection string
@@ -217,7 +215,6 @@ func (f *Filer) CreateEntry(ctx context.Context, entry *Entry, o_excl bool, isFr
}
}
f.maybeAddBucket(entry)
f.NotifyUpdateEvent(ctx, oldEntry, entry, true, isFromOtherCluster, signatures)
f.deleteChunksIfNotNew(oldEntry, entry)
@@ -254,15 +251,13 @@ func (f *Filer) ensureParentDirecotryEntry(ctx context.Context, entry *Entry, di
dirEntry = &Entry{
FullPath: util.FullPath(dirPath),
Attr: Attr{
Mtime: now,
Crtime: now,
Mode: os.ModeDir | entry.Mode | 0111,
Uid: entry.Uid,
Gid: entry.Gid,
Collection: entry.Collection,
Replication: entry.Replication,
UserName: entry.UserName,
GroupNames: entry.GroupNames,
Mtime: now,
Crtime: now,
Mode: os.ModeDir | entry.Mode | 0111,
Uid: entry.Uid,
Gid: entry.Gid,
UserName: entry.UserName,
GroupNames: entry.GroupNames,
},
}
@@ -274,7 +269,6 @@ func (f *Filer) ensureParentDirecotryEntry(ctx context.Context, entry *Entry, di
return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr)
}
} else {
f.maybeAddBucket(dirEntry)
f.NotifyUpdateEvent(ctx, nil, dirEntry, false, isFromOtherCluster, nil)
}

106
weed/filer/filer_buckets.go
View File

@@ -1,76 +1,9 @@
package filer
import (
"context"
"math"
"strings"
"sync"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
)
type BucketName string
type BucketOption struct {
Name BucketName
Replication string
fsync bool
}
type FilerBuckets struct {
dirBucketsPath string
buckets map[BucketName]*BucketOption
sync.RWMutex
}
func (f *Filer) LoadBuckets() {
f.buckets = &FilerBuckets{
buckets: make(map[BucketName]*BucketOption),
}
limit := int64(math.MaxInt32)
entries, _, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit, "", "", "")
if err != nil {
glog.V(1).Infof("no buckets found: %v", err)
return
}
shouldFsyncMap := make(map[string]bool)
for _, bucket := range f.FsyncBuckets {
shouldFsyncMap[bucket] = true
}
glog.V(1).Infof("buckets found: %d", len(entries))
f.buckets.Lock()
for _, entry := range entries {
_, shouldFsnyc := shouldFsyncMap[entry.Name()]
f.buckets.buckets[BucketName(entry.Name())] = &BucketOption{
Name: BucketName(entry.Name()),
Replication: entry.Replication,
fsync: shouldFsnyc,
}
}
f.buckets.Unlock()
}
func (f *Filer) ReadBucketOption(buketName string) (replication string, fsync bool) {
f.buckets.RLock()
defer f.buckets.RUnlock()
option, found := f.buckets.buckets[BucketName(buketName)]
if !found {
return "", false
}
return option.Replication, option.fsync
}
func (f *Filer) isBucket(entry *Entry) bool {
if !entry.IsDirectory() {
return false
@@ -83,43 +16,6 @@ func (f *Filer) isBucket(entry *Entry) bool {
return false
}
f.buckets.RLock()
defer f.buckets.RUnlock()
_, found := f.buckets.buckets[BucketName(dirName)]
return found
}
func (f *Filer) maybeAddBucket(entry *Entry) {
if !entry.IsDirectory() {
return
}
parent, dirName := entry.FullPath.DirAndName()
if parent != f.DirBucketsPath {
return
}
f.addBucket(dirName, &BucketOption{
Name: BucketName(dirName),
Replication: entry.Replication,
})
}
func (f *Filer) addBucket(buketName string, bucketOption *BucketOption) {
f.buckets.Lock()
defer f.buckets.Unlock()
f.buckets.buckets[BucketName(buketName)] = bucketOption
}
func (f *Filer) deleteBucket(buketName string) {
f.buckets.Lock()
defer f.buckets.Unlock()
delete(f.buckets.buckets, BucketName(buketName))
return true
}

1
weed/filer/filer_delete_entry.go
View File

@@ -60,7 +60,6 @@ func (f *Filer) DeleteEntryMetaAndData(ctx context.Context, p util.FullPath, isR
if isDeleteCollection {
collectionName := entry.Name()
f.doDeleteCollection(collectionName)
f.deleteBucket(collectionName)
}
return nil

10
weed/filer/read_write.go
View File

@@ -53,12 +53,10 @@ func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, conte
Name: name,
IsDirectory: false,
Attributes: &filer_pb.FuseAttributes{
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
Collection: "",
Replication: "",
FileSize: uint64(len(content)),
Mtime: time.Now().Unix(),
Crtime: time.Now().Unix(),
FileMode: uint32(0644),
FileSize: uint64(len(content)),
},
Content: content,
},