gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactoring for supporing cassandra as filer meta data store

Browse Source
This commit is contained in:
Chris Lu
2015-01-05 14:58:30 -08:00
parent a3e4145e8a
commit 165734ce11
8 changed files with 53 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
go/filer/directory.go
View File

@@ -1,18 +0,0 @@
package filer
type DirectoryId int32
type DirectoryEntry struct {
Name string //dir name without path
Id DirectoryId
}
type DirectoryManager interface {
FindDirectory(dirPath string) (DirectoryId, error)
ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error)
MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
DeleteDirectory(dirPath string) error
//functions used by FUSE
FindDirectoryById(DirectoryId, error)
}

15
go/filer/embedded_filer/directory.go Normal file
View File

@@ -0,0 +1,15 @@
package embedded_filer
import (
"github.com/chrislusf/weed-fs/go/filer"
)
type DirectoryManager interface {
FindDirectory(dirPath string) (filer.DirectoryId, error)
ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error)
MakeDirectory(currentDirPath string, dirName string) (filer.DirectoryId, error)
MoveUnderDirectory(oldDirPath string, newParentDirPath string) error
DeleteDirectory(dirPath string) error
//functions used by FUSE
FindDirectoryById(filer.DirectoryId, error)
}

19
go/filer/directory_in_map.go → go/filer/embedded_filer/directory_in_map.go
View File

@@ -1,4 +1,4 @@
package filer package embedded_filer
import ( import (
"bufio" "bufio"
@@ -10,6 +10,7 @@ import (
"strings" "strings"
"sync" "sync"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/util" "github.com/chrislusf/weed-fs/go/util"
) )
@@ -19,12 +20,12 @@ type DirectoryEntryInMap struct {
Name string Name string
Parent *DirectoryEntryInMap Parent *DirectoryEntryInMap
SubDirectories map[string]*DirectoryEntryInMap SubDirectories map[string]*DirectoryEntryInMap
Id DirectoryId Id filer.DirectoryId
} }
type DirectoryManagerInMap struct { type DirectoryManagerInMap struct {
Root *DirectoryEntryInMap Root *DirectoryEntryInMap
max DirectoryId max filer.DirectoryId
logFile *os.File logFile *os.File
isLoading bool isLoading bool
} }
@@ -83,7 +84,7 @@ func (dm *DirectoryManagerInMap) processEachLine(line string) error {
if pe != nil { if pe != nil {
return pe return pe
} }
if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil { if e := dm.loadDirectory(parts[1], filer.DirectoryId(v)); e != nil {
return e return e
} }
case "mov": case "mov":
@@ -142,7 +143,7 @@ func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryI
} }
return dir, nil return dir, nil
} }
func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (DirectoryId, error) { func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (filer.DirectoryId, error) {
d, e := dm.findDirectory(dirPath) d, e := dm.findDirectory(dirPath)
if e == nil { if e == nil {
return d.Id, nil return d.Id, nil
@@ -150,7 +151,7 @@ func (dm *DirectoryManagerInMap) FindDirectory(dirPath string) (DirectoryId, err
return dm.Root.Id, e return dm.Root.Id, e
} }
func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error { func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId filer.DirectoryId) error {
dirPath = filepath.Clean(dirPath) dirPath = filepath.Clean(dirPath)
if dirPath == "/" { if dirPath == "/" {
return nil return nil
@@ -201,7 +202,7 @@ func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEn
return dir, created return dir, created
} }
func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) { func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (filer.DirectoryId, error) {
dir, _ := dm.makeDirectory(dirPath) dir, _ := dm.makeDirectory(dirPath)
return dir.Id, nil return dir.Id, nil
} }
@@ -228,13 +229,13 @@ func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParent
return nil return nil
} }
func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []DirectoryEntry, err error) { func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryEntry, err error) {
d, e := dm.findDirectory(dirPath) d, e := dm.findDirectory(dirPath)
if e != nil { if e != nil {
return dirNames, e return dirNames, e
} }
for k, v := range d.SubDirectories { for k, v := range d.SubDirectories {
dirNames = append(dirNames, DirectoryEntry{Name: k, Id: v.Id}) dirNames = append(dirNames, filer.DirectoryEntry{Name: k, Id: v.Id})
} }
return dirNames, nil return dirNames, nil
} }

2
go/filer/directory_test.go → go/filer/embedded_filer/directory_test.go
View File

@@ -1,4 +1,4 @@
package filer package embedded_filer
import ( import (
"os" "os"

9
go/filer/filer_embedded.go → go/filer/embedded_filer/filer_embedded.go
View File

@@ -1,4 +1,4 @@
package filer package embedded_filer
import ( import (
"errors" "errors"
@@ -6,6 +6,7 @@ import (
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/operation" "github.com/chrislusf/weed-fs/go/operation"
) )
@@ -48,13 +49,13 @@ func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) {
} }
return filer.files.FindFile(dirId, file) return filer.files.FindFile(dirId, file)
} }
func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId DirectoryId, err error) { func (filer *FilerEmbedded) FindDirectory(dirPath string) (dirId filer.DirectoryId, err error) {
return filer.directories.FindDirectory(dirPath) return filer.directories.FindDirectory(dirPath)
} }
func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []DirectoryEntry, err error) { func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryEntry, err error) {
return filer.directories.ListDirectories(dirPath) return filer.directories.ListDirectories(dirPath)
} }
func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error) { func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) {
dirId, e := filer.directories.FindDirectory(dirPath) dirId, e := filer.directories.FindDirectory(dirPath)
if e != nil { if e != nil {
return nil, e return nil, e

15
go/filer/files_in_leveldb.go → go/filer/embedded_filer/files_in_leveldb.go
View File

@@ -1,8 +1,9 @@
package filer package embedded_filer
import ( import (
"bytes" "bytes"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/glog"
"github.com/syndtr/goleveldb/leveldb" "github.com/syndtr/goleveldb/leveldb"
"github.com/syndtr/goleveldb/leveldb/util" "github.com/syndtr/goleveldb/leveldb/util"
@@ -27,7 +28,7 @@ func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) {
return return
} }
func genKey(dirId DirectoryId, fileName string) []byte { func genKey(dirId filer.DirectoryId, fileName string) []byte {
ret := make([]byte, 0, 4+len(fileName)) ret := make([]byte, 0, 4+len(fileName))
for i := 3; i >= 0; i-- { for i := 3; i >= 0; i-- {
ret = append(ret, byte(dirId>>(uint(i)*8))) ret = append(ret, byte(dirId>>(uint(i)*8)))
@@ -36,25 +37,25 @@ func genKey(dirId DirectoryId, fileName string) []byte {
return ret return ret
} }
func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) { func (fl *FileListInLevelDb) CreateFile(dirId filer.DirectoryId, fileName string, fid string) (err error) {
glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid) glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid)
return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil) return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil)
} }
func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) { func (fl *FileListInLevelDb) DeleteFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
if fid, err = fl.FindFile(dirId, fileName); err != nil { if fid, err = fl.FindFile(dirId, fileName); err != nil {
return return
} }
err = fl.db.Delete(genKey(dirId, fileName), nil) err = fl.db.Delete(genKey(dirId, fileName), nil)
return fid, err return fid, err
} }
func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) { func (fl *FileListInLevelDb) FindFile(dirId filer.DirectoryId, fileName string) (fid string, err error) {
data, e := fl.db.Get(genKey(dirId, fileName), nil) data, e := fl.db.Get(genKey(dirId, fileName), nil)
if e != nil { if e != nil {
return "", e return "", e
} }
return string(data), nil return string(data), nil
} }
func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []FileEntry) { func (fl *FileListInLevelDb) ListFiles(dirId filer.DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) {
glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit) glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit)
dirKey := genKey(dirId, "") dirKey := genKey(dirId, "")
iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil) iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil)
@@ -74,7 +75,7 @@ func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, l
break break
} }
} }
files = append(files, FileEntry{Name: fileName, Id: FileId(string(iter.Value()))}) files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))})
} }
iter.Release() iter.Release()
return return

15
go/filer/filer.go
View File

@@ -7,13 +7,22 @@ type FileEntry struct {
Id FileId `json:"fid,omitempty"` Id FileId `json:"fid,omitempty"`
} }
type DirectoryId int32
type DirectoryEntry struct {
Name string //dir name without path
Id DirectoryId
}
type Filer interface { type Filer interface {
CreateFile(filePath string, fid string) (err error) CreateFile(fullFileName string, fid string) (err error)
FindFile(filePath string) (fid string, err error) FindFile(fullFileName string) (fid string, err error)
DeleteFile(fullFileName string) (fid string, err error)
//Optional functions. embedded filer support these
FindDirectory(dirPath string) (dirId DirectoryId, err error) FindDirectory(dirPath string) (dirId DirectoryId, err error)
ListDirectories(dirPath string) (dirs []DirectoryEntry, err error) ListDirectories(dirPath string) (dirs []DirectoryEntry, err error)
ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error) ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error)
DeleteDirectory(dirPath string, recursive bool) (err error) DeleteDirectory(dirPath string, recursive bool) (err error)
DeleteFile(filePath string) (fid string, err error)
Move(fromPath string, toPath string) (err error) Move(fromPath string, toPath string) (err error)
} }

3
go/weed/weed_server/filer_server.go
View File

@@ -5,6 +5,7 @@ import (
"strconv" "strconv"
"github.com/chrislusf/weed-fs/go/filer" "github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/filer/embedded_filer"
"github.com/chrislusf/weed-fs/go/glog" "github.com/chrislusf/weed-fs/go/glog"
) )
@@ -28,7 +29,7 @@ func NewFilerServer(r *http.ServeMux, port int, master string, dir string, colle
port: ":" + strconv.Itoa(port), port: ":" + strconv.Itoa(port),
} }
if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil { if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil {
glog.Fatal("Can not start filer in dir", dir, ": ", err.Error()) glog.Fatal("Can not start filer in dir", dir, ": ", err.Error())
return return
} }