gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add leveldb store

Browse Source 
1. switch to viper for filer store configuration
2. simplify FindEntry() return values, removing “found”
3. add leveldb store
This commit is contained in:
Chris Lu
2018-05-26 03:49:46 -07:00
parent c34feca59c
commit 9e77563c99
17 changed files with 382 additions and 193 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
weed/filer2/configuration.go Normal file
View File

@@ -0,0 +1,87 @@
package filer2
import (
"os"
"github.com/spf13/viper"
"github.com/chrislusf/seaweedfs/weed/glog"
)
const (
FILER_TOML_EXAMPLE = `
# A sample TOML config file for SeaweedFS filer store
# local in memory, mostly for testing purpose
[memory]
enabled = false
[leveldb]
enabled = false
dir = "." # directory to store level db files
[mysql]
enabled = true
server = "192.168.1.1"
port = 8080
username = ""
password = ""
database = ""
connection_max_idle = 100
connection_max_open = 100
[postgres]
enabled = false
server = "192.168.1.1"
port = 8080
username = ""
password = ""
database = ""
connection_max_idle = 100
connection_max_open = 100
`
)
var (
Stores []FilerStore
)
func (f *Filer) LoadConfiguration() {
// find a filer store
viper.SetConfigName("filer") // name of config file (without extension)
viper.AddConfigPath(".") // optionally look for config in the working directory
viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in
if err := viper.ReadInConfig(); err != nil { // Handle errors reading the config file
glog.Fatalf("Failed to load filer.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/" +
"\n\nPlease follow this example and add a filer.toml file to " +
"current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/:\n" + FILER_TOML_EXAMPLE)
}
glog.V(0).Infof("Reading filer configuration from %s", viper.ConfigFileUsed())
for _, store := range Stores {
if viper.GetBool(store.GetName() + ".enabled") {
viperSub := viper.Sub(store.GetName())
if err := store.Initialize(viperSub); err != nil {
glog.Fatalf("Failed to initialize store for %s: %+v",
store.GetName(), err)
}
f.SetStore(store)
glog.V(0).Infof("Configure filer for %s from %s", store.GetName(), viper.ConfigFileUsed())
return
}
}
println()
println("Supported filer stores are:")
for _, store := range Stores {
println(" " + store.GetName())
}
println()
println("Please configure a supported filer store in", viper.ConfigFileUsed())
println()
os.Exit(-1)
}

38
weed/filer2/embedded/embedded_store.go
View File

@@ -1,38 +0,0 @@
package embedded
import (
"github.com/syndtr/goleveldb/leveldb"
"github.com/chrislusf/seaweedfs/weed/filer2"
)
type EmbeddedStore struct {
db *leveldb.DB
}
func NewEmbeddedStore(dir string) (filer *EmbeddedStore, err error) {
filer = &EmbeddedStore{}
if filer.db, err = leveldb.OpenFile(dir, nil); err != nil {
return
}
return
}
func (filer *EmbeddedStore) InsertEntry(entry *filer2.Entry) (err error) {
return nil
}
func (filer *EmbeddedStore) UpdateEntry(entry *filer2.Entry) (err error) {
return nil
}
func (filer *EmbeddedStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) {
return false, nil, nil
}
func (filer *EmbeddedStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
return nil, nil
}
func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) {
return nil, nil
}

5
weed/filer2/entry.go
View File

@@ -28,15 +28,14 @@ type Entry struct {
Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"`
}
func (entry Entry) Size() uint64 {
func (entry *Entry) Size() uint64 {
return TotalSize(entry.Chunks)
}
func (entry Entry) Timestamp() time.Time {
func (entry *Entry) Timestamp() time.Time {
if entry.IsDirectory() {
return entry.Crtime
} else {
return entry.Mtime
}
}

4
weed/filer2/entry_codec.go
View File

@@ -9,7 +9,7 @@ import (
"fmt"
)
func (entry Entry) EncodeAttributesAndChunks() ([]byte, error) {
func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) {
message := &filer_pb.Entry{
Attributes: &filer_pb.FuseAttributes{
Crtime: entry.Attr.Crtime.Unix(),
@@ -23,7 +23,7 @@ func (entry Entry) EncodeAttributesAndChunks() ([]byte, error) {
return proto.Marshal(message)
}
func (entry Entry) DecodeAttributesAndChunks(blob []byte) (error) {
func (entry *Entry) DecodeAttributesAndChunks(blob []byte) (error) {
message := &filer_pb.Entry{}

12
weed/filer2/filer.go
View File

@@ -50,11 +50,7 @@ func (f *Filer) CreateEntry(entry *Entry) (error) {
// not found, check the store directly
if dirEntry == nil {
glog.V(4).Infof("find uncached directory: %s", dirPath)
var dirFindErr error
_, dirEntry, dirFindErr = f.FindEntry(FullPath(dirPath))
if dirFindErr != nil {
return fmt.Errorf("findDirectory %s: %v", dirPath, dirFindErr)
}
dirEntry, _ = f.FindEntry(FullPath(dirPath))
} else {
glog.V(4).Infof("found cached directory: %s", dirPath)
}
@@ -116,13 +112,13 @@ func (f *Filer) UpdateEntry(entry *Entry) (err error) {
return f.store.UpdateEntry(entry)
}
func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) {
func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) {
return f.store.FindEntry(p)
}
func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) {
found, entry, err := f.FindEntry(p)
if err != nil || !found {
entry, err := f.FindEntry(p)
if err != nil {
return nil, err
}
if entry.IsDirectory() {

9
weed/filer2/filerstore.go
View File

@@ -1,11 +1,16 @@
package filer2
import "errors"
import (
"errors"
"github.com/spf13/viper"
)
type FilerStore interface {
GetName() string
Initialize(viper *viper.Viper) (error)
InsertEntry(*Entry) (error)
UpdateEntry(*Entry) (err error)
FindEntry(FullPath) (found bool, entry *Entry, err error)
FindEntry(FullPath) (entry *Entry, err error)
DeleteEntry(FullPath) (fileEntry *Entry, err error)
ListDirectoryEntries(dirPath FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error)
}

171
weed/filer2/leveldb/leveldb_store.go Normal file
View File

@@ -0,0 +1,171 @@
package leveldb
import (
"fmt"
"bytes"
"github.com/syndtr/goleveldb/leveldb"
"github.com/chrislusf/seaweedfs/weed/filer2"
leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/spf13/viper"
weed_util "github.com/chrislusf/seaweedfs/weed/util"
)
const (
DIR_FILE_SEPARATOR = byte(0x00)
)
func init() {
filer2.Stores = append(filer2.Stores, &LevelDBStore{})
}
type LevelDBStore struct {
db *leveldb.DB
}
func (filer *LevelDBStore) GetName() string {
return "leveldb"
}
func (filer *LevelDBStore) Initialize(viper *viper.Viper) (err error) {
dir := viper.GetString("dir")
return filer.initialize(dir)
}
func (filer *LevelDBStore) initialize(dir string) (err error) {
if err := weed_util.TestFolderWritable(dir); err != nil {
return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
}
if filer.db, err = leveldb.OpenFile(dir, nil); err != nil {
return
}
return
}
func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) {
key := genKey(entry.DirAndName())
value, err := entry.EncodeAttributesAndChunks()
if err != nil {
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
}
err = store.db.Put(key, value, nil)
if err != nil {
return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
}
// println("saved", entry.FullPath, "chunks", len(entry.Chunks))
return nil
}
func (store *LevelDBStore) UpdateEntry(entry *filer2.Entry) (err error) {
return store.InsertEntry(entry)
}
func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
key := genKey(fullpath.DirAndName())
data, err := store.db.Get(key, nil)
if err == leveldb.ErrNotFound {
return nil, filer2.ErrNotFound
}
if err != nil {
return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)
}
entry = &filer2.Entry{
FullPath: fullpath,
}
err = entry.DecodeAttributesAndChunks(data)
if err != nil {
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
}
// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))
return entry, nil
}
func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
key := genKey(fullpath.DirAndName())
entry, _ = store.FindEntry(fullpath)
err = store.db.Delete(key, nil)
if err != nil {
return entry, fmt.Errorf("delete %s : %v", entry.FullPath, err)
}
return entry, nil
}
func (store *LevelDBStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool,
limit int) (entries []*filer2.Entry, err error) {
directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
iter := store.db.NewIterator(&leveldb_util.Range{Start: genDirectoryKeyPrefix(fullpath, startFileName)}, nil)
for iter.Next() {
key := iter.Key()
if !bytes.HasPrefix(key, directoryPrefix) {
break
}
fileName := getNameFromKey(key)
if fileName == "" {
continue
}
if fileName == startFileName && !inclusive {
continue
}
limit--
if limit < 0 {
break
}
entry := &filer2.Entry{
FullPath: filer2.NewFullPath(string(fullpath), fileName),
}
if decodeErr := entry.DecodeAttributesAndChunks(iter.Value()); decodeErr != nil {
err = decodeErr
glog.V(0).Infof("list %s : %v", entry.FullPath, err)
break
}
entries = append(entries, entry)
}
iter.Release()
return entries, err
}
func genKey(dirPath, fileName string) (key []byte) {
key = []byte(dirPath)
key = append(key, DIR_FILE_SEPARATOR)
key = append(key, []byte(fileName)...)
return key
}
func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string) (keyPrefix []byte) {
keyPrefix = []byte(string(fullpath))
keyPrefix = append(keyPrefix, DIR_FILE_SEPARATOR)
if len(startFileName) > 0 {
keyPrefix = append(keyPrefix, []byte(startFileName)...)
}
return keyPrefix
}
func getNameFromKey(key []byte) (string) {
sepIndex := len(key) - 1
for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR {
sepIndex--
}
return string(key[sepIndex+1:])
}

61
weed/filer2/leveldb/leveldb_store_test.go Normal file
View File

@@ -0,0 +1,61 @@
package leveldb
import (
"testing"
"github.com/chrislusf/seaweedfs/weed/filer2"
"io/ioutil"
"os"
)
func TestCreateAndFind(t *testing.T) {
filer := filer2.NewFiler("")
dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
defer os.RemoveAll(dir)
store := &LevelDBStore{}
store.initialize(dir)
filer.SetStore(store)
filer.DisableDirectoryCache()
fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
entry1 := &filer2.Entry{
FullPath: fullpath,
Attr: filer2.Attr{
Mode: 0440,
Uid: 1234,
Gid: 5678,
},
}
if err := filer.CreateEntry(entry1); err != nil {
t.Errorf("create entry %v: %v", entry1.FullPath, err)
return
}
entry, err := filer.FindEntry(fullpath)
if err != nil {
t.Errorf("find entry: %v", err)
return
}
if entry.FullPath != entry1.FullPath {
t.Errorf("find wrong entry: %v", entry.FullPath)
return
}
// checking one upper directory
entries, _ := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
}
// checking one upper directory
entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100)
if len(entries) != 1 {
t.Errorf("list entries count: %v", len(entries))
return
}
}

25
weed/filer2/memdb/memdb_store.go
View File

@@ -6,8 +6,13 @@ import (
"strings"
"fmt"
"time"
"github.com/spf13/viper"
)
func init() {
filer2.Stores = append(filer2.Stores, &MemDbStore{})
}
type MemDbStore struct {
tree *btree.BTree
}
@@ -20,10 +25,13 @@ func (a Entry) Less(b btree.Item) bool {
return strings.Compare(string(a.FullPath), string(b.(Entry).FullPath)) < 0
}
func NewMemDbStore() (filer *MemDbStore) {
filer = &MemDbStore{}
func (filer *MemDbStore) GetName() string {
return "memory"
}
func (filer *MemDbStore) Initialize(viper *viper.Viper) (err error) {
filer.tree = btree.New(8)
return
return nil
}
func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
@@ -34,22 +42,21 @@ func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
}
func (filer *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) {
found, _, err := filer.FindEntry(entry.FullPath)
if !found {
return fmt.Errorf("No such file: %s", entry.FullPath)
if _, err = filer.FindEntry(entry.FullPath); err != nil {
return fmt.Errorf("no such file %s : %v", entry.FullPath, err)
}
entry.Mtime = time.Now()
filer.tree.ReplaceOrInsert(Entry{entry})
return nil
}
func (filer *MemDbStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) {
func (filer *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
item := filer.tree.Get(Entry{&filer2.Entry{FullPath: fullpath}})
if item == nil {
return false, nil, nil
return nil, nil
}
entry = item.(Entry).Entry
return true, entry, nil
return entry, nil
}
func (filer *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) {

8
weed/filer2/memdb/memdb_store_test.go
View File

@@ -7,7 +7,9 @@ import (
func TestCreateAndFind(t *testing.T) {
filer := filer2.NewFiler("")
filer.SetStore(NewMemDbStore())
store := &MemDbStore{}
store.Initialize(nil)
filer.SetStore(store)
filer.DisableDirectoryCache()
fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
@@ -47,7 +49,9 @@ func TestCreateAndFind(t *testing.T) {
func TestCreateFileAndList(t *testing.T) {
filer := filer2.NewFiler("")
filer.SetStore(NewMemDbStore())
store := &MemDbStore{}
store.Initialize(nil)
filer.SetStore(store)
filer.DisableDirectoryCache()
entry1 := &filer2.Entry{