gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplifying dirhash to 64bit integer

Browse Source
This commit is contained in:
Chris Lu
2018-05-26 21:24:03 -07:00
parent 955eae3500
commit 87b3b84471
3 changed files with 40 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
weed/filer2/abstract_sql/abstract_sql_store.go
View File

@@ -21,7 +21,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) {
} }
res, err := store.DB.Exec("INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)", res, err := store.DB.Exec("INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)",
md5hash(dir), name, dir, meta) hashToLong(dir), name, dir, meta)
if err != nil { if err != nil {
return fmt.Errorf("mysql insert %s: %s", entry.FullPath, err) return fmt.Errorf("mysql insert %s: %s", entry.FullPath, err)
} }
@@ -42,7 +42,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) {
} }
res, err := store.DB.Exec("UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?", res, err := store.DB.Exec("UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?",
meta, md5hash(dir), name, dir) meta, hashToLong(dir), name, dir)
if err != nil { if err != nil {
return fmt.Errorf("mysql update %s: %s", entry.FullPath, err) return fmt.Errorf("mysql update %s: %s", entry.FullPath, err)
} }
@@ -58,7 +58,7 @@ func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entr
dir, name := fullpath.DirAndName() dir, name := fullpath.DirAndName()
row := store.DB.QueryRow("SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?", row := store.DB.QueryRow("SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?",
md5hash(dir), name, dir) hashToLong(dir), name, dir)
var data []byte var data []byte
if err := row.Scan(&data); err != nil { if err := row.Scan(&data); err != nil {
return nil, fmt.Errorf("mysql read entry %s: %v", fullpath, err) return nil, fmt.Errorf("mysql read entry %s: %v", fullpath, err)
@@ -81,7 +81,7 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En
dir, name := fullpath.DirAndName() dir, name := fullpath.DirAndName()
res, err := store.DB.Exec("DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?", res, err := store.DB.Exec("DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?",
md5hash(dir), name, dir) hashToLong(dir), name, dir)
if err != nil { if err != nil {
return nil, fmt.Errorf("mysql delete %s: %s", fullpath, err) return nil, fmt.Errorf("mysql delete %s: %s", fullpath, err)
} }
@@ -101,7 +101,7 @@ func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, st
sqlText = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? LIMIT ?" sqlText = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? LIMIT ?"
} }
rows, err := store.DB.Query(sqlText, md5hash(string(fullpath)), startFileName, string(fullpath), limit) rows, err := store.DB.Query(sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit)
if err != nil { if err != nil {
return nil, fmt.Errorf("mysql list %s : %v", fullpath, err) return nil, fmt.Errorf("mysql list %s : %v", fullpath, err)
} }

25
weed/filer2/abstract_sql/hashing.go
View File

@@ -5,9 +5,28 @@ import (
"io" "io"
) )
// returns a 128 bit hash // returns a 64 bit big int
func md5hash(dir string) []byte { func hashToLong(dir string) (v int64) {
h := md5.New() h := md5.New()
io.WriteString(h, dir) io.WriteString(h, dir)
return h.Sum(nil)
b := h.Sum(nil)
v += int64(b[0])
v <<= 8
v += int64(b[1])
v <<= 8
v += int64(b[2])
v <<= 8
v += int64(b[3])
v <<= 8
v += int64(b[4])
v <<= 8
v += int64(b[5])
v <<= 8
v += int64(b[6])
v <<= 8
v += int64(b[7])
return
} }

19
weed/filer2/configuration.go
View File

@@ -27,9 +27,9 @@ dir = "." # directory to store level db files
# need to manually create a table "filemeta". # need to manually create a table "filemeta".
# #
# CREATE TABLE IF NOT EXISTS filemeta ( # CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BINARY(16) COMMENT 'MD5 hash value of directory field', # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field',
# name VARCHAR(1000) NOT NULL DEFAULT "" COMMENT 'directory or file name', # name VARCHAR(1000) COMMENT 'directory or file name',
# directory VARCHAR(4096) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', # directory VARCHAR(4096) COMMENT 'full path to parent directory',
# meta BLOB, # meta BLOB,
# PRIMARY KEY (dirhash, name) # PRIMARY KEY (dirhash, name)
# ) DEFAULT CHARSET=utf8; # ) DEFAULT CHARSET=utf8;
@@ -44,10 +44,17 @@ connection_max_idle = 2
connection_max_open = 100 connection_max_open = 100
[postgres] [postgres]
# CREATE TABLE IF NOT EXISTS filemeta (
# dirhash BIGINT,
# name VARCHAR(1000),
# directory VARCHAR(4096),
# meta bytea,
# PRIMARY KEY (dirhash, name)
# );
enabled = false enabled = false
server = "192.168.1.1" server = "localhost"
port = 8080 port = 5432
username = "" username = "postgres"
password = "" password = ""
database = "" database = ""
connection_max_idle = 100 connection_max_idle = 100