gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

simplifying volume id mechanism, removing automatic volume id generation and discovering

Browse Source 
periodically report machine status

git-svn-id: https://weed-fs.googlecode.com/svn/trunk@25 282b0af5-e82d-9cf1-ede4-77906d7719d0
This commit is contained in:
chris.lu@gmail.com
2011-12-26 09:43:17 +00:00
parent ae3a53388f
commit 6913fabc64
4 changed files with 99 additions and 149 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
weed-fs/src/cmd/weedc.go
View File

@@ -10,15 +10,17 @@ import (
"mime"
"strconv"
"strings"
"time"
)
var (
port = flag.Int("port", 8080, "http listen port")
chunkFolder = flag.String("dir", "/tmp", "data directory to store files")
chunkCount = flag.Int("chunks", 5, "data chunks to store files")
volumes = flag.String("volumes", "0,1-3,4", "comma-separated list of volume ids or range of ids")
publicServer = flag.String("pserver", "localhost:8080", "public server to serve data read")
metaServer = flag.String("mserver", "localhost:9333", "metadata server to store mappings")
IsDebug = flag.Bool("debug", false, "enable debug mode")
pulse = flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
store *storage.Store
)
@@ -38,8 +40,8 @@ func storeHandler(w http.ResponseWriter, r *http.Request) {
}
func GetHandler(w http.ResponseWriter, r *http.Request) {
n := new(storage.Needle)
vid, fid, ext := parseURLPath(r.URL.Path)
volumeId, _ := strconv.Atoui64(vid)
vid, fid, ext := parseURLPath(r.URL.Path)
volumeId, _ := strconv.Atoui64(vid)
n.ParsePath(fid)
if *IsDebug {
@@ -54,14 +56,14 @@ func GetHandler(w http.ResponseWriter, r *http.Request) {
log.Println("request with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
return
}
if ext!="" {
if ext != "" {
w.Header().Set("Content-Type", mime.TypeByExtension(ext))
}
w.Write(n.Data)
}
func PostHandler(w http.ResponseWriter, r *http.Request) {
vid, _, _ := parseURLPath(r.URL.Path)
volumeId, e := strconv.Atoui64(vid)
vid, _, _ := parseURLPath(r.URL.Path)
volumeId, e := strconv.Atoui64(vid)
if e != nil {
writeJson(w, r, e)
} else {
@@ -73,19 +75,19 @@ func PostHandler(w http.ResponseWriter, r *http.Request) {
}
func DeleteHandler(w http.ResponseWriter, r *http.Request) {
n := new(storage.Needle)
vid, fid, _ := parseURLPath(r.URL.Path)
volumeId, _ := strconv.Atoui64(vid)
n.ParsePath(fid)
vid, fid, _ := parseURLPath(r.URL.Path)
volumeId, _ := strconv.Atoui64(vid)
n.ParsePath(fid)
cookie := n.Cookie
count, _ := store.Read(volumeId, n)
cookie := n.Cookie
count, _ := store.Read(volumeId, n)
if n.Cookie != cookie {
log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
return
}
n.Size = 0
if n.Cookie != cookie {
log.Println("delete with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
return
}
n.Size = 0
store.Write(volumeId, n)
m := make(map[string]uint32)
m["size"] = uint32(count)
@@ -105,33 +107,38 @@ func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) {
}
//log.Println("JSON Response", string(bytes))
}
func parseURLPath(path string)(vid, fid, ext string){
sepIndex := strings.LastIndex(path, "/")
commaIndex := strings.LastIndex(path[sepIndex:], ",")
dotIndex := strings.LastIndex(path[sepIndex:], ".")
vid = path[sepIndex+1 : commaIndex]
fid = path[commaIndex+1:]
ext = ""
if dotIndex > 0 {
fid = path[commaIndex+1 : dotIndex]
ext = path[dotIndex+1:]
}
if commaIndex <= 0 {
log.Println("unknown file id", path[sepIndex+1:commaIndex])
return
}
return
func parseURLPath(path string) (vid, fid, ext string) {
sepIndex := strings.LastIndex(path, "/")
commaIndex := strings.LastIndex(path[sepIndex:], ",")
dotIndex := strings.LastIndex(path[sepIndex:], ".")
vid = path[sepIndex+1 : commaIndex]
fid = path[commaIndex+1:]
ext = ""
if dotIndex > 0 {
fid = path[commaIndex+1 : dotIndex]
ext = path[dotIndex+1:]
}
if commaIndex <= 0 {
log.Println("unknown file id", path[sepIndex+1:commaIndex])
return
}
return
}
func main() {
flag.Parse()
//TODO: now default to 1G, this value should come from server?
store = storage.NewStore(*port, *publicServer, *chunkFolder, 1024*1024*1024, *chunkCount)
store = storage.NewStore(*port, *publicServer, *chunkFolder, *volumes)
defer store.Close()
http.HandleFunc("/", storeHandler)
http.HandleFunc("/status", statusHandler)
store.Join(*metaServer)
go func() {
for {
store.Join(*metaServer)
time.Sleep(int64(*pulse) * 1e9)
}
}()
log.Println("store joined at", *metaServer)
log.Println("Start storage service at http://127.0.0.1:" + strconv.Itoa(*port))

10
weed-fs/src/cmd/weeds.go
View File

@@ -17,6 +17,7 @@ var (
metaFolder = flag.String("mdir", "/tmp", "data directory to store mappings")
capacity = flag.Int("capacity", 100, "maximum number of volumes to hold")
mapper *directory.Mapper
IsDebug = flag.Bool("debug", false, "verbose debug information")
)
func dirLookupHandler(w http.ResponseWriter, r *http.Request) {
@@ -42,10 +43,8 @@ func dirJoinHandler(w http.ResponseWriter, r *http.Request) {
publicServer := r.FormValue("publicServer")
volumes := new([]storage.VolumeInfo)
json.Unmarshal([]byte(r.FormValue("volumes")), volumes)
capacity, _ := strconv.Atoi(r.FormValue("capacity"))
log.Println("Recieved joining request from remote address", s, "capacity=", capacity, "volumes", r.FormValue("volumes"))
vids := mapper.Add(*directory.NewMachine(s, publicServer, *volumes, capacity))
writeJson(w, r, vids)
log.Println("Recieved updates from", s, "volumes", r.FormValue("volumes"))
mapper.Add(*directory.NewMachine(s, publicServer, *volumes))
}
func dirStatusHandler(w http.ResponseWriter, r *http.Request) {
writeJson(w, r, mapper)
@@ -67,8 +66,7 @@ func writeJson(w http.ResponseWriter, r *http.Request, obj interface{}) {
func main() {
flag.Parse()
mapper = directory.NewMapper(*metaFolder, "directory", *capacity)
defer mapper.Save()
mapper = directory.NewMapper(*metaFolder, "directory")
http.HandleFunc("/dir/assign", dirAssignHandler)
http.HandleFunc("/dir/lookup", dirLookupHandler)
http.HandleFunc("/dir/write", dirWriteHandler)