gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

split into server and clients

Browse Source 
git-svn-id: https://weed-fs.googlecode.com/svn/trunk@9 282b0af5-e82d-9cf1-ede4-77906d7719d0
This commit is contained in:
chris.lu@gmail.com
2011-12-16 14:51:26 +00:00
parent 02f4e7b82c
commit 23ecd7bb33
9 changed files with 466 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

145
weed-fs/src/cmd/gstore.go
View File

@@ -1,145 +0,0 @@
package main
import (
"store"
"directory"
"flag"
"fmt"
"http"
"json"
"log"
"mime"
"rand"
"strconv"
"strings"
"time"
)
var (
port = flag.Int("port", 8080, "http listen port")
chunkFolder = flag.String("dir", "/tmp", "data directory to store files")
chunkCount = flag.Int("chunks", 5, "data chunks to store files")
chunkEnabled = flag.Bool("data", false, "act as a store server")
chunkServer = flag.String("cserver", "localhost:8080", "chunk server to store data")
publicServer = flag.String("pserver", "localhost:8080", "public server to serve data read")
metaServer = flag.String("mserver", "localhost:8080", "metadata server to store mappings")
metaEnabled = flag.Bool("meta", false, "act as a directory server")
metaFolder = flag.String("mdir", "/tmp", "data directory to store mappings")
)
type Haystack struct {
store *store.Store
directory *directory.Mapper
}
func storeHandler(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
server.GetHandler(w, r)
case "DELETE":
server.DeleteHandler(w, r)
case "POST":
server.PostHandler(w, r)
}
}
func (s *Haystack) GetHandler(w http.ResponseWriter, r *http.Request) {
n := new(store.Needle)
path := r.URL.Path
sepIndex := strings.Index(path[1:], "/") + 1
volumeId, _ := strconv.Atoui64(path[1:sepIndex])
dotIndex := strings.LastIndex(path, ".")
n.ParsePath(path[sepIndex+1 : dotIndex])
ext := path[dotIndex:]
s.store.Read(volumeId, n)
w.Header().Set("Content-Type", mime.TypeByExtension(ext))
w.Write(n.Data)
}
func (s *Haystack) PostHandler(w http.ResponseWriter, r *http.Request) {
volumeId, _ := strconv.Atoui64(r.FormValue("volumeId"))
s.store.Write(volumeId, store.NewNeedle(r))
w.Header().Set("Content-Type", "text/plain")
fmt.Fprint(w, "volumeId=", volumeId, "\n")
}
func (s *Haystack) DeleteHandler(w http.ResponseWriter, r *http.Request) {
}
func dirReadHandler(w http.ResponseWriter, r *http.Request) {
volumeId, _ := strconv.Atoui64(r.FormValue("volumeId"))
machineList := server.directory.Get((uint32)(volumeId))
x := rand.Intn(len(machineList))
machine := machineList[x]
bytes, _ := json.Marshal(machine)
callback := r.FormValue("callback")
w.Header().Set("Content-Type", "application/javascript")
if callback == "" {
w.Write(bytes)
} else {
w.Write([]uint8(callback))
w.Write([]uint8("("))
w.Write(bytes)
w.Write([]uint8(")"))
}
}
func dirWriteHandler(w http.ResponseWriter, r *http.Request) {
machineList := server.directory.PickForWrite()
bytes, _ := json.Marshal(machineList)
callback := r.FormValue("callback")
w.Header().Set("Content-Type", "application/javascript")
if callback == "" {
w.Write(bytes)
} else {
w.Write([]uint8(callback))
w.Write([]uint8("("))
w.Write(bytes)
w.Write([]uint8(")"))
}
}
func dirJoinHandler(w http.ResponseWriter, r *http.Request) {
s := r.FormValue("server")
publicServer := r.FormValue("publicServer")
volumes := make([]store.VolumeStat, 0)
json.Unmarshal([]byte(r.FormValue("volumes")), volumes)
server.directory.Add(directory.NewMachine(s, publicServer), volumes)
}
func dirStatusHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
bytes, _ := json.Marshal(server.directory)
fmt.Fprint(w, bytes)
}
var server *Haystack
func main() {
flag.Parse()
bothEnabled := false
if !*chunkEnabled && !*metaEnabled {
bothEnabled = true
log.Println("Act as both a store server and a directory server")
}
server = new(Haystack)
if *chunkEnabled || bothEnabled {
log.Println("data stored in", *chunkFolder)
server.store = store.NewStore(*chunkServer, *publicServer, *chunkFolder)
defer server.store.Close()
http.HandleFunc("/", storeHandler)
}
if *metaEnabled || bothEnabled {
server.directory = directory.NewMapper(*metaFolder, "directory")
defer server.directory.Save()
http.HandleFunc("/dir/read", dirReadHandler)
http.HandleFunc("/dir/write", dirWriteHandler)
http.HandleFunc("/dir/join", dirJoinHandler)
http.HandleFunc("/dir/status", dirStatusHandler)
}
go func() {
time.Sleep(3000 * 1000)
server.store.Join(*metaServer)
log.Println("store joined at", *metaServer)
}()
log.Println("Serving at http://127.0.0.1:" + strconv.Itoa(*port))
http.ListenAndServe(":"+strconv.Itoa(*port), nil)
}

73
weed-fs/src/cmd/weedc.go Normal file
View File

@@ -0,0 +1,73 @@
package main
import (
"storage"
"flag"
"fmt"
"http"
"log"
"mime"
"strconv"
"strings"
)
var (
port = flag.Int("port", 8080, "http listen port")
chunkFolder = flag.String("dir", "/tmp", "data directory to store files")
chunkCount = flag.Int("chunks", 5, "data chunks to store files")
publicServer = flag.String("pserver", "localhost:8080", "public server to serve data read")
metaServer = flag.String("mserver", "localhost:9333", "metadata server to store mappings")
store *storage.Store
)
func storeHandler(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "GET":
GetHandler(w, r)
case "DELETE":
DeleteHandler(w, r)
case "POST":
PostHandler(w, r)
}
}
func GetHandler(w http.ResponseWriter, r *http.Request) {
n := new(storage.Needle)
path := r.URL.Path
sepIndex := strings.Index(path[1:], "/") + 1
volumeId, _ := strconv.Atoui64(path[1:sepIndex])
dotIndex := strings.LastIndex(path, ".")
n.ParsePath(path[sepIndex+1 : dotIndex])
ext := path[dotIndex:]
store.Read(volumeId, n)
w.Header().Set("Content-Type", mime.TypeByExtension(ext))
w.Write(n.Data)
}
func PostHandler(w http.ResponseWriter, r *http.Request) {
volumeId, _ := strconv.Atoui64(r.FormValue("volumeId"))
store.Write(volumeId, storage.NewNeedle(r))
w.Header().Set("Content-Type", "text/plain")
fmt.Fprint(w, "volumeId=", volumeId, "\n")
}
func DeleteHandler(w http.ResponseWriter, r *http.Request) {
}
func main() {
flag.Parse()
store = storage.NewStore(*port, *publicServer, *chunkFolder)
defer store.Close()
http.HandleFunc("/", storeHandler)
store.Join(*metaServer)
log.Println("store joined at", *metaServer)
log.Println("Start storage service at http://127.0.0.1:" + strconv.Itoa(*port))
e := http.ListenAndServe(":"+strconv.Itoa(*port), nil)
if e!=nil {
log.Fatalf("Fail to start:",e.String())
}
}

80
weed-fs/src/cmd/weeds.go Normal file
View File

@@ -0,0 +1,80 @@
package main
import (
"storage"
"directory"
"flag"
"fmt"
"http"
"json"
"log"
"rand"
"strconv"
)
var (
port = flag.Int("port", 9333, "http listen port")
metaFolder = flag.String("mdir", "/tmp", "data directory to store mappings")
mapper *directory.Mapper
)
func dirReadHandler(w http.ResponseWriter, r *http.Request) {
volumeId, _ := strconv.Atoui64(r.FormValue("volumeId"))
machineList := mapper.Get((uint32)(volumeId))
x := rand.Intn(len(machineList))
machine := machineList[x]
bytes, _ := json.Marshal(machine)
callback := r.FormValue("callback")
w.Header().Set("Content-Type", "application/javascript")
if callback == "" {
w.Write(bytes)
} else {
w.Write([]uint8(callback))
w.Write([]uint8("("))
w.Write(bytes)
w.Write([]uint8(")"))
}
}
func dirWriteHandler(w http.ResponseWriter, r *http.Request) {
machineList := mapper.PickForWrite()
bytes, _ := json.Marshal(machineList)
callback := r.FormValue("callback")
w.Header().Set("Content-Type", "application/javascript")
if callback == "" {
w.Write(bytes)
} else {
w.Write([]uint8(callback))
w.Write([]uint8("("))
w.Write(bytes)
w.Write([]uint8(")"))
}
}
func dirJoinHandler(w http.ResponseWriter, r *http.Request) {
s := r.FormValue("server")
publicServer := r.FormValue("publicServer")
volumes := make([]storage.VolumeStat, 0)
json.Unmarshal([]byte(r.FormValue("volumes")), volumes)
mapper.Add(directory.NewMachine(s, publicServer), volumes)
}
func dirStatusHandler(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/plain")
bytes, _ := json.Marshal(mapper)
fmt.Fprint(w, bytes)
}
func main() {
flag.Parse()
mapper = directory.NewMapper(*metaFolder, "directory")
defer mapper.Save()
http.HandleFunc("/dir/read", dirReadHandler)
http.HandleFunc("/dir/write", dirWriteHandler)
http.HandleFunc("/dir/join", dirJoinHandler)
http.HandleFunc("/dir/status", dirStatusHandler)
log.Println("Start directory service at http://127.0.0.1:" + strconv.Itoa(*port))
e := http.ListenAndServe(":"+strconv.Itoa(*port), nil)
if e!=nil {
log.Fatalf("Fail to start:",e.String())
}
}