gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/chrislusf/weed-fs

Browse Source
This commit is contained in:
Brian McQueen
2014-12-14 00:13:51 -08:00
parent bd664def45 e431d4121e
commit a3583e4e7c
96 changed files with 567 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
go/weed/benchmark.go
View File

@@ -2,9 +2,6 @@ package main
import (
"bufio"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/util"
"fmt"
"io"
"math"
@@ -16,6 +13,10 @@ import (
"strings"
"sync"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/util"
)
type BenchmarkOptions struct {
@@ -30,11 +31,14 @@ type BenchmarkOptions struct {
sequentialRead *bool
collection *string
cpuprofile *string
maxCpu *int
vid2server map[string]string //cache for vid locations
}
var (
b BenchmarkOptions
b BenchmarkOptions
sharedBytes []byte
)
func init() {
@@ -50,33 +54,35 @@ func init() {
b.read = cmdBenchmark.Flag.Bool("read", true, "enable read")
b.sequentialRead = cmdBenchmark.Flag.Bool("readSequentially", false, "randomly read by ids from \"-list\" specified file")
b.collection = cmdBenchmark.Flag.String("collection", "benchmark", "write data to this collection")
b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "write cpu profile to file")
b.cpuprofile = cmdBenchmark.Flag.String("cpuprofile", "", "cpu profile output file")
b.maxCpu = cmdBenchmark.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
b.vid2server = make(map[string]string)
sharedBytes = make([]byte, 1024)
}
var cmdBenchmark = &Command{
UsageLine: "benchmark -server=localhost:9333 -c=10 -n=100000",
Short: "benchmark on writing millions of files and read out",
Long: `benchmark on an empty weed file system.
Two tests during benchmark:
1) write lots of small files to the system
2) read the files out
The file content is mostly zero, but no compression is done.
You can choose to only benchmark read or write.
During write, the list of uploaded file ids is stored in "-list" specified file.
You can also use your own list of file ids to run read test.
Write speed and read speed will be collected.
The numbers are used to get a sense of the system.
Usually your network or the hard drive is the real bottleneck.
Another thing to watch is whether the volumes are evenly distributed
to each volume server. Because the 7 more benchmark volumes are randomly distributed
to servers with free slots, it's highly possible some servers have uneven amount of
benchmark volumes. To remedy this, you can use this to grow the benchmark volumes
benchmark volumes. To remedy this, you can use this to grow the benchmark volumes
before starting the benchmark command:
http://localhost:9333/vol/grow?collection=benchmark&count=5
@@ -87,18 +93,17 @@ var cmdBenchmark = &Command{
}
var (
wait sync.WaitGroup
writeStats *stats
readStats *stats
serverLimitChan map[string]chan bool
wait sync.WaitGroup
writeStats *stats
readStats *stats
)
func init() {
serverLimitChan = make(map[string]chan bool)
}
func runbenchmark(cmd *Command, args []string) bool {
fmt.Printf("This is Seaweed File System version %s %s %s\n", util.VERSION, runtime.GOOS, runtime.GOARCH)
if *b.maxCpu < 1 {
*b.maxCpu = runtime.NumCPU()
}
runtime.GOMAXPROCS(*b.maxCpu)
if *b.cpuprofile != "" {
f, err := os.Create(*b.cpuprofile)
if err != nil {
@@ -122,12 +127,12 @@ func runbenchmark(cmd *Command, args []string) bool {
func bench_write() {
fileIdLineChan := make(chan string)
finishChan := make(chan bool)
writeStats = newStats()
writeStats = newStats(*b.concurrency)
idChan := make(chan int)
wait.Add(*b.concurrency)
go writeFileIds(*b.idListFile, fileIdLineChan, finishChan)
for i := 0; i < *b.concurrency; i++ {
go writeFiles(idChan, fileIdLineChan, writeStats)
wait.Add(1)
go writeFiles(idChan, fileIdLineChan, &writeStats.localStats[i])
}
writeStats.start = time.Now()
writeStats.total = *b.numberOfFiles
@@ -138,28 +143,30 @@ func bench_write() {
close(idChan)
wait.Wait()
writeStats.end = time.Now()
wait.Add(1)
wait.Add(2)
finishChan <- true
finishChan <- true
close(finishChan)
wait.Wait()
close(finishChan)
writeStats.printStats()
}
func bench_read() {
fileIdLineChan := make(chan string)
finishChan := make(chan bool)
readStats = newStats()
wait.Add(*b.concurrency)
readStats = newStats(*b.concurrency)
go readFileIds(*b.idListFile, fileIdLineChan)
readStats.start = time.Now()
readStats.total = *b.numberOfFiles
go readStats.checkProgress("Randomly Reading Benchmark", finishChan)
for i := 0; i < *b.concurrency; i++ {
go readFiles(fileIdLineChan, readStats)
wait.Add(1)
go readFiles(fileIdLineChan, &readStats.localStats[i])
}
wait.Wait()
wait.Add(1)
finishChan <- true
wait.Wait()
close(finishChan)
readStats.end = time.Now()
readStats.printStats()
@@ -170,126 +177,102 @@ type delayedFile struct {
fp *operation.FilePart
}
func writeFiles(idChan chan int, fileIdLineChan chan string, s *stats) {
func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) {
defer wait.Done()
delayedDeleteChan := make(chan *delayedFile, 100)
var waitForDeletions sync.WaitGroup
for i := 0; i < 7; i++ {
waitForDeletions.Add(1)
go func() {
waitForDeletions.Add(1)
defer waitForDeletions.Done()
for df := range delayedDeleteChan {
if df == nil {
break
}
if df.enterTime.After(time.Now()) {
time.Sleep(df.enterTime.Sub(time.Now()))
}
fp := df.fp
serverLimitChan[fp.Server] <- true
if e := util.Delete("http://" + fp.Server + "/" + fp.Fid); e == nil {
if e := util.Delete("http://" + df.fp.Server + "/" + df.fp.Fid); e == nil {
s.completed++
} else {
s.failed++
}
<-serverLimitChan[fp.Server]
}
waitForDeletions.Done()
}()
}
for {
if id, ok := <-idChan; ok {
start := time.Now()
fileSize := int64(*b.fileSize + rand.Intn(64))
fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize}
if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil {
fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection
if _, ok := serverLimitChan[fp.Server]; !ok {
serverLimitChan[fp.Server] = make(chan bool, 7)
}
serverLimitChan[fp.Server] <- true
if _, err := fp.Upload(0, *b.server); err == nil {
if rand.Intn(100) < *b.deletePercentage {
s.total++
delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
} else {
fileIdLineChan <- fp.Fid
}
s.completed++
s.transferred += fileSize
for id := range idChan {
start := time.Now()
fileSize := int64(*b.fileSize + rand.Intn(64))
fp := &operation.FilePart{Reader: &FakeReader{id: uint64(id), size: fileSize}, FileSize: fileSize}
if assignResult, err := operation.Assign(*b.server, 1, "", *b.collection, ""); err == nil {
fp.Server, fp.Fid, fp.Collection = assignResult.PublicUrl, assignResult.Fid, *b.collection
if _, err := fp.Upload(0, *b.server); err == nil {
if rand.Intn(100) < *b.deletePercentage {
s.total++
delayedDeleteChan <- &delayedFile{time.Now().Add(time.Second), fp}
} else {
s.failed++
}
writeStats.addSample(time.Now().Sub(start))
<-serverLimitChan[fp.Server]
if *cmdBenchmark.IsDebug {
fmt.Printf("writing %d file %s\n", id, fp.Fid)
fileIdLineChan <- fp.Fid
}
s.completed++
s.transferred += fileSize
} else {
s.failed++
println("writing file error:", err.Error())
fmt.Printf("Failed to write with error:%v\n", err)
}
writeStats.addSample(time.Now().Sub(start))
if *cmdBenchmark.IsDebug {
fmt.Printf("writing %d file %s\n", id, fp.Fid)
}
} else {
break
s.failed++
println("writing file error:", err.Error())
}
}
close(delayedDeleteChan)
waitForDeletions.Wait()
wait.Done()
}
func readFiles(fileIdLineChan chan string, s *stats) {
serverLimitChan := make(map[string]chan bool)
func readFiles(fileIdLineChan chan string, s *stat) {
defer wait.Done()
masterLimitChan := make(chan bool, 1)
for {
if fid, ok := <-fileIdLineChan; ok {
if len(fid) == 0 {
continue
}
if fid[0] == '#' {
continue
}
if *cmdBenchmark.IsDebug {
fmt.Printf("reading file %s\n", fid)
}
parts := strings.SplitN(fid, ",", 2)
vid := parts[0]
start := time.Now()
if server, ok := b.vid2server[vid]; !ok {
masterLimitChan <- true
if _, now_ok := b.vid2server[vid]; !now_ok {
if ret, err := operation.Lookup(*b.server, vid); err == nil {
if len(ret.Locations) > 0 {
server = ret.Locations[0].PublicUrl
b.vid2server[vid] = server
}
for fid := range fileIdLineChan {
if len(fid) == 0 {
continue
}
if fid[0] == '#' {
continue
}
if *cmdBenchmark.IsDebug {
fmt.Printf("reading file %s\n", fid)
}
parts := strings.SplitN(fid, ",", 2)
vid := parts[0]
start := time.Now()
if server, ok := b.vid2server[vid]; !ok {
masterLimitChan <- true
if _, now_ok := b.vid2server[vid]; !now_ok {
if ret, err := operation.Lookup(*b.server, vid); err == nil {
if len(ret.Locations) > 0 {
server = ret.Locations[0].PublicUrl
b.vid2server[vid] = server
}
}
<-masterLimitChan
}
if server, ok := b.vid2server[vid]; ok {
if _, ok := serverLimitChan[server]; !ok {
serverLimitChan[server] = make(chan bool, 7)
}
serverLimitChan[server] <- true
url := "http://" + server + "/" + fid
if bytesRead, err := util.Get(url); err == nil {
s.completed++
s.transferred += int64(len(bytesRead))
readStats.addSample(time.Now().Sub(start))
} else {
s.failed++
println("!!!! Failed to read from ", url, " !!!!!")
}
<-serverLimitChan[server]
<-masterLimitChan
}
if server, ok := b.vid2server[vid]; ok {
url := "http://" + server + "/" + fid
if bytesRead, err := util.Get(url); err == nil {
s.completed++
s.transferred += int64(len(bytesRead))
readStats.addSample(time.Now().Sub(start))
} else {
s.failed++
println("!!!! volume id ", vid, " location not found!!!!!")
fmt.Printf("Failed to read %s error:%v\n", url, err)
}
} else {
break
s.failed++
println("!!!! volume id ", vid, " location not found!!!!!")
}
}
wait.Done()
}
func writeFileIds(fileName string, fileIdLineChan chan string, finishChan chan bool) {
@@ -353,20 +336,28 @@ const (
// An efficient statics collecting and rendering
type stats struct {
data []int
overflow []int
data []int
overflow []int
localStats []stat
start time.Time
end time.Time
total int
}
type stat struct {
completed int
failed int
total int
transferred int64
start time.Time
end time.Time
}
var percentages = []int{50, 66, 75, 80, 90, 95, 98, 99, 100}
func newStats() *stats {
return &stats{data: make([]int, benchResolution), overflow: make([]int, 0)}
func newStats(n int) *stats {
return &stats{
data: make([]int, benchResolution),
overflow: make([]int, 0),
localStats: make([]stat, n),
}
}
func (s *stats) addSample(d time.Duration) {
@@ -387,28 +378,41 @@ func (s *stats) checkProgress(testName string, finishChan chan bool) {
for {
select {
case <-finishChan:
wait.Done()
return
case t := <-ticker:
completed, transferred, taken := s.completed-lastCompleted, s.transferred-lastTransferred, t.Sub(lastTime)
completed, transferred, taken, total := 0, int64(0), t.Sub(lastTime), s.total
for _, localStat := range s.localStats {
completed += localStat.completed
transferred += localStat.transferred
total += localStat.total
}
fmt.Printf("Completed %d of %d requests, %3.1f%% %3.1f/s %3.1fMB/s\n",
s.completed, s.total, float64(s.completed)*100/float64(s.total),
float64(completed)*float64(int64(time.Second))/float64(int64(taken)),
float64(transferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024),
completed, total, float64(completed)*100/float64(total),
float64(completed-lastCompleted)*float64(int64(time.Second))/float64(int64(taken)),
float64(transferred-lastTransferred)*float64(int64(time.Second))/float64(int64(taken))/float64(1024*1024),
)
lastCompleted, lastTransferred, lastTime = s.completed, s.transferred, t
lastCompleted, lastTransferred, lastTime = completed, transferred, t
}
}
}
func (s *stats) printStats() {
completed, failed, transferred, total := 0, 0, int64(0), s.total
for _, localStat := range s.localStats {
completed += localStat.completed
failed += localStat.failed
transferred += localStat.transferred
total += localStat.total
}
timeTaken := float64(int64(s.end.Sub(s.start))) / 1000000000
fmt.Printf("\nConcurrency Level: %d\n", *b.concurrency)
fmt.Printf("Time taken for tests: %.3f seconds\n", timeTaken)
fmt.Printf("Complete requests: %d\n", s.completed)
fmt.Printf("Failed requests: %d\n", s.failed)
fmt.Printf("Total transferred: %d bytes\n", s.transferred)
fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(s.completed)/timeTaken)
fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(s.transferred)/1024/timeTaken)
fmt.Printf("Complete requests: %d\n", completed)
fmt.Printf("Failed requests: %d\n", failed)
fmt.Printf("Total transferred: %d bytes\n", transferred)
fmt.Printf("Requests per second: %.2f [#/sec]\n", float64(completed)/timeTaken)
fmt.Printf("Transfer rate: %.2f [Kbytes/sec]\n", float64(transferred)/1024/timeTaken)
n, sum := 0, 0
min, max := 10000000, 0
for i := 0; i < len(s.data); i++ {
@@ -496,15 +500,32 @@ func (l *FakeReader) Read(p []byte) (n int, err error) {
} else {
n = len(p)
}
for i := 0; i < n-8; i += 8 {
for s := uint(0); s < 8; s++ {
p[i] = byte(l.id >> (s * 8))
if n >= 8 {
for i := 0; i < 8; i++ {
p[i] = byte(l.id >> uint(i*8))
}
}
l.size -= int64(n)
return
}
func (l *FakeReader) WriteTo(w io.Writer) (n int64, err error) {
size := int(l.size)
bufferSize := len(sharedBytes)
for size > 0 {
tempBuffer := sharedBytes
if size < bufferSize {
tempBuffer = sharedBytes[0:size]
}
count, e := w.Write(tempBuffer)
if e != nil {
return int64(size), e
}
size -= count
}
return l.size, nil
}
func Readln(r *bufio.Reader) ([]byte, error) {
var (
isPrefix bool = true

2
go/weed/compact.go
View File

@@ -12,7 +12,7 @@ func init() {
var cmdCompact = &Command{
UsageLine: "compact -dir=/tmp -volumeId=234",
Short: "run weed tool compact on volume file if corrupted",
Short: "run weed tool compact on volume file",
Long: `Force an compaction to remove deleted files from volume files.
The compacted .dat file is stored as .cpd file.
The compacted .idx file is stored as .cpx file.

5
go/weed/download.go
View File

@@ -1,14 +1,15 @@
package main
import (
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/util"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"strings"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/util"
)
var (

7
go/weed/export.go
View File

@@ -3,8 +3,6 @@ package main
import (
"archive/tar"
"bytes"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
"fmt"
"os"
"path"
@@ -12,6 +10,9 @@ import (
"strings"
"text/template"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
)
func init() {
@@ -36,7 +37,7 @@ var cmdExport = &Command{
var (
exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files")
exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name")
exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
exportVolumeId = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume .dat and .idx files should already exist in the dir.")
dest = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
format = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
tarFh *tar.Writer

15
go/weed/filer.go
View File

@@ -1,13 +1,14 @@
package main
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/util"
"github.com/chrislusf/weed-fs/go/weed/weed_server"
"net/http"
"os"
"strconv"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/util"
"github.com/chrislusf/weed-fs/go/weed/weed_server"
)
var (
@@ -20,6 +21,7 @@ type FilerOptions struct {
collection *string
defaultReplicaPlacement *string
dir *string
redirectOnRead *bool
}
func init() {
@@ -28,7 +30,8 @@ func init() {
f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection")
f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data")
f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "Default replication type if not specified.")
f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified")
f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
}
var cmdFiler = &Command{
@@ -59,7 +62,9 @@ func runFiler(cmd *Command, args []string) bool {
}
r := http.NewServeMux()
_, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection)
_, nfs_err := weed_server.NewFilerServer(r, *f.port, *f.master, *f.dir, *f.collection,
*f.defaultReplicaPlacement, *f.redirectOnRead,
)
if nfs_err != nil {
glog.Fatalf(nfs_err.Error())
}

7
go/weed/fix.go
View File

@@ -1,11 +1,12 @@
package main
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
"os"
"path"
"strconv"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
)
func init() {
@@ -16,7 +17,7 @@ func init() {
var cmdFix = &Command{
UsageLine: "fix -dir=/tmp -volumeId=234",
Short: "run weed tool fix on index file if corrupted",
Long: `Fix runs the WeedFS fix command to re-create the index .idx file.
Long: `Fix runs the SeeweedFS fix command to re-create the index .idx file.
`,
}

12
go/weed/master.go
View File

@@ -1,16 +1,17 @@
package main
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/util"
"github.com/chrislusf/weed-fs/go/weed/weed_server"
"github.com/gorilla/mux"
"net/http"
"os"
"runtime"
"strconv"
"strings"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/util"
"github.com/chrislusf/weed-fs/go/weed/weed_server"
"github.com/gorilla/mux"
)
func init() {
@@ -29,6 +30,7 @@ var cmdMaster = &Command{
var (
mport = cmdMaster.Flag.Int("port", 9333, "http listen port")
masterIp = cmdMaster.Flag.String("ip", "", "master listening ip address, default to listen on all network interfaces")
masterBindIp = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
mPublicIp = cmdMaster.Flag.String("publicIp", "", "peer accessible <ip>|<server_name>")
metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
masterPeers = cmdMaster.Flag.String("peers", "", "other master nodes in comma separated ip:port list")
@@ -61,7 +63,7 @@ func runMaster(cmd *Command, args []string) bool {
*volumeSizeLimitMB, *mpulse, *confFile, *defaultReplicaPlacement, *garbageThreshold, masterWhiteList,
)
listeningAddress := *masterIp + ":" + strconv.Itoa(*mport)
listeningAddress := *masterBindIp + ":" + strconv.Itoa(*mport)
glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", listeningAddress)

2
go/weed/mount.go
View File

@@ -1,7 +1,5 @@
package main
import ()
type MountOptions struct {
filer *string
dir *string

7
go/weed/mount_std.go
View File

@@ -3,15 +3,16 @@
package main
import (
"fmt"
"os"
"runtime"
"bazil.org/fuse"
"bazil.org/fuse/fs"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/util"
"fmt"
"os"
"runtime"
)
func runMount(cmd *Command, args []string) bool {

25
go/weed/server.go
View File

@@ -1,10 +1,6 @@
package main
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/util"
"github.com/chrislusf/weed-fs/go/weed/weed_server"
"github.com/gorilla/mux"
"net/http"
"os"
"runtime"
@@ -13,6 +9,11 @@ import (
"strings"
"sync"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/util"
"github.com/chrislusf/weed-fs/go/weed/weed_server"
"github.com/gorilla/mux"
)
type ServerOptions struct {
@@ -31,17 +32,17 @@ func init() {
var cmdServer = &Command{
UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
Short: "start a server, including volume server, and automatically elect a master server",
Long: `start both a volume server to provide storage spaces
Long: `start both a volume server to provide storage spaces
and a master server to provide volume=>location mapping service and sequence number of file ids
This is provided as a convenient way to start both volume server and master server.
The servers are exactly the same as starting them separately.
So other volume servers can use this embedded master server also.
Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
They run with meta data on disk, not shared. So each filer server is different.
`,
}
@@ -72,12 +73,14 @@ var (
)
func init() {
serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "write cpu profile to file")
serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file")
filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request")
}
func runServer(cmd *Command, args []string) bool {
@@ -149,7 +152,9 @@ func runServer(cmd *Command, args []string) bool {
if *isStartingFiler {
go func() {
r := http.NewServeMux()
_, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection)
_, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection,
*filerOptions.defaultReplicaPlacement, *filerOptions.redirectOnRead,
)
if nfs_err != nil {
glog.Fatalf(nfs_err.Error())
}

3
go/weed/shell.go
View File

@@ -2,9 +2,10 @@ package main
import (
"bufio"
"github.com/chrislusf/weed-fs/go/glog"
"fmt"
"os"
"github.com/chrislusf/weed-fs/go/glog"
)
func init() {

2
go/weed/signal_handling_notsupported.go
View File

@@ -2,7 +2,5 @@
package main
import ()
func OnInterrupt(fn func()) {
}

3
go/weed/upload.go
View File

@@ -1,11 +1,12 @@
package main
import (
"github.com/chrislusf/weed-fs/go/operation"
"encoding/json"
"fmt"
"os"
"path/filepath"
"github.com/chrislusf/weed-fs/go/operation"
)
var (

3
go/weed/version.go
View File

@@ -1,9 +1,10 @@
package main
import (
"github.com/chrislusf/weed-fs/go/util"
"fmt"
"runtime"
"github.com/chrislusf/weed-fs/go/util"
)
var cmdVersion = &Command{

11
go/weed/volume.go
View File

@@ -1,15 +1,16 @@
package main
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/util"
"github.com/chrislusf/weed-fs/go/weed/weed_server"
"net/http"
"os"
"runtime"
"strconv"
"strings"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/util"
"github.com/chrislusf/weed-fs/go/weed/weed_server"
)
func init() {
@@ -30,7 +31,7 @@ var (
maxVolumeCounts = cmdVolume.Flag.String("max", "7", "maximum numbers of volumes, count[,count]...")
ip = cmdVolume.Flag.String("ip", "", "ip or server name")
publicIp = cmdVolume.Flag.String("publicIp", "", "Publicly accessible <ip|server_name>")
bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
volumeBindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
masterNode = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location")
vpulse = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
vTimeout = cmdVolume.Flag.Int("idleTimeout", 10, "connection idle seconds")
@@ -85,7 +86,7 @@ func runVolume(cmd *Command, args []string) bool {
*fixJpgOrientation,
)
listeningAddress := *bindIp + ":" + strconv.Itoa(*vport)
listeningAddress := *volumeBindIp + ":" + strconv.Itoa(*vport)
glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", listeningAddress)

3
go/weed/volume_test.go
View File

@@ -1,10 +1,11 @@
package main
import (
"github.com/chrislusf/weed-fs/go/glog"
"net/http"
"testing"
"time"
"github.com/chrislusf/weed-fs/go/glog"
)
func TestXYZ(t *testing.T) {

3
go/weed/weed.go
View File

@@ -1,7 +1,6 @@
package main
import (
"github.com/chrislusf/weed-fs/go/glog"
"flag"
"fmt"
"io"
@@ -13,6 +12,8 @@ import (
"time"
"unicode"
"unicode/utf8"
"github.com/chrislusf/weed-fs/go/glog"
)
var IsDebug *bool

11
go/weed/weed_server/common.go
View File

@@ -2,11 +2,6 @@ package weed_server
import (
"bytes"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/util"
"encoding/json"
"fmt"
"net"
@@ -14,6 +9,12 @@ import (
"path/filepath"
"strconv"
"strings"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/util"
)
var serverStats *stats.ServerStats

27
go/weed/weed_server/filer_server.go
View File

@@ -1,24 +1,31 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/glog"
"net/http"
"strconv"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/glog"
)
type FilerServer struct {
port string
master string
collection string
filer filer.Filer
port string
master string
collection string
defaultReplication string
redirectOnRead bool
filer filer.Filer
}
func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string) (fs *FilerServer, err error) {
func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string,
replication string, redirectOnRead bool,
) (fs *FilerServer, err error) {
fs = &FilerServer{
master: master,
collection: collection,
port: ":" + strconv.Itoa(port),
master: master,
collection: collection,
defaultReplication: replication,
redirectOnRead: redirectOnRead,
port: ":" + strconv.Itoa(port),
}
if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil {

22
go/weed/weed_server/filer_server_handlers.go
View File

@@ -1,12 +1,8 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/util"
"encoding/json"
"errors"
"github.com/syndtr/goleveldb/leveldb"
"io"
"io/ioutil"
"math/rand"
@@ -14,6 +10,11 @@ import (
"net/url"
"strconv"
"strings"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/util"
"github.com/syndtr/goleveldb/leveldb"
)
func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
@@ -80,7 +81,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
return
}
urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl
u, _ := url.Parse("http://" + urlLocation + "/" + fileId)
urlString := "http://" + urlLocation + "/" + fileId
if fs.redirectOnRead {
http.Redirect(w, r, urlString, http.StatusFound)
return
}
u, _ := url.Parse(urlString)
request := &http.Request{
Method: r.Method,
URL: u,
@@ -109,7 +115,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
assignResult, ae := operation.Assign(fs.master, 1, query.Get("replication"), fs.collection, query.Get("ttl"))
replication := query.Get("replication")
if replication == "" {
replication = fs.defaultReplication
}
assignResult, ae := operation.Assign(fs.master, 1, replication, fs.collection, query.Get("ttl"))
if ae != nil {
glog.V(0).Infoln("failing to assign a file id", ae.Error())
writeJsonError(w, r, ae)

3
go/weed/weed_server/filer_server_handlers_admin.go
View File

@@ -1,8 +1,9 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"net/http"
"github.com/chrislusf/weed-fs/go/glog"
)
/*

9
go/weed/weed_server/master_server.go
View File

@@ -1,16 +1,17 @@
package weed_server
import (
"net/http"
"net/http/httputil"
"net/url"
"sync"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/sequence"
"github.com/chrislusf/weed-fs/go/topology"
"github.com/chrislusf/weed-fs/go/util"
"github.com/goraft/raft"
"github.com/gorilla/mux"
"net/http"
"net/http/httputil"
"net/url"
"sync"
)
type MasterServer struct {

11
go/weed/weed_server/master_server_handlers.go
View File

@@ -1,12 +1,13 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"net/http"
"strconv"
"strings"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
)
func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) {
@@ -78,7 +79,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
return
}
if !ms.Topo.HasWriableVolume(option) {
if !ms.Topo.HasWritableVolume(option) {
if ms.Topo.FreeSpace() <= 0 {
w.WriteHeader(http.StatusNotFound)
writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"})
@@ -86,7 +87,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
} else {
ms.vgLock.Lock()
defer ms.vgLock.Unlock()
if !ms.Topo.HasWriableVolume(option) {
if !ms.Topo.HasWritableVolume(option) {
if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil {
writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()})
return

15
go/weed/weed_server/master_server_handlers_admin.go
View File

@@ -1,18 +1,19 @@
package weed_server
import (
proto "code.google.com/p/goprotobuf/proto"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/topology"
"github.com/chrislusf/weed-fs/go/util"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"strconv"
"strings"
proto "code.google.com/p/goprotobuf/proto"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/topology"
"github.com/chrislusf/weed-fs/go/util"
)
func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) {
@@ -143,7 +144,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r *
}
}
func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool {
func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool {
vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl)
return vl.GetActiveVolumeCount(option) > 0
}

55
go/weed/weed_server/raft_server.go
View File

@@ -2,19 +2,22 @@ package weed_server
import (
"bytes"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/topology"
"encoding/json"
"errors"
"fmt"
"github.com/goraft/raft"
"github.com/gorilla/mux"
"io/ioutil"
"math/rand"
"net/http"
"net/url"
"os"
"path"
"strings"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/topology"
"github.com/goraft/raft"
"github.com/gorilla/mux"
)
type RaftServer struct {
@@ -46,6 +49,13 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin
transporter.Transport.MaxIdleConnsPerHost = 1024
glog.V(1).Infof("Starting RaftServer with IP:%v:", httpAddr)
// Clear old cluster configurations if peers are set
if len(s.peers) > 0 {
os.RemoveAll(path.Join(s.dataDir, "conf"))
os.RemoveAll(path.Join(s.dataDir, "log"))
os.RemoveAll(path.Join(s.dataDir, "snapshot"))
}
s.raftServer, err = raft.NewServer(s.httpAddr, s.dataDir, transporter, nil, topo, "")
if err != nil {
glog.V(0).Infoln(err)
@@ -53,35 +63,30 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin
}
transporter.Install(s.raftServer, s)
s.raftServer.SetHeartbeatInterval(1 * time.Second)
s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 1150 * time.Millisecond)
s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 3450 * time.Millisecond)
s.raftServer.Start()
s.router.HandleFunc("/cluster/join", s.joinHandler).Methods("POST")
s.router.HandleFunc("/cluster/status", s.statusHandler).Methods("GET")
// Join to leader if specified.
if len(s.peers) > 0 {
if !s.raftServer.IsLogEmpty() {
glog.V(0).Infoln("Starting cluster with existing logs.")
} else {
glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ","))
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
firstJoinError := s.Join(s.peers)
if firstJoinError != nil {
glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: "http://" + s.httpAddr,
})
if err != nil {
glog.V(0).Infoln(err)
return nil
}
// Join to leader if specified.
glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ","))
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
firstJoinError := s.Join(s.peers)
if firstJoinError != nil {
glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: "http://" + s.httpAddr,
})
if err != nil {
glog.V(0).Infoln(err)
return nil
}
}
// Initialize the server by joining itself.
} else if s.raftServer.IsLogEmpty() {
// Initialize the server by joining itself.
glog.V(0).Infoln("Initializing new cluster")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
@@ -95,7 +100,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin
}
} else {
glog.V(0).Infoln("Recovered from log")
glog.V(0).Infoln("Old conf,log,snapshot should have been removed.")
}
return s

7
go/weed/weed_server/raft_server_handlers.go
View File

@@ -1,13 +1,14 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"encoding/json"
"github.com/goraft/raft"
"io/ioutil"
"net/http"
"strings"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/goraft/raft"
)
// Handles incoming RAFT joins.

5
go/weed/weed_server/volume_server.go
View File

@@ -1,12 +1,13 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
"math/rand"
"net/http"
"strconv"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
)
type VolumeServer struct {

13
go/weed/weed_server/volume_server_handlers.go
View File

@@ -1,12 +1,6 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/images"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/topology"
"io"
"mime"
"mime/multipart"
@@ -14,6 +8,13 @@ import (
"strconv"
"strings"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/images"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/topology"
)
var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

5
go/weed/weed_server/volume_server_handlers_admin.go
View File

@@ -1,11 +1,12 @@
package weed_server
import (
"net/http"
"path/filepath"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/util"
"net/http"
"path/filepath"
)
func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) {

3
go/weed/weed_server/volume_server_handlers_vacuum.go
View File

@@ -1,8 +1,9 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"net/http"
"github.com/chrislusf/weed-fs/go/glog"
)
func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) {