gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'master' of https://github.com/chrislusf/weed-fs

Browse Source
This commit is contained in:
Brian McQueen
2014-12-14 00:13:51 -08:00
parent bd664def45 e431d4121e
commit a3583e4e7c
96 changed files with 567 additions and 421 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
go/weed/weed_server/common.go
View File

@@ -2,11 +2,6 @@ package weed_server
import (
"bytes"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/util"
"encoding/json"
"fmt"
"net"
@@ -14,6 +9,12 @@ import (
"path/filepath"
"strconv"
"strings"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/util"
)
var serverStats *stats.ServerStats

27
go/weed/weed_server/filer_server.go
View File

@@ -1,24 +1,31 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/glog"
"net/http"
"strconv"
"github.com/chrislusf/weed-fs/go/filer"
"github.com/chrislusf/weed-fs/go/glog"
)
type FilerServer struct {
port string
master string
collection string
filer filer.Filer
port string
master string
collection string
defaultReplication string
redirectOnRead bool
filer filer.Filer
}
func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string) (fs *FilerServer, err error) {
func NewFilerServer(r *http.ServeMux, port int, master string, dir string, collection string,
replication string, redirectOnRead bool,
) (fs *FilerServer, err error) {
fs = &FilerServer{
master: master,
collection: collection,
port: ":" + strconv.Itoa(port),
master: master,
collection: collection,
defaultReplication: replication,
redirectOnRead: redirectOnRead,
port: ":" + strconv.Itoa(port),
}
if fs.filer, err = filer.NewFilerEmbedded(master, dir); err != nil {

22
go/weed/weed_server/filer_server_handlers.go
View File

@@ -1,12 +1,8 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/util"
"encoding/json"
"errors"
"github.com/syndtr/goleveldb/leveldb"
"io"
"io/ioutil"
"math/rand"
@@ -14,6 +10,11 @@ import (
"net/url"
"strconv"
"strings"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/util"
"github.com/syndtr/goleveldb/leveldb"
)
func (fs *FilerServer) filerHandler(w http.ResponseWriter, r *http.Request) {
@@ -80,7 +81,12 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
return
}
urlLocation := lookup.Locations[rand.Intn(len(lookup.Locations))].PublicUrl
u, _ := url.Parse("http://" + urlLocation + "/" + fileId)
urlString := "http://" + urlLocation + "/" + fileId
if fs.redirectOnRead {
http.Redirect(w, r, urlString, http.StatusFound)
return
}
u, _ := url.Parse(urlString)
request := &http.Request{
Method: r.Method,
URL: u,
@@ -109,7 +115,11 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
query := r.URL.Query()
assignResult, ae := operation.Assign(fs.master, 1, query.Get("replication"), fs.collection, query.Get("ttl"))
replication := query.Get("replication")
if replication == "" {
replication = fs.defaultReplication
}
assignResult, ae := operation.Assign(fs.master, 1, replication, fs.collection, query.Get("ttl"))
if ae != nil {
glog.V(0).Infoln("failing to assign a file id", ae.Error())
writeJsonError(w, r, ae)

3
go/weed/weed_server/filer_server_handlers_admin.go
View File

@@ -1,8 +1,9 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"net/http"
"github.com/chrislusf/weed-fs/go/glog"
)
/*

9
go/weed/weed_server/master_server.go
View File

@@ -1,16 +1,17 @@
package weed_server
import (
"net/http"
"net/http/httputil"
"net/url"
"sync"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/sequence"
"github.com/chrislusf/weed-fs/go/topology"
"github.com/chrislusf/weed-fs/go/util"
"github.com/goraft/raft"
"github.com/gorilla/mux"
"net/http"
"net/http/httputil"
"net/url"
"sync"
)
type MasterServer struct {

11
go/weed/weed_server/master_server_handlers.go
View File

@@ -1,12 +1,13 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"net/http"
"strconv"
"strings"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
)
func (ms *MasterServer) lookupVolumeId(vids []string, collection string) (volumeLocations map[string]operation.LookupResult) {
@@ -78,7 +79,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
return
}
if !ms.Topo.HasWriableVolume(option) {
if !ms.Topo.HasWritableVolume(option) {
if ms.Topo.FreeSpace() <= 0 {
w.WriteHeader(http.StatusNotFound)
writeJsonQuiet(w, r, operation.AssignResult{Error: "No free volumes left!"})
@@ -86,7 +87,7 @@ func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request)
} else {
ms.vgLock.Lock()
defer ms.vgLock.Unlock()
if !ms.Topo.HasWriableVolume(option) {
if !ms.Topo.HasWritableVolume(option) {
if _, err = ms.vg.AutomaticGrowByType(option, ms.Topo); err != nil {
writeJsonQuiet(w, r, operation.AssignResult{Error: "Cannot grow volume group! " + err.Error()})
return

15
go/weed/weed_server/master_server_handlers_admin.go
View File

@@ -1,18 +1,19 @@
package weed_server
import (
proto "code.google.com/p/goprotobuf/proto"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/topology"
"github.com/chrislusf/weed-fs/go/util"
"encoding/json"
"errors"
"io/ioutil"
"net/http"
"strconv"
"strings"
proto "code.google.com/p/goprotobuf/proto"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/topology"
"github.com/chrislusf/weed-fs/go/util"
)
func (ms *MasterServer) collectionDeleteHandler(w http.ResponseWriter, r *http.Request) {
@@ -143,7 +144,7 @@ func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r *
}
}
func (ms *MasterServer) hasWriableVolume(option *topology.VolumeGrowOption) bool {
func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool {
vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl)
return vl.GetActiveVolumeCount(option) > 0
}

55
go/weed/weed_server/raft_server.go
View File

@@ -2,19 +2,22 @@ package weed_server
import (
"bytes"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/topology"
"encoding/json"
"errors"
"fmt"
"github.com/goraft/raft"
"github.com/gorilla/mux"
"io/ioutil"
"math/rand"
"net/http"
"net/url"
"os"
"path"
"strings"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/topology"
"github.com/goraft/raft"
"github.com/gorilla/mux"
)
type RaftServer struct {
@@ -46,6 +49,13 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin
transporter.Transport.MaxIdleConnsPerHost = 1024
glog.V(1).Infof("Starting RaftServer with IP:%v:", httpAddr)
// Clear old cluster configurations if peers are set
if len(s.peers) > 0 {
os.RemoveAll(path.Join(s.dataDir, "conf"))
os.RemoveAll(path.Join(s.dataDir, "log"))
os.RemoveAll(path.Join(s.dataDir, "snapshot"))
}
s.raftServer, err = raft.NewServer(s.httpAddr, s.dataDir, transporter, nil, topo, "")
if err != nil {
glog.V(0).Infoln(err)
@@ -53,35 +63,30 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin
}
transporter.Install(s.raftServer, s)
s.raftServer.SetHeartbeatInterval(1 * time.Second)
s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 1150 * time.Millisecond)
s.raftServer.SetElectionTimeout(time.Duration(pulseSeconds) * 3450 * time.Millisecond)
s.raftServer.Start()
s.router.HandleFunc("/cluster/join", s.joinHandler).Methods("POST")
s.router.HandleFunc("/cluster/status", s.statusHandler).Methods("GET")
// Join to leader if specified.
if len(s.peers) > 0 {
if !s.raftServer.IsLogEmpty() {
glog.V(0).Infoln("Starting cluster with existing logs.")
} else {
glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ","))
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
firstJoinError := s.Join(s.peers)
if firstJoinError != nil {
glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: "http://" + s.httpAddr,
})
if err != nil {
glog.V(0).Infoln(err)
return nil
}
// Join to leader if specified.
glog.V(0).Infoln("Joining cluster:", strings.Join(s.peers, ","))
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
firstJoinError := s.Join(s.peers)
if firstJoinError != nil {
glog.V(0).Infoln("No existing server found. Starting as leader in the new cluster.")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
Name: s.raftServer.Name(),
ConnectionString: "http://" + s.httpAddr,
})
if err != nil {
glog.V(0).Infoln(err)
return nil
}
}
// Initialize the server by joining itself.
} else if s.raftServer.IsLogEmpty() {
// Initialize the server by joining itself.
glog.V(0).Infoln("Initializing new cluster")
_, err := s.raftServer.Do(&raft.DefaultJoinCommand{
@@ -95,7 +100,7 @@ func NewRaftServer(r *mux.Router, peers []string, httpAddr string, dataDir strin
}
} else {
glog.V(0).Infoln("Recovered from log")
glog.V(0).Infoln("Old conf,log,snapshot should have been removed.")
}
return s

7
go/weed/weed_server/raft_server_handlers.go
View File

@@ -1,13 +1,14 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"encoding/json"
"github.com/goraft/raft"
"io/ioutil"
"net/http"
"strings"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/goraft/raft"
)
// Handles incoming RAFT joins.

5
go/weed/weed_server/volume_server.go
View File

@@ -1,12 +1,13 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
"math/rand"
"net/http"
"strconv"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/storage"
)
type VolumeServer struct {

13
go/weed/weed_server/volume_server_handlers.go
View File

@@ -1,12 +1,6 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/images"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/topology"
"io"
"mime"
"mime/multipart"
@@ -14,6 +8,13 @@ import (
"strconv"
"strings"
"time"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/images"
"github.com/chrislusf/weed-fs/go/operation"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/storage"
"github.com/chrislusf/weed-fs/go/topology"
)
var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")

5
go/weed/weed_server/volume_server_handlers_admin.go
View File

@@ -1,11 +1,12 @@
package weed_server
import (
"net/http"
"path/filepath"
"github.com/chrislusf/weed-fs/go/glog"
"github.com/chrislusf/weed-fs/go/stats"
"github.com/chrislusf/weed-fs/go/util"
"net/http"
"path/filepath"
)
func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) {

3
go/weed/weed_server/volume_server_handlers_vacuum.go
View File

@@ -1,8 +1,9 @@
package weed_server
import (
"github.com/chrislusf/weed-fs/go/glog"
"net/http"
"github.com/chrislusf/weed-fs/go/glog"
)
func (vs *VolumeServer) vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) {