gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

compatible with Go1

Browse Source 
git-svn-id: https://weed-fs.googlecode.com/svn/trunk@46 282b0af5-e82d-9cf1-ede4-77906d7719d0
This commit is contained in:
chris.lu@gmail.com
2012-06-29 07:53:47 +00:00
parent 8edf12f026
commit bb01324482
17 changed files with 148 additions and 693 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
weed-fs/src/pkg/directory/Makefile
View File

@@ -1,36 +0,0 @@
# Makefile generated by gb: http://go-gb.googlecode.com
# gb provides configuration-free building and distributing
include $(GOROOT)/src/Make.inc
TARG=directory
GOFILES=\
file_id.go\
volume_mapping.go\
# gb: this is the local install
GBROOT=../../..
# gb: compile/link against local install
GCIMPORTS+= -I $(GBROOT)/_obj
LDIMPORTS+= -L $(GBROOT)/_obj
# gb: compile/link against GOPATH entries
GOPATHSEP=:
ifeq ($(GOHOSTOS),windows)
GOPATHSEP=;
endif
GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
# gb: copy to local install
$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a
mkdir -p $(dir $@); cp -f $< $@
package: $(GBROOT)/_obj/$(TARG).a
include $(GOROOT)/src/Make.pkg
# gb: local dependencies
_obj/$(TARG).a: $(GBROOT)/_obj/storage.a
_obj/$(TARG).a: $(GBROOT)/_obj/util.a

12
weed-fs/src/pkg/directory/file_id.go
View File

@@ -3,10 +3,10 @@ package directory
import (
"encoding/hex"
"log"
"storage"
"pkg/storage"
"strconv"
"strings"
"util"
"pkg/util"
)
type FileId struct {
@@ -18,14 +18,14 @@ type FileId struct {
func NewFileId(VolumeId uint32, Key uint64, Hashcode uint32) *FileId {
return &FileId{VolumeId: VolumeId, Key: Key, Hashcode: Hashcode}
}
func ParseFileId(fid string) *FileId {
func ParseFileId(fid string) *FileId{
a := strings.Split(fid, ",")
if len(a) != 2 {
log.Println("Invalid fid", fid, ", split length", len(a))
log.Println("Invalid fid", fid, ", split length", len(a))
return nil
}
vid_string, key_hash_string := a[0], a[1]
vid, _ := strconv.Atoui64(vid_string)
vid, _ := strconv.ParseUint(vid_string, 10, 64)
key, hash := storage.ParseKeyHash(key_hash_string)
return &FileId{VolumeId: uint32(vid), Key: key, Hashcode: hash}
}
@@ -36,5 +36,5 @@ func (n *FileId) String() string {
nonzero_index := 0
for ; bytes[nonzero_index] == 0; nonzero_index++ {
}
return strconv.Uitoa64(uint64(n.VolumeId)) + "," + hex.EncodeToString(bytes[nonzero_index:])
return strconv.FormatUint(uint64(n.VolumeId), 10) + "," + hex.EncodeToString(bytes[nonzero_index:])
}

23
weed-fs/src/pkg/directory/volume_mapping.go
View File

@@ -1,12 +1,13 @@
package directory
import (
"gob"
"encoding/gob"
"errors"
"log"
"math/rand"
"os"
"path"
"rand"
"log"
"storage"
"pkg/storage"
"strconv"
"sync"
)
@@ -31,7 +32,7 @@ type Mapper struct {
lock sync.Mutex
Machines []*Machine
vid2machineId map[uint32]int //machineId is +1 of the index of []*Machine, to detect not found entries
Writers []uint32 // transient array of Writers volume id
Writers []uint32 // transient array of Writers volume id
FileIdSequence uint64
fileIdCounter uint64
@@ -64,11 +65,11 @@ func NewMapper(dirname string, filename string, volumeSizeLimit uint64) (m *Mapp
}
return
}
func (m *Mapper) PickForWrite() (string, MachineInfo, os.Error) {
func (m *Mapper) PickForWrite() (string, MachineInfo, error) {
len_writers := len(m.Writers)
if len_writers <= 0 {
log.Println("No more writable volumes!")
return "", m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("No more writable volumes!")
return "", m.Machines[rand.Intn(len(m.Machines))].Server, errors.New("No more writable volumes!")
}
vid := m.Writers[rand.Intn(len_writers)]
machine_id := m.vid2machineId[vid]
@@ -76,7 +77,7 @@ func (m *Mapper) PickForWrite() (string, MachineInfo, os.Error) {
machine := m.Machines[machine_id-1]
return NewFileId(vid, m.NextFileId(), rand.Uint32()).String(), machine.Server, nil
}
return "", m.Machines[rand.Intn(len(m.Machines))].Server, os.NewError("Strangely vid " + strconv.Uitoa64(uint64(vid)) + " is on no machine!")
return "", m.Machines[rand.Intn(len(m.Machines))].Server, errors.New("Strangely vid " + strconv.FormatUint(uint64(vid), 10) + " is on no machine!")
}
func (m *Mapper) NextFileId() uint64 {
if m.fileIdCounter <= 0 {
@@ -87,16 +88,16 @@ func (m *Mapper) NextFileId() uint64 {
m.fileIdCounter--
return m.FileIdSequence - m.fileIdCounter
}
func (m *Mapper) Get(vid uint32) (*Machine, os.Error) {
func (m *Mapper) Get(vid uint32) (*Machine, error) {
machineId := m.vid2machineId[vid]
if machineId <= 0 {
return nil, os.NewError("invalid volume id " + strconv.Uitob64(uint64(vid), 10))
return nil, errors.New("invalid volume id " + strconv.FormatUint(uint64(vid), 10))
}
return m.Machines[machineId-1], nil
}
func (m *Mapper) Add(machine Machine) {
//check existing machine, linearly
//log.Println("Adding machine", machine.Server.Url)
//log.Println("Adding machine", machine.Server.Url)
m.lock.Lock()
foundExistingMachineId := -1
for index, entry := range m.Machines {

37
weed-fs/src/pkg/storage/Makefile
View File

@@ -1,37 +0,0 @@
# Makefile generated by gb: http://go-gb.googlecode.com
# gb provides configuration-free building and distributing
include $(GOROOT)/src/Make.inc
TARG=storage
GOFILES=\
needle.go\
needle_map.go\
store.go\
volume.go\
# gb: this is the local install
GBROOT=../../..
# gb: compile/link against local install
GCIMPORTS+= -I $(GBROOT)/_obj
LDIMPORTS+= -L $(GBROOT)/_obj
# gb: compile/link against GOPATH entries
GOPATHSEP=:
ifeq ($(GOHOSTOS),windows)
GOPATHSEP=;
endif
GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
# gb: copy to local install
$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a
mkdir -p $(dir $@); cp -f $< $@
package: $(GBROOT)/_obj/$(TARG).a
include $(GOROOT)/src/Make.pkg
# gb: local dependencies
_obj/$(TARG).a: $(GBROOT)/_obj/util.a

75
weed-fs/src/pkg/storage/needle.go
View File

@@ -1,14 +1,14 @@
package storage
import (
"encoding/hex"
"encoding/hex"
"io"
"io/ioutil"
"http"
"log"
"net/http"
"os"
"strings"
. "util"
"pkg/util"
)
type Needle struct {
@@ -21,6 +21,7 @@ type Needle struct {
}
func NewNeedle(r *http.Request) (n *Needle) {
n = new(Needle)
form, fe := r.MultipartReader()
if fe != nil {
@@ -30,11 +31,11 @@ func NewNeedle(r *http.Request) (n *Needle) {
data, _ := ioutil.ReadAll(part)
n.Data = data
commaSep := strings.LastIndex(r.URL.Path, ",")
commaSep := strings.LastIndex(r.URL.Path, ",")
dotSep := strings.LastIndex(r.URL.Path, ".")
fid := r.URL.Path[commaSep+1:]
if dotSep > 0 {
fid = r.URL.Path[commaSep+1:dotSep]
fid = r.URL.Path[commaSep+1 : dotSep]
}
n.ParsePath(fid)
@@ -49,51 +50,51 @@ func (n *Needle) ParsePath(fid string) {
}
n.Key, n.Cookie = ParseKeyHash(fid)
}
func (n *Needle) Append(w io.Writer) (uint32){
func (n *Needle) Append(w io.Writer) uint32 {
header := make([]byte, 16)
Uint32toBytes(header[0:4], n.Cookie)
Uint64toBytes(header[4:12], n.Key)
util.Uint32toBytes(header[0:4], n.Cookie)
util.Uint64toBytes(header[4:12], n.Key)
n.Size = uint32(len(n.Data))
Uint32toBytes(header[12:16], n.Size)
util.Uint32toBytes(header[12:16], n.Size)
w.Write(header)
w.Write(n.Data)
rest := 8 - ((n.Size + 16 + 4) % 8)
Uint32toBytes(header[0:4], uint32(n.Checksum))
util.Uint32toBytes(header[0:4], uint32(n.Checksum))
w.Write(header[0 : rest+4])
return n.Size
}
func (n *Needle) Read(r io.Reader, size uint32)(int, os.Error) {
func (n *Needle) Read(r io.Reader, size uint32) (int, error) {
bytes := make([]byte, size+16+4)
ret, e := r.Read(bytes)
n.Cookie = BytesToUint32(bytes[0:4])
n.Key = BytesToUint64(bytes[4:12])
n.Size = BytesToUint32(bytes[12:16])
n.Cookie = util.BytesToUint32(bytes[0:4])
n.Key = util.BytesToUint64(bytes[4:12])
n.Size = util.BytesToUint32(bytes[12:16])
n.Data = bytes[16 : 16+size]
n.Checksum = int32(BytesToUint32(bytes[16+size : 16+size+4]))
n.Checksum = int32(util.BytesToUint32(bytes[16+size : 16+size+4]))
return ret, e
}
func ReadNeedle(r *os.File) (*Needle, uint32) {
n := new(Needle)
bytes := make([]byte, 16)
count, e := r.Read(bytes)
if count<=0 || e!=nil {
return nil, 0
}
n.Cookie = BytesToUint32(bytes[0:4])
n.Key = BytesToUint64(bytes[4:12])
n.Size = BytesToUint32(bytes[12:16])
rest := 8 - ((n.Size + 16 + 4) % 8)
r.Seek(int64(n.Size+4+rest), 1)
return n, 16+n.Size+4+rest
n := new(Needle)
bytes := make([]byte, 16)
count, e := r.Read(bytes)
if count <= 0 || e != nil {
return nil, 0
}
n.Cookie = util.BytesToUint32(bytes[0:4])
n.Key = util.BytesToUint64(bytes[4:12])
n.Size = util.BytesToUint32(bytes[12:16])
rest := 8 - ((n.Size + 16 + 4) % 8)
r.Seek(int64(n.Size+4+rest), 1)
return n, 16 + n.Size + 4 + rest
}
func ParseKeyHash(key_hash_string string)(uint64,uint32) {
key_hash_bytes, khe := hex.DecodeString(key_hash_string)
key_hash_len := len(key_hash_bytes)
if khe != nil || key_hash_len <= 4 {
log.Println("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe)
return 0, 0
}
key := BytesToUint64(key_hash_bytes[0 : key_hash_len-4])
hash := BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len])
return key, hash
func ParseKeyHash(key_hash_string string) (uint64, uint32) {
key_hash_bytes, khe := hex.DecodeString(key_hash_string)
key_hash_len := len(key_hash_bytes)
if khe != nil || key_hash_len <= 4 {
log.Println("Invalid key_hash", key_hash_string, "length:", key_hash_len, "error", khe)
return 0, 0
}
key := util.BytesToUint64(key_hash_bytes[0 : key_hash_len-4])
hash := util.BytesToUint32(key_hash_bytes[key_hash_len-4 : key_hash_len])
return key, hash
}

42
weed-fs/src/pkg/storage/needle_map.go
View File

@@ -3,7 +3,7 @@ package storage
import (
"log"
"os"
. "util"
"pkg/util"
)
type NeedleValue struct {
@@ -36,24 +36,30 @@ func LoadNeedleMap(file *os.File) *NeedleMap {
count, e := nm.indexFile.Read(bytes)
if count > 0 {
fstat, _ := file.Stat()
log.Println("Loading index file", fstat.Name, "size", fstat.Size)
log.Println("Loading index file", fstat.Name(), "size", fstat.Size())
}
for count > 0 && e == nil {
for i := 0; i < count; i += 16 {
key := BytesToUint64(bytes[i : i+8])
offset := BytesToUint32(bytes[i+8 : i+12])
size := BytesToUint32(bytes[i+12 : i+16])
nm.m[key] = &NeedleValue{Offset: offset, Size: size}, offset > 0
}
key := util.BytesToUint64(bytes[i : i+8])
offset := util.BytesToUint32(bytes[i+8 : i+12])
size := util.BytesToUint32(bytes[i+12 : i+16])
if offset>0 {
nm.m[key] = &NeedleValue{util.Offset: offset, Size: size}
}else{
delete(nm.m, key)
}
}
count, e = nm.indexFile.Read(bytes)
}
}
return nm
}
func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, os.Error) {
nm.m[key] = &NeedleValue{Offset: offset, Size: size}
Uint64toBytes(nm.bytes[0:8], key)
Uint32toBytes(nm.bytes[8:12], offset)
Uint32toBytes(nm.bytes[12:16], size)
}
func (nm *NeedleMap) Put(key uint64, offset uint32, size uint32) (int, error) {
nm.m[key] = &NeedleValue{Offset: offset, Size: size}
util.Uint64toBytes(nm.bytes[0:8], key)
util.Uint32toBytes(nm.bytes[8:12], offset)
util.Uint32toBytes(nm.bytes[12:16], size)
return nm.indexFile.Write(nm.bytes)
}
func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
@@ -61,10 +67,10 @@ func (nm *NeedleMap) Get(key uint64) (element *NeedleValue, ok bool) {
return
}
func (nm *NeedleMap) Delete(key uint64) {
nm.m[key] = nil, false
Uint64toBytes(nm.bytes[0:8], key)
Uint32toBytes(nm.bytes[8:12], 0)
Uint32toBytes(nm.bytes[12:16], 0)
delete(nm.m, key)
util.Uint64toBytes(nm.bytes[0:8], key)
util.Uint32toBytes(nm.bytes[8:12], 0)
util.Uint32toBytes(nm.bytes[12:16], 0)
nm.indexFile.Write(nm.bytes)
}
func (nm *NeedleMap) Close() {

86
weed-fs/src/pkg/storage/store.go
View File

@@ -1,19 +1,19 @@
package storage
import (
"encoding/json"
"errors"
"log"
"json"
"os"
"strings"
"net/url"
"strconv"
"url"
"util"
"strings"
"pkg/util"
)
type Store struct {
volumes map[uint64]*Volume
dir string
Port int
volumes map[uint64]*Volume
dir string
Port int
PublicUrl string
}
type VolumeInfo struct {
@@ -24,44 +24,44 @@ type VolumeInfo struct {
func NewStore(port int, publicUrl, dirname string, volumeListString string) (s *Store) {
s = &Store{Port: port, PublicUrl: publicUrl, dir: dirname}
s.volumes = make(map[uint64]*Volume)
s.AddVolume(volumeListString)
log.Println("Store started on dir:", dirname, "with", len(s.volumes), "volumes")
return
}
func (s *Store) AddVolume(volumeListString string) os.Error{
for _, range_string := range strings.Split(volumeListString, ",") {
if strings.Index(range_string, "-") < 0 {
id_string := range_string
id, err := strconv.Atoui64(id_string)
if err != nil {
return os.NewError("Volume Id " + id_string+ " is not a valid unsigned integer!")
}
s.addVolume(id)
} else {
pair := strings.Split(range_string, "-")
start, start_err := strconv.Atoui64(pair[0])
if start_err != nil {
return os.NewError("Volume Start Id" + pair[0] + " is not a valid unsigned integer!")
}
end, end_err := strconv.Atoui64(pair[1])
if end_err != nil {
return os.NewError("Volume End Id" + pair[1] + " is not a valid unsigned integer!")
}
for id := start; id<=end; id++ {
s.addVolume(id)
}
}
}
return nil
func (s *Store) AddVolume(volumeListString string) error {
for _, range_string := range strings.Split(volumeListString, ",") {
if strings.Index(range_string, "-") < 0 {
id_string := range_string
id, err := strconv.ParseUint(id_string, 10, 64)
if err != nil {
return errors.New("Volume Id " + id_string + " is not a valid unsigned integer!")
}
s.addVolume(id)
} else {
pair := strings.Split(range_string, "-")
start, start_err := strconv.ParseUint(pair[0], 10, 64)
if start_err != nil {
return errors.New("Volume Start Id" + pair[0] + " is not a valid unsigned integer!")
}
end, end_err := strconv.ParseUint(pair[1], 10, 64)
if end_err != nil {
return errors.New("Volume End Id" + pair[1] + " is not a valid unsigned integer!")
}
for id := start; id <= end; id++ {
s.addVolume(id)
}
}
}
return nil
}
func (s *Store) addVolume(vid uint64) os.Error{
if s.volumes[vid]!=nil {
return os.NewError("Volume Id "+strconv.Uitoa64(vid)+" already exists!")
}
s.volumes[vid] = NewVolume(s.dir, uint32(vid))
return nil
func (s *Store) addVolume(vid uint64) error {
if s.volumes[vid] != nil {
return errors.New("Volume Id " + strconv.FormatUint(vid, 10) + " already exists!")
}
s.volumes[vid] = NewVolume(s.dir, uint32(vid))
return nil
}
func (s *Store) Status() *[]*VolumeInfo {
stats := new([]*VolumeInfo)
@@ -92,11 +92,11 @@ func (s *Store) Close() {
}
}
func (s *Store) Write(i uint64, n *Needle) uint32 {
return s.volumes[i].write(n)
return s.volumes[i].write(n)
}
func (s *Store) Delete(i uint64, n *Needle) uint32 {
return s.volumes[i].delete(n)
}
func (s *Store) Read(i uint64, n *Needle) (int, os.Error) {
func (s *Store) Read(i uint64, n *Needle) (int, error) {
return s.volumes[i].read(n)
}

15
weed-fs/src/pkg/storage/volume.go
View File

@@ -1,11 +1,12 @@
package storage
import (
"io"
"log"
"os"
"path"
"strconv"
"sync"
"log"
)
const (
@@ -22,9 +23,9 @@ type Volume struct {
}
func NewVolume(dirname string, id uint32) (v *Volume) {
var e os.Error
var e error
v = &Volume{dir: dirname, Id: id}
fileName := strconv.Uitoa64(uint64(v.Id))
fileName := strconv.FormatUint(uint64(v.Id), 10)
v.dataFile, e = os.OpenFile(path.Join(v.dir, fileName+".dat"), os.O_RDWR|os.O_CREATE, 0644)
if e != nil {
log.Fatalf("New Volume [ERROR] %s\n", e)
@@ -41,7 +42,7 @@ func NewVolume(dirname string, id uint32) (v *Volume) {
func (v *Volume) Size() int64 {
stat, e := v.dataFile.Stat()
if e == nil {
return stat.Size
return stat.Size()
}
return -1
}
@@ -51,7 +52,7 @@ func (v *Volume) Close() {
}
func (v *Volume) maybeWriteSuperBlock() {
stat, _ := v.dataFile.Stat()
if stat.Size == 0 {
if stat.Size() == 0 {
header := make([]byte, SuperBlockSize)
header[0] = 1
v.dataFile.Write(header)
@@ -82,7 +83,7 @@ func (v *Volume) delete(n *Needle) uint32 {
}
return 0
}
func (v *Volume) read(n *Needle) (int, os.Error) {
func (v *Volume) read(n *Needle) (int, error) {
v.accessLock.Lock()
defer v.accessLock.Unlock()
nv, ok := v.nm.Get(n.Key)
@@ -90,5 +91,5 @@ func (v *Volume) read(n *Needle) (int, os.Error) {
v.dataFile.Seek(int64(nv.Offset)*8, 0)
return n.Read(v.dataFile, nv.Size)
}
return -1, os.EOF
return -1, io.EOF
}

32
weed-fs/src/pkg/util/Makefile
View File

@@ -1,32 +0,0 @@
# Makefile generated by gb: http://go-gb.googlecode.com
# gb provides configuration-free building and distributing
include $(GOROOT)/src/Make.inc
TARG=util
GOFILES=\
bytes.go\
post.go\
# gb: this is the local install
GBROOT=../../..
# gb: compile/link against local install
GCIMPORTS+= -I $(GBROOT)/_obj
LDIMPORTS+= -L $(GBROOT)/_obj
# gb: compile/link against GOPATH entries
GOPATHSEP=:
ifeq ($(GOHOSTOS),windows)
GOPATHSEP=;
endif
GCIMPORTS+=-I $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -I , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
LDIMPORTS+=-L $(subst $(GOPATHSEP),/pkg/$(GOOS)_$(GOARCH) -L , $(GOPATH))/pkg/$(GOOS)_$(GOARCH)
# gb: copy to local install
$(GBROOT)/_obj/$(TARG).a: _obj/$(TARG).a
mkdir -p $(dir $@); cp -f $< $@
package: $(GBROOT)/_obj/$(TARG).a
include $(GOROOT)/src/Make.pkg

34
weed-fs/src/pkg/util/post.go
View File

@@ -1,23 +1,23 @@
package util
import (
"http"
"io/ioutil"
"url"
"log"
"io/ioutil"
"log"
"net/http"
"net/url"
)
func Post(url string, values url.Values)[]byte{
r, err := http.PostForm(url, values)
if err != nil {
log.Println("post:", err)
return nil
}
defer r.Body.Close()
b, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println("post:", err)
return nil
}
return b
func Post(url string, values url.Values) []byte {
r, err := http.PostForm(url, values)
if err != nil {
log.Println("post:", err)
return nil
}
defer r.Body.Close()
b, err := ioutil.ReadAll(r.Body)
if err != nil {
log.Println("post:", err)
return nil
}
return b
}