gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

support for collections!

Browse Source
This commit is contained in:
Chris Lu
2013-11-12 02:21:22 -08:00
parent 8f0e2f31af
commit 3b68711139
16 changed files with 163 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
go/storage/store.go
View File

@@ -39,7 +39,7 @@ func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts
}
return
}
func (s *Store) AddVolume(volumeListString string, replicationType string) error {
func (s *Store) AddVolume(volumeListString string, collection string, replicationType string) error {
rt, e := NewReplicationTypeFromString(replicationType)
if e != nil {
return e
@@ -51,7 +51,7 @@ func (s *Store) AddVolume(volumeListString string, replicationType string) error
if err != nil {
return fmt.Errorf("Volume Id %s is not a valid unsigned integer!", id_string)
}
e = s.addVolume(VolumeId(id), rt)
e = s.addVolume(VolumeId(id), collection, rt)
} else {
pair := strings.Split(range_string, "-")
start, start_err := strconv.ParseUint(pair[0], 10, 64)
@@ -63,7 +63,7 @@ func (s *Store) AddVolume(volumeListString string, replicationType string) error
return fmt.Errorf("Volume End Id %s is not a valid unsigned integer!", pair[1])
}
for id := start; id <= end; id++ {
if err := s.addVolume(VolumeId(id), rt); err != nil {
if err := s.addVolume(VolumeId(id), collection, rt); err != nil {
e = err
}
}
@@ -90,13 +90,13 @@ func (s *Store) findFreeLocation() (ret *DiskLocation) {
}
return ret
}
func (s *Store) addVolume(vid VolumeId, replicationType ReplicationType) error {
func (s *Store) addVolume(vid VolumeId, collection string, replicationType ReplicationType) error {
if s.findVolume(vid) != nil {
return fmt.Errorf("Volume Id %s already exists!", vid)
}
if location := s.findFreeLocation(); location != nil {
glog.V(0).Infoln("In dir", location.directory, "adds volume =", vid, ", replicationType =", replicationType)
if volume, err := NewVolume(location.directory, vid, replicationType); err == nil {
glog.V(0).Infoln("In dir", location.directory, "adds volume =", vid, ", collection =", collection, ", replicationType =", replicationType)
if volume, err := NewVolume(location.directory, collection, vid, replicationType); err == nil {
location.volumes[vid] = volume
return nil
} else {
@@ -158,12 +158,17 @@ func (l *DiskLocation) loadExistingVolumes() {
for _, dir := range dirs {
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
collection := ""
base := name[:len(name)-len(".dat")]
i := strings.Index(base, "_")
if i > 0 {
collection, base = base[0:i], base[i+1:]
}
if vid, err := NewVolumeId(base); err == nil {
if l.volumes[vid] == nil {
if v, e := NewVolume(l.directory, vid, CopyNil); e == nil {
if v, e := NewVolume(l.directory, collection, vid, CopyNil); e == nil {
l.volumes[vid] = v
glog.V(0).Infoln("In dir", l.directory, "read volume =", vid, "replicationType =", v.ReplicaType, "version =", v.Version(), "size =", v.Size())
glog.V(0).Infoln("data file", l.directory+"/"+name, "replicationType =", v.ReplicaType, "version =", v.Version(), "size =", v.Size())
}
}
}
@@ -177,7 +182,9 @@ func (s *Store) Status() []*VolumeInfo {
for _, location := range s.locations {
for k, v := range location.volumes {
s := &VolumeInfo{Id: VolumeId(k), Size: v.ContentSize(),
RepType: v.ReplicaType, Version: v.Version(),
Collection: v.Collection,
RepType: v.ReplicaType,
Version: v.Version(),
FileCount: v.nm.FileCount(),
DeleteCount: v.nm.DeletedCount(),
DeletedByteCount: v.nm.DeletedSize(),
@@ -208,7 +215,9 @@ func (s *Store) Join() error {
maxVolumeCount = maxVolumeCount + location.maxVolumeCount
for k, v := range location.volumes {
s := &VolumeInfo{Id: VolumeId(k), Size: uint64(v.Size()),
RepType: v.ReplicaType, Version: v.Version(),
Collection: v.Collection,
RepType: v.ReplicaType,
Version: v.Version(),
FileCount: v.nm.FileCount(),
DeleteCount: v.nm.DeletedCount(),
DeletedByteCount: v.nm.DeletedSize(),

32
go/storage/volume.go
View File

@@ -29,32 +29,38 @@ func (s *SuperBlock) Bytes() []byte {
}
type Volume struct {
Id VolumeId
dir string
dataFile *os.File
nm NeedleMapper
readOnly bool
Id VolumeId
dir string
Collection string
dataFile *os.File
nm NeedleMapper
readOnly bool
SuperBlock
accessLock sync.Mutex
}
func NewVolume(dirname string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
v = &Volume{dir: dirname, Id: id}
func NewVolume(dirname string, collection string, id VolumeId, replicationType ReplicationType) (v *Volume, e error) {
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{ReplicaType: replicationType}
e = v.load(true)
return
}
func loadVolumeWithoutIndex(dirname string, id VolumeId) (v *Volume, e error) {
v = &Volume{dir: dirname, Id: id}
func loadVolumeWithoutIndex(dirname string, collection string, id VolumeId) (v *Volume, e error) {
v = &Volume{dir: dirname, Collection: collection, Id: id}
v.SuperBlock = SuperBlock{ReplicaType: CopyNil}
e = v.load(false)
return
}
func (v *Volume) load(alsoLoadIndex bool) error {
var e error
fileName := path.Join(v.dir, v.Id.String())
var fileName string
if v.Collection == "" {
fileName = path.Join(v.dir, v.Id.String())
} else {
fileName = path.Join(v.dir, v.Collection+"_"+v.Id.String())
}
if exists, canRead, canWrite, _ := checkFile(fileName + ".dat"); exists && !canRead {
return fmt.Errorf("cannot read Volume Data file %s.dat", fileName)
} else if !exists || canWrite {
@@ -309,11 +315,11 @@ func (v *Volume) freeze() error {
return nil
}
func ScanVolumeFile(dirname string, id VolumeId,
func ScanVolumeFile(dirname string, collection string, id VolumeId,
visitSuperBlock func(SuperBlock) error,
visitNeedle func(n *Needle, offset int64) error) (err error) {
var v *Volume
if v, err = loadVolumeWithoutIndex(dirname, id); err != nil {
if v, err = loadVolumeWithoutIndex(dirname, collection, id); err != nil {
return
}
if err = visitSuperBlock(v.SuperBlock); err != nil {
@@ -365,7 +371,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string) (err erro
nm := NewNeedleMap(idx)
new_offset := int64(SuperBlockSize)
err = ScanVolumeFile(v.dir, v.Id, func(superBlock SuperBlock) error {
err = ScanVolumeFile(v.dir, v.Collection, v.Id, func(superBlock SuperBlock) error {
_, err = dst.Write(superBlock.Bytes())
return err
}, func(n *Needle, offset int64) error {

1
go/storage/volume_info.go
View File

@@ -6,6 +6,7 @@ type VolumeInfo struct {
Id VolumeId
Size uint64
RepType ReplicationType
Collection string
Version Version
FileCount int
DeleteCount int