Set volumes ReadOnly if low free disk space

weed/storage/disk_location.go

@@ -1,11 +1,14 @@
 package storage
 
 import (
-	"fmt"
-	"io/ioutil"
-	"os"
-	"strings"
-	"sync"
+ 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/stats"
+ 	"io/ioutil"
+ 	"os"
+	"path/filepath"
+ 	"strings"
+ 	"sync"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
@@ -15,6 +18,7 @@ import (
 type DiskLocation struct {
 	Directory      string
 	MaxVolumeCount int
+	FreeDiskSpaceWatermark float32
 	volumes        map[needle.VolumeId]*Volume
 	volumesLock    sync.RWMutex
 
@@ -23,10 +27,11 @@ type DiskLocation struct {
 	ecVolumesLock sync.RWMutex
 }
 
-func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation {
-	location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount}
+func NewDiskLocation(dir string, maxVolumeCount int, freeDiskSpaceWatermark float32) *DiskLocation {
+	location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount, FreeDiskSpaceWatermark: freeDiskSpaceWatermark}
 	location.volumes = make(map[needle.VolumeId]*Volume)
 	location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume)
+	go location.CheckDiskSpace()
 	return location
 }
 
@@ -293,3 +298,21 @@ func (l *DiskLocation) UnUsedSpace(volumeSizeLimit uint64) (unUsedSpace uint64)
 
 	return
 }
+
+func (l *DiskLocation) CheckDiskSpace() {
+	lastStat := false
+	t := time.NewTicker(time.Minute)
+	for _ = range t.C {
+		if dir, e := filepath.Abs(l.Directory); e == nil {
+			s := stats.NewDiskStatus(dir)
+			if (s.PercentFree < l.FreeDiskSpaceWatermark) != lastStat {
+				lastStat = !lastStat
+				for _, v := range l.volumes {
+					v.SetLowDiskSpace(lastStat)
+				}
+
+			}
+		}
+	}
+
+}

weed/storage/store.go

@@ -48,11 +48,11 @@ func (s *Store) String() (str string) {
 	return
 }
 
-func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, needleMapKind NeedleMapType) (s *Store) {
+func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, freeDiskSpaceWatermark []float32, needleMapKind NeedleMapType) (s *Store) {
 	s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapType: needleMapKind}
 	s.Locations = make([]*DiskLocation, 0)
 	for i := 0; i < len(dirnames); i++ {
-		location := NewDiskLocation(dirnames[i], maxVolumeCounts[i])
+		location := NewDiskLocation(dirnames[i], maxVolumeCounts[i], freeDiskSpaceWatermark[i])
 		location.loadExistingVolumes(needleMapKind)
 		s.Locations = append(s.Locations, location)
 		stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i]))

weed/storage/volume.go

@@ -27,6 +27,7 @@ type Volume struct {
 	needleMapKind      NeedleMapType
 	noWriteOrDelete    bool // if readonly, either noWriteOrDelete or noWriteCanDelete
 	noWriteCanDelete   bool // if readonly, either noWriteOrDelete or noWriteCanDelete
+	lowDiskSpace       bool
 	hasRemoteFile      bool // if the volume has a remote file
 	MemoryMapMaxSizeMb uint32
 
@@ -45,6 +46,11 @@ type Volume struct {
 	volumeInfo *volume_server_pb.VolumeInfo
 }
 
+func (v *Volume) SetLowDiskSpace(lowDiskSpace bool) {
+	glog.V(0).Infof("SetLowDiskSpace id %d value %t", v.Id, lowDiskSpace)
+	v.lowDiskSpace = lowDiskSpace
+}
+
 func NewVolume(dirname string, collection string, id needle.VolumeId, needleMapKind NeedleMapType, replicaPlacement *super_block.ReplicaPlacement, ttl *needle.TTL, preallocate int64, memoryMapMaxSizeMb uint32) (v *Volume, e error) {
 	// if replicaPlacement is nil, the superblock will be loaded from disk
 	v = &Volume{dir: dirname, Collection: collection, Id: id, MemoryMapMaxSizeMb: memoryMapMaxSizeMb,
@@ -244,5 +250,5 @@ func (v *Volume) RemoteStorageNameKey() (storageName, storageKey string) {
 }
 
 func (v *Volume) IsReadOnly() bool {
-	return v.noWriteOrDelete || v.noWriteCanDelete
+	return v.noWriteOrDelete || v.noWriteCanDelete || v.lowDiskSpace
 }