gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ui add ec shard statuses

Browse Source
This commit is contained in:
Chris Lu
2019-06-04 21:52:37 -07:00
parent 0e52862586
commit 2215e81be7
8 changed files with 81 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/server/master_ui/templates.go
View File

@@ -76,6 +76,7 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
<th>Rack</th> <th>Rack</th>
<th>RemoteAddr</th> <th>RemoteAddr</th>
<th>#Volumes</th> <th>#Volumes</th>
<th>#ErasureCodingShards</th>
<th>Max</th> <th>Max</th>
</tr> </tr>
</thead> </thead>
@@ -88,6 +89,7 @@ var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>
<td>{{ $rack.Id }}</td> <td>{{ $rack.Id }}</td>
<td><a href="http://{{ $dn.Url }}/ui/index.html">{{ $dn.Url }}</a></td> <td><a href="http://{{ $dn.Url }}/ui/index.html">{{ $dn.Url }}</a></td>
<td>{{ $dn.Volumes }}</td> <td>{{ $dn.Volumes }}</td>
<td>{{ $dn.EcShards }}</td>
<td>{{ $dn.Max }}</td> <td>{{ $dn.Max }}</td>
</tr> </tr>
{{ end }} {{ end }}

2
weed/server/volume_server_handlers_ui.go
View File

@@ -24,6 +24,7 @@ func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request)
Version string Version string
Masters []string Masters []string
Volumes interface{} Volumes interface{}
EcVolumes interface{}
DiskStatuses interface{} DiskStatuses interface{}
Stats interface{} Stats interface{}
Counters *stats.ServerStats Counters *stats.ServerStats
@@ -31,6 +32,7 @@ func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request)
util.VERSION, util.VERSION,
vs.SeedMasterNodes, vs.SeedMasterNodes,
vs.store.Status(), vs.store.Status(),
vs.store.EcVolumes(),
ds, ds,
infos, infos,
serverStats, serverStats,

26
weed/server/volume_server_ui/templates.go
View File

@@ -128,6 +128,32 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOC
</table> </table>
</div> </div>
<div class="row">
<h2>Erasure Coding Shards</h2>
<table class="table table-striped">
<thead>
<tr>
<th>Id</th>
<th>Collection</th>
<th>Shard Size</th>
<th>Shards</th>
<th>CreatedAt</th>
</tr>
</thead>
<tbody>
{{ range .EcVolumes }}
<tr>
<td><code>{{ .VolumeId }}</code></td>
<td>{{ .Collection }}</td>
<td>{{ .ShardSize }} Bytes</td>
<td>{{ .ShardIdList }}</td>
<td>{{ .CreatedAt.Format "02 Jan 06 15:04 -0700" }}</td>
</tr>
{{ end }}
</tbody>
</table>
</div>
</div> </div>
</body> </body>
</html> </html>

4
weed/storage/erasure_coding/ec_shard.go
View File

@@ -39,6 +39,10 @@ func NewEcVolumeShard(dirname string, collection string, id needle.VolumeId, sha
return return
} }
func (shard *EcVolumeShard) Size() int64 {
return shard.ecdFileSize
}
func (shard *EcVolumeShard) String() string { func (shard *EcVolumeShard) String() string {
return fmt.Sprintf("ec shard %v:%v, dir:%s, Collection:%s", shard.VolumeId, shard.ShardId, shard.dir, shard.Collection) return fmt.Sprintf("ec shard %v:%v, dir:%s, Collection:%s", shard.VolumeId, shard.ShardId, shard.dir, shard.Collection)
} }

20
weed/storage/erasure_coding/ec_volume.go
View File

@@ -20,6 +20,7 @@ type EcVolume struct {
dir string dir string
ecxFile *os.File ecxFile *os.File
ecxFileSize int64 ecxFileSize int64
ecxCreatedAt time.Time
Shards []*EcVolumeShard Shards []*EcVolumeShard
ShardLocations map[ShardId][]string ShardLocations map[ShardId][]string
ShardLocationsRefreshTime time.Time ShardLocationsRefreshTime time.Time
@@ -41,6 +42,7 @@ func NewEcVolume(dir string, collection string, vid needle.VolumeId) (ev *EcVolu
return nil, fmt.Errorf("can not stat ec volume index %s.ecx: %v", baseFileName, statErr) return nil, fmt.Errorf("can not stat ec volume index %s.ecx: %v", baseFileName, statErr)
} }
ev.ecxFileSize = ecxFi.Size() ev.ecxFileSize = ecxFi.Size()
ev.ecxCreatedAt = ecxFi.ModTime()
ev.ShardLocations = make(map[ShardId][]string) ev.ShardLocations = make(map[ShardId][]string)
@@ -113,6 +115,24 @@ func (ev *EcVolume) FileName() string {
} }
func (ev *EcVolume) ShardSize() int64 {
if len(ev.Shards) > 0 {
return ev.Shards[0].Size()
}
return 0
}
func (ev *EcVolume) CreatedAt() time.Time {
return ev.ecxCreatedAt
}
func (ev *EcVolume) ShardIdList() (shardIds []ShardId) {
for _, s := range ev.Shards {
shardIds = append(shardIds, s.ShardId)
}
return
}
func (ev *EcVolume) ToVolumeEcShardInformationMessage() (messages []*master_pb.VolumeEcShardInformationMessage) { func (ev *EcVolume) ToVolumeEcShardInformationMessage() (messages []*master_pb.VolumeEcShardInformationMessage) {
prevVolumeId := needle.VolumeId(math.MaxUint32) prevVolumeId := needle.VolumeId(math.MaxUint32)
var m *master_pb.VolumeEcShardInformationMessage var m *master_pb.VolumeEcShardInformationMessage

15
weed/storage/store_ec.go
View File

@@ -4,6 +4,7 @@ import (
"context" "context"
"fmt" "fmt"
"io" "io"
"sort"
"sync" "sync"
"time" "time"
@@ -365,3 +366,17 @@ func (s *Store) recoverOneRemoteEcShardInterval(ctx context.Context, ecVolume *e
return len(buf), nil return len(buf), nil
} }
func (s *Store) EcVolumes() (ecVolumes []*erasure_coding.EcVolume) {
for _, location := range s.Locations {
location.ecVolumesLock.RLock()
for _, v := range location.ecVolumes {
ecVolumes = append(ecVolumes, v)
}
location.ecVolumesLock.RUnlock()
}
sort.Slice(ecVolumes, func(i, j int) bool {
return ecVolumes[i].VolumeId > ecVolumes[j].VolumeId
})
return ecVolumes
}

1
weed/topology/data_node.go
View File

@@ -146,6 +146,7 @@ func (dn *DataNode) ToMap() interface{} {
ret := make(map[string]interface{}) ret := make(map[string]interface{})
ret["Url"] = dn.Url() ret["Url"] = dn.Url()
ret["Volumes"] = dn.GetVolumeCount() ret["Volumes"] = dn.GetVolumeCount()
ret["EcShards"] = dn.GetEcShardsCount()
ret["Max"] = dn.GetMaxVolumeCount() ret["Max"] = dn.GetMaxVolumeCount()
ret["Free"] = dn.FreeSpace() ret["Free"] = dn.FreeSpace()
ret["PublicUrl"] = dn.PublicUrl ret["PublicUrl"] = dn.PublicUrl

11
weed/topology/data_node_ec.go
View File

@@ -14,6 +14,17 @@ func (dn *DataNode) GetEcShards() (ret []*erasure_coding.EcVolumeInfo) {
return ret return ret
} }
func (dn *DataNode) GetEcShardsCount() (count int) {
dn.RLock()
defer dn.RUnlock()
for _, ecVolumeInfo := range dn.ecShards {
count += ecVolumeInfo.ShardBits.ShardIdCount()
}
return count
}
func (dn *DataNode) UpdateEcShards(actualShards []*erasure_coding.EcVolumeInfo) (newShards, deletedShards []*erasure_coding.EcVolumeInfo) { func (dn *DataNode) UpdateEcShards(actualShards []*erasure_coding.EcVolumeInfo) (newShards, deletedShards []*erasure_coding.EcVolumeInfo) {
// prepare the new ec shard map // prepare the new ec shard map
actualEcShardMap := make(map[needle.VolumeId]*erasure_coding.EcVolumeInfo) actualEcShardMap := make(map[needle.VolumeId]*erasure_coding.EcVolumeInfo)