gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

go fmt

Browse Source
This commit is contained in:
chrislu
2024-02-29 09:38:52 -08:00
parent 2a7028373d
commit 1b4484bf0a
31 changed files with 116 additions and 117 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
weed/mq/broker/broker_grpc_sub.go
View File

@@ -68,7 +68,7 @@ func (b *MessageQueueBroker) SubscribeMessage(req *mq_pb.SubscribeMessageRequest
}() }()
var startPosition log_buffer.MessagePosition var startPosition log_buffer.MessagePosition
if req.GetInit()!=nil && req.GetInit().GetPartitionOffset() != nil { if req.GetInit() != nil && req.GetInit().GetPartitionOffset() != nil {
offset := req.GetInit().GetPartitionOffset() offset := req.GetInit().GetPartitionOffset()
if offset.StartTsNs != 0 { if offset.StartTsNs != 0 {
startPosition = log_buffer.NewMessagePosition(offset.StartTsNs, -2) startPosition = log_buffer.NewMessagePosition(offset.StartTsNs, -2)

8
weed/mq/broker/broker_topic_partition_read_write.go
View File

@@ -26,7 +26,7 @@ func (b *MessageQueueBroker) genLogFlushFunc(t topic.Topic, partition *mq_pb.Par
startTime, stopTime = startTime.UTC(), stopTime.UTC() startTime, stopTime = startTime.UTC(), stopTime.UTC()
targetFile := fmt.Sprintf("%s/%s",partitionDir, startTime.Format(topic.TIME_FORMAT)) targetFile := fmt.Sprintf("%s/%s", partitionDir, startTime.Format(topic.TIME_FORMAT))
// TODO append block with more metadata // TODO append block with more metadata
@@ -50,7 +50,7 @@ func (b *MessageQueueBroker) genLogOnDiskReadFunc(t topic.Topic, partition *mq_p
return b.MasterClient.LookupFileId(fileId) return b.MasterClient.LookupFileId(fileId)
} }
eachChunkFn := func (buf []byte, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64) (processedTsNs int64, err error) { eachChunkFn := func(buf []byte, eachLogEntryFn log_buffer.EachLogEntryFuncType, starTsNs, stopTsNs int64) (processedTsNs int64, err error) {
for pos := 0; pos+4 < len(buf); { for pos := 0; pos+4 < len(buf); {
size := util.BytesToUint32(buf[pos : pos+4]) size := util.BytesToUint32(buf[pos : pos+4])
@@ -99,7 +99,7 @@ func (b *MessageQueueBroker) genLogOnDiskReadFunc(t topic.Topic, partition *mq_p
if chunk.Size == 0 { if chunk.Size == 0 {
continue continue
} }
if chunk.IsChunkManifest{ if chunk.IsChunkManifest {
glog.Warningf("this should not happen. unexpected chunk manifest in %s/%s", partitionDir, entry.Name) glog.Warningf("this should not happen. unexpected chunk manifest in %s/%s", partitionDir, entry.Name)
return return
} }
@@ -145,7 +145,7 @@ func (b *MessageQueueBroker) genLogOnDiskReadFunc(t topic.Topic, partition *mq_p
if entry.IsDirectory { if entry.IsDirectory {
return nil return nil
} }
if stopTsNs!=0 && entry.Name > stopTime.UTC().Format(topic.TIME_FORMAT) { if stopTsNs != 0 && entry.Name > stopTime.UTC().Format(topic.TIME_FORMAT) {
isDone = true isDone = true
return nil return nil
} }

1
weed/mq/client/pub_client/publish.go
View File

@@ -7,7 +7,6 @@ import (
"github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
) )
func (p *TopicPublisher) Publish(key, value []byte) error { func (p *TopicPublisher) Publish(key, value []byte) error {
hashKey := util.HashToInt32(key) % pub_balancer.MaxPartitionCount hashKey := util.HashToInt32(key) % pub_balancer.MaxPartitionCount
if hashKey < 0 { if hashKey < 0 {

7
weed/mq/sub_coordinator/consumer_group.go
View File

@@ -40,13 +40,13 @@ func NewConsumerGroupInstance(instanceId string) *ConsumerGroupInstance {
} }
} }
func (cg *ConsumerGroup) OnAddConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic) { func (cg *ConsumerGroup) OnAddConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic) {
cg.onConsumerGroupInstanceChange("add consumer instance "+ consumerGroupInstance) cg.onConsumerGroupInstanceChange("add consumer instance " + consumerGroupInstance)
} }
func (cg *ConsumerGroup) OnRemoveConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic) { func (cg *ConsumerGroup) OnRemoveConsumerGroupInstance(consumerGroupInstance string, topic *mq_pb.Topic) {
cg.onConsumerGroupInstanceChange("remove consumer instance "+ consumerGroupInstance) cg.onConsumerGroupInstanceChange("remove consumer instance " + consumerGroupInstance)
} }
func (cg *ConsumerGroup) onConsumerGroupInstanceChange(reason string){ func (cg *ConsumerGroup) onConsumerGroupInstanceChange(reason string) {
if cg.reBalanceTimer != nil { if cg.reBalanceTimer != nil {
cg.reBalanceTimer.Stop() cg.reBalanceTimer.Stop()
cg.reBalanceTimer = nil cg.reBalanceTimer = nil
@@ -126,5 +126,4 @@ func (cg *ConsumerGroup) RebalanceConsumberGroupInstances(knownPartitionSlotToBr
consumerGroupInstance.ResponseChan <- response consumerGroupInstance.ResponseChan <- response
} }
} }

6
weed/mq/sub_coordinator/coordinator.go
View File

@@ -31,7 +31,7 @@ func NewCoordinator(balancer *pub_balancer.Balancer) *Coordinator {
func (c *Coordinator) GetTopicConsumerGroups(topic *mq_pb.Topic, createIfMissing bool) *TopicConsumerGroups { func (c *Coordinator) GetTopicConsumerGroups(topic *mq_pb.Topic, createIfMissing bool) *TopicConsumerGroups {
topicName := toTopicName(topic) topicName := toTopicName(topic)
tcg, _ := c.TopicSubscribers.Get(topicName) tcg, _ := c.TopicSubscribers.Get(topicName)
if tcg == nil && createIfMissing{ if tcg == nil && createIfMissing {
tcg = &TopicConsumerGroups{ tcg = &TopicConsumerGroups{
ConsumerGroups: cmap.New[*ConsumerGroup](), ConsumerGroups: cmap.New[*ConsumerGroup](),
} }
@@ -56,14 +56,14 @@ func (c *Coordinator) AddSubscriber(consumerGroup, consumerGroupInstance string,
cg, _ := tcg.ConsumerGroups.Get(consumerGroup) cg, _ := tcg.ConsumerGroups.Get(consumerGroup)
if cg == nil { if cg == nil {
cg = NewConsumerGroup(topic, c.balancer) cg = NewConsumerGroup(topic, c.balancer)
if !tcg.ConsumerGroups.SetIfAbsent(consumerGroup, cg){ if !tcg.ConsumerGroups.SetIfAbsent(consumerGroup, cg) {
cg, _ = tcg.ConsumerGroups.Get(consumerGroup) cg, _ = tcg.ConsumerGroups.Get(consumerGroup)
} }
} }
cgi, _ := cg.ConsumerGroupInstances.Get(consumerGroupInstance) cgi, _ := cg.ConsumerGroupInstances.Get(consumerGroupInstance)
if cgi == nil { if cgi == nil {
cgi = NewConsumerGroupInstance(consumerGroupInstance) cgi = NewConsumerGroupInstance(consumerGroupInstance)
if !cg.ConsumerGroupInstances.SetIfAbsent(consumerGroupInstance, cgi){ if !cg.ConsumerGroupInstances.SetIfAbsent(consumerGroupInstance, cgi) {
cgi, _ = cg.ConsumerGroupInstances.Get(consumerGroupInstance) cgi, _ = cg.ConsumerGroupInstances.Get(consumerGroupInstance)
} }
} }

1
weed/mq/topic/local_partition.go
View File

@@ -22,6 +22,7 @@ type LocalPartition struct {
} }
var TIME_FORMAT = "2006-01-02-15-04-05" var TIME_FORMAT = "2006-01-02-15-04-05"
func NewLocalPartition(partition Partition, isLeader bool, followerBrokers []pb.ServerAddress, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition { func NewLocalPartition(partition Partition, isLeader bool, followerBrokers []pb.ServerAddress, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition {
return &LocalPartition{ return &LocalPartition{
Partition: partition, Partition: partition,

4
weed/pb/filer_pb/filer.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.32.0 // protoc-gen-go v1.31.0
// protoc v4.25.2 // protoc v4.25.3
// source: filer.proto // source: filer.proto
package filer_pb package filer_pb

2
weed/pb/filer_pb/filer_grpc.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT. // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions: // versions:
// - protoc-gen-go-grpc v1.3.0 // - protoc-gen-go-grpc v1.3.0
// - protoc v4.25.2 // - protoc v4.25.3
// source: filer.proto // source: filer.proto
package filer_pb package filer_pb

4
weed/pb/iam_pb/iam.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.32.0 // protoc-gen-go v1.31.0
// protoc v4.25.2 // protoc v4.25.3
// source: iam.proto // source: iam.proto
package iam_pb package iam_pb

2
weed/pb/iam_pb/iam_grpc.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT. // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions: // versions:
// - protoc-gen-go-grpc v1.3.0 // - protoc-gen-go-grpc v1.3.0
// - protoc v4.25.2 // - protoc v4.25.3
// source: iam.proto // source: iam.proto
package iam_pb package iam_pb

4
weed/pb/master_pb/master.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.32.0 // protoc-gen-go v1.31.0
// protoc v4.25.2 // protoc v4.25.3
// source: master.proto // source: master.proto
package master_pb package master_pb

2
weed/pb/master_pb/master_grpc.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT. // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions: // versions:
// - protoc-gen-go-grpc v1.3.0 // - protoc-gen-go-grpc v1.3.0
// - protoc v4.25.2 // - protoc v4.25.3
// source: master.proto // source: master.proto
package master_pb package master_pb

4
weed/pb/mount_pb/mount.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.32.0 // protoc-gen-go v1.31.0
// protoc v4.25.2 // protoc v4.25.3
// source: mount.proto // source: mount.proto
package mount_pb package mount_pb

2
weed/pb/mount_pb/mount_grpc.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT. // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions: // versions:
// - protoc-gen-go-grpc v1.3.0 // - protoc-gen-go-grpc v1.3.0
// - protoc v4.25.2 // - protoc v4.25.3
// source: mount.proto // source: mount.proto
package mount_pb package mount_pb

2
weed/pb/mq_pb/mq_grpc.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT. // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions: // versions:
// - protoc-gen-go-grpc v1.3.0 // - protoc-gen-go-grpc v1.3.0
// - protoc v4.25.2 // - protoc v4.25.3
// source: mq.proto // source: mq.proto
package mq_pb package mq_pb

4
weed/pb/remote_pb/remote.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.32.0 // protoc-gen-go v1.31.0
// protoc v4.25.2 // protoc v4.25.3
// source: remote.proto // source: remote.proto
package remote_pb package remote_pb

4
weed/pb/s3_pb/s3.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.32.0 // protoc-gen-go v1.31.0
// protoc v4.25.2 // protoc v4.25.3
// source: s3.proto // source: s3.proto
package s3_pb package s3_pb

2
weed/pb/s3_pb/s3_grpc.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT. // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions: // versions:
// - protoc-gen-go-grpc v1.3.0 // - protoc-gen-go-grpc v1.3.0
// - protoc v4.25.2 // - protoc v4.25.3
// source: s3.proto // source: s3.proto
package s3_pb package s3_pb

4
weed/pb/volume_server_pb/volume_server.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.32.0 // protoc-gen-go v1.31.0
// protoc v4.25.2 // protoc v4.25.3
// source: volume_server.proto // source: volume_server.proto
package volume_server_pb package volume_server_pb

2
weed/pb/volume_server_pb/volume_server_grpc.pb.go
View File

@@ -1,7 +1,7 @@
// Code generated by protoc-gen-go-grpc. DO NOT EDIT. // Code generated by protoc-gen-go-grpc. DO NOT EDIT.
// versions: // versions:
// - protoc-gen-go-grpc v1.3.0 // - protoc-gen-go-grpc v1.3.0
// - protoc v4.25.2 // - protoc v4.25.3
// source: volume_server.proto // source: volume_server.proto
package volume_server_pb package volume_server_pb

10
weed/util/log_buffer/log_buffer.go
View File

@@ -199,10 +199,10 @@ func (logBuffer *LogBuffer) copyToFlush() *dataToFlush {
return nil return nil
} }
func (logBuffer *LogBuffer) GetEarliestTime() time.Time{ func (logBuffer *LogBuffer) GetEarliestTime() time.Time {
return logBuffer.startTime return logBuffer.startTime
} }
func (logBuffer *LogBuffer) GetEarliestPosition() MessagePosition{ func (logBuffer *LogBuffer) GetEarliestPosition() MessagePosition {
return MessagePosition{ return MessagePosition{
Time: logBuffer.startTime, Time: logBuffer.startTime,
BatchIndex: logBuffer.batchIndex, BatchIndex: logBuffer.batchIndex,
@@ -241,8 +241,8 @@ func (logBuffer *LogBuffer) ReadFromBuffer(lastReadPosition MessagePosition) (bu
} }
if tsMemory.IsZero() { // case 2.2 if tsMemory.IsZero() { // case 2.2
println("2.2 no data") println("2.2 no data")
return nil, -2,nil return nil, -2, nil
} else if lastReadPosition.Before(tsMemory) && lastReadPosition.BatchIndex +1 < tsBatchIndex { // case 2.3 } else if lastReadPosition.Before(tsMemory) && lastReadPosition.BatchIndex+1 < tsBatchIndex { // case 2.3
if !logBuffer.lastFlushTime.IsZero() { if !logBuffer.lastFlushTime.IsZero() {
glog.V(0).Infof("resume with last flush time: %v", logBuffer.lastFlushTime) glog.V(0).Infof("resume with last flush time: %v", logBuffer.lastFlushTime)
return nil, -2, ResumeFromDiskError return nil, -2, ResumeFromDiskError
@@ -273,7 +273,7 @@ func (logBuffer *LogBuffer) ReadFromBuffer(lastReadPosition MessagePosition) (bu
} }
} }
// glog.V(4).Infof("%s return the current buf %v", m.name, lastReadPosition) // glog.V(4).Infof("%s return the current buf %v", m.name, lastReadPosition)
return copiedBytes(logBuffer.buf[:logBuffer.pos]), logBuffer.batchIndex,nil return copiedBytes(logBuffer.buf[:logBuffer.pos]), logBuffer.batchIndex, nil
} }
lastTs := lastReadPosition.UnixNano() lastTs := lastReadPosition.UnixNano()