add a snowflake sequencer as more robust fid generator, but less compressable than small auto-inc id
This commit is contained in:
@@ -506,7 +506,7 @@ default = "localhost:8888" # used by maintenance scripts if the scripts needs
|
||||
|
||||
|
||||
[master.sequencer]
|
||||
type = "raft" # Choose [raft|etcd] type for storing the file id sequence
|
||||
type = "raft" # Choose [raft|etcd|snowflake] type for storing the file id sequence
|
||||
# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
|
||||
# example : http://127.0.0.1:2379,http://127.0.0.1:2389
|
||||
sequencer_etcd_urls = "http://127.0.0.1:2379"
|
||||
|
||||
43
weed/sequence/snowflake_sequencer.go
Normal file
43
weed/sequence/snowflake_sequencer.go
Normal file
@@ -0,0 +1,43 @@
|
||||
package sequence
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
|
||||
"github.com/bwmarrin/snowflake"
|
||||
)
|
||||
|
||||
// a simple snowflake Sequencer
|
||||
type SnowflakeSequencer struct {
|
||||
node *snowflake.Node
|
||||
}
|
||||
|
||||
func NewSnowflakeSequencer(nodeid string) (*SnowflakeSequencer, error) {
|
||||
node, err := snowflake.NewNode(int64(hash(nodeid) & 0x3ff))
|
||||
if err != nil {
|
||||
fmt.Println(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
sequencer := &SnowflakeSequencer{node: node}
|
||||
return sequencer, nil
|
||||
}
|
||||
|
||||
func hash(s string) uint32 {
|
||||
h := fnv.New32a()
|
||||
h.Write([]byte(s))
|
||||
return h.Sum32()
|
||||
}
|
||||
|
||||
func (m *SnowflakeSequencer) NextFileId(count uint64) uint64 {
|
||||
return uint64(m.node.Generate().Int64())
|
||||
}
|
||||
|
||||
// ignore setmax as we are snowflake
|
||||
func (m *SnowflakeSequencer) SetMax(seenValue uint64) {
|
||||
}
|
||||
|
||||
// return a new id as no Peek is stored
|
||||
func (m *SnowflakeSequencer) Peek() uint64 {
|
||||
return uint64(m.node.Generate().Int64())
|
||||
}
|
||||
@@ -277,6 +277,14 @@ func (ms *MasterServer) createSequencer(option *MasterOption) sequence.Sequencer
|
||||
glog.Error(err)
|
||||
seq = nil
|
||||
}
|
||||
case "snowflake":
|
||||
var err error
|
||||
glog.V(0).Infof("use a snowfalke seq id, nodeid %s:%d", option.Host, option.Port)
|
||||
seq, err = sequence.NewSnowflakeSequencer(fmt.Sprintf("%s:%d", option.Host, option.Port))
|
||||
if err != nil {
|
||||
glog.Error(err)
|
||||
seq = nil
|
||||
}
|
||||
default:
|
||||
seq = sequence.NewMemorySequencer()
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user