gsh-digital-services/seaweedFS
5
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactoring

Browse Source
This commit is contained in:
Chris Lu
2019-04-18 21:43:36 -07:00
parent 33c92b819a
commit e5506152c0
72 changed files with 384 additions and 328 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
weed/storage/needle/btree_map.go
View File

@@ -1,53 +0,0 @@
package needle
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/google/btree"
)
//This map assumes mostly inserting increasing keys
type BtreeMap struct {
tree *btree.BTree
}
func NewBtreeMap() *BtreeMap {
return &BtreeMap{
tree: btree.New(32),
}
}
func (cm *BtreeMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
found := cm.tree.ReplaceOrInsert(NeedleValue{key, offset, size})
if found != nil {
old := found.(NeedleValue)
return old.Offset, old.Size
}
return
}
func (cm *BtreeMap) Delete(key NeedleId) (oldSize uint32) {
found := cm.tree.Delete(NeedleValue{key, Offset{}, 0})
if found != nil {
old := found.(NeedleValue)
return old.Size
}
return
}
func (cm *BtreeMap) Get(key NeedleId) (*NeedleValue, bool) {
found := cm.tree.Get(NeedleValue{key, Offset{}, 0})
if found != nil {
old := found.(NeedleValue)
return &old, true
}
return nil, false
}
// Visit visits all entries or stop if any error when visiting
func (cm *BtreeMap) Visit(visit func(NeedleValue) error) (ret error) {
cm.tree.Ascend(func(item btree.Item) bool {
needle := item.(NeedleValue)
ret = visit(needle)
return ret == nil
})
return ret
}

288
weed/storage/needle/compact_map.go
View File

@@ -1,288 +0,0 @@
package needle
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"sort"
"sync"
)
const (
batch = 100000
)
type SectionalNeedleId uint32
const SectionalNeedleIdLimit = 1<<32 - 1
type SectionalNeedleValue struct {
Key SectionalNeedleId
OffsetLower OffsetLower `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
Size uint32 `comment:"Size of the data portion"`
}
type SectionalNeedleValueExtra struct {
OffsetHigher OffsetHigher
}
type CompactSection struct {
sync.RWMutex
values []SectionalNeedleValue
valuesExtra []SectionalNeedleValueExtra
overflow Overflow
overflowExtra OverflowExtra
start NeedleId
end NeedleId
counter int
}
type Overflow []SectionalNeedleValue
type OverflowExtra []SectionalNeedleValueExtra
func NewCompactSection(start NeedleId) *CompactSection {
return &CompactSection{
values: make([]SectionalNeedleValue, batch),
valuesExtra: make([]SectionalNeedleValueExtra, batch),
overflow: Overflow(make([]SectionalNeedleValue, 0)),
overflowExtra: OverflowExtra(make([]SectionalNeedleValueExtra, 0)),
start: start,
}
}
//return old entry size
func (cs *CompactSection) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
cs.Lock()
if key > cs.end {
cs.end = key
}
skey := SectionalNeedleId(key - cs.start)
if i := cs.binarySearchValues(skey); i >= 0 {
oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size
//println("key", key, "old size", ret)
cs.valuesExtra[i].OffsetHigher, cs.values[i].OffsetLower, cs.values[i].Size = offset.OffsetHigher, offset.OffsetLower, size
} else {
needOverflow := cs.counter >= batch
needOverflow = needOverflow || cs.counter > 0 && cs.values[cs.counter-1].Key > skey
if needOverflow {
//println("start", cs.start, "counter", cs.counter, "key", key)
if oldValueExtra, oldValue, found := cs.findOverflowEntry(skey); found {
oldOffset.OffsetHigher, oldOffset.OffsetLower, oldSize = oldValueExtra.OffsetHigher, oldValue.OffsetLower, oldValue.Size
}
cs.setOverflowEntry(skey, offset, size)
} else {
p := &cs.values[cs.counter]
p.Key, cs.valuesExtra[cs.counter].OffsetHigher, p.OffsetLower, p.Size = skey, offset.OffsetHigher, offset.OffsetLower, size
//println("added index", cs.counter, "key", key, cs.values[cs.counter].Key)
cs.counter++
}
}
cs.Unlock()
return
}
func (cs *CompactSection) setOverflowEntry(skey SectionalNeedleId, offset Offset, size uint32) {
needleValue := SectionalNeedleValue{Key: skey, OffsetLower: offset.OffsetLower, Size: size}
needleValueExtra := SectionalNeedleValueExtra{OffsetHigher: OffsetHigher{}}
insertCandidate := sort.Search(len(cs.overflow), func(i int) bool {
return cs.overflow[i].Key >= needleValue.Key
})
if insertCandidate != len(cs.overflow) && cs.overflow[insertCandidate].Key == needleValue.Key {
cs.overflow[insertCandidate] = needleValue
} else {
cs.overflow = append(cs.overflow, needleValue)
cs.overflowExtra = append(cs.overflowExtra, needleValueExtra)
for i := len(cs.overflow) - 1; i > insertCandidate; i-- {
cs.overflow[i] = cs.overflow[i-1]
cs.overflowExtra[i] = cs.overflowExtra[i-1]
}
cs.overflow[insertCandidate] = needleValue
}
}
func (cs *CompactSection) findOverflowEntry(key SectionalNeedleId) (nve SectionalNeedleValueExtra, nv SectionalNeedleValue, found bool) {
foundCandidate := sort.Search(len(cs.overflow), func(i int) bool {
return cs.overflow[i].Key >= key
})
if foundCandidate != len(cs.overflow) && cs.overflow[foundCandidate].Key == key {
return cs.overflowExtra[foundCandidate], cs.overflow[foundCandidate], true
}
return nve, nv, false
}
func (cs *CompactSection) deleteOverflowEntry(key SectionalNeedleId) {
length := len(cs.overflow)
deleteCandidate := sort.Search(length, func(i int) bool {
return cs.overflow[i].Key >= key
})
if deleteCandidate != length && cs.overflow[deleteCandidate].Key == key {
for i := deleteCandidate; i < length-1; i++ {
cs.overflow[i] = cs.overflow[i+1]
cs.overflowExtra[i] = cs.overflowExtra[i+1]
}
cs.overflow = cs.overflow[0 : length-1]
cs.overflowExtra = cs.overflowExtra[0 : length-1]
}
}
//return old entry size
func (cs *CompactSection) Delete(key NeedleId) uint32 {
skey := SectionalNeedleId(key - cs.start)
cs.Lock()
ret := uint32(0)
if i := cs.binarySearchValues(skey); i >= 0 {
if cs.values[i].Size > 0 && cs.values[i].Size != TombstoneFileSize {
ret = cs.values[i].Size
cs.values[i].Size = TombstoneFileSize
}
}
if _, v, found := cs.findOverflowEntry(skey); found {
cs.deleteOverflowEntry(skey)
ret = v.Size
}
cs.Unlock()
return ret
}
func (cs *CompactSection) Get(key NeedleId) (*NeedleValue, bool) {
cs.RLock()
skey := SectionalNeedleId(key - cs.start)
if ve, v, ok := cs.findOverflowEntry(skey); ok {
cs.RUnlock()
nv := toNeedleValue(ve, v, cs)
return &nv, true
}
if i := cs.binarySearchValues(skey); i >= 0 {
cs.RUnlock()
nv := toNeedleValue(cs.valuesExtra[i], cs.values[i], cs)
return &nv, true
}
cs.RUnlock()
return nil, false
}
func (cs *CompactSection) binarySearchValues(key SectionalNeedleId) int {
x := sort.Search(cs.counter, func(i int) bool {
return cs.values[i].Key >= key
})
if x == cs.counter {
return -1
}
if cs.values[x].Key > key {
return -2
}
return x
}
//This map assumes mostly inserting increasing keys
//This map assumes mostly inserting increasing keys
type CompactMap struct {
list []*CompactSection
}
func NewCompactMap() *CompactMap {
return &CompactMap{}
}
func (cm *CompactMap) Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32) {
x := cm.binarySearchCompactSection(key)
if x < 0 || (key-cm.list[x].start) > SectionalNeedleIdLimit {
// println(x, "adding to existing", len(cm.list), "sections, starting", key)
cs := NewCompactSection(key)
cm.list = append(cm.list, cs)
x = len(cm.list) - 1
//keep compact section sorted by start
for x >= 0 {
if x > 0 && cm.list[x-1].start > key {
cm.list[x] = cm.list[x-1]
// println("shift", x, "start", cs.start, "to", x-1)
x = x - 1
} else {
cm.list[x] = cs
// println("cs", x, "start", cs.start)
break
}
}
}
// println(key, "set to section[", x, "].start", cm.list[x].start)
return cm.list[x].Set(key, offset, size)
}
func (cm *CompactMap) Delete(key NeedleId) uint32 {
x := cm.binarySearchCompactSection(key)
if x < 0 {
return uint32(0)
}
return cm.list[x].Delete(key)
}
func (cm *CompactMap) Get(key NeedleId) (*NeedleValue, bool) {
x := cm.binarySearchCompactSection(key)
if x < 0 {
return nil, false
}
return cm.list[x].Get(key)
}
func (cm *CompactMap) binarySearchCompactSection(key NeedleId) int {
l, h := 0, len(cm.list)-1
if h < 0 {
return -5
}
if cm.list[h].start <= key {
if cm.list[h].counter < batch || key <= cm.list[h].end {
return h
}
return -4
}
for l <= h {
m := (l + h) / 2
if key < cm.list[m].start {
h = m - 1
} else { // cm.list[m].start <= key
if cm.list[m+1].start <= key {
l = m + 1
} else {
return m
}
}
}
return -3
}
// Visit visits all entries or stop if any error when visiting
func (cm *CompactMap) Visit(visit func(NeedleValue) error) error {
for _, cs := range cm.list {
cs.RLock()
for i, v := range cs.overflow {
if err := visit(toNeedleValue(cs.overflowExtra[i], v, cs)); err != nil {
cs.RUnlock()
return err
}
}
for i, v := range cs.values {
if i >= cs.counter {
break
}
if _, _, found := cs.findOverflowEntry(v.Key); !found {
if err := visit(toNeedleValue(cs.valuesExtra[i], v, cs)); err != nil {
cs.RUnlock()
return err
}
}
}
cs.RUnlock()
}
return nil
}
func toNeedleValue(snve SectionalNeedleValueExtra, snv SectionalNeedleValue, cs *CompactSection) NeedleValue {
offset := Offset{
OffsetHigher: snve.OffsetHigher,
OffsetLower: snv.OffsetLower,
}
return NeedleValue{Key: NeedleId(snv.Key) + cs.start, Offset: offset, Size: snv.Size}
}
func (nv NeedleValue) toSectionalNeedleValue(cs *CompactSection) (SectionalNeedleValue, SectionalNeedleValueExtra) {
return SectionalNeedleValue{
SectionalNeedleId(nv.Key - cs.start),
nv.Offset.OffsetLower,
nv.Size,
}, SectionalNeedleValueExtra{
nv.Offset.OffsetHigher,
}
}

93
weed/storage/needle/compact_map_perf_test.go
View File

@@ -1,93 +0,0 @@
package needle
import (
"fmt"
"log"
"os"
"runtime"
"testing"
"time"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
/*
To see the memory usage:
go test -run TestMemoryUsage
The TotalAlloc section shows the memory increase for each iteration.
go test -run TestMemoryUsage -memprofile=mem.out
go tool pprof --alloc_space needle.test mem.out
*/
func TestMemoryUsage(t *testing.T) {
var maps []*CompactMap
totalRowCount := uint64(0)
startTime := time.Now()
for i := 0; i < 10; i++ {
indexFile, ie := os.OpenFile("../../../test/sample.idx", os.O_RDWR|os.O_RDONLY, 0644)
if ie != nil {
log.Fatalln(ie)
}
m, rowCount := loadNewNeedleMap(indexFile)
maps = append(maps, m)
totalRowCount += rowCount
indexFile.Close()
PrintMemUsage(totalRowCount)
now := time.Now()
fmt.Printf("\tTaken = %v\n", now.Sub(startTime))
startTime = now
}
}
func loadNewNeedleMap(file *os.File) (*CompactMap, uint64) {
m := NewCompactMap()
bytes := make([]byte, NeedleEntrySize)
rowCount := uint64(0)
count, e := file.Read(bytes)
for count > 0 && e == nil {
for i := 0; i < count; i += NeedleEntrySize {
rowCount++
key := BytesToNeedleId(bytes[i : i+NeedleIdSize])
offset := BytesToOffset(bytes[i+NeedleIdSize : i+NeedleIdSize+OffsetSize])
size := util.BytesToUint32(bytes[i+NeedleIdSize+OffsetSize : i+NeedleIdSize+OffsetSize+SizeSize])
if !offset.IsZero() {
m.Set(NeedleId(key), offset, size)
} else {
m.Delete(key)
}
}
count, e = file.Read(bytes)
}
return m, rowCount
}
func PrintMemUsage(totalRowCount uint64) {
runtime.GC()
var m runtime.MemStats
runtime.ReadMemStats(&m)
// For info on each, see: https://golang.org/pkg/runtime/#MemStats
fmt.Printf("Each %.2f Bytes", float64(m.TotalAlloc)/float64(totalRowCount))
fmt.Printf("\tAlloc = %v MiB", bToMb(m.Alloc))
fmt.Printf("\tTotalAlloc = %v MiB", bToMb(m.TotalAlloc))
fmt.Printf("\tSys = %v MiB", bToMb(m.Sys))
fmt.Printf("\tNumGC = %v", m.NumGC)
}
func bToMb(b uint64) uint64 {
return b / 1024 / 1024
}

159
weed/storage/needle/compact_map_test.go
View File

@@ -1,159 +0,0 @@
package needle
import (
"fmt"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"testing"
)
func TestOverflow2(t *testing.T) {
m := NewCompactMap()
m.Set(NeedleId(150088), ToOffset(8), 3000073)
m.Set(NeedleId(150073), ToOffset(8), 3000073)
m.Set(NeedleId(150089), ToOffset(8), 3000073)
m.Set(NeedleId(150076), ToOffset(8), 3000073)
m.Set(NeedleId(150124), ToOffset(8), 3000073)
m.Set(NeedleId(150137), ToOffset(8), 3000073)
m.Set(NeedleId(150147), ToOffset(8), 3000073)
m.Set(NeedleId(150145), ToOffset(8), 3000073)
m.Set(NeedleId(150158), ToOffset(8), 3000073)
m.Set(NeedleId(150162), ToOffset(8), 3000073)
m.Visit(func(value NeedleValue) error {
println("needle key:", value.Key)
return nil
})
}
func TestIssue52(t *testing.T) {
m := NewCompactMap()
m.Set(NeedleId(10002), ToOffset(10002), 10002)
if element, ok := m.Get(NeedleId(10002)); ok {
fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
}
m.Set(NeedleId(10001), ToOffset(10001), 10001)
if element, ok := m.Get(NeedleId(10002)); ok {
fmt.Printf("key %d ok %v %d, %v, %d\n", 10002, ok, element.Key, element.Offset, element.Size)
} else {
t.Fatal("key 10002 missing after setting 10001")
}
}
func TestCompactMap(t *testing.T) {
m := NewCompactMap()
for i := uint32(0); i < 100*batch; i += 2 {
m.Set(NeedleId(i), ToOffset(int64(i)), i)
}
for i := uint32(0); i < 100*batch; i += 37 {
m.Delete(NeedleId(i))
}
for i := uint32(0); i < 10*batch; i += 3 {
m.Set(NeedleId(i), ToOffset(int64(i+11)), i+5)
}
// for i := uint32(0); i < 100; i++ {
// if v := m.Get(Key(i)); v != nil {
// glog.V(4).Infoln(i, "=", v.Key, v.Offset, v.Size)
// }
// }
for i := uint32(0); i < 10*batch; i++ {
v, ok := m.Get(NeedleId(i))
if i%3 == 0 {
if !ok {
t.Fatal("key", i, "missing!")
}
if v.Size != i+5 {
t.Fatal("key", i, "size", v.Size)
}
} else if i%37 == 0 {
if ok && v.Size != TombstoneFileSize {
t.Fatal("key", i, "should have been deleted needle value", v)
}
} else if i%2 == 0 {
if v.Size != i {
t.Fatal("key", i, "size", v.Size)
}
}
}
for i := uint32(10 * batch); i < 100*batch; i++ {
v, ok := m.Get(NeedleId(i))
if i%37 == 0 {
if ok && v.Size != TombstoneFileSize {
t.Fatal("key", i, "should have been deleted needle value", v)
}
} else if i%2 == 0 {
if v == nil {
t.Fatal("key", i, "missing")
}
if v.Size != i {
t.Fatal("key", i, "size", v.Size)
}
}
}
}
func TestOverflow(t *testing.T) {
cs := NewCompactSection(1)
cs.setOverflowEntry(1, ToOffset(12), 12)
cs.setOverflowEntry(2, ToOffset(12), 12)
cs.setOverflowEntry(3, ToOffset(12), 12)
cs.setOverflowEntry(4, ToOffset(12), 12)
cs.setOverflowEntry(5, ToOffset(12), 12)
if cs.overflow[2].Key != 3 {
t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
}
cs.setOverflowEntry(3, ToOffset(24), 24)
if cs.overflow[2].Key != 3 {
t.Fatalf("expecting o[2] has key 3: %+v", cs.overflow[2].Key)
}
if cs.overflow[2].Size != 24 {
t.Fatalf("expecting o[2] has size 24: %+v", cs.overflow[2].Size)
}
cs.deleteOverflowEntry(4)
if len(cs.overflow) != 4 {
t.Fatalf("expecting 4 entries now: %+v", cs.overflow)
}
_, x, _ := cs.findOverflowEntry(5)
if x.Key != 5 {
t.Fatalf("expecting entry 5 now: %+v", x)
}
for i, x := range cs.overflow {
println("overflow[", i, "]:", x.Key)
}
println()
cs.deleteOverflowEntry(1)
for i, x := range cs.overflow {
println("overflow[", i, "]:", x.Key)
}
println()
cs.setOverflowEntry(4, ToOffset(44), 44)
for i, x := range cs.overflow {
println("overflow[", i, "]:", x.Key)
}
println()
cs.setOverflowEntry(1, ToOffset(11), 11)
for i, x := range cs.overflow {
println("overflow[", i, "]:", x.Key)
}
println()
}

41
weed/storage/needle/crc.go Normal file
View File

@@ -0,0 +1,41 @@
package needle
import (
"crypto/md5"
"fmt"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/klauspost/crc32"
)
var table = crc32.MakeTable(crc32.Castagnoli)
type CRC uint32
func NewCRC(b []byte) CRC {
return CRC(0).Update(b)
}
func (c CRC) Update(b []byte) CRC {
return CRC(crc32.Update(uint32(c), table, b))
}
func (c CRC) Value() uint32 {
return uint32(c>>15|c<<17) + 0xa282ead8
}
func (n *Needle) Etag() string {
bits := make([]byte, 4)
util.Uint32toBytes(bits, uint32(n.Checksum))
return fmt.Sprintf("%x", bits)
}
func (n *Needle) MD5() string {
hash := md5.New()
hash.Write(n.Data)
return fmt.Sprintf("%x", hash.Sum(nil))
}

35
weed/storage/needle/file_id.go Normal file
View File

@@ -0,0 +1,35 @@
package needle
import (
"encoding/hex"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
type FileId struct {
VolumeId VolumeId
Key NeedleId
Cookie Cookie
}
func NewFileIdFromNeedle(VolumeId VolumeId, n *Needle) *FileId {
return &FileId{VolumeId: VolumeId, Key: n.Id, Cookie: n.Cookie}
}
func NewFileId(VolumeId VolumeId, key uint64, cookie uint32) *FileId {
return &FileId{VolumeId: VolumeId, Key: Uint64ToNeedleId(key), Cookie: Uint32ToCookie(cookie)}
}
func (n *FileId) String() string {
return n.VolumeId.String() + "," + formatNeedleIdCookie(n.Key, n.Cookie)
}
func formatNeedleIdCookie(key NeedleId, cookie Cookie) string {
bytes := make([]byte, NeedleIdSize+CookieSize)
NeedleIdToBytes(bytes[0:NeedleIdSize], key)
CookieToBytes(bytes[NeedleIdSize:NeedleIdSize+CookieSize], cookie)
nonzero_index := 0
for ; bytes[nonzero_index] == 0; nonzero_index++ {
}
return hex.EncodeToString(bytes[nonzero_index:])
}

191
weed/storage/needle/needle.go Normal file
View File

@@ -0,0 +1,191 @@
package needle
import (
"encoding/json"
"fmt"
"net/http"
"strconv"
"strings"
"time"
"io/ioutil"
"github.com/chrislusf/seaweedfs/weed/images"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
const (
NeedleChecksumSize = 4
PairNamePrefix = "Seaweed-"
)
/*
* A Needle means a uploaded and stored file.
* Needle file size is limited to 4GB for now.
*/
type Needle struct {
Cookie Cookie `comment:"random number to mitigate brute force lookups"`
Id NeedleId `comment:"needle id"`
Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
DataSize uint32 `comment:"Data size"` //version2
Data []byte `comment:"The actual file data"`
Flags byte `comment:"boolean flags"` //version2
NameSize uint8 //version2
Name []byte `comment:"maximum 256 characters"` //version2
MimeSize uint8 //version2
Mime []byte `comment:"maximum 256 characters"` //version2
PairsSize uint16 //version2
Pairs []byte `comment:"additional name value pairs, json format, maximum 64kB"`
LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes to disk
Ttl *TTL
Checksum CRC `comment:"CRC32 to check integrity"`
AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
Padding []byte `comment:"Aligned to 8 bytes"`
}
func (n *Needle) String() (str string) {
str = fmt.Sprintf("%s Size:%d, DataSize:%d, Name:%s, Mime:%s", formatNeedleIdCookie(n.Id, n.Cookie), n.Size, n.DataSize, n.Name, n.Mime)
return
}
func ParseUpload(r *http.Request) (
fileName string, data []byte, mimeType string, pairMap map[string]string, isGzipped bool, originalDataSize int,
modifiedTime uint64, ttl *TTL, isChunkedFile bool, e error) {
pairMap = make(map[string]string)
for k, v := range r.Header {
if len(v) > 0 && strings.HasPrefix(k, PairNamePrefix) {
pairMap[k] = v[0]
}
}
if r.Method == "POST" {
fileName, data, mimeType, isGzipped, originalDataSize, isChunkedFile, e = parseMultipart(r)
} else {
isGzipped = false
mimeType = r.Header.Get("Content-Type")
fileName = ""
data, e = ioutil.ReadAll(r.Body)
originalDataSize = len(data)
}
if e != nil {
return
}
modifiedTime, _ = strconv.ParseUint(r.FormValue("ts"), 10, 64)
ttl, _ = ReadTTL(r.FormValue("ttl"))
return
}
func CreateNeedleFromRequest(r *http.Request, fixJpgOrientation bool) (n *Needle, originalSize int, e error) {
var pairMap map[string]string
fname, mimeType, isGzipped, isChunkedFile := "", "", false, false
n = new(Needle)
fname, n.Data, mimeType, pairMap, isGzipped, originalSize, n.LastModified, n.Ttl, isChunkedFile, e = ParseUpload(r)
if e != nil {
return
}
if len(fname) < 256 {
n.Name = []byte(fname)
n.SetHasName()
}
if len(mimeType) < 256 {
n.Mime = []byte(mimeType)
n.SetHasMime()
}
if len(pairMap) != 0 {
trimmedPairMap := make(map[string]string)
for k, v := range pairMap {
trimmedPairMap[k[len(PairNamePrefix):]] = v
}
pairs, _ := json.Marshal(trimmedPairMap)
if len(pairs) < 65536 {
n.Pairs = pairs
n.PairsSize = uint16(len(pairs))
n.SetHasPairs()
}
}
if isGzipped {
n.SetGzipped()
}
if n.LastModified == 0 {
n.LastModified = uint64(time.Now().Unix())
}
n.SetHasLastModifiedDate()
if n.Ttl != EMPTY_TTL {
n.SetHasTtl()
}
if isChunkedFile {
n.SetIsChunkManifest()
}
if fixJpgOrientation {
loweredName := strings.ToLower(fname)
if mimeType == "image/jpeg" || strings.HasSuffix(loweredName, ".jpg") || strings.HasSuffix(loweredName, ".jpeg") {
n.Data = images.FixJpgOrientation(n.Data)
}
}
n.Checksum = NewCRC(n.Data)
commaSep := strings.LastIndex(r.URL.Path, ",")
dotSep := strings.LastIndex(r.URL.Path, ".")
fid := r.URL.Path[commaSep+1:]
if dotSep > 0 {
fid = r.URL.Path[commaSep+1 : dotSep]
}
e = n.ParsePath(fid)
return
}
func (n *Needle) ParsePath(fid string) (err error) {
length := len(fid)
if length <= CookieSize*2 {
return fmt.Errorf("Invalid fid: %s", fid)
}
delta := ""
deltaIndex := strings.LastIndex(fid, "_")
if deltaIndex > 0 {
fid, delta = fid[0:deltaIndex], fid[deltaIndex+1:]
}
n.Id, n.Cookie, err = ParseNeedleIdCookie(fid)
if err != nil {
return err
}
if delta != "" {
if d, e := strconv.ParseUint(delta, 10, 64); e == nil {
n.Id += NeedleId(d)
} else {
return e
}
}
return err
}
func ParseNeedleIdCookie(key_hash_string string) (NeedleId, Cookie, error) {
if len(key_hash_string) <= CookieSize*2 {
return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too short.")
}
if len(key_hash_string) > (NeedleIdSize+CookieSize)*2 {
return NeedleIdEmpty, 0, fmt.Errorf("KeyHash is too long.")
}
split := len(key_hash_string) - CookieSize*2
needleId, err := ParseNeedleId(key_hash_string[:split])
if err != nil {
return NeedleIdEmpty, 0, fmt.Errorf("Parse needleId error: %v", err)
}
cookie, err := ParseCookie(key_hash_string[split:])
if err != nil {
return NeedleIdEmpty, 0, fmt.Errorf("Parse cookie error: %v", err)
}
return needleId, cookie, nil
}
func (n *Needle) LastModifiedString() string {
return time.Unix(int64(n.LastModified), 0).Format("2006-01-02T15:04:05")
}

109
weed/storage/needle/needle_parse_multipart.go Normal file
View File

@@ -0,0 +1,109 @@
package needle
import (
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/util"
"io"
"io/ioutil"
"mime"
"net/http"
"path"
"strconv"
"strings"
)
func parseMultipart(r *http.Request) (
fileName string, data []byte, mimeType string, isGzipped bool, originalDataSize int, isChunkedFile bool, e error) {
defer func() {
if e != nil && r.Body != nil {
io.Copy(ioutil.Discard, r.Body)
r.Body.Close()
}
}()
form, fe := r.MultipartReader()
if fe != nil {
glog.V(0).Infoln("MultipartReader [ERROR]", fe)
e = fe
return
}
//first multi-part item
part, fe := form.NextPart()
if fe != nil {
glog.V(0).Infoln("Reading Multi part [ERROR]", fe)
e = fe
return
}
fileName = part.FileName()
if fileName != "" {
fileName = path.Base(fileName)
}
data, e = ioutil.ReadAll(part)
if e != nil {
glog.V(0).Infoln("Reading Content [ERROR]", e)
return
}
//if the filename is empty string, do a search on the other multi-part items
for fileName == "" {
part2, fe := form.NextPart()
if fe != nil {
break // no more or on error, just safely break
}
fName := part2.FileName()
//found the first <file type> multi-part has filename
if fName != "" {
data2, fe2 := ioutil.ReadAll(part2)
if fe2 != nil {
glog.V(0).Infoln("Reading Content [ERROR]", fe2)
e = fe2
return
}
//update
data = data2
fileName = path.Base(fName)
break
}
}
originalDataSize = len(data)
isChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
if !isChunkedFile {
dotIndex := strings.LastIndex(fileName, ".")
ext, mtype := "", ""
if dotIndex > 0 {
ext = strings.ToLower(fileName[dotIndex:])
mtype = mime.TypeByExtension(ext)
}
contentType := part.Header.Get("Content-Type")
if contentType != "" && mtype != contentType {
mimeType = contentType //only return mime type if not deductable
mtype = contentType
}
if part.Header.Get("Content-Encoding") == "gzip" {
if unzipped, e := util.UnGzipData(data); e == nil {
originalDataSize = len(unzipped)
}
isGzipped = true
} else if util.IsGzippable(ext, mtype, data) {
if compressedData, err := util.GzipData(data); err == nil {
if len(data) > len(compressedData) {
data = compressedData
isGzipped = true
}
}
}
}
return
}

391
weed/storage/needle/needle_read_write.go Normal file
View File

@@ -0,0 +1,391 @@
package needle
import (
"errors"
"fmt"
"io"
"os"
"math"
"github.com/chrislusf/seaweedfs/weed/glog"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
const (
FlagGzip = 0x01
FlagHasName = 0x02
FlagHasMime = 0x04
FlagHasLastModifiedDate = 0x08
FlagHasTtl = 0x10
FlagHasPairs = 0x20
FlagIsChunkManifest = 0x80
LastModifiedBytesLength = 5
TtlBytesLength = 2
)
func (n *Needle) DiskSize(version Version) int64 {
return getActualSize(n.Size, version)
}
func (n *Needle) Append(w *os.File, version Version) (offset uint64, size uint32, actualSize int64, err error) {
if end, e := w.Seek(0, io.SeekEnd); e == nil {
defer func(w *os.File, off int64) {
if err != nil {
if te := w.Truncate(end); te != nil {
glog.V(0).Infof("Failed to truncate %s back to %d with error: %v", w.Name(), end, te)
}
}
}(w, end)
offset = uint64(end)
} else {
err = fmt.Errorf("Cannot Read Current Volume Position: %v", e)
return
}
switch version {
case Version1:
header := make([]byte, NeedleEntrySize)
CookieToBytes(header[0:CookieSize], n.Cookie)
NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
n.Size = uint32(len(n.Data))
size = n.Size
util.Uint32toBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
if _, err = w.Write(header); err != nil {
return
}
if _, err = w.Write(n.Data); err != nil {
return
}
actualSize = NeedleEntrySize + int64(n.Size)
padding := PaddingLength(n.Size, version)
util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
_, err = w.Write(header[0 : NeedleChecksumSize+padding])
return
case Version2, Version3:
header := make([]byte, NeedleEntrySize+TimestampSize) // adding timestamp to reuse it and avoid extra allocation
CookieToBytes(header[0:CookieSize], n.Cookie)
NeedleIdToBytes(header[CookieSize:CookieSize+NeedleIdSize], n.Id)
if len(n.Name) >= math.MaxUint8 {
n.NameSize = math.MaxUint8
} else {
n.NameSize = uint8(len(n.Name))
}
n.DataSize, n.MimeSize = uint32(len(n.Data)), uint8(len(n.Mime))
if n.DataSize > 0 {
n.Size = 4 + n.DataSize + 1
if n.HasName() {
n.Size = n.Size + 1 + uint32(n.NameSize)
}
if n.HasMime() {
n.Size = n.Size + 1 + uint32(n.MimeSize)
}
if n.HasLastModifiedDate() {
n.Size = n.Size + LastModifiedBytesLength
}
if n.HasTtl() {
n.Size = n.Size + TtlBytesLength
}
if n.HasPairs() {
n.Size += 2 + uint32(n.PairsSize)
}
} else {
n.Size = 0
}
size = n.DataSize
util.Uint32toBytes(header[CookieSize+NeedleIdSize:CookieSize+NeedleIdSize+SizeSize], n.Size)
if _, err = w.Write(header[0:NeedleEntrySize]); err != nil {
return
}
if n.DataSize > 0 {
util.Uint32toBytes(header[0:4], n.DataSize)
if _, err = w.Write(header[0:4]); err != nil {
return
}
if _, err = w.Write(n.Data); err != nil {
return
}
util.Uint8toBytes(header[0:1], n.Flags)
if _, err = w.Write(header[0:1]); err != nil {
return
}
if n.HasName() {
util.Uint8toBytes(header[0:1], n.NameSize)
if _, err = w.Write(header[0:1]); err != nil {
return
}
if _, err = w.Write(n.Name[:n.NameSize]); err != nil {
return
}
}
if n.HasMime() {
util.Uint8toBytes(header[0:1], n.MimeSize)
if _, err = w.Write(header[0:1]); err != nil {
return
}
if _, err = w.Write(n.Mime); err != nil {
return
}
}
if n.HasLastModifiedDate() {
util.Uint64toBytes(header[0:8], n.LastModified)
if _, err = w.Write(header[8-LastModifiedBytesLength : 8]); err != nil {
return
}
}
if n.HasTtl() && n.Ttl != nil {
n.Ttl.ToBytes(header[0:TtlBytesLength])
if _, err = w.Write(header[0:TtlBytesLength]); err != nil {
return
}
}
if n.HasPairs() {
util.Uint16toBytes(header[0:2], n.PairsSize)
if _, err = w.Write(header[0:2]); err != nil {
return
}
if _, err = w.Write(n.Pairs); err != nil {
return
}
}
}
padding := PaddingLength(n.Size, version)
util.Uint32toBytes(header[0:NeedleChecksumSize], n.Checksum.Value())
if version == Version2 {
_, err = w.Write(header[0 : NeedleChecksumSize+padding])
} else {
// version3
util.Uint64toBytes(header[NeedleChecksumSize:NeedleChecksumSize+TimestampSize], n.AppendAtNs)
_, err = w.Write(header[0 : NeedleChecksumSize+TimestampSize+padding])
}
return offset, n.DataSize, getActualSize(n.Size, version), err
}
return 0, 0, 0, fmt.Errorf("Unsupported Version! (%d)", version)
}
func ReadNeedleBlob(r *os.File, offset int64, size uint32, version Version) (dataSlice []byte, err error) {
dataSlice = make([]byte, int(getActualSize(size, version)))
_, err = r.ReadAt(dataSlice, offset)
return dataSlice, err
}
func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version) (err error) {
bytes, err := ReadNeedleBlob(r, offset, size, version)
if err != nil {
return err
}
n.ParseNeedleHeader(bytes)
if n.Size != size {
return fmt.Errorf("File Entry Not Found. offset %d, Needle id %d expected size %d Memory %d", offset, n.Id, n.Size, size)
}
switch version {
case Version1:
n.Data = bytes[NeedleEntrySize : NeedleEntrySize+size]
case Version2, Version3:
err = n.readNeedleDataVersion2(bytes[NeedleEntrySize : NeedleEntrySize+int(n.Size)])
}
if size == 0 || err != nil {
return err
}
checksum := util.BytesToUint32(bytes[NeedleEntrySize+size : NeedleEntrySize+size+NeedleChecksumSize])
newChecksum := NewCRC(n.Data)
if checksum != newChecksum.Value() {
return errors.New("CRC error! Data On Disk Corrupted")
}
n.Checksum = newChecksum
if version == Version3 {
tsOffset := NeedleEntrySize + size + NeedleChecksumSize
n.AppendAtNs = util.BytesToUint64(bytes[tsOffset : tsOffset+TimestampSize])
}
return nil
}
func (n *Needle) ParseNeedleHeader(bytes []byte) {
n.Cookie = BytesToCookie(bytes[0:CookieSize])
n.Id = BytesToNeedleId(bytes[CookieSize : CookieSize+NeedleIdSize])
n.Size = util.BytesToUint32(bytes[CookieSize+NeedleIdSize : NeedleEntrySize])
}
func (n *Needle) readNeedleDataVersion2(bytes []byte) (err error) {
index, lenBytes := 0, len(bytes)
if index < lenBytes {
n.DataSize = util.BytesToUint32(bytes[index : index+4])
index = index + 4
if int(n.DataSize)+index > lenBytes {
return fmt.Errorf("index out of range %d", 1)
}
n.Data = bytes[index : index+int(n.DataSize)]
index = index + int(n.DataSize)
n.Flags = bytes[index]
index = index + 1
}
if index < lenBytes && n.HasName() {
n.NameSize = uint8(bytes[index])
index = index + 1
if int(n.NameSize)+index > lenBytes {
return fmt.Errorf("index out of range %d", 2)
}
n.Name = bytes[index : index+int(n.NameSize)]
index = index + int(n.NameSize)
}
if index < lenBytes && n.HasMime() {
n.MimeSize = uint8(bytes[index])
index = index + 1
if int(n.MimeSize)+index > lenBytes {
return fmt.Errorf("index out of range %d", 3)
}
n.Mime = bytes[index : index+int(n.MimeSize)]
index = index + int(n.MimeSize)
}
if index < lenBytes && n.HasLastModifiedDate() {
if LastModifiedBytesLength+index > lenBytes {
return fmt.Errorf("index out of range %d", 4)
}
n.LastModified = util.BytesToUint64(bytes[index : index+LastModifiedBytesLength])
index = index + LastModifiedBytesLength
}
if index < lenBytes && n.HasTtl() {
if TtlBytesLength+index > lenBytes {
return fmt.Errorf("index out of range %d", 5)
}
n.Ttl = LoadTTLFromBytes(bytes[index : index+TtlBytesLength])
index = index + TtlBytesLength
}
if index < lenBytes && n.HasPairs() {
if 2+index > lenBytes {
return fmt.Errorf("index out of range %d", 6)
}
n.PairsSize = util.BytesToUint16(bytes[index : index+2])
index += 2
if int(n.PairsSize)+index > lenBytes {
return fmt.Errorf("index out of range %d", 7)
}
end := index + int(n.PairsSize)
n.Pairs = bytes[index:end]
index = end
}
return nil
}
func ReadNeedleHeader(r *os.File, version Version, offset int64) (n *Needle, bytes []byte, bodyLength int64, err error) {
n = new(Needle)
if version == Version1 || version == Version2 || version == Version3 {
bytes = make([]byte, NeedleEntrySize)
var count int
count, err = r.ReadAt(bytes, offset)
if count <= 0 || err != nil {
return nil, bytes, 0, err
}
n.ParseNeedleHeader(bytes)
bodyLength = NeedleBodyLength(n.Size, version)
}
return
}
func PaddingLength(needleSize uint32, version Version) uint32 {
if version == Version3 {
// this is same value as version2, but just listed here for clarity
return NeedlePaddingSize - ((NeedleEntrySize + needleSize + NeedleChecksumSize + TimestampSize) % NeedlePaddingSize)
}
return NeedlePaddingSize - ((NeedleEntrySize + needleSize + NeedleChecksumSize) % NeedlePaddingSize)
}
func NeedleBodyLength(needleSize uint32, version Version) int64 {
if version == Version3 {
return int64(needleSize) + NeedleChecksumSize + TimestampSize + int64(PaddingLength(needleSize, version))
}
return int64(needleSize) + NeedleChecksumSize + int64(PaddingLength(needleSize, version))
}
//n should be a needle already read the header
//the input stream will read until next file entry
func (n *Needle) ReadNeedleBody(r *os.File, version Version, offset int64, bodyLength int64) (bytes []byte, err error) {
if bodyLength <= 0 {
return nil, nil
}
bytes = make([]byte, bodyLength)
if _, err = r.ReadAt(bytes, offset); err != nil {
return
}
err = n.ReadNeedleBodyBytes(bytes, version)
return
}
func (n *Needle) ReadNeedleBodyBytes(needleBody []byte, version Version) (err error) {
if len(needleBody) <= 0 {
return nil
}
switch version {
case Version1:
n.Data = needleBody[:n.Size]
n.Checksum = NewCRC(n.Data)
case Version2, Version3:
err = n.readNeedleDataVersion2(needleBody[0:n.Size])
n.Checksum = NewCRC(n.Data)
if version == Version3 {
tsOffset := n.Size + NeedleChecksumSize
n.AppendAtNs = util.BytesToUint64(needleBody[tsOffset : tsOffset+TimestampSize])
}
default:
err = fmt.Errorf("unsupported version %d!", version)
}
return
}
func (n *Needle) IsGzipped() bool {
return n.Flags&FlagGzip > 0
}
func (n *Needle) SetGzipped() {
n.Flags = n.Flags | FlagGzip
}
func (n *Needle) HasName() bool {
return n.Flags&FlagHasName > 0
}
func (n *Needle) SetHasName() {
n.Flags = n.Flags | FlagHasName
}
func (n *Needle) HasMime() bool {
return n.Flags&FlagHasMime > 0
}
func (n *Needle) SetHasMime() {
n.Flags = n.Flags | FlagHasMime
}
func (n *Needle) HasLastModifiedDate() bool {
return n.Flags&FlagHasLastModifiedDate > 0
}
func (n *Needle) SetHasLastModifiedDate() {
n.Flags = n.Flags | FlagHasLastModifiedDate
}
func (n *Needle) HasTtl() bool {
return n.Flags&FlagHasTtl > 0
}
func (n *Needle) SetHasTtl() {
n.Flags = n.Flags | FlagHasTtl
}
func (n *Needle) IsChunkedManifest() bool {
return n.Flags&FlagIsChunkManifest > 0
}
func (n *Needle) SetIsChunkManifest() {
n.Flags = n.Flags | FlagIsChunkManifest
}
func (n *Needle) HasPairs() bool {
return n.Flags&FlagHasPairs != 0
}
func (n *Needle) SetHasPairs() {
n.Flags = n.Flags | FlagHasPairs
}
func getActualSize(size uint32, version Version) int64 {
return NeedleEntrySize + NeedleBodyLength(size, version)
}

61
weed/storage/needle/needle_read_write_test.go Normal file
View File

@@ -0,0 +1,61 @@
package needle
import (
"io/ioutil"
"os"
"testing"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
func TestAppend(t *testing.T) {
n := &Needle{
Cookie: types.Cookie(123), // Cookie Cookie `comment:"random number to mitigate brute force lookups"`
Id: types.NeedleId(123), // Id NeedleId `comment:"needle id"`
Size: 8, // Size uint32 `comment:"sum of DataSize,Data,NameSize,Name,MimeSize,Mime"`
DataSize: 4, // DataSize uint32 `comment:"Data size"` //version2
Data: []byte("abcd"), // Data []byte `comment:"The actual file data"`
Flags: 0, // Flags byte `comment:"boolean flags"` //version2
NameSize: 0, // NameSize uint8 //version2
Name: nil, // Name []byte `comment:"maximum 256 characters"` //version2
MimeSize: 0, // MimeSize uint8 //version2
Mime: nil, // Mime []byte `comment:"maximum 256 characters"` //version2
PairsSize: 0, // PairsSize uint16 //version2
Pairs: nil, // Pairs []byte `comment:"additional name value pairs, json format, maximum 6
LastModified: 123, // LastModified uint64 //only store LastModifiedBytesLength bytes, which is 5 bytes
Ttl: nil, // Ttl *TTL
Checksum: 123, // Checksum CRC `comment:"CRC32 to check integrity"`
AppendAtNs: 123, // AppendAtNs uint64 `comment:"append timestamp in nano seconds"` //version3
Padding: nil, // Padding []byte `comment:"Aligned to 8 bytes"`
}
tempFile, err := ioutil.TempFile("", ".dat")
if err != nil {
t.Errorf("Fail TempFile. %v", err)
return
}
/*
uint8 : 0 to 255
uint16 : 0 to 65535
uint32 : 0 to 4294967295
uint64 : 0 to 18446744073709551615
int8 : -128 to 127
int16 : -32768 to 32767
int32 : -2147483648 to 2147483647
int64 : -9223372036854775808 to 9223372036854775807
*/
fileSize := int64(4294967295) + 10000
tempFile.Truncate(fileSize)
defer func() {
tempFile.Close()
os.Remove(tempFile.Name())
}()
offset, _, _, _ := n.Append(tempFile, CurrentVersion)
if offset != uint64(fileSize) {
t.Errorf("Fail to Append Needle.")
}
}

49
weed/storage/needle/needle_test.go Normal file
View File

@@ -0,0 +1,49 @@
package needle
import (
"testing"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
func TestParseKeyHash(t *testing.T) {
testcases := []struct {
KeyHash string
ID types.NeedleId
Cookie types.Cookie
Err bool
}{
// normal
{"4ed4c8116e41", 0x4ed4, 0xc8116e41, false},
// cookie with leading zeros
{"4ed401116e41", 0x4ed4, 0x01116e41, false},
// odd length
{"ed400116e41", 0xed4, 0x00116e41, false},
// uint
{"fed4c8114ed4c811f0116e41", 0xfed4c8114ed4c811, 0xf0116e41, false},
// err: too short
{"4ed4c811", 0, 0, true},
// err: too long
{"4ed4c8114ed4c8114ed4c8111", 0, 0, true},
// err: invalid character
{"helloworld", 0, 0, true},
}
for _, tc := range testcases {
if id, cookie, err := ParseNeedleIdCookie(tc.KeyHash); err != nil && !tc.Err {
t.Fatalf("Parse %s error: %v", tc.KeyHash, err)
} else if err == nil && tc.Err {
t.Fatalf("Parse %s expected error got nil", tc.KeyHash)
} else if id != tc.ID || cookie != tc.Cookie {
t.Fatalf("Parse %s wrong result. Expected: (%d, %d) got: (%d, %d)", tc.KeyHash, tc.ID, tc.Cookie, id, cookie)
}
}
}
func BenchmarkParseKeyHash(b *testing.B) {
b.ReportAllocs()
for i := 0; i < b.N; i++ {
ParseNeedleIdCookie("4ed44ed44ed44ed4c8116e41")
}
}

17
weed/storage/needle/needle_value.go
View File

@@ -1,17 +0,0 @@
package needle
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/google/btree"
)
type NeedleValue struct {
Key NeedleId
Offset Offset `comment:"Volume offset"` //since aligned to 8 bytes, range is 4G*8=32G
Size uint32 `comment:"Size of the data portion"`
}
func (this NeedleValue) Less(than btree.Item) bool {
that := than.(NeedleValue)
return this.Key < that.Key
}

12
weed/storage/needle/needle_value_map.go
View File

@@ -1,12 +0,0 @@
package needle
import (
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
type NeedleValueMap interface {
Set(key NeedleId, offset Offset, size uint32) (oldOffset Offset, oldSize uint32)
Delete(key NeedleId) uint32
Get(key NeedleId) (*NeedleValue, bool)
Visit(visit func(NeedleValue) error) error
}

18
weed/storage/needle/volume_id.go Normal file
View File

@@ -0,0 +1,18 @@
package needle
import (
"strconv"
)
type VolumeId uint32
func NewVolumeId(vid string) (VolumeId, error) {
volumeId, err := strconv.ParseUint(vid, 10, 64)
return VolumeId(volumeId), err
}
func (vid *VolumeId) String() string {
return strconv.FormatUint(uint64(*vid), 10)
}
func (vid *VolumeId) Next() VolumeId {
return VolumeId(uint32(*vid) + 1)
}

135
weed/storage/needle/volume_ttl.go Normal file
View File

@@ -0,0 +1,135 @@
package needle
import (
"strconv"
)
const (
//stored unit types
Empty byte = iota
Minute
Hour
Day
Week
Month
Year
)
type TTL struct {
Count byte
Unit byte
}
var EMPTY_TTL = &TTL{}
// translate a readable ttl to internal ttl
// Supports format example:
// 3m: 3 minutes
// 4h: 4 hours
// 5d: 5 days
// 6w: 6 weeks
// 7M: 7 months
// 8y: 8 years
func ReadTTL(ttlString string) (*TTL, error) {
if ttlString == "" {
return EMPTY_TTL, nil
}
ttlBytes := []byte(ttlString)
unitByte := ttlBytes[len(ttlBytes)-1]
countBytes := ttlBytes[0 : len(ttlBytes)-1]
if '0' <= unitByte && unitByte <= '9' {
countBytes = ttlBytes
unitByte = 'm'
}
count, err := strconv.Atoi(string(countBytes))
unit := toStoredByte(unitByte)
return &TTL{Count: byte(count), Unit: unit}, err
}
// read stored bytes to a ttl
func LoadTTLFromBytes(input []byte) (t *TTL) {
return &TTL{Count: input[0], Unit: input[1]}
}
// read stored bytes to a ttl
func LoadTTLFromUint32(ttl uint32) (t *TTL) {
input := make([]byte, 2)
input[1] = byte(ttl)
input[0] = byte(ttl >> 8)
return LoadTTLFromBytes(input)
}
// save stored bytes to an output with 2 bytes
func (t *TTL) ToBytes(output []byte) {
output[0] = t.Count
output[1] = t.Unit
}
func (t *TTL) ToUint32() (output uint32) {
output = uint32(t.Count) << 8
output += uint32(t.Unit)
return output
}
func (t *TTL) String() string {
if t == nil || t.Count == 0 {
return ""
}
if t.Unit == Empty {
return ""
}
countString := strconv.Itoa(int(t.Count))
switch t.Unit {
case Minute:
return countString + "m"
case Hour:
return countString + "h"
case Day:
return countString + "d"
case Week:
return countString + "w"
case Month:
return countString + "M"
case Year:
return countString + "y"
}
return ""
}
func toStoredByte(readableUnitByte byte) byte {
switch readableUnitByte {
case 'm':
return Minute
case 'h':
return Hour
case 'd':
return Day
case 'w':
return Week
case 'M':
return Month
case 'y':
return Year
}
return 0
}
func (t TTL) Minutes() uint32 {
switch t.Unit {
case Empty:
return 0
case Minute:
return uint32(t.Count)
case Hour:
return uint32(t.Count) * 60
case Day:
return uint32(t.Count) * 60 * 24
case Week:
return uint32(t.Count) * 60 * 24 * 7
case Month:
return uint32(t.Count) * 60 * 24 * 31
case Year:
return uint32(t.Count) * 60 * 24 * 365
}
return 0
}

60
weed/storage/needle/volume_ttl_test.go Normal file
View File

@@ -0,0 +1,60 @@
package needle
import (
"testing"
)
func TestTTLReadWrite(t *testing.T) {
ttl, _ := ReadTTL("")
if ttl.Minutes() != 0 {
t.Errorf("empty ttl:%v", ttl)
}
ttl, _ = ReadTTL("9")
if ttl.Minutes() != 9 {
t.Errorf("9 ttl:%v", ttl)
}
ttl, _ = ReadTTL("8m")
if ttl.Minutes() != 8 {
t.Errorf("8m ttl:%v", ttl)
}
ttl, _ = ReadTTL("5h")
if ttl.Minutes() != 300 {
t.Errorf("5h ttl:%v", ttl)
}
ttl, _ = ReadTTL("5d")
if ttl.Minutes() != 5*24*60 {
t.Errorf("5d ttl:%v", ttl)
}
ttl, _ = ReadTTL("5w")
if ttl.Minutes() != 5*7*24*60 {
t.Errorf("5w ttl:%v", ttl)
}
ttl, _ = ReadTTL("5M")
if ttl.Minutes() != 5*31*24*60 {
t.Errorf("5M ttl:%v", ttl)
}
ttl, _ = ReadTTL("5y")
if ttl.Minutes() != 5*365*24*60 {
t.Errorf("5y ttl:%v", ttl)
}
output := make([]byte, 2)
ttl.ToBytes(output)
ttl2 := LoadTTLFromBytes(output)
if ttl.Minutes() != ttl2.Minutes() {
t.Errorf("ttl:%v ttl2:%v", ttl, ttl2)
}
ttl3 := LoadTTLFromUint32(ttl.ToUint32())
if ttl.Minutes() != ttl3.Minutes() {
t.Errorf("ttl:%v ttl3:%v", ttl, ttl3)
}
}

10
weed/storage/needle/volume_version.go Normal file
View File

@@ -0,0 +1,10 @@
package needle
type Version uint8
const (
Version1 = Version(1)
Version2 = Version(2)
Version3 = Version(3)
CurrentVersion = Version3
)