Optimiz leveldb metric (#3830)

* optimiz updating mapmetric for leveldb

* import loading leveldb

* add comments

weed/storage/needle_map_leveldb.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/syndtr/goleveldb/leveldb/errors"
 	"github.com/syndtr/goleveldb/leveldb/opt"
@@ -179,6 +180,7 @@ func levelDbWrite(db *leveldb.DB, key NeedleId, offset Offset, size Size, update
 	}
 	return nil
 }
+
 func levelDbDelete(db *leveldb.DB, key NeedleId) error {
 	bytes := make([]byte, NeedleIdSize)
 	NeedleIdToBytes(bytes, key)
@@ -305,23 +307,45 @@ func (m *LevelDbNeedleMap) DoOffsetLoading(v *Volume, indexFile *os.File, startF
 	}
 
 	err = idx.WalkIndexFile(indexFile, startFrom, func(key NeedleId, offset Offset, size Size) (e error) {
-		if !offset.IsZero() && size.IsValid() {
+		m.mapMetric.FileCounter++
+		bytes := make([]byte, NeedleIdSize)
+		NeedleIdToBytes(bytes[0:NeedleIdSize], key)
+		// fresh loading
+		if startFrom == 0 {
+			m.mapMetric.FileByteCounter += uint64(size)
+			e = levelDbWrite(db, key, offset, size, false, 0)
+			return e
+		}
+		// increment loading
+		data, err := db.Get(bytes, nil)
+		if err != nil {
+			if !strings.Contains(strings.ToLower(err.Error()), "not found") {
+				// unexpected error
+				return err
+			}
+			// new needle, unlikely happen
+			m.mapMetric.FileByteCounter += uint64(size)
 			e = levelDbWrite(db, key, offset, size, false, 0)
 		} else {
-			e = levelDbDelete(db, key)
+			// needle is found
+			oldSize := BytesToSize(data[OffsetSize : OffsetSize+SizeSize])
+			oldOffset := BytesToOffset(data[0:OffsetSize])
+			if !offset.IsZero() && size.IsValid() {
+				// updated needle
+				m.mapMetric.FileByteCounter += uint64(size)
+				if !oldOffset.IsZero() && oldSize.IsValid() {
+					m.mapMetric.DeletionCounter++
+					m.mapMetric.DeletionByteCounter += uint64(oldSize)
+				}
+				e = levelDbWrite(db, key, offset, size, false, 0)
+			} else {
+				// deleted needle
+				m.mapMetric.DeletionCounter++
+				m.mapMetric.DeletionByteCounter += uint64(oldSize)
+				e = levelDbDelete(db, key)
+			}
 		}
 		return e
 	})
-	if err != nil {
-		return err
-	}
-
-	if startFrom != 0 {
-		return needleMapMetricFromIndexFile(indexFile, &m.mapMetric)
-	}
-	return nil
-}
-
-func (m *LevelDbNeedleMap) UpdateNeedleMapMetric(indexFile *os.File) error {
-	return needleMapMetricFromIndexFile(indexFile, &m.mapMetric)
+	return err
 }