able to update file content

having some issue when vi reports file changed.

weed/filer2/embedded/embedded_store.go

@@ -26,7 +26,7 @@ func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int3
 	return nil
 }
 
-func (filer *EmbeddedStore) AppendFileChunk(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) {
+func (filer *EmbeddedStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) {
 	return nil
 }
 

weed/filer2/filechunks.go

@@ -19,12 +19,27 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
 }
 
 func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) {
+
+	visibles := nonOverlappingVisibleIntervals(chunks)
+
+	fileIds := make(map[string]bool)
+	for _, interval := range visibles {
+		fileIds[interval.fileId] = true
+	}
+	for _, chunk := range chunks {
+		if found := fileIds[chunk.FileId]; found {
+			compacted = append(compacted, chunk)
+		} else {
+			garbage = append(garbage, chunk)
+		}
+	}
+
 	return
 }
 
 func logPrintf(name string, visibles []*visibleInterval) {
 
-	return
+	// return
 
 	log.Printf("%s len %d", name, len(visibles))
 	for _, v := range visibles {
@@ -52,7 +67,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v
 	var minStopInterval, upToDateInterval *visibleInterval
 	watermarkStart := chunks[0].Offset
 	for _, chunk := range chunks {
-		// log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
+		log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size))
 		logPrintf("parallelIntervals", parallelIntervals)
 		for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset {
 			logPrintf("parallelIntervals loop 1", parallelIntervals)
@@ -72,12 +87,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v
 			var remaining []*visibleInterval
 			for _, interval := range parallelIntervals {
 				if interval.stop != watermarkStart {
-					remaining = append(remaining, newVisibleInterval(
-						interval.start,
-						interval.stop,
-						interval.fileId,
-						interval.modifiedTime,
-					))
+					remaining = append(remaining, interval)
 				}
 			}
 			parallelIntervals = remaining
@@ -108,12 +118,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v
 		var remaining []*visibleInterval
 		for _, interval := range parallelIntervals {
 			if interval.stop != watermarkStart {
-				remaining = append(remaining, newVisibleInterval(
-					interval.start,
-					interval.stop,
-					interval.fileId,
-					interval.modifiedTime,
-				))
+				remaining = append(remaining, interval)
 			}
 		}
 		parallelIntervals = remaining
@@ -122,11 +127,12 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v
 	logPrintf("intervals", intervals)
 
 	// merge connected intervals, now the intervals are non-intersecting
-	var lastInterval *visibleInterval
+	var lastIntervalIndex int
 	var prevIntervalIndex int
 	for i, interval := range intervals {
 		if i == 0 {
 			prevIntervalIndex = i
+			lastIntervalIndex = i
 			continue
 		}
 		if intervals[i-1].fileId != interval.fileId ||
@@ -139,18 +145,16 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v
 			))
 			prevIntervalIndex = i
 		}
-		lastInterval = intervals[i]
+		lastIntervalIndex = i
 		logPrintf("intervals loop 1 visibles", visibles)
 	}
 
-	if lastInterval != nil {
-		visibles = append(visibles, newVisibleInterval(
-			intervals[prevIntervalIndex].start,
-			lastInterval.stop,
-			intervals[prevIntervalIndex].fileId,
-			intervals[prevIntervalIndex].modifiedTime,
-		))
-	}
+	visibles = append(visibles, newVisibleInterval(
+		intervals[prevIntervalIndex].start,
+		intervals[lastIntervalIndex].stop,
+		intervals[prevIntervalIndex].fileId,
+		intervals[prevIntervalIndex].modifiedTime,
+	))
 
 	logPrintf("visibles", visibles)
 

weed/filer2/filechunks_test.go

@@ -7,6 +7,28 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 )
 
+func TestCompactFileChunks(t *testing.T) {
+	chunks := []*filer_pb.FileChunk{
+		{Offset:10, Size:100, FileId:"abc", Mtime:50},
+		{Offset:100, Size:100, FileId:"def", Mtime:100},
+		{Offset:200, Size:100, FileId:"ghi", Mtime:200},
+		{Offset:110, Size:200, FileId:"jkl", Mtime:300},
+	}
+
+	compacted, garbarge := CompactFileChunks(chunks)
+
+	log.Printf("Compacted: %+v", compacted)
+	log.Printf("Garbage  : %+v", garbarge)
+
+	if len(compacted) != 3 {
+		t.Fatalf("unexpected compacted: %d", len(compacted))
+	}
+	if len(garbarge) != 1 {
+		t.Fatalf("unexpected garbarge: %d", len(garbarge))
+	}
+
+}
+
 func TestIntervalMerging(t *testing.T) {
 
 	testcases := []struct {
@@ -84,6 +106,17 @@ func TestIntervalMerging(t *testing.T) {
 				{start: 200, stop: 220, fileId: "abc"},
 			},
 		},
+		// case 6: same updates
+		{
+			Chunks: []*filer_pb.FileChunk{
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+				{Offset: 0, Size: 100, FileId: "abc", Mtime: 123},
+			},
+			Expected: []*visibleInterval{
+				{start: 0, stop: 100, fileId: "abc"},
+			},
+		},
 	}
 
 	for i, testcase := range testcases {

weed/filer2/filer.go

@@ -113,8 +113,8 @@ func (f *Filer) CreateEntry(entry *Entry) (error) {
 	return nil
 }
 
-func (f *Filer) AppendFileChunk(p FullPath, chunks []*filer_pb.FileChunk) (err error) {
-	return f.store.AppendFileChunk(p, chunks)
+func (f *Filer) SetFileChunks(p FullPath, chunks []*filer_pb.FileChunk) (err error) {
+	return f.store.SetFileChunks(p, chunks)
 }
 
 func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) {

weed/filer2/filer_structure.go

@@ -68,7 +68,7 @@ var ErrNotFound = errors.New("filer: no entry is found in filer store")
 
 type FilerStore interface {
 	InsertEntry(*Entry) (error)
-	AppendFileChunk(FullPath, []*filer_pb.FileChunk) (err error)
+	SetFileChunks(FullPath, []*filer_pb.FileChunk) (err error)
 	FindEntry(FullPath) (found bool, entry *Entry, err error)
 	DeleteEntry(FullPath) (fileEntry *Entry, err error)
 

weed/filer2/memdb/memdb_store.go

@@ -33,14 +33,13 @@ func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
 	return nil
 }
 
-func (filer *MemDbStore) AppendFileChunk(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) {
+func (filer *MemDbStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) {
 	found, entry, err := filer.FindEntry(fullpath)
 	if !found {
 		return fmt.Errorf("No such file: %s", fullpath)
 	}
-	entry.Chunks = append(entry.Chunks, fileChunks...)
+	entry.Chunks = fileChunks
 	entry.Mtime = time.Now()
-	println("appending to entry", entry.Name(), len(entry.Chunks))
 	filer.tree.ReplaceOrInsert(Entry{entry})
 	return nil
 }