filer: option to encrypt data on volume server

weed/filer2/filechunks.go

@@ -71,6 +71,7 @@ type ChunkView struct {
 	Size        uint64
 	LogicOffset int64
 	IsFullChunk bool
+	CipherKey   []byte
 }
 
 func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) {
@@ -94,6 +95,7 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int
 				Size:        uint64(min(chunk.stop, stop) - offset),
 				LogicOffset: offset,
 				IsFullChunk: isFullChunk,
+				CipherKey:   chunk.cipherKey,
 			})
 			offset = min(chunk.stop, stop)
 		}
@@ -120,13 +122,7 @@ var bufPool = sync.Pool{
 
 func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
 
-	newV := newVisibleInterval(
-		chunk.Offset,
-		chunk.Offset+int64(chunk.Size),
-		chunk.GetFileIdString(),
-		chunk.Mtime,
-		true,
-	)
+	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey)
 
 	length := len(visibles)
 	if length == 0 {
@@ -140,23 +136,11 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.
 	logPrintf("  before", visibles)
 	for _, v := range visibles {
 		if v.start < chunk.Offset && chunk.Offset < v.stop {
-			newVisibles = append(newVisibles, newVisibleInterval(
-				v.start,
-				chunk.Offset,
-				v.fileId,
-				v.modifiedTime,
-				false,
-			))
+			newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey))
 		}
 		chunkStop := chunk.Offset + int64(chunk.Size)
 		if v.start < chunkStop && chunkStop < v.stop {
-			newVisibles = append(newVisibles, newVisibleInterval(
-				chunkStop,
-				v.stop,
-				v.fileId,
-				v.modifiedTime,
-				false,
-			))
+			newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey))
 		}
 		if chunkStop <= v.start || v.stop <= chunk.Offset {
 			newVisibles = append(newVisibles, v)
@@ -208,15 +192,17 @@ type VisibleInterval struct {
 	modifiedTime int64
 	fileId       string
 	isFullChunk  bool
+	cipherKey    []byte
 }
 
-func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool) VisibleInterval {
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte) VisibleInterval {
 	return VisibleInterval{
 		start:        start,
 		stop:         stop,
 		fileId:       fileId,
 		modifiedTime: modifiedTime,
 		isFullChunk:  isFullChunk,
+		cipherKey:    cipherKey,
 	}
 }
 

weed/filer2/filer.go

@@ -33,6 +33,7 @@ type Filer struct {
 	DirBucketsPath      string
 	DirQueuesPath       string
 	buckets             *FilerBuckets
+	Cipher              bool
 }
 
 func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32) *Filer {

weed/filer2/filer_client_util.go

@@ -70,12 +70,7 @@ func ReadIntoBuffer(filerClient FilerClient, fullFilePath FullPath, buff []byte,
 
 			volumeServerAddress := filerClient.AdjustedUrl(locations.Locations[0].Url)
 			var n int64
-			n, err = util.ReadUrl(
-				fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId),
-				chunkView.Offset,
-				int(chunkView.Size),
-				buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)],
-				!chunkView.IsFullChunk)
+			n, err = util.ReadUrl(fmt.Sprintf("http://%s/%s", volumeServerAddress, chunkView.FileId), chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), buff[chunkView.LogicOffset-baseOffset:chunkView.LogicOffset-baseOffset+int64(chunkView.Size)])
 
 			if err != nil {
 

weed/filer2/stream.go

@@ -27,7 +27,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f
 
 	for _, chunkView := range chunkViews {
 		urlString := fileId2Url[chunkView.FileId]
-		_, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) {
+		err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) {
 			w.Write(data)
 		})
 		if err != nil {