Merge branch 'master' into mq

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -35,10 +35,16 @@ spec:
       {{ with .Values.podLabels }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.filer.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       annotations:
       {{ with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.filer.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.filer.restartPolicy }}
       {{- if .Values.filer.affinity }}

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -34,10 +34,16 @@ spec:
       {{ with .Values.podLabels }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.master.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       annotations:
       {{ with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.master.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.master.restartPolicy }}
       {{- if .Values.master.affinity }}

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -27,10 +27,16 @@ spec:
       {{ with .Values.podLabels }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.s3.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       annotations:
       {{ with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.s3.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       restartPolicy: {{ default .Values.global.restartPolicy .Values.s3.restartPolicy }}
       {{- if .Values.s3.tolerations }}

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -28,10 +28,16 @@ spec:
       {{ with .Values.podLabels }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.volume.podLabels }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
       annotations:
       {{ with .Values.podAnnotations }}
       {{- toYaml . | nindent 8 }}
       {{- end }}
+      {{- with .Values.volume.podAnnotations }}
+      {{- toYaml . | nindent 8 }}
+      {{- end }}
     spec:
       {{- if .Values.volume.affinity }}
       affinity:

k8s/charts/seaweedfs/values.yaml

@@ -127,6 +127,12 @@ master:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Labels to be added to the master pods
+  podLabels: {}
+
+  # Annotations to be added to the master pods
+  podAnnotations: {}
+
   ## Set podManagementPolicy
   podManagementPolicy: Parallel
 
@@ -369,6 +375,12 @@ volume:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Labels to be added to the volume pods
+  podLabels: {}
+
+  # Annotations to be added to the volume pods
+  podAnnotations: {}
+
   ## Set podManagementPolicy
   podManagementPolicy: Parallel
 
@@ -551,6 +563,12 @@ filer:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Labels to be added to the filer pods
+  podLabels: {}
+
+  # Annotations to be added to the filer pods
+  podAnnotations: {}
+
   ## Set podManagementPolicy
   podManagementPolicy: Parallel
 
@@ -761,6 +779,12 @@ s3:
   extraVolumes: ""
   extraVolumeMounts: ""
 
+  # Labels to be added to the s3 pods
+  podLabels: {}
+
+  # Annotations to be added to the s3 pods
+  podAnnotations: {}
+
   # Resource requests, limits, etc. for the server cluster placement. This
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request

weed/filer/filer_deletion.go

@@ -2,7 +2,6 @@ package filer
 
 import (
 	"github.com/seaweedfs/seaweedfs/weed/storage"
-	"math"
 	"strings"
 	"time"
 
@@ -138,47 +137,19 @@ func (f *Filer) DeleteChunksNotRecursive(chunks []*filer_pb.FileChunk) {
 }
 
 func (f *Filer) deleteChunksIfNotNew(oldEntry, newEntry *Entry) {
-
-	if oldEntry == nil {
-		return
+	var oldChunks, newChunks []*filer_pb.FileChunk
+	if oldEntry != nil {
+		oldChunks = oldEntry.GetChunks()
 	}
-	if newEntry == nil {
-		f.DeleteChunks(oldEntry.GetChunks())
-		return
+	if newEntry != nil {
+		newChunks = newEntry.GetChunks()
 	}
 
-	var toDelete []*filer_pb.FileChunk
-	newChunkIds := make(map[string]bool)
-	newDataChunks, newManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(),
-		newEntry.GetChunks(), 0, math.MaxInt64)
-	if err != nil {
-		glog.Errorf("Failed to resolve new entry chunks when delete old entry chunks. new: %s, old: %s",
-			newEntry.GetChunks(), oldEntry.Chunks)
-		return
-	}
-	for _, newChunk := range newDataChunks {
-		newChunkIds[newChunk.GetFileIdString()] = true
-	}
-	for _, newChunk := range newManifestChunks {
-		newChunkIds[newChunk.GetFileIdString()] = true
-	}
-
-	oldDataChunks, oldManifestChunks, err := ResolveChunkManifest(f.MasterClient.GetLookupFileIdFunction(),
-		oldEntry.GetChunks(), 0, math.MaxInt64)
+	toDelete, err := MinusChunks(f.MasterClient.GetLookupFileIdFunction(), oldChunks, newChunks)
 	if err != nil {
 		glog.Errorf("Failed to resolve old entry chunks when delete old entry chunks. new: %s, old: %s",
-			newEntry.GetChunks(), oldEntry.GetChunks())
+			newChunks, oldChunks)
 		return
 	}
-	for _, oldChunk := range oldDataChunks {
-		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
-			toDelete = append(toDelete, oldChunk)
-		}
-	}
-	for _, oldChunk := range oldManifestChunks {
-		if _, found := newChunkIds[oldChunk.GetFileIdString()]; !found {
-			toDelete = append(toDelete, oldChunk)
-		}
-	}
 	f.DeleteChunksNotRecursive(toDelete)
 }

weed/server/master_grpc_server.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/google/uuid"
 	"github.com/seaweedfs/seaweedfs/weed/cluster"
 	"net"
 	"sort"
@@ -260,7 +261,12 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 		return ms.informNewLeader(stream)
 	}
 
-	peerAddress := pb.ServerAddress(req.ClientAddress)
+	clientAddress := req.ClientAddress
+	// Ensure that the clientAddress is unique.
+	if clientAddress == "" {
+		clientAddress = uuid.New().String()
+	}
+	peerAddress := pb.ServerAddress(clientAddress)
 
 	// buffer by 1 so we don't end up getting stuck writing to stopChan forever
 	stopChan := make(chan bool, 1)