use grpc to replace http APIs for batch volume id lookup and batch delete

1. remove batch volume id lookup http API /vol/lookup
2. remove batch delete http API /delete

weed/server/common.go

@@ -132,17 +132,6 @@ func submitForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl st
 	return
 }
 
-func deleteForClientHandler(w http.ResponseWriter, r *http.Request, masterUrl string) {
-	r.ParseForm()
-	fids := r.Form["fid"]
-	ret, err := operation.DeleteFiles(masterUrl, fids)
-	if err != nil {
-		writeJsonError(w, r, http.StatusInternalServerError, err)
-		return
-	}
-	writeJsonQuiet(w, r, http.StatusAccepted, ret)
-}
-
 func parseURLPath(path string) (vid, fid, filename, ext string, isVolumeIdOnly bool) {
 	switch strings.Count(path, "/") {
 	case 3:

weed/server/filer_grpc_server.go

@@ -110,10 +110,7 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
 	fullpath := filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name))
 	chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
 
-	for _, garbage := range garbages {
-		glog.V(0).Infof("deleting %s garbage chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size))
-		fs.filer.DeleteFileByFileId(garbage.FileId)
-	}
+	fs.filer.DeleteChunks(garbages)
 
 	err = fs.filer.CreateEntry(&filer2.Entry{
 		FullPath: fullpath,
@@ -168,14 +165,8 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
 	}
 
 	if err = fs.filer.UpdateEntry(newEntry); err == nil {
-		for _, garbage := range unusedChunks {
-			glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size))
-			fs.filer.DeleteFileByFileId(garbage.FileId)
-		}
-		for _, garbage := range garbages {
-			glog.V(0).Infof("deleting %s garbage chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size))
-			fs.filer.DeleteFileByFileId(garbage.FileId)
-		}
+		fs.filer.DeleteChunks(unusedChunks)
+		fs.filer.DeleteChunks(garbages)
 	}
 
 	fs.filer.NotifyUpdateEvent(entry, newEntry, true)

weed/server/master_grpc_server_lookup.go

@@ -0,0 +1,28 @@
+package weed_server
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
+	"context"
+)
+
+func (ms *MasterServer) LookupVolume(ctx context.Context, req *master_pb.LookupVolumeRequest) (*master_pb.LookupVolumeResponse, error) {
+	resp := &master_pb.LookupVolumeResponse{}
+	volumeLocations := ms.lookupVolumeId(req.VolumeIds, req.Collection)
+
+	for _, result := range volumeLocations {
+		var locations []*master_pb.Location
+		for _, loc := range result.Locations {
+			locations = append(locations, &master_pb.Location{
+				Url:       loc.Url,
+				PublicUrl: loc.PublicUrl,
+			})
+		}
+		resp.VolumeIdLocations = append(resp.VolumeIdLocations, &master_pb.LookupVolumeResponse_VolumeIdLocation{
+			VolumeId:  result.VolumeId,
+			Locations: locations,
+			Error:     result.Error,
+		})
+	}
+
+	return resp, nil
+}

weed/server/master_server.go

@@ -76,12 +76,10 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 	r.HandleFunc("/dir/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.dirLookupHandler)))
 	r.HandleFunc("/dir/status", ms.proxyToLeader(ms.guard.WhiteList(ms.dirStatusHandler)))
 	r.HandleFunc("/col/delete", ms.proxyToLeader(ms.guard.WhiteList(ms.collectionDeleteHandler)))
-	r.HandleFunc("/vol/lookup", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeLookupHandler)))
 	r.HandleFunc("/vol/grow", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeGrowHandler)))
 	r.HandleFunc("/vol/status", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeStatusHandler)))
 	r.HandleFunc("/vol/vacuum", ms.proxyToLeader(ms.guard.WhiteList(ms.volumeVacuumHandler)))
 	r.HandleFunc("/submit", ms.guard.WhiteList(ms.submitFromMasterServerHandler))
-	r.HandleFunc("/delete", ms.guard.WhiteList(ms.deleteFromMasterServerHandler))
 	r.HandleFunc("/stats/health", ms.guard.WhiteList(statsHealthHandler))
 	r.HandleFunc("/stats/counter", ms.guard.WhiteList(statsCounterHandler))
 	r.HandleFunc("/stats/memory", ms.guard.WhiteList(statsMemoryHandler))

weed/server/master_server_handlers.go

@@ -58,15 +58,6 @@ func (ms *MasterServer) dirLookupHandler(w http.ResponseWriter, r *http.Request)
 	writeJsonQuiet(w, r, httpStatus, location)
 }
 
-// This can take batched volumeIds, &volumeId=x&volumeId=y&volumeId=z
-func (ms *MasterServer) volumeLookupHandler(w http.ResponseWriter, r *http.Request) {
-	r.ParseForm()
-	vids := r.Form["volumeId"]
-	collection := r.FormValue("collection") //optional, but can be faster if too many collections
-	volumeLocations := ms.lookupVolumeId(vids, collection)
-	writeJsonQuiet(w, r, http.StatusOK, volumeLocations)
-}
-
 func (ms *MasterServer) dirAssignHandler(w http.ResponseWriter, r *http.Request) {
 	stats.AssignRequest()
 	requestedCount, e := strconv.ParseUint(r.FormValue("count"), 10, 64)

weed/server/master_server_handlers_admin.go

@@ -119,14 +119,6 @@ func (ms *MasterServer) submitFromMasterServerHandler(w http.ResponseWriter, r *
 	}
 }
 
-func (ms *MasterServer) deleteFromMasterServerHandler(w http.ResponseWriter, r *http.Request) {
-	if ms.Topo.IsLeader() {
-		deleteForClientHandler(w, r, ms.selfUrl(r))
-	} else {
-		deleteForClientHandler(w, r, ms.Topo.RaftServer.Leader())
-	}
-}
-
 func (ms *MasterServer) HasWritableVolume(option *topology.VolumeGrowOption) bool {
 	vl := ms.Topo.GetVolumeLayout(option.Collection, option.ReplicaPlacement, option.Ttl)
 	return vl.GetActiveVolumeCount(option) > 0

weed/server/volume_server.go

@@ -62,7 +62,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	adminMux.HandleFunc("/stats/counter", vs.guard.WhiteList(statsCounterHandler))
 	adminMux.HandleFunc("/stats/memory", vs.guard.WhiteList(statsMemoryHandler))
 	adminMux.HandleFunc("/stats/disk", vs.guard.WhiteList(vs.statsDiskHandler))
-	adminMux.HandleFunc("/delete", vs.guard.WhiteList(vs.batchDeleteHandler))
 	adminMux.HandleFunc("/", vs.privateStoreHandler)
 	if publicMux != adminMux {
 		// separated admin and public port

weed/server/volume_server_handlers_write.go

@@ -109,71 +109,6 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
 
 }
 
-//Experts only: takes multiple fid parameters. This function does not propagate deletes to replicas.
-func (vs *VolumeServer) batchDeleteHandler(w http.ResponseWriter, r *http.Request) {
-	r.ParseForm()
-	var ret []operation.DeleteResult
-	now := uint64(time.Now().Unix())
-	for _, fid := range r.Form["fid"] {
-		vid, id_cookie, err := operation.ParseFileId(fid)
-		if err != nil {
-			ret = append(ret, operation.DeleteResult{
-				Fid:    fid,
-				Status: http.StatusBadRequest,
-				Error:  err.Error()})
-			continue
-		}
-		n := new(storage.Needle)
-		volumeId, _ := storage.NewVolumeId(vid)
-		n.ParsePath(id_cookie)
-		// glog.V(4).Infoln("batch deleting", n)
-		cookie := n.Cookie
-		if _, err := vs.store.ReadVolumeNeedle(volumeId, n); err != nil {
-			ret = append(ret, operation.DeleteResult{
-				Fid:    fid,
-				Status: http.StatusNotFound,
-				Error:  err.Error(),
-			})
-			continue
-		}
-
-		if n.IsChunkedManifest() {
-			ret = append(ret, operation.DeleteResult{
-				Fid:    fid,
-				Status: http.StatusNotAcceptable,
-				Error:  "ChunkManifest: not allowed in batch delete mode.",
-			})
-			continue
-		}
-
-		if n.Cookie != cookie {
-			ret = append(ret, operation.DeleteResult{
-				Fid:    fid,
-				Status: http.StatusBadRequest,
-				Error:  "File Random Cookie does not match.",
-			})
-			glog.V(0).Infoln("deleting", fid, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
-			return
-		}
-		n.LastModified = now
-		if size, err := vs.store.Delete(volumeId, n); err != nil {
-			ret = append(ret, operation.DeleteResult{
-				Fid:    fid,
-				Status: http.StatusInternalServerError,
-				Error:  err.Error()},
-			)
-		} else {
-			ret = append(ret, operation.DeleteResult{
-				Fid:    fid,
-				Status: http.StatusAccepted,
-				Size:   int(size)},
-			)
-		}
-	}
-
-	writeJsonQuiet(w, r, http.StatusAccepted, ret)
-}
-
 func setEtag(w http.ResponseWriter, etag string) {
 	if etag != "" {
 		if strings.HasPrefix(etag, "\"") {