support for collections!

go/weed/compact.go

@@ -21,8 +21,9 @@ var cmdCompact = &Command{
 }
 
 var (
-	compactVolumePath = cmdCompact.Flag.String("dir", "/tmp", "data directory to store files")
-	compactVolumeId   = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
+	compactVolumePath       = cmdCompact.Flag.String("dir", "/tmp", "data directory to store files")
+	compactVolumeCollection = cmdCompact.Flag.String("collection", "", "volume collection name")
+	compactVolumeId         = cmdCompact.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir.")
 )
 
 func runCompact(cmd *Command, args []string) bool {
@@ -32,7 +33,7 @@ func runCompact(cmd *Command, args []string) bool {
 	}
 
 	vid := storage.VolumeId(*compactVolumeId)
-	v, err := storage.NewVolume(*compactVolumePath, vid, storage.CopyNil)
+	v, err := storage.NewVolume(*compactVolumePath, *compactVolumeCollection, vid, storage.CopyNil)
 	if err != nil {
 		glog.Fatalf("Load Volume [ERROR] %s\n", err)
 	}

go/weed/export.go

@@ -35,6 +35,7 @@ var cmdExport = &Command{
 
 var (
 	exportVolumePath = cmdExport.Flag.String("dir", "/tmp", "input data directory to store volume data files")
+	exportCollection = cmdExport.Flag.String("collection", "", "the volume collection name")
 	exportVolumeId   = cmdExport.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
 	dest             = cmdExport.Flag.String("o", "", "output tar file name, must ends with .tar, or just a \"-\" for stdout")
 	format           = cmdExport.Flag.String("fileNameFormat", defaultFnFormat, "filename format, default to {{.Mime}}/{{.Id}}:{{.Name}}")
@@ -95,7 +96,7 @@ func runExport(cmd *Command, args []string) bool {
 
 	var version storage.Version
 
-	err = storage.ScanVolumeFile(*exportVolumePath, vid, func(superBlock storage.SuperBlock) error {
+	err = storage.ScanVolumeFile(*exportVolumePath, *exportCollection, vid, func(superBlock storage.SuperBlock) error {
 		version = superBlock.Version
 		return nil
 	}, func(n *storage.Needle, offset int64) error {

go/weed/fix.go

@@ -22,8 +22,9 @@ var cmdFix = &Command{
 }
 
 var (
-	fixVolumePath = cmdFix.Flag.String("dir", "/tmp", "data directory to store files")
-	fixVolumeId   = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
+	fixVolumePath       = cmdFix.Flag.String("dir", "/tmp", "data directory to store files")
+	fixVolumeCollection = cmdFix.Flag.String("collection", "", "the volume collection name")
+	fixVolumeId         = cmdFix.Flag.Int("volumeId", -1, "a volume id. The volume should already exist in the dir. The volume index file should not exist.")
 )
 
 func runFix(cmd *Command, args []string) bool {
@@ -33,6 +34,9 @@ func runFix(cmd *Command, args []string) bool {
 	}
 
 	fileName := strconv.Itoa(*fixVolumeId)
+	if *fixVolumeCollection != "" {
+		fileName = *fixVolumeCollection + "_" + fileName
+	}
 	indexFile, err := os.OpenFile(path.Join(*fixVolumePath, fileName+".idx"), os.O_WRONLY|os.O_CREATE, 0644)
 	if err != nil {
 		glog.Fatalf("Create Volume Index [ERROR] %s\n", err)
@@ -43,7 +47,7 @@ func runFix(cmd *Command, args []string) bool {
 	defer nm.Close()
 
 	vid := storage.VolumeId(*fixVolumeId)
-	err = storage.ScanVolumeFile(*fixVolumePath, vid, func(superBlock storage.SuperBlock) error {
+	err = storage.ScanVolumeFile(*fixVolumePath, *fixVolumeCollection, vid, func(superBlock storage.SuperBlock) error {
 		return nil
 	}, func(n *storage.Needle, offset int64) error {
 		debug("key", n.Id, "offset", offset, "size", n.Size, "disk_size", n.DiskSize(), "gzip", n.IsGzipped())

go/weed/master.go

@@ -56,13 +56,14 @@ var vgLock sync.Mutex
 
 func dirLookupHandler(w http.ResponseWriter, r *http.Request) {
 	vid := r.FormValue("volumeId")
+	collection := r.FormValue("collection") //optional, but can be faster if too many collections
 	commaSep := strings.Index(vid, ",")
 	if commaSep > 0 {
 		vid = vid[0:commaSep]
 	}
 	volumeId, err := storage.NewVolumeId(vid)
 	if err == nil {
-		machines := topo.Lookup(volumeId)
+		machines := topo.Lookup(collection, volumeId)
 		if machines != nil {
 			ret := []map[string]string{}
 			for _, dn := range machines {
@@ -88,6 +89,7 @@ func dirAssignHandler(w http.ResponseWriter, r *http.Request) {
 	if repType == "" {
 		repType = *defaultRepType
 	}
+	collection := r.FormValue("collection")
 	dataCenter := r.FormValue("dataCenter")
 	rt, err := storage.NewReplicationTypeFromString(repType)
 	if err != nil {
@@ -96,7 +98,7 @@ func dirAssignHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if topo.GetVolumeLayout(rt).GetActiveVolumeCount(dataCenter) <= 0 {
+	if topo.GetVolumeLayout(collection, rt).GetActiveVolumeCount(dataCenter) <= 0 {
 		if topo.FreeSpace() <= 0 {
 			w.WriteHeader(http.StatusNotFound)
 			writeJsonQuiet(w, r, map[string]string{"error": "No free volumes left!"})
@@ -104,15 +106,15 @@ func dirAssignHandler(w http.ResponseWriter, r *http.Request) {
 		} else {
 			vgLock.Lock()
 			defer vgLock.Unlock()
-			if topo.GetVolumeLayout(rt).GetActiveVolumeCount(dataCenter) <= 0 {
-				if _, err = vg.AutomaticGrowByType(rt, dataCenter, topo); err != nil {
+			if topo.GetVolumeLayout(collection, rt).GetActiveVolumeCount(dataCenter) <= 0 {
+				if _, err = vg.AutomaticGrowByType(collection, rt, dataCenter, topo); err != nil {
 					writeJsonQuiet(w, r, map[string]string{"error": "Cannot grow volume group! " + err.Error()})
 					return
 				}
 			}
 		}
 	}
-	fid, count, dn, err := topo.PickForWrite(rt, c, dataCenter)
+	fid, count, dn, err := topo.PickForWrite(collection, rt, c, dataCenter)
 	if err == nil {
 		writeJsonQuiet(w, r, map[string]interface{}{"fid": fid, "url": dn.Url(), "publicUrl": dn.PublicUrl, "count": count})
 	} else {
@@ -168,7 +170,7 @@ func volumeGrowHandler(w http.ResponseWriter, r *http.Request) {
 			if topo.FreeSpace() < count*rt.GetCopyCount() {
 				err = errors.New("Only " + strconv.Itoa(topo.FreeSpace()) + " volumes left! Not enough for " + strconv.Itoa(count*rt.GetCopyCount()))
 			} else {
-				count, err = vg.GrowByCountAndType(count, rt, r.FormValue("dataCneter"), topo)
+				count, err = vg.GrowByCountAndType(count, r.FormValue("collection"), rt, r.FormValue("dataCneter"), topo)
 			}
 		} else {
 			err = errors.New("parameter count is not found")
@@ -197,7 +199,7 @@ func redirectHandler(w http.ResponseWriter, r *http.Request) {
 		debug("parsing error:", err, r.URL.Path)
 		return
 	}
-	machines := topo.Lookup(volumeId)
+	machines := topo.Lookup("", volumeId)
 	if machines != nil && len(machines) > 0 {
 		http.Redirect(w, r, "http://"+machines[0].PublicUrl+r.URL.Path, http.StatusMovedPermanently)
 	} else {

go/weed/volume.go

@@ -56,13 +56,13 @@ func statusHandler(w http.ResponseWriter, r *http.Request) {
 	writeJsonQuiet(w, r, m)
 }
 func assignVolumeHandler(w http.ResponseWriter, r *http.Request) {
-	err := store.AddVolume(r.FormValue("volume"), r.FormValue("replicationType"))
+	err := store.AddVolume(r.FormValue("volume"), r.FormValue("collection"), r.FormValue("replicationType"))
 	if err == nil {
 		writeJsonQuiet(w, r, map[string]string{"error": ""})
 	} else {
 		writeJsonQuiet(w, r, map[string]string{"error": err.Error()})
 	}
-	debug("assign volume =", r.FormValue("volume"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
+	debug("assign volume =", r.FormValue("volume"), ", collection =", r.FormValue("collection"), ", replicationType =", r.FormValue("replicationType"), ", error =", err)
 }
 func vacuumVolumeCheckHandler(w http.ResponseWriter, r *http.Request) {
 	err, ret := store.CheckCompactVolume(r.FormValue("volume"), r.FormValue("garbageThreshold"))
@@ -112,6 +112,8 @@ func storeHandler(w http.ResponseWriter, r *http.Request) {
 		GetOrHeadHandler(w, r, false)
 	case "DELETE":
 		secure(volumeWhiteList, DeleteHandler)(w, r)
+	case "PUT":
+		secure(volumeWhiteList, PostHandler)(w, r)
 	case "POST":
 		secure(volumeWhiteList, PostHandler)(w, r)
 	}