Fix/parse upload filename (#6241)

* fix: parse filename in PUT + refactor

* fix: master iu public url with http

* fix: better parsing and handle disposition header

* fix: take mime type from file extension if not set

weed/operation/needle_parse_test.go

@@ -44,7 +44,7 @@ func TestCreateNeedleFromRequest(t *testing.T) {
 	{
 		mockClient.needleHandling = func(n *needle.Needle, originalSize int, err error) {
 			assert.Equal(t, nil, err, "upload: %v", err)
-			assert.Equal(t, "", string(n.Mime), "mime detection failed: %v", string(n.Mime))
+			assert.Equal(t, "text/plain; charset=utf-8", string(n.Mime), "mime detection failed: %v", string(n.Mime))
 			assert.Equal(t, true, n.IsCompressed(), "this should be compressed")
 			assert.Equal(t, true, util.IsGzippedContent(n.Data), "this should be gzip")
 			fmt.Printf("needle: %v, originalSize: %d\n", n, originalSize)

weed/server/master_ui/master.html

@@ -33,14 +33,14 @@
                 {{ with .RaftServer }}
                 <tr>
                     <th>Leader</th>
-                    <td><a href="http://{{ .Leader }}">{{ .Leader }}</a></td>
+                    <td><a href="{{ url .Leader }}">{{ .Leader }}</a></td>
                 </tr>
                 <tr>
                     <th>Other Masters</th>
                     <td class="col-sm-5">
                         <ul class="list-unstyled">
                             {{ range $k, $p := .Peers }}
-                            <li><a href="http://{{ $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
+                            <li><a href="{{ url $p.Name }}/ui/index.html">{{ $p.Name }}</a></li>
                             {{ end }}
                         </ul>
                     </td>
@@ -88,9 +88,9 @@
             <tr>
                 <td><code>{{ $dc.Id }}</code></td>
                 <td>{{ $rack.Id }}</td>
-                <td><a href="http://{{ $dn.Url }}/ui/index.html">{{ $dn.Url }}</a>
+                <td><a href="{{ url $dn.Url }}/ui/index.html">{{ $dn.Url }}</a>
                     {{ if ne $dn.PublicUrl $dn.Url }}
-                    / <a href="http://{{ $dn.PublicUrl }}/ui/index.html">{{ $dn.PublicUrl }}</a>
+                    / <a href="{{ url $dn.PublicUrl }}/ui/index.html">{{ $dn.PublicUrl }}</a>
                     {{ end }}
                 </td>
                 <td>{{ $dn.Volumes }}</td>

weed/server/master_ui/templates.go

@@ -3,6 +3,7 @@ package master_ui
 import (
 	_ "embed"
 	"html/template"
+	"strings"
 )
 
 //go:embed master.html
@@ -11,5 +12,17 @@ var masterHtml string
 //go:embed masterNewRaft.html
 var masterNewRaftHtml string
 
-var StatusTpl = template.Must(template.New("status").Parse(masterHtml))
+var templateFunctions = template.FuncMap{
+	"url": func(input string) string {
+
+		if !strings.HasPrefix(input, "http://") && !strings.HasPrefix(input, "https://") {
+			return "http://" + input
+		}
+
+		return input
+	},
+}
+
+var StatusTpl = template.Must(template.New("status").Funcs(templateFunctions).Parse(masterHtml))
+
 var StatusNewRaftTpl = template.Must(template.New("status").Parse(masterNewRaftHtml))

weed/storage/needle/needle_parse_upload.go

@@ -43,19 +43,8 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
 		}
 	}
 
-	if r.Method == http.MethodPost {
+	e = parseUpload(r, sizeLimit, pu)
-		contentType := r.Header.Get("Content-Type")
 
-		// If content-type is explicitly set, upload the file without parsing form-data
-		if contentType != "" && !strings.Contains(contentType, "form-data") {
-			e = parseRawPost(r, sizeLimit, pu)
-		} else {
-			e = parseMultipart(r, sizeLimit, pu)
-		}
-
-	} else {
-		e = parsePut(r, sizeLimit, pu)
-	}
 	if e != nil {
 		return
 	}
@@ -108,28 +97,21 @@ func ParseUpload(r *http.Request, sizeLimit int64, bytesBuffer *bytes.Buffer) (p
 	return
 }
 
-func parsePut(r *http.Request, sizeLimit int64, pu *ParsedUpload) error {
+func parseUpload(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
-	pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
-	// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
-	pu.MimeType = r.Header.Get("Content-Type")
-	pu.FileName = ""
-	dataSize, err := pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
-	if err == io.EOF || dataSize == sizeLimit+1 {
-		io.Copy(io.Discard, r.Body)
-	}
-	pu.Data = pu.bytesBuffer.Bytes()
-	r.Body.Close()
-	return nil
-}
 
-func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
 	defer func() {
 		if e != nil && r.Body != nil {
 			io.Copy(io.Discard, r.Body)
 			r.Body.Close()
 		}
 	}()
+
+	contentType := r.Header.Get("Content-Type")
+	var dataSize int64
+
+	if r.Method == http.MethodPost && (contentType == "" || strings.Contains(contentType, "form-data")) {
 		form, fe := r.MultipartReader()
+
 		if fe != nil {
 			glog.V(0).Infoln("MultipartReader [ERROR]", fe)
 			e = fe
@@ -149,7 +131,6 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
 			pu.FileName = path.Base(pu.FileName)
 		}
 
-	var dataSize int64
 		dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(part, sizeLimit+1))
 		if e != nil {
 			glog.V(0).Infoln("Reading Content [ERROR]", e)
@@ -161,6 +142,8 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
 		}
 		pu.Data = pu.bytesBuffer.Bytes()
 
+		contentType = part.Header.Get("Content-Type")
+
 		// if the filename is empty string, do a search on the other multi-part items
 		for pu.FileName == "" {
 			part2, fe := form.NextPart()
@@ -187,49 +170,38 @@ func parseMultipart(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error
 				// update
 				pu.Data = pu.bytesBuffer.Bytes()
 				pu.FileName = path.Base(fName)
+				contentType = part.Header.Get("Content-Type")
+				part = part2
 				break
 			}
 		}
 
-	pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
-
-	if !pu.IsChunkedFile {
-
-		dotIndex := strings.LastIndex(pu.FileName, ".")
-		ext, mtype := "", ""
-		if dotIndex > 0 {
-			ext = strings.ToLower(pu.FileName[dotIndex:])
-			mtype = mime.TypeByExtension(ext)
-		}
-		contentType := part.Header.Get("Content-Type")
-		if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
-			pu.MimeType = contentType // only return mime type if not deducible
-			mtype = contentType
-		}
-
-	}
 		pu.IsGzipped = part.Header.Get("Content-Encoding") == "gzip"
 		// pu.IsZstd = part.Header.Get("Content-Encoding") == "zstd"
 
-	return
+	} else {
-}
+		disposition := r.Header.Get("Content-Disposition")
 
-func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error) {
+		if strings.Contains(disposition, "name=") {
 
-	defer func() {
+			if !strings.HasPrefix(disposition, "inline") && !strings.HasPrefix(disposition, "attachment") {
-		if e != nil && r.Body != nil {
+				disposition = "attachment; " + disposition
-			io.Copy(io.Discard, r.Body)
-			r.Body.Close()
 			}
-	}()
 
-	pu.FileName = r.Header.Get("Content-Disposition")
+			_, mediaTypeParams, err := mime.ParseMediaType(disposition)
 
-	if pu.FileName != "" && strings.Contains(pu.FileName, "filename=") {
+			if err == nil {
-		parts := strings.Split(pu.FileName, "filename=")
+				dpFilename, hasFilename := mediaTypeParams["filename"]
-		parts = strings.Split(parts[1], "\"")
+				dpName, hasName := mediaTypeParams["name"]
+
+				if hasFilename {
+					pu.FileName = dpFilename
+				} else if hasName {
+					pu.FileName = dpName
+				}
+
+			}
 
-		pu.FileName = parts[1]
 		} else {
 			pu.FileName = ""
 		}
@@ -240,7 +212,6 @@ func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error)
 			pu.FileName = path.Base(r.URL.Path)
 		}
 
-	var dataSize int64
 		dataSize, e = pu.bytesBuffer.ReadFrom(io.LimitReader(r.Body, sizeLimit+1))
 
 		if e != nil {
@@ -251,27 +222,32 @@ func parseRawPost(r *http.Request, sizeLimit int64, pu *ParsedUpload) (e error)
 			e = fmt.Errorf("file over the limited %d bytes", sizeLimit)
 			return
 		}
+
 		pu.Data = pu.bytesBuffer.Bytes()
+		pu.MimeType = contentType
+		pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
+		// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
+	}
+
 	pu.IsChunkedFile, _ = strconv.ParseBool(r.FormValue("cm"))
 
 	if !pu.IsChunkedFile {
 
 		dotIndex := strings.LastIndex(pu.FileName, ".")
 		ext, mtype := "", ""
+
 		if dotIndex > 0 {
 			ext = strings.ToLower(pu.FileName[dotIndex:])
 			mtype = mime.TypeByExtension(ext)
 		}
-		contentType := r.Header.Get("Content-Type")
 
 		if contentType != "" && contentType != "application/octet-stream" && mtype != contentType {
 			pu.MimeType = contentType // only return mime type if not deducible
-			mtype = contentType
+		} else if mtype != "" && pu.MimeType == "" && mtype != "application/octet-stream" {
+			pu.MimeType = mtype
 		}
 
 	}
-	pu.IsGzipped = r.Header.Get("Content-Encoding") == "gzip"
-	// pu.IsZstd = r.Header.Get("Content-Encoding") == "zstd"
 
 	return
 }