Added tls for http clients (#5766)

* Added global http client

* Added Do func for global http client

* Changed the code to use the global http client

* Fix http client in volume uploader

* Fixed pkg name

* Fixed http util funcs

* Fixed http client for bench_filer_upload

* Fixed http client for stress_filer_upload

* Fixed http client for filer_server_handlers_proxy

* Fixed http client for command_fs_merge_volumes

* Fixed http client for command_fs_merge_volumes and command_volume_fsck

* Fixed http client for s3api_server

* Added init global client for main funcs

* Rename global_client to client

* Changed:
- fixed NewHttpClient;
- added CheckIsHttpsClientEnabled func
- updated security.toml in scaffold

* Reduce the visibility of some functions in the util/http/client pkg

* Added the loadSecurityConfig function

* Use util.LoadSecurityConfiguration() in NewHttpClient func

weed/shell/command_fs_merge_volumes.go

@@ -19,14 +19,10 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
 	"github.com/seaweedfs/seaweedfs/weed/util"
-)
-
-var (
-	client *http.Client
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 func init() {
-	client = &http.Client{}
 	Commands = append(Commands, &commandFsMergeVolumes{})
 }
 
@@ -104,7 +100,7 @@ func (c *commandFsMergeVolumes) Do(args []string, commandEnv *CommandEnv, writer
 		return nil
 	}
 
-	defer client.CloseIdleConnections()
+	defer util_http.GetGlobalHttpClient().CloseIdleConnections()
 
 	return commandEnv.WithFilerClient(false, func(filerClient filer_pb.SeaweedFilerClient) error {
 		return filer_pb.TraverseBfs(commandEnv, util.FullPath(dir), func(parentPath util.FullPath, entry *filer_pb.Entry) {
@@ -304,7 +300,7 @@ func moveChunk(chunk *filer_pb.FileChunk, toVolumeId needle.VolumeId, masterClie
 	if err != nil {
 		return err
 	}
-	defer util.CloseResponse(resp)
+	defer util_http.CloseResponse(resp)
 	defer reader.Close()
 
 	var filename string
@@ -322,7 +318,12 @@ func moveChunk(chunk *filer_pb.FileChunk, toVolumeId needle.VolumeId, masterClie
 	isCompressed := resp.Header.Get("Content-Encoding") == "gzip"
 	md5 := resp.Header.Get("Content-MD5")
 
-	_, err, _ = operation.Upload(reader, &operation.UploadOption{
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return err
+	}
+	
+	_, err, _ = uploader.Upload(reader, &operation.UploadOption{
 		UploadUrl:         uploadURL,
 		Filename:          filename,
 		IsInputCompressed: isCompressed,
@@ -348,12 +349,12 @@ func readUrl(fileUrl string) (*http.Response, io.ReadCloser, error) {
 	}
 	req.Header.Add("Accept-Encoding", "gzip")
 
-	r, err := client.Do(req)
+	r, err := util_http.GetGlobalHttpClient().Do(req)
 	if err != nil {
 		return nil, nil, err
 	}
 	if r.StatusCode >= 400 {
-		util.CloseResponse(r)
+		util_http.CloseResponse(r)
 		return nil, nil, fmt.Errorf("%s: %s", fileUrl, r.Status)
 	}
 

weed/shell/command_s3_clean_uploads.go

@@ -12,6 +12,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 func init() {
@@ -90,7 +91,7 @@ func (c *commandS3CleanUploads) cleanupUploads(commandEnv *CommandEnv, writer io
 		deleteUrl := fmt.Sprintf("http://%s%s/%s?recursive=true&ignoreRecursiveError=true", commandEnv.option.FilerAddress.ToHttpAddress(), uploadsDir, staleUpload)
 		fmt.Fprintf(writer, "purge %s\n", deleteUrl)
 
-		err = util.Delete(deleteUrl, string(encodedJwt))
+		err = util_http.Delete(deleteUrl, string(encodedJwt))
 		if err != nil && err.Error() != "" {
 			return fmt.Errorf("purge %s/%s: %v", uploadsDir, staleUpload, err)
 		}

weed/shell/command_volume_fsck.go

@@ -31,6 +31,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 func init() {
@@ -552,9 +553,7 @@ func (c *commandVolumeFsck) httpDelete(path util.FullPath) {
 		fmt.Fprintf(c.writer, "HTTP delete request error: %v\n", err)
 	}
 
-	client := &http.Client{}
-
-	resp, err := client.Do(req)
+	resp, err := util_http.GetGlobalHttpClient().Do(req)
 	if err != nil {
 		fmt.Fprintf(c.writer, "DELETE fetch error: %v\n", err)
 	}