Added tls for http clients (#5766)

* Added global http client

* Added Do func for global http client

* Changed the code to use the global http client

* Fix http client in volume uploader

* Fixed pkg name

* Fixed http util funcs

* Fixed http client for bench_filer_upload

* Fixed http client for stress_filer_upload

* Fixed http client for filer_server_handlers_proxy

* Fixed http client for command_fs_merge_volumes

* Fixed http client for command_fs_merge_volumes and command_volume_fsck

* Fixed http client for s3api_server

* Added init global client for main funcs

* Rename global_client to client

* Changed:
- fixed NewHttpClient;
- added CheckIsHttpsClientEnabled func
- updated security.toml in scaffold

* Reduce the visibility of some functions in the util/http/client pkg

* Added the loadSecurityConfig function

* Use util.LoadSecurityConfiguration() in NewHttpClient func

weed/mq/broker/broker_topic_partition_read_write.go

@@ -13,6 +13,7 @@ import (
 	"math"
 	"sync/atomic"
 	"time"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 func (b *MessageQueueBroker) genLogFlushFunc(t topic.Topic, partition *mq_pb.Partition) log_buffer.LogFlushFuncType {
@@ -131,7 +132,7 @@ func (b *MessageQueueBroker) genLogOnDiskReadFunc(t topic.Topic, partition *mq_p
 			for _, urlString := range urlStrings {
 				// TODO optimization opportunity: reuse the buffer
 				var data []byte
-				if data, _, err = util.Get(urlString); err == nil {
+				if data, _, err = util_http.Get(urlString); err == nil {
 					processed = true
 					if processedTsNs, err = eachChunkFn(data, eachLogEntryFn, starTsNs, stopTsNs); err != nil {
 						return

weed/mq/broker/broker_write.go

@@ -55,7 +55,13 @@ func (b *MessageQueueBroker) appendToFile(targetFile string, data []byte) error
 func (b *MessageQueueBroker) assignAndUpload(targetFile string, data []byte) (fileId string, uploadResult *operation.UploadResult, err error) {
 
 	reader := util.NewBytesReader(data)
-	fileId, uploadResult, err, _ = operation.UploadWithRetry(
+
+	uploader, err := operation.NewUploader()
+	if err != nil {
+		return
+	}
+
+	fileId, uploadResult, err, _ = uploader.UploadWithRetry(
 		b,
 		&filer_pb.AssignVolumeRequest{
 			Count:       1,

weed/mq/client/cmd/weed_pub_kv/publisher_kv.go

@@ -9,6 +9,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 var (
@@ -45,6 +46,7 @@ func doPublish(publisher *pub_client.TopicPublisher, id int) {
 
 func main() {
 	flag.Parse()
+	util_http.InitGlobalHttpClient()
 
 	config := &pub_client.PublisherConfiguration{
 		Topic:          topic.NewTopic(*namespace, *t),

weed/mq/client/cmd/weed_pub_record/publisher_record.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 var (
@@ -86,6 +87,7 @@ func (r *MyRecord) ToRecordValue() *schema_pb.RecordValue {
 
 func main() {
 	flag.Parse()
+	util_http.InitGlobalHttpClient()
 
 	recordType := schema.RecordTypeBegin().
 		WithField("key", schema.TypeBytes).

weed/mq/client/cmd/weed_sub_kv/subscriber_kv.go

@@ -11,6 +11,7 @@ import (
 	"google.golang.org/grpc/credentials/insecure"
 	"strings"
 	"time"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 var (
@@ -23,6 +24,7 @@ var (
 
 func main() {
 	flag.Parse()
+	util_http.InitGlobalHttpClient()
 
 	subscriberConfig := &sub_client.SubscriberConfiguration{
 		ClientId:                fmt.Sprintf("client-%d", *clientId),

weed/mq/client/cmd/weed_sub_record/subscriber_record.go

@@ -13,6 +13,7 @@ import (
 	"google.golang.org/protobuf/proto"
 	"strings"
 	"time"
+	util_http "github.com/seaweedfs/seaweedfs/weed/util/http"
 )
 
 var (
@@ -49,6 +50,7 @@ func FromSchemaRecordValue(recordValue *schema_pb.RecordValue) *MyRecord {
 
 func main() {
 	flag.Parse()
+	util_http.InitGlobalHttpClient()
 
 	subscriberConfig := &sub_client.SubscriberConfiguration{
 		ClientId:                fmt.Sprintf("client-%d", *clientId),