* chore: remove unreachable dead code across the codebase Remove ~50,000 lines of unreachable code identified by static analysis. Major removals: - weed/filer/redis_lua: entire unused Redis Lua filer store implementation - weed/wdclient/net2, resource_pool: unused connection/resource pool packages - weed/plugin/worker/lifecycle: unused lifecycle plugin worker - weed/s3api: unused S3 policy templates, presigned URL IAM, streaming copy, multipart IAM, key rotation, and various SSE helper functions - weed/mq/kafka: unused partition mapping, compression, schema, and protocol functions - weed/mq/offset: unused SQL storage and migration code - weed/worker: unused registry, task, and monitoring functions - weed/query: unused SQL engine, parquet scanner, and type functions - weed/shell: unused EC proportional rebalance functions - weed/storage/erasure_coding/distribution: unused distribution analysis functions - Individual unreachable functions removed from 150+ files across admin, credential, filer, iam, kms, mount, mq, operation, pb, s3api, server, shell, storage, topology, and util packages * fix(s3): reset shared memory store in IAM test to prevent flaky failure TestLoadIAMManagerFromConfig_EmptyConfigWithFallbackKey was flaky because the MemoryStore credential backend is a singleton registered via init(). Earlier tests that create anonymous identities pollute the shared store, causing LookupAnonymous() to unexpectedly return true. Fix by calling Reset() on the memory store before the test runs. * style: run gofmt on changed files * fix: restore KMS functions used by integration tests * fix(plugin): prevent panic on send to closed worker session channel The Plugin.sendToWorker method could panic with "send on closed channel" when a worker disconnected while a message was being sent. The race was between streamSession.close() closing the outgoing channel and sendToWorker writing to it concurrently. Add a done channel to streamSession that is closed before the outgoing channel, and check it in sendToWorker's select to safely detect closed sessions without panicking.
217 lines
6.1 KiB
Go
217 lines
6.1 KiB
Go
package s3api
|
|
|
|
import (
|
|
"net/http"
|
|
|
|
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
|
|
)
|
|
|
|
// CopySizeCalculator handles size calculations for different copy scenarios
|
|
type CopySizeCalculator struct {
|
|
srcSize int64
|
|
srcEncrypted bool
|
|
dstEncrypted bool
|
|
srcType EncryptionType
|
|
dstType EncryptionType
|
|
isCompressed bool
|
|
}
|
|
|
|
// EncryptionType represents different encryption types
|
|
type EncryptionType int
|
|
|
|
const (
|
|
EncryptionTypeNone EncryptionType = iota
|
|
EncryptionTypeSSEC
|
|
EncryptionTypeSSEKMS
|
|
EncryptionTypeSSES3
|
|
)
|
|
|
|
// NewCopySizeCalculator creates a new size calculator for copy operations
|
|
func NewCopySizeCalculator(entry *filer_pb.Entry, r *http.Request) *CopySizeCalculator {
|
|
calc := &CopySizeCalculator{
|
|
srcSize: int64(entry.Attributes.FileSize),
|
|
isCompressed: isCompressedEntry(entry),
|
|
}
|
|
|
|
// Determine source encryption type
|
|
calc.srcType, calc.srcEncrypted = getSourceEncryptionType(entry.Extended)
|
|
|
|
// Determine destination encryption type
|
|
calc.dstType, calc.dstEncrypted = getDestinationEncryptionType(r)
|
|
|
|
return calc
|
|
}
|
|
|
|
// CalculateTargetSize calculates the expected size of the target object
|
|
func (calc *CopySizeCalculator) CalculateTargetSize() int64 {
|
|
// For compressed objects, size calculation is complex
|
|
if calc.isCompressed {
|
|
return -1 // Indicates unknown size
|
|
}
|
|
|
|
switch {
|
|
case !calc.srcEncrypted && !calc.dstEncrypted:
|
|
// Plain → Plain: no size change
|
|
return calc.srcSize
|
|
|
|
case !calc.srcEncrypted && calc.dstEncrypted:
|
|
// Plain → Encrypted: no overhead since IV is in metadata
|
|
return calc.srcSize
|
|
|
|
case calc.srcEncrypted && !calc.dstEncrypted:
|
|
// Encrypted → Plain: no overhead since IV is in metadata
|
|
return calc.srcSize
|
|
|
|
case calc.srcEncrypted && calc.dstEncrypted:
|
|
// Encrypted → Encrypted: no overhead since IV is in metadata
|
|
return calc.srcSize
|
|
|
|
default:
|
|
return calc.srcSize
|
|
}
|
|
}
|
|
|
|
// CalculateActualSize calculates the actual unencrypted size of the content
|
|
func (calc *CopySizeCalculator) CalculateActualSize() int64 {
|
|
// With IV in metadata, encrypted and unencrypted sizes are the same
|
|
return calc.srcSize
|
|
}
|
|
|
|
// getSourceEncryptionType determines the encryption type of the source object
|
|
func getSourceEncryptionType(metadata map[string][]byte) (EncryptionType, bool) {
|
|
if IsSSECEncrypted(metadata) {
|
|
return EncryptionTypeSSEC, true
|
|
}
|
|
if IsSSEKMSEncrypted(metadata) {
|
|
return EncryptionTypeSSEKMS, true
|
|
}
|
|
if IsSSES3EncryptedInternal(metadata) {
|
|
return EncryptionTypeSSES3, true
|
|
}
|
|
return EncryptionTypeNone, false
|
|
}
|
|
|
|
// getDestinationEncryptionType determines the encryption type for the destination
|
|
func getDestinationEncryptionType(r *http.Request) (EncryptionType, bool) {
|
|
if IsSSECRequest(r) {
|
|
return EncryptionTypeSSEC, true
|
|
}
|
|
if IsSSEKMSRequest(r) {
|
|
return EncryptionTypeSSEKMS, true
|
|
}
|
|
if IsSSES3RequestInternal(r) {
|
|
return EncryptionTypeSSES3, true
|
|
}
|
|
return EncryptionTypeNone, false
|
|
}
|
|
|
|
// isCompressedEntry checks if the entry represents a compressed object
|
|
func isCompressedEntry(entry *filer_pb.Entry) bool {
|
|
// Check for compression indicators in metadata
|
|
if compressionType, exists := entry.Extended["compression"]; exists {
|
|
return string(compressionType) != ""
|
|
}
|
|
|
|
// Check MIME type for compressed formats
|
|
mimeType := entry.Attributes.Mime
|
|
compressedMimeTypes := []string{
|
|
"application/gzip",
|
|
"application/x-gzip",
|
|
"application/zip",
|
|
"application/x-compress",
|
|
"application/x-compressed",
|
|
}
|
|
|
|
for _, compressedType := range compressedMimeTypes {
|
|
if mimeType == compressedType {
|
|
return true
|
|
}
|
|
}
|
|
|
|
return false
|
|
}
|
|
|
|
// SizeTransitionInfo provides detailed information about size changes during copy
|
|
type SizeTransitionInfo struct {
|
|
SourceSize int64
|
|
TargetSize int64
|
|
ActualSize int64
|
|
SizeChange int64
|
|
SourceType EncryptionType
|
|
TargetType EncryptionType
|
|
IsCompressed bool
|
|
RequiresResize bool
|
|
}
|
|
|
|
// GetSizeTransitionInfo returns detailed size transition information
|
|
func (calc *CopySizeCalculator) GetSizeTransitionInfo() *SizeTransitionInfo {
|
|
targetSize := calc.CalculateTargetSize()
|
|
actualSize := calc.CalculateActualSize()
|
|
|
|
info := &SizeTransitionInfo{
|
|
SourceSize: calc.srcSize,
|
|
TargetSize: targetSize,
|
|
ActualSize: actualSize,
|
|
SizeChange: targetSize - calc.srcSize,
|
|
SourceType: calc.srcType,
|
|
TargetType: calc.dstType,
|
|
IsCompressed: calc.isCompressed,
|
|
RequiresResize: targetSize != calc.srcSize,
|
|
}
|
|
|
|
return info
|
|
}
|
|
|
|
// OptimizedSizeCalculation provides size calculations optimized for different scenarios
|
|
type OptimizedSizeCalculation struct {
|
|
Strategy UnifiedCopyStrategy
|
|
SourceSize int64
|
|
TargetSize int64
|
|
ActualContentSize int64
|
|
EncryptionOverhead int64
|
|
CanPreallocate bool
|
|
RequiresStreaming bool
|
|
}
|
|
|
|
// CalculateOptimizedSizes calculates sizes optimized for the copy strategy
|
|
func CalculateOptimizedSizes(entry *filer_pb.Entry, r *http.Request, strategy UnifiedCopyStrategy) *OptimizedSizeCalculation {
|
|
calc := NewCopySizeCalculator(entry, r)
|
|
info := calc.GetSizeTransitionInfo()
|
|
|
|
result := &OptimizedSizeCalculation{
|
|
Strategy: strategy,
|
|
SourceSize: info.SourceSize,
|
|
TargetSize: info.TargetSize,
|
|
ActualContentSize: info.ActualSize,
|
|
CanPreallocate: !info.IsCompressed && info.TargetSize > 0,
|
|
RequiresStreaming: info.IsCompressed || info.TargetSize < 0,
|
|
}
|
|
|
|
// Calculate encryption overhead for the target
|
|
// With IV in metadata, all encryption overhead is 0
|
|
result.EncryptionOverhead = 0
|
|
|
|
// Adjust based on strategy
|
|
switch strategy {
|
|
case CopyStrategyDirect:
|
|
// Direct copy: no size change
|
|
result.TargetSize = result.SourceSize
|
|
result.CanPreallocate = true
|
|
|
|
case CopyStrategyKeyRotation:
|
|
// Key rotation: size might change slightly due to different IVs
|
|
if info.SourceType == EncryptionTypeSSEC && info.TargetType == EncryptionTypeSSEC {
|
|
// SSE-C key rotation: same overhead
|
|
result.TargetSize = result.SourceSize
|
|
}
|
|
result.CanPreallocate = true
|
|
|
|
case CopyStrategyEncrypt, CopyStrategyDecrypt, CopyStrategyReencrypt:
|
|
// Size changes based on encryption transition
|
|
result.TargetSize = info.TargetSize
|
|
result.CanPreallocate = !info.IsCompressed
|
|
}
|
|
|
|
return result
|
|
}
|